Merge pull request #1651 from mandiant/williballenthin-patch-1

v6.0.0a3

.devcontainer/Dockerfile

@@ -0,0 +1,21 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3/.devcontainer/base.Dockerfile
+
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+ARG VARIANT="3.10-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+# [Optional] Uncomment this line to install global node packages.
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.devcontainer/devcontainer.json

@@ -0,0 +1,51 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3
+{
+	"name": "Python 3",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": "..",
+		"args": { 
+			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+			// Append -bullseye or -buster to pin to an OS version.
+			// Use -bullseye variants on local on arm64/Apple Silicon.
+			"VARIANT": "3.10",
+			// Options
+			"NODE_VERSION": "none"
+		}
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": { 
+		"python.defaultInterpreterPath": "/usr/local/bin/python",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python",
+		"ms-python.vscode-pylance"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "git submodule update --init && pip3 install --user -e .[dev]",
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"features": {
+		"git": "latest"
+	}
+}

.github/CONTRIBUTING.md

@@ -31,7 +31,7 @@ This project and everyone participating in it is governed by the [Capa Code of C
 
 ### Capa and its repositories
 
-We host the capa project as three Github repositories:
+We host the capa project as three GitHub repositories:
   - [capa](https://github.com/mandiant/capa)
   - [capa-rules](https://github.com/mandiant/capa-rules)
   - [capa-testfiles](https://github.com/mandiant/capa-testfiles)
@@ -159,12 +159,25 @@ The process described here has several goals:
 
 Please follow these steps to have your contribution considered by the maintainers:
 
+0. Sign the [Contributor License Agreement](#contributor-license-agreement)
 1. Follow the [styleguides](#styleguides)
 2. Update the CHANGELOG and add tests and documentation. In case they are not needed, indicate it in [the PR template](pull_request_template.md).
 3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing? </summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
 
 While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
 
+### Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution,
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to <https://cla.developers.google.com/> to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
 ## Styleguides
 
 ### Git Commit Messages

.github/flake8.ini

@@ -0,0 +1,41 @@
+[flake8]
+max-line-length = 120
+
+extend-ignore =
+    # E203: whitespace before ':'  (black does this)
+    E203,
+    # F401: `foo` imported but unused  (prefer ruff)
+    F401,
+    # F811 Redefinition of unused `foo`  (prefer ruff)
+    F811,
+    # E501 line too long  (prefer black)
+    E501,
+    # B010 Do not call setattr with a constant attribute value
+    B010,
+    # G200 Logging statement uses exception in arguments
+    G200,
+    # SIM102 Use a single if-statement instead of nested if-statements
+    # doesn't provide a space for commenting or logical separation of conditions
+    SIM102,
+    # SIM114 Use logical or and a single body
+    # makes logic trees too complex
+    SIM114,
+    # SIM117 Use 'with Foo, Bar:' instead of multiple with statements
+    # makes lines too long
+    SIM117
+
+per-file-ignores =
+    # T201 print found.
+    #
+    # scripts are meant to print output
+    scripts/*: T201
+    # capa.exe is meant to print output
+    capa/main.py: T201
+    # IDA tests emit results to output window so need to print
+    tests/test_ida_features.py: T201
+    # utility used to find the Binary Ninja API via invoking python.exe
+    capa/features/extractors/binja/find_binja_api.py: T201
+
+copyright-check = True
+copyright-min-file-size = 1 
+copyright-regexp = Copyright \(C\) 2023 Mandiant, Inc. All Rights Reserved.

.github/mypy/mypy.ini

@@ -21,9 +21,6 @@ ignore_missing_imports = True
 [mypy-flirt.*]
 ignore_missing_imports = True
 
-[mypy-smda.*]
-ignore_missing_imports = True
-
 [mypy-lief.*]
 ignore_missing_imports = True
 
@@ -45,9 +42,15 @@ ignore_missing_imports = True
 [mypy-idautils.*]
 ignore_missing_imports = True
 
+[mypy-ida_auto.*]
+ignore_missing_imports = True
+
 [mypy-ida_bytes.*]
 ignore_missing_imports = True
 
+[mypy-ida_nalt.*]
+ignore_missing_imports = True
+
 [mypy-ida_kernwin.*]
 ignore_missing_imports = True
 
@@ -60,6 +63,9 @@ ignore_missing_imports = True
 [mypy-ida_loader.*]
 ignore_missing_imports = True
 
+[mypy-ida_segment.*]
+ignore_missing_imports = True
+
 [mypy-PyQt5.*]
 ignore_missing_imports = True
 
@@ -74,3 +80,9 @@ ignore_missing_imports = True
 
 [mypy-elftools.*]
 ignore_missing_imports = True
+
+[mypy-dncil.*]
+ignore_missing_imports = True
+
+[mypy-netnode.*]
+ignore_missing_imports = True

.github/pyinstaller/hooks/hook-smda.py

@@ -1,5 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-import PyInstaller.utils.hooks
-
-# ref: https://groups.google.com/g/pyinstaller/c/amWi0-66uZI/m/miPoKfWjBAAJ
-binaries = PyInstaller.utils.hooks.collect_dynamic_libs("capstone")

.github/pyinstaller/hooks/hook-vivisect.py

@@ -38,39 +38,36 @@ hiddenimports = [
     "vivisect",
     "vivisect.analysis",
     "vivisect.analysis.amd64",
-    "vivisect.analysis.amd64",
     "vivisect.analysis.amd64.emulation",
     "vivisect.analysis.amd64.golang",
     "vivisect.analysis.crypto",
-    "vivisect.analysis.crypto",
     "vivisect.analysis.crypto.constants",
     "vivisect.analysis.elf",
     "vivisect.analysis.elf.elfplt",
     "vivisect.analysis.elf.elfplt_late",
     "vivisect.analysis.elf.libc_start_main",
     "vivisect.analysis.generic",
-    "vivisect.analysis.generic",
     "vivisect.analysis.generic.codeblocks",
     "vivisect.analysis.generic.emucode",
     "vivisect.analysis.generic.entrypoints",
     "vivisect.analysis.generic.funcentries",
     "vivisect.analysis.generic.impapi",
+    "vivisect.analysis.generic.linker",
     "vivisect.analysis.generic.mkpointers",
+    "vivisect.analysis.generic.noret",
     "vivisect.analysis.generic.pointers",
     "vivisect.analysis.generic.pointertables",
     "vivisect.analysis.generic.relocations",
     "vivisect.analysis.generic.strconst",
     "vivisect.analysis.generic.switchcase",
+    "vivisect.analysis.generic.symswitchcase",
     "vivisect.analysis.generic.thunks",
-    "vivisect.analysis.generic.noret",
-    "vivisect.analysis.i386",
     "vivisect.analysis.i386",
     "vivisect.analysis.i386.calling",
     "vivisect.analysis.i386.golang",
     "vivisect.analysis.i386.importcalls",
     "vivisect.analysis.i386.instrhook",
-    "vivisect.analysis.i386.thunk_bx",
-    "vivisect.analysis.ms",
+    "vivisect.analysis.i386.thunk_reg",
     "vivisect.analysis.ms",
     "vivisect.analysis.ms.hotpatch",
     "vivisect.analysis.ms.localhints",
@@ -81,8 +78,40 @@ hiddenimports = [
     "vivisect.impapi.posix.amd64",
     "vivisect.impapi.posix.i386",
     "vivisect.impapi.windows",
+    "vivisect.impapi.windows.advapi_32",
+    "vivisect.impapi.windows.advapi_64",
     "vivisect.impapi.windows.amd64",
+    "vivisect.impapi.windows.gdi_32",
+    "vivisect.impapi.windows.gdi_64",
     "vivisect.impapi.windows.i386",
+    "vivisect.impapi.windows.kernel_32",
+    "vivisect.impapi.windows.kernel_64",
+    "vivisect.impapi.windows.msvcr100_32",
+    "vivisect.impapi.windows.msvcr100_64",
+    "vivisect.impapi.windows.msvcr110_32",
+    "vivisect.impapi.windows.msvcr110_64",
+    "vivisect.impapi.windows.msvcr120_32",
+    "vivisect.impapi.windows.msvcr120_64",
+    "vivisect.impapi.windows.msvcr71_32",
+    "vivisect.impapi.windows.msvcr80_32",
+    "vivisect.impapi.windows.msvcr80_64",
+    "vivisect.impapi.windows.msvcr90_32",
+    "vivisect.impapi.windows.msvcr90_64",
+    "vivisect.impapi.windows.msvcrt_32",
+    "vivisect.impapi.windows.msvcrt_64",
+    "vivisect.impapi.windows.ntdll_32",
+    "vivisect.impapi.windows.ntdll_64",
+    "vivisect.impapi.windows.ole_32",
+    "vivisect.impapi.windows.ole_64",
+    "vivisect.impapi.windows.rpcrt4_32",
+    "vivisect.impapi.windows.rpcrt4_64",
+    "vivisect.impapi.windows.shell_32",
+    "vivisect.impapi.windows.shell_64",
+    "vivisect.impapi.windows.user_32",
+    "vivisect.impapi.windows.user_64",
+    "vivisect.impapi.windows.ws2plus_32",
+    "vivisect.impapi.windows.ws2plus_64",
+    "vivisect.impapi.winkern",
     "vivisect.impapi.winkern.i386",
     "vivisect.impapi.winkern.amd64",
     "vivisect.parsers.blob",

.github/pyinstaller/pyinstaller.spec

@@ -6,51 +6,35 @@ import subprocess
 import wcwidth
 
 
-# when invoking pyinstaller from the project root,
-# this gets run from the project root.
-with open('./capa/version.py', 'wb') as f:
-    # git output will look like:
-    #
-    #     tags/v1.0.0-0-g3af38dc
-    #         ------- tag
-    #                 - commits since
-    #                   g------- git hash fragment
-    version = (subprocess.check_output(["git", "describe", "--always", "--tags", "--long"])
-               .decode("utf-8")
-               .strip()
-               .replace("tags/", ""))
-    f.write(("__version__ = '%s'" % version).encode("utf-8"))
-
 a = Analysis(
     # when invoking pyinstaller from the project root,
     # this gets invoked from the directory of the spec file,
     # i.e. ./.github/pyinstaller
-    ['../../capa/main.py'],
-    pathex=['capa'],
+    ["../../capa/main.py"],
+    pathex=["capa"],
     binaries=None,
     datas=[
         # when invoking pyinstaller from the project root,
         # this gets invoked from the directory of the spec file,
         # i.e. ./.github/pyinstaller
-        ('../../rules', 'rules'),
-        ('../../sigs', 'sigs'),
-
+        ("../../rules", "rules"),
+        ("../../sigs", "sigs"),
+        ("../../cache", "cache"),
         # capa.render.default uses tabulate that depends on wcwidth.
         # it seems wcwidth uses a json file `version.json`
         # and this doesn't get picked up by pyinstaller automatically.
         # so we manually embed the wcwidth resources here.
         #
         # ref: https://stackoverflow.com/a/62278462/87207
-        (os.path.dirname(wcwidth.__file__), 'wcwidth')
+        (os.path.dirname(wcwidth.__file__), "wcwidth"),
     ],
     # when invoking pyinstaller from the project root,
     # this gets run from the project root.
-    hookspath=['.github/pyinstaller/hooks'],
+    hookspath=[".github/pyinstaller/hooks"],
     runtime_hooks=None,
     excludes=[
         # ignore packages that would otherwise be bundled with the .exe.
         # review: build/pyinstaller/xref-pyinstaller.html
-
         # we don't do any GUI stuff, so ignore these modules
         "tkinter",
         "_tkinter",
@@ -60,7 +44,6 @@ a = Analysis(
         # since we don't spawn a notebook, we can safely remove these.
         "IPython",
         "ipywidgets",
-
         # these are pulled in by networkx
         # but we don't need to compute the strongly connected components.
         "numpy",
@@ -68,7 +51,6 @@ a = Analysis(
         "matplotlib",
         "pandas",
         "pytest",
-
         # deps from viv that we don't use.
         # this duplicates the entries in `hook-vivisect`,
         # but works better this way.
@@ -78,35 +60,36 @@ a = Analysis(
         "PyQt5",
         "qt5",
         "pyqtwebengine",
-        "pyasn1"
-    ])
+        "pyasn1",
+        "binaryninja",
+    ],
+)
 
-a.binaries = a.binaries - TOC([
- ('tcl85.dll', None, None),
- ('tk85.dll', None, None),
- ('_tkinter', None, None)])
+a.binaries = a.binaries - TOC([("tcl85.dll", None, None), ("tk85.dll", None, None), ("_tkinter", None, None)])
 
 pyz = PYZ(a.pure, a.zipped_data)
 
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          exclude_binaries=False,
-          name='capa',
-          icon='logo.ico',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    exclude_binaries=False,
+    name="capa",
+    icon="logo.ico",
+    debug=False,
+    strip=None,
+    upx=True,
+    console=True,
+)
 
 # enable the following to debug the contents of the .exe
 #
-#coll = COLLECT(exe,
+# coll = COLLECT(exe,
 #               a.binaries,
 #               a.zipfiles,
 #               a.datas,
 #               strip=None,
 #               upx=True,
-#               name='capa-dat')
+#               name='capa-dat')

.github/ruff.toml

@@ -0,0 +1,43 @@
+# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E", "F"]
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# E402 module level import not at top of file
+# E722 do not use bare 'except'
+# E501 line too long
+ignore = ["E402", "E722", "E501"]
+
+line-length = 120
+
+exclude = [
+    # Exclude a variety of commonly ignored directories.
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+    # protobuf generated files
+    "*_pb2.py", 
+    "*_pb2.pyi"
+]

.github/tox.ini

@@ -1,10 +0,0 @@
-[pycodestyle]
-; E402: module level import not at top of file
-; W503: line break before binary operator
-; E231 missing whitespace after ',' (emitted by black)
-; E203 whitespace before ':' (emitted by black)
-ignore = E402,W503,E203,E231
-max-line-length = 160
-statistics = True
-count = True
-exclude = .*

.github/workflows/build.yml

@@ -1,60 +1,66 @@
 name: build
 
 on:
-  push:
-    branches: [master]
+  pull_request:
+    branches: [ master ]
   release:
     types: [edited, published]
 
+permissions:
+  contents: write
+
 jobs:
   build:
     name: PyInstaller for ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
+      # set to false for debugging
+      fail-fast: true
       matrix:
         include:
-          - os: ubuntu-18.04
+          - os: ubuntu-20.04
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
           - os: windows-2019
             artifact_name: capa.exe
             asset_name: windows
-          - os: macos-10.15
+          - os: macos-11
+            # use older macOS for assumed better portability
             artifact_name: capa
             asset_name: macos
     steps:
       - name: Checkout capa
-        uses: actions/checkout@v2
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: true
       # using Python 3.8 to support running across multiple operating systems including Windows 7
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
           python-version: 3.8
-      - if: matrix.os == 'ubuntu-18.04'
+      - if: matrix.os == 'ubuntu-20.04'
         run: sudo apt-get install -y libyaml-dev
-      - name: Install PyInstaller
-        run: pip install 'pyinstaller==4.2'
-      - name: Install capa
-        run: pip install -e .
+      - name: Upgrade pip, setuptools
+        run: python -m pip install --upgrade pip setuptools
+      - name: Install capa with build requirements
+        run: pip install -e .[build]
+      - name: Cache the rule set
+        run: python ./scripts/cache-ruleset.py ./rules/ ./cache/
       - name: Build standalone executable
-        run: pyinstaller .github/pyinstaller/pyinstaller.spec
+        run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
       - name: Does it run (PE)?
         run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_"
       - name: Does it run (Shellcode)?
         run: dist/capa "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
       - name: Does it run (ELF)?
         run: dist/capa "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: ${{ matrix.asset_name }}
           path: dist/${{ matrix.artifact_name }}
 
   test_run:
-    # test that binaries run on push to master
-    if: github.event_name == 'push'
     name: Test run on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     needs: [build]
@@ -62,22 +68,19 @@ jobs:
       matrix:
         include:
           # OSs not already tested above
-          - os: ubuntu-18.04
+          - os: ubuntu-22.04
             artifact_name: capa
             asset_name: linux
-          - os: ubuntu-20.04
-            artifact_name: capa
-            asset_name: linux
-          - os: windows-2016
+          - os: windows-2022
             artifact_name: capa.exe
             asset_name: windows
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           name: ${{ matrix.asset_name }}
       - name: Set executable flag
-        if: matrix.os != 'windows-2016'
+        if: matrix.os != 'windows-2022'
         run: chmod +x ${{ matrix.artifact_name }}
       - name: Run capa
         run: ./${{ matrix.artifact_name }} -h
@@ -99,7 +102,7 @@ jobs:
             artifact_name: capa
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           name: ${{ matrix.asset_name }}
       - name: Set executable flag
@@ -109,7 +112,7 @@ jobs:
       - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
         run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
       - name: Upload ${{ env.zip_name }} to GH Release
-        uses: svenstaro/upload-release-action@v2
+        uses: svenstaro/upload-release-action@2728235f7dc9ff598bd86ce3c274b74f802d2208 # v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN}}
           file: ${{ env.zip_name }}

.github/workflows/changelog.yml

@@ -7,6 +7,8 @@ on:
   pull_request_target:
     types: [opened, edited, synchronize]
 
+permissions: read-all
+
 jobs:
   check_changelog:
     # no need to check for dependency updates via dependabot
@@ -17,7 +19,7 @@ jobs:
     steps:
     - name: Get changed files
       id: files
-      uses: Ana06/get-changed-files@v1.2
+      uses: Ana06/get-changed-files@e0c398b7065a8d84700c471b6afc4116d1ba4e96 # v2.2.0
     - name: check changelog updated
       id: changelog_updated
       env:
@@ -27,14 +29,14 @@ jobs:
         echo $FILES | grep -qF 'CHANGELOG.md' || echo $PR_BODY | grep -qiF "$NO_CHANGELOG"
     - name: Reject pull request if no CHANGELOG update
       if: ${{ always() && steps.changelog_updated.outcome == 'failure' }}
-      uses: Ana06/automatic-pull-request-review@v0.1.0
+      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: REQUEST_CHANGES
         body: "Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning to the `master (unreleased)` section of CHANGELOG.md. If no CHANGELOG update is needed add the following to the PR description: `${{ env.NO_CHANGELOG }}`"
         allow_duplicate: false
     - name: Dismiss previous review if CHANGELOG update
-      uses: Ana06/automatic-pull-request-review@v0.1.0
+      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: DISMISS

.github/workflows/publish.yml

@@ -1,30 +1,49 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
+# use PyPI trusted publishing, as described here: 
+# https://blog.trailofbits.com/2023/05/23/trusted-publishing-a-new-benchmark-for-packaging-security/
 name: publish to pypi
 
 on:
   release:
     types: [published]
 
+permissions:
+  contents: write
+
 jobs:
-  deploy:
-    runs-on: ubuntu-20.04
+  pypi-publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+    permissions:
+      id-token: write
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
-          python-version: '3.6'
+          python-version: '3.8'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+          pip install -e .[build]
+      - name: build package
         run: |
-          python setup.py sdist bdist_wheel
-          twine upload --skip-existing dist/*
-
+          python -m build
+      - name: upload package artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          path: dist/*
+      - name: upload package to GitHub release
+        uses: svenstaro/upload-release-action@2728235f7dc9ff598bd86ce3c274b74f802d2208 # v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN}}
+          file: dist/*
+          tag: ${{ github.ref }}
+          overwrite: true
+          file_glob: true
+      - name: publish package
+        uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0  # release/v1
+        with:
+          skip-existing: true
+          verbose: true
+          print-hash: true

.github/workflows/scorecard.yml

@@ -0,0 +1,72 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '43 4 * * 3'
+  push:
+    branches: [ "master" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27
+        with:
+          sarif_file: results.sarif

.github/workflows/tag.yml

@@ -4,13 +4,15 @@ on:
   release:
     types: [published]
 
+permissions: read-all
+
 jobs:
   tag:
     name: Tag capa rules
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa-rules
-      uses: actions/checkout@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       with:
         repository: mandiant/capa-rules
         token: ${{ secrets.CAPA_TOKEN }}
@@ -21,8 +23,9 @@ jobs:
         git config user.name 'Capa Bot'
         name=${{ github.event.release.tag_name }}
         git tag $name -m "https://github.com/mandiant/capa/releases/$name"
+        # TODO update branch name-major=${name%%.*}
     - name: Push tag to capa-rules
-      uses: ad-m/github-push-action@master
+      uses: ad-m/github-push-action@0fafdd62b84042d49ec0cb92d9cac7f7ce4ec79e # master
       with:
         repository: mandiant/capa-rules
         github_token: ${{ secrets.CAPA_TOKEN }}

.github/workflows/tests.yml

@@ -6,6 +6,8 @@ on:
   pull_request:
     branches: [ master ]
 
+permissions: read-all
+
 # save workspaces to speed up testing
 env:
   CAPA_SAVE_WORKSPACE: "True"
@@ -15,7 +17,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa
-      uses: actions/checkout@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
     # The sync GH action in capa-rules relies on a single '- *$' in the CHANGELOG file
     - name: Ensure CHANGELOG has '- *$'
       run: |
@@ -26,33 +28,38 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa
-      uses: actions/checkout@v2
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+    # use latest available python to take advantage of best performance
+    - name: Set up Python 3.11
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
-        python-version: "3.8"
+        python-version: "3.11"
     - name: Install dependencies
       run: pip install -e .[dev]
+    - name: Lint with ruff
+      run: pre-commit run ruff
     - name: Lint with isort
-      run: isort --profile black --length-sort --line-width 120 -c .
+      run: pre-commit run isort
     - name: Lint with black
-      run: black -l 120 --check .
+      run: pre-commit run black
+    - name: Lint with flake8
+      run: pre-commit run flake8
     - name: Check types with mypy
-      run: mypy --config-file .github/mypy/mypy.ini capa/ scripts/ tests/
+      run:  pre-commit run mypy
 
   rule_linter:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       with:
-        submodules: true
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+        submodules: recursive
+    - name: Set up Python 3.11
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
-        python-version: "3.8"
+        python-version: "3.11"
     - name: Install capa
-      run: pip install -e .
+      run: pip install -e .[dev]
     - name: Run rule linter
       run: python scripts/lint.py rules/
 
@@ -63,24 +70,24 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-10.15]
+        os: [ubuntu-20.04, windows-2019, macos-11]
         # across all operating systems
-        python-version: ["3.6", "3.10"]
+        python-version: ["3.8", "3.11"]
         include:
           # on Ubuntu run these as well
-          - os: ubuntu-20.04
-            python-version: "3.7"
           - os: ubuntu-20.04
             python-version: "3.8"
           - os: ubuntu-20.04
             python-version: "3.9"
+          - os: ubuntu-20.04
+            python-version: "3.10"
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@v2
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       with:
-        submodules: true
+        submodules: recursive
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pyyaml
@@ -90,3 +97,45 @@ jobs:
       run: pip install -e .[dev]
     - name: Run tests
       run: pytest -v tests/
+
+  binja-tests:
+    name: Binary Ninja tests for ${{ matrix.python-version }}
+    env:
+      BN_SERIAL: ${{ secrets.BN_SERIAL }}
+    runs-on: ubuntu-20.04
+    needs: [code_style, rule_linter]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.11"]
+    steps:
+    - name: Checkout capa with submodules
+      # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
+      if: ${{ env.BN_SERIAL != 0 }}
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      with:
+        submodules: recursive
+    - name: Set up Python ${{ matrix.python-version }}
+      if: ${{ env.BN_SERIAL != 0 }}
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install pyyaml
+      if: ${{ env.BN_SERIAL != 0 }}
+      run: sudo apt-get install -y libyaml-dev
+    - name: Install capa
+      if: ${{ env.BN_SERIAL != 0 }}
+      run: pip install -e .[dev]
+    - name: install Binary Ninja
+      if: ${{ env.BN_SERIAL != 0 }}
+      run: |
+        mkdir ./.github/binja
+        curl "https://raw.githubusercontent.com/Vector35/binaryninja-api/6812c97/scripts/download_headless.py" -o ./.github/binja/download_headless.py
+        python ./.github/binja/download_headless.py --serial ${{ env.BN_SERIAL }} --output .github/binja/BinaryNinja-headless.zip
+        unzip .github/binja/BinaryNinja-headless.zip -d .github/binja/
+        python .github/binja/binaryninja/scripts/install_api.py --install-on-root --silent
+    - name: Run tests
+      if: ${{ env.BN_SERIAL != 0 }}
+      env:
+        BN_LICENSE: ${{ secrets.BN_LICENSE }}
+      run: pytest -v tests/test_binja_features.py  # explicitly refer to the binja tests for performance. other tests run above.

.gitignore

@@ -108,13 +108,21 @@ venv.bak/
 *.viv
 *.idb
 *.i64
+.vscode
+
 !rules/lib
 
-# hooks/ci.sh output
-isort-output.log
-black-output.log
-rule-linter-output.log
-.vscode
 scripts/perf/*.txt
 scripts/perf/*.svg
 scripts/perf/*.zip
+
+.direnv
+.envrc
+.DS_Store
+*/.DS_Store
+Pipfile
+Pipfile.lock
+/cache/
+.github/binja/binaryninja
+.github/binja/download_headless.py
+.github/binja/BinaryNinja-headless.zip

.pre-commit-config.yaml

@@ -0,0 +1,111 @@
+# install the pre-commit hooks:
+#
+#    ❯ pre-commit install --hook-type pre-commit
+#    pre-commit installed at .git/hooks/pre-commit
+#
+#    ❯ pre-commit install --hook-type pre-push
+#    pre-commit installed at .git/hooks/pre-push
+#
+# run all linters liks:
+#
+#    ❯ pre-commit run --all-files
+#    isort....................................................................Passed
+#    black....................................................................Passed
+#    ruff.....................................................................Passed
+#    flake8...................................................................Passed
+#    mypy.....................................................................Passed
+#
+# run a single linter like:
+#
+#    ❯ pre-commit run --all-files isort
+#    isort....................................................................Passed
+
+repos:
+-   repo: local
+    hooks:
+    -   id: isort
+        name: isort
+        stages: [commit, push]
+        language: system
+        entry: isort
+        args: 
+        -   "--length-sort"
+        -   "--profile"
+        -   "black"
+        -   "--line-length=120"
+        -   "--skip-glob"
+        -   "*_pb2.py"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: black
+        name: black
+        stages: [commit, push]
+        language: system
+        entry: black
+        args: 
+        -   "--line-length=120"
+        -   "--extend-exclude"
+        -   ".*_pb2.py"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: ruff
+        name: ruff
+        stages: [commit, push]
+        language: system
+        entry: ruff
+        args: 
+        -   "check"
+        -   "--config"
+        -   ".github/ruff.toml"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: flake8
+        name: flake8
+        stages: [commit, push]
+        language: system
+        entry: flake8
+        args: 
+        -   "--config"
+        -   ".github/flake8.ini"
+        -   "--extend-exclude"
+        -   "capa/render/proto/capa_pb2.py"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: mypy
+        name: mypy
+        stages: [commit, push]
+        language: system
+        entry: mypy
+        args: 
+        -   "--check-untyped-defs"
+        -   "--ignore-missing-imports"
+        -   "--config-file=.github/mypy/mypy.ini"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false

CHANGELOG.md

@@ -12,15 +12,433 @@
 
 ### Bug Fixes
 
-- elf: fix OS detection for Linux kernel modules #867 @williballenthin
-
 ### capa explorer IDA Pro plugin
 
 ### Development
 
 ### Raw diffs
-- [capa v3.1.0...master](https://github.com/mandiant/capa/compare/v3.1.0...master)
-- [capa-rules v3.1.0...master](https://github.com/mandiant/capa-rules/compare/v3.1.0...master)
+- [capa v6.0.0...master](https://github.com/mandiant/capa/compare/v6.0.0...master)
+- [capa-rules v6.0.0...master](https://github.com/mandiant/capa-rules/compare/v6.0.0...master)
+
+## v6.0.0
+
+capa v6.0 brings many bug fixes and quality improvements, including 64 rule updates and 26 new rules. We're now publishing to PyPI via [Trusted Publishing](https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/) and have migrated to using a `pyproject.toml` file. @Aayush-Goel-04 contributed a lot of new code across many files, so please welcome them to the project, along with @anders-v @crowface28 @dkelly2e @RonnieSalomonsen and @ejfocampo as first-time rule contributors!
+
+For those that use capa as a library, we've introduced some limited breaking changes that better represent data types (versus less-structured data like dictionaries and strings). With the recent deprecation, we've also dropped support for Python 3.7.
+
+### New Features
+- add script to detect feature overlap between new and existing capa rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
+- extract forwarded exports from PE files #1624 @williballenthin
+- extract function and API names from ELF symtab entries @yelhamer https://github.com/mandiant/capa-rules/issues/736
+- use fancy box drawing characters for default output #1586 @williballenthin
+
+### Breaking Changes
+- use a class to represent Metadata (not dict) #1411 @Aayush-Goel-04 @manasghandat
+- use pathlib.Path to represent file paths #1534 @Aayush-Goel-04
+- Python 3.8 is now the minimum supported Python version #1578 @williballenthin
+- Require a Contributor License Agreement (CLA) for PRs going forward #1642 @williballenthin
+
+### New Rules (26)
+
+- load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
+- nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
+- data-manipulation/encryption/aes/encrypt-data-using-aes-mixcolumns-step @mr-tz
+- linking/static/aplib/linked-against-aplib still@teamt5.org
+- communication/mailslot/read-from-mailslot nick.simonian@mandiant.com
+- nursery/hash-data-using-sha512managed-in-dotnet jonathanlepore@google.com
+- nursery/compiled-with-exescript jonathanlepore@google.com
+- nursery/check-for-sandbox-via-mac-address-ouis-in-dotnet jonathanlepore@google.com
+- host-interaction/hardware/enumerate-devices-by-category @mr-tz
+- host-interaction/service/continue-service @mr-tz
+- host-interaction/service/pause-service @mr-tz
+- persistence/exchange/act-as-exchange-transport-agent jakub.jozwiak@mandiant.com
+- host-interaction/file-system/create-virtual-file-system-in-dotnet jakub.jozwiak@mandiant.com
+- compiler/cx_freeze/compiled-with-cx_freeze @mr-tz jakub.jozwiak@mandiant.com
+- communication/socket/create-vmci-socket jakub.jozwiak@mandiant.com
+- persistence/office/act-as-excel-xll-add-in jakub.jozwiak@mandiant.com
+- persistence/office/act-as-office-com-add-in jakub.jozwiak@mandiant.com
+- persistence/office/act-as-word-wll-add-in jakub.jozwiak@mandiant.com
+- anti-analysis/anti-debugging/debugger-evasion/hide-thread-from-debugger michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com
+- host-interaction/memory/create-new-application-domain-in-dotnet jakub.jozwiak@mandiant.com
+- host-interaction/gui/switch-active-desktop jakub.jozwiak@mandiant.com
+- host-interaction/service/query-service-configuration @mr-tz
+- anti-analysis/anti-av/patch-event-tracing-for-windows-function jakub.jozwiak@mandiant.com
+- data-manipulation/encoding/xor/covertly-decode-and-write-data-to-windows-directory-using-indirect-calls dan.kelly@mandiant.com
+- linking/runtime-linking/resolve-function-by-brute-ratel-badger-hash jakub.jozwiak@mandiant.com
+
+### Bug Fixes
+- extractor: add a Binary Ninja test that asserts its version #1487 @xusheng6
+- extractor: update Binary Ninja stack string detection after the new constant outlining feature #1473 @xusheng6
+- extractor: update vivisect Arch extraction #1334 @mr-tz
+- extractor: avoid Binary Ninja exception when analyzing certain files #1441 @xusheng6 
+- symtab: fix struct.unpack() format for 64-bit ELF files @yelhamer
+- symtab: safeguard against ZeroDivisionError for files containing a symtab with a null entry size @yelhamer
+- improve ELF strtab and needed parsing @mr-tz
+- better handle exceptional cases when parsing ELF files #1458 @Aayush-Goel-04
+- improved testing coverage for Binary Ninja backend #1446 @Aayush-Goel-04
+- add logging and print redirect to tqdm for capa main #749 @Aayush-Goel-04
+- extractor: fix binja installation path detection does not work with Python 3.11
+- tests: refine the IDA test runner script #1513 @williballenthin
+- output: don't leave behind traces of progress bar @williballenthin
+- import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin
+- main: don't show spinner when emitting debug messages #1636 @williballenthin
+
+### capa explorer IDA Pro plugin
+
+### Development
+- update ATT&CK/MBC data for linting #1568 @mr-tz
+- log time taken to analyze each function #1290 @williballenthin
+- tests: make fixture available via conftest.py #1592 @williballenthin
+- publish via PyPI trusted publishing #1491 @williballenthin
+- migrate to pyproject.toml #1301 @williballenthin
+- use [pre-commit](https://pre-commit.com/) to invoke linters #1579 @williballenthin
+
+
+### Raw diffs
+- [capa v5.1.0...v6.0.0](https://github.com/mandiant/capa/compare/v5.1.0...v6.0.0a1)
+- [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0a1)
+
+## v5.1.0
+capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and a output color format that's easier to read.
+
+Over 25 capa rules have been added and improved.
+
+Thanks for all the support, especially to @xusheng6, @captainGeech42, @ggold7046, @manasghandat, @ooprathamm, @linpeiyu164, @yelhamer, @HongThatCong, @naikordian, @stevemk14ebr, @emtuls, @raymondlleong, @bkojusner, @joren485, and everyone else who submitted bugs and provided feedback!
+
+### New Features
+- add protobuf format for result documents #1219 @williballenthin @mr-tz 
+- extractor: add Binary Ninja feature extractor @xusheng6
+- new cli flag `--os` to override auto-detected operating system for a sample @captainGeech42
+- change colour/highlight to "cyan" instead of "blue" for better readability #1384 @ggold7046
+- add new format to parse output json back to capa #1396 @ooprathamm
+- parse ELF symbols' names to guess OS #1403 @yelhamer
+
+### New Rules (26)
+
+- persistence/scheduled-tasks/schedule-task-via-at joren485
+- data-manipulation/prng/generate-random-numbers-via-rtlgenrandom william.ballenthin@mandiant.com
+- communication/ip/convert-ip-address-from-string @mr-tz
+- data-manipulation/compression/compress-data-via-zlib-inflate-or-deflate blas.kojusner@mandiant.com
+- executable/installer/dotnet/packaged-as-single-file-dotnet-application michael.hunhoff@mandiant.com
+- communication/socket/create-raw-socket blas.kojusner@mandiant.com
+- communication/http/reference-http-user-agent-string @mr-tz
+- communication/http/get-http-content-length william.ballenthin@mandiant.com
+- nursery/move-directory michael.hunhoff@mandiant.com
+- nursery/get-http-request-uri william.ballenthin@mandiant.com
+- nursery/create-zip-archive-in-dotnet michael.hunhoff@mandiant.com
+- nursery/extract-zip-archive-in-dotnet anushka.virgaonkar@mandiant.com michael.hunhoff@mandiant.com
+- data-manipulation/encryption/tea/decrypt-data-using-tea william.ballenthin@mandiant.com raymond.leong@mandiant.com
+- data-manipulation/encryption/tea/encrypt-data-using-tea william.ballenthin@mandiant.com raymond.leong@mandiant.com
+- data-manipulation/encryption/xtea/encrypt-data-using-xtea raymond.leong@mandiant.com
+- data-manipulation/encryption/xxtea/encrypt-data-using-xxtea raymond.leong@mandiant.com
+- nursery/hash-data-using-ripemd128 raymond.leong@mandiant.com
+- nursery/hash-data-using-ripemd256 raymond.leong@mandiant.com
+- nursery/hash-data-using-ripemd320 raymond.leong@mandiant.com
+- nursery/set-web-proxy-in-dotnet michael.hunhoff@mandiant.com
+- nursery/check-for-windows-sandbox-via-subdirectory echernofsky@google.com
+- nursery/enumerate-pe-sections-in-dotnet @mr-tz
+- nursery/destroy-software-breakpoint-capability echernofsky@google.com
+- nursery/send-data-to-internet michael.hunhoff@mandiant.com
+- nursery/compiled-with-cx_freeze @mr-tz
+- nursery/contain-a-thread-local-storage-tls-section-in-dotnet michael.hunhoff@mandiant.com
+
+### Bug Fixes
+- extractor: interface of cache modified to prevent extracting file and global features multiple times @stevemk14ebr
+- extractor: removed '.dynsym' as the library name for ELF imports #1318 @stevemk14ebr 
+- extractor: fix vivisect loop detection corner case #1310 @mr-tz
+- match: extend OS characteristic to match OS_ANY to all supported OSes #1324 @mike-hunhoff
+- extractor: fix IDA and vivisect string and bytes features overlap and tests #1327 #1336 @xusheng6
+
+### capa explorer IDA Pro plugin
+- rule generator plugin now loads faster when jumping between functions @stevemk14ebr
+- fix exception when plugin loaded in IDA hosted under idat #1341 @mike-hunhoff
+- improve embedded PE detection performance and reduce FP potential #1344 @mike-hunhoff
+
+### Raw diffs
+- [capa v5.0.0...v5.1.0](https://github.com/mandiant/capa/compare/v5.0.0...v5.1.0)
+- [capa-rules v5.0.0...v5.1.0](https://github.com/mandiant/capa-rules/compare/v5.0.0...v5.1.0)
+
+
+## v5.0.0 (2023-02-08)
+This capa version comes with major improvements and additions to better handle .NET binaries. To showcase this we've updated and added over 30 .NET rules.  
+
+Additionally, capa now caches its rule set for better performance. The capa explorer also caches its analysis results, so that multiple IDA Pro or plugin invocations don't need to repeat the same analysis.
+
+We have removed the SMDA backend and changed the program return codes to be positive numbers.
+
+Other improvements to highlight include better ELF OS detection, various rendering bug fixes, and enhancements to the feature extraction. We've also added support for Python 3.11.
+
+Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardweiss80, @joren485, @ryantxu1, @mwilliams31, @anushkavirgaonkar, @MalwareMechanic, @Still34, @dzbeck, @johnk3r, and everyone else who submitted bugs and provided feedback!
+
+### New Features
+- verify rule metadata format on load #1160 @mr-tz
+- dotnet: emit property features #1168 @anushkavirgaonkar
+- dotnet: emit API features for objects created via the newobj instruction #1186 @mike-hunhoff
+- dotnet: emit API features for generic methods #1231 @mike-hunhoff
+- Python 3.11 support #1192 @williballenthin
+- dotnet: emit calls to/from MethodDef methods #1236 @mike-hunhoff
+- dotnet: emit namespace/class features for ldvirtftn/ldftn instructions #1241 @mike-hunhoff
+- dotnet: emit namespace/class features for type references #1242 @mike-hunhoff
+- dotnet: extract dotnet and pe format #1187 @mr-tz
+- don't render all library rule matches in vverbose output #1174 @mr-tz
+- cache the rule set across invocations for better performance #1212 @williballenthin
+- update ATT&CK/MBC data for linting #1297 @mr-tz
+
+### Breaking Changes
+- remove SMDA backend #1062 @williballenthin
+- error return codes are now positive numbers #1269 @mr-tz
+
+### New Rules (77)
+
+- collection/use-dotnet-library-sharpclipboard @johnk3r
+- data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r
+- data-manipulation/json/use-dotnet-library-newtonsoftjson @johnk3r
+- data-manipulation/svg/use-dotnet-library-sharpvectors @johnk3r
+- executable/resource/embed-dependencies-as-resources-using-fodycostura @johnk3r @mr-tz
+- communication/ftp/send/send-file-using-ftp michael.hunhof@mandiant.com anushka.virgaonkar@mandiant.com
+- nursery/extract-zip-archive anushka.virgaonkar@mandiant.com
+- nursery/allocate-unmanaged-memory-in-dotnet michael.hunhoff@mandiant.com
+- nursery/check-file-extension-in-dotnet michael.hunhoff@mandiant.com
+- nursery/decode-data-using-base64-in-dotnet michael.hunhoff@mandiant.com
+- nursery/deserialize-json-in-dotnet michael.hunhoff@mandiant.com
+- nursery/find-data-using-regex-in-dotnet michael.hunhoff@mandiant.com
+- nursery/generate-random-filename-in-dotnet michael.hunhoff@mandiant.com
+- nursery/get-os-version-in-dotnet michael.hunhoff@mandiant.com
+- nursery/load-xml-in-dotnet michael.hunhoff@mandiant.com
+- nursery/manipulate-unmanaged-memory-in-dotnet michael.hunhoff@mandiant.com
+- nursery/save-image-in-dotnet michael.hunhoff@mandiant.com
+- nursery/send-email-in-dotnet michael.hunhoff@mandiant.com
+- nursery/serialize-json-in-dotnet michael.hunhoff@mandiant.com
+- nursery/set-http-user-agent-in-dotnet michael.hunhoff@mandiant.com
+- nursery/compile-csharp-in-dotnet michael.hunhoff@mandiant.com
+- nursery/compile-visual-basic-in-dotnet michael.hunhoff@mandiant.com
+- nursery/compress-data-using-gzip-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-sqlite-statement-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-via-asynchronous-task-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-via-timer-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-wmi-query-in-dotnet michael.hunhoff@mandiant.com
+- nursery/manipulate-network-credentials-in-dotnet michael.hunhoff@mandiant.com
+- nursery/encrypt-data-using-aes william.ballenthin@mandiant.com Ivan Kwiatkowski (@JusticeRage)
+- host-interaction/uac/bypass/bypass-uac-via-rpc david.cannings@pwc.com david@edeca.net
+- nursery/check-for-vm-using-instruction-vpcext richard.weiss@mandiant.com
+- nursery/get-windows-directory-from-kuser_shared_data david.cannings@pwc.com
+- nursery/encrypt-data-using-openssl-dsa Ana06
+- nursery/encrypt-data-using-openssl-ecdsa Ana06
+- nursery/encrypt-data-using-openssl-rsa Ana06
+- runtime/dotnet/execute-via-dotnet-startup-hook william.ballenthin@mandiant.com
+- host-interaction/console/manipulate-console-buffer william.ballenthin@mandiant.com michael.hunhoff@mandiant.com
+- nursery/access-wmi-data-in-dotnet michael.hunhoff@mandiant.com
+- nursery/allocate-unmanaged-memory-via-dotnet michael.hunhoff@mandiant.com
+- nursery/generate-random-bytes-in-dotnet michael.hunhoff@mandiant.com
+- nursery/manipulate-console-window michael.hunhoff@mandiant.com
+- nursery/obfuscated-with-koivm michael.hunhoff@mandiant.com
+- nursery/implement-com-dll moritz.raabe@mandiant.com
+- nursery/linked-against-libsodium @mr-tz
+- compiler/nuitka/compiled-with-nuitka @williballenthin
+- nursery/authenticate-data-with-md5-mac william.ballenthin@mandiant.com
+- nursery/resolve-function-by-djb2-hash still@teamt5.org
+- host-interaction/mutex/create-semaphore-on-linux @ramen0x3f
+- host-interaction/mutex/lock-semaphore-on-linux @ramen0x3f
+- host-interaction/mutex/unlock-semaphore-on-linux @ramen0x3f
+- data-manipulation/hashing/sha384/hash-data-using-sha384 william.ballenthin@mandiant.com
+- data-manipulation/hashing/sha512/hash-data-using-sha512 william.ballenthin@mandiant.com
+- nursery/decode-data-using-url-encoding michael.hunhoff@mandiant.com
+- nursery/manipulate-user-privileges michael.hunhoff@mandiant.com
+- lib/get-os-version @mr-tz
+- nursery/decrypt-data-using-tea william.ballenthin@mandiant.com
+- nursery/encrypt-data-using-tea william.ballenthin@mandiant.com
+- nursery/hash-data-using-whirlpool william.ballenthin@mandiant.com
+- nursery/reference-base58-string william.ballenthin@mandiant.com
+- communication/mailslot/create-mailslot william.ballenthin@mandiant.com
+- executable/resource/access-dotnet-resource @mr-tz
+- linking/static/linked-against-cpp-standard-library @mr-tz
+- data-manipulation/compression/compress-data-using-lzo david@edeca.net david.cannings@pwc.com
+- data-manipulation/compression/decompress-data-using-lzo david@edeca.net david.cannings@pwc.com
+- communication/socket/tcp/create-tcp-socket-via-raw-afd-driver william.ballenthin@mandiant.com
+- host-interaction/process/map-section-object william.ballenthin@mandiant.com
+- lib/create-or-open-section-object william.ballenthin@mandiant.com
+- load-code/dotnet/execute-dotnet-assembly-via-clr-host blas.kojusner@mandiant.com
+- load-code/execute-vbscript-javascript-or-jscript-in-memory blas.kojusner@mandiant.com
+- host-interaction/file-system/reference-absolute-stream-path-on-windows blas.kojusner@mandiant.com
+- nursery/generate-method-via-reflection-in-dotnet michael.hunhoff@mandiant.com
+- nursery/unmanaged-call-via-dynamic-pinvoke-in-dotnet michael.hunhoff@mandiant.com
+
+### Bug Fixes
+- render: convert feature attributes to aliased dictionary for vverbose #1152 @mike-hunhoff
+- decouple Token dependency / extractor and features #1139 @mr-tz
+- update pydantic model to guarantee type coercion #1176 @mike-hunhoff
+- do not overwrite version in version.py during PyInstaller build #1169 @mr-tz
+- render: fix vverbose rendering of offsets #1215 @williballenthin
+- elf: better detect OS via GLIBC ABI version needed and dependencies #1221 @williballenthin
+- dotnet: address unhandled exceptions with improved type checking #1230 @mike-hunhoff
+- fix import-to-ida script formatting #1208 @williballenthin
+- render: fix verbose rendering of scopes #1263 @williballenthin
+- rules: better detect invalid rules #1282 @williballenthin
+- show-features: better render strings with embedded whitespace #1267 @williballenthin
+- handle vivisect bug around strings at instruction level, use min length 4 #1271 @williballenthin @mr-tz
+- extractor: guard against invalid "calls from" features #1177 @mr-tz
+- extractor: add format to global features #1258 @mr-tz
+- extractor: discover all strings with length >= 4 #1280 @mr-tz
+- extractor: don't extract byte features for strings #1293 @mr-tz
+
+### capa explorer IDA Pro plugin
+- fix: display instruction items #1154 @mr-tz
+- fix: accept only plaintext pasted content #1194 @williballenthin
+- fix: UnboundLocalError #1217 @williballenthin
+- extractor: add support for COFF files and extern functions #1223 @mike-hunhoff
+- doc: improve error messaging and documentation related to capa rule set #1249 @mike-hunhoff
+- fix: assume 32-bit displacement for offsets #1250 @mike-hunhoff
+- generator: refactor caching and matching #1251 @mike-hunhoff
+- fix: improve exception handling to prevent IDA from locking up when errors occur #1262 @mike-hunhoff
+- verify rule metadata using Pydantic #1167 @mr-tz
+- extractor: make read consistent with file object behavior #1254 @mr-tz
+- fix: UnboundLocalError x2 #1302 @mike-hunhoff
+- cache capa results across IDA sessions #1279 @mr-tz
+
+### Raw diffs
+- [capa v4.0.1...v5.0.0](https://github.com/mandiant/capa/compare/v4.0.1...v5.0.0)
+- [capa-rules v4.0.1...v5.0.0](https://github.com/mandiant/capa-rules/compare/v4.0.1...v5.0.0)
+
+
+## v4.0.1 (2022-08-15)
+Some rules contained invalid metadata fields that caused an error when rendering rule hits. We've updated all rules and enhanced the rule linter to catch such issues.
+
+### New Rules (1)
+
+- anti-analysis/obfuscation/obfuscated-with-vs-obfuscation jakub.jozwiak@mandiant.com
+
+
+### Bug Fixes
+- linter: use pydantic to validate rule metadata #1141 @mike-hunhoff
+- build binaries using PyInstaller no longer overwrites functions in version.py #1136 @mr-tz
+
+### Raw diffs
+- [capa v4.0.0...v4.0.1](https://github.com/mandiant/capa/compare/v4.0.0...v4.0.1)
+- [capa-rules v4.0.0...v4.0.1](https://github.com/mandiant/capa-rules/compare/v4.0.0...v4.0.1)
+
+## v4.0.0 (2022-08-10)
+Version 4 adds support for analyzing .NET executables. capa will autodetect .NET modules, or you can explicitly invoke the new feature extractor via `--format dotnet`. We've also extended the rule syntax for .NET features including `namespace` and `class`.
+
+Additionally, new `instruction` scope and `operand` features enable users to create more explicit rules. These features are not backwards compatible. We removed the previously used `/x32` and `/x64` flavors of number and operand features.
+
+We updated 49 existing rules and added 22 new rules leveraging these new features and characteristics to detect capabilities seen in .NET malware.
+
+More breaking changes include updates to the JSON results document, freeze file format schema (now format version v2), and the internal handling of addresses.
+
+Thanks for all the support, especially to @htnhan, @jtothej, @sara-rn, @anushkavirgaonkar, and @_re_fox!
+
+*Deprecation warning: v4.0 will be the last capa version to support the SMDA backend.*
+
+### New Features
+
+ - add new scope "instruction" for matching mnemonics and operands #767 @williballenthin
+ - add new feature "operand[{0, 1, 2}].number" for matching instruction operand immediate values #767 @williballenthin
+ - add new feature "operand[{0, 1, 2}].offset" for matching instruction operand offsets #767 @williballenthin
+ - extract additional offset/number features in certain circumstances #320 @williballenthin
+ - add detection and basic feature extraction for dotnet #987 @mr-tz, @mike-hunhoff, @williballenthin
+ - add file string extraction for dotnet files #1012 @mike-hunhoff
+ - add file function-name extraction for dotnet files #1015 @mike-hunhoff
+ - add unmanaged call characteristic for dotnet files #1023 @mike-hunhoff
+ - add mixed mode characteristic feature extraction for dotnet files #1024 @mike-hunhoff
+ - emit class and namespace features for dotnet files #1030 @mike-hunhoff
+ - render: support Addresses that aren't simple integers, like .NET token+offset #981 @williballenthin
+ - document rule tags and branches #1006 @williballenthin, @mr-tz
+
+### Breaking Changes
+
+  - instruction scope and operand feature are new and are not backwards compatible with older versions of capa
+  - Python 3.7 is now the minimum supported Python version #866 @williballenthin
+  - remove /x32 and /x64 flavors of number and operand features #932 @williballenthin
+  - the tool now accepts multiple paths to rules, and JSON doc updated accordingly @williballenthin
+  - extractors must use handles to identify functions/basic blocks/instructions #981 @williballenthin
+  - the freeze file format schema was updated, including format version bump to v2 #986 @williballenthin
+
+Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issues/937), we plan to remove the SMDA backend for v5. If you rely on this backend, please reach out so we can discuss extending the support for SMDA or transitioning your workflow to use vivisect.
+
+### New Rules (30)
+
+- data-manipulation/encryption/aes/manually-build-aes-constants huynh.t.nhan@gmail.com
+- nursery/get-process-image-filename michael.hunhoff@mandiant.com
+- compiler/v/compiled-with-v jakub.jozwiak@mandiant.com
+- compiler/zig/compiled-with-zig jakub.jozwiak@mandiant.com
+- anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com
+- internal/limitation/file/internal-dotnet-file-limitation william.ballenthin@mandiant.com
+- nursery/get-os-information-via-kuser_shared_data @mr-tz
+- load-code/pe/resolve-function-by-parsing-PE-exports @sara-rn
+- anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com
+- nursery/execute-dotnet-assembly anushka.virgaonkar@mandiant.com
+- nursery/invoke-dotnet-assembly-method anushka.virgaonkar@mandiant.com
+- collection/screenshot/capture-screenshot-via-keybd-event @_re_fox
+- collection/browser/gather-chrome-based-browser-login-information @_re_fox
+- nursery/power-down-monitor michael.hunhoff@mandiant.com
+- nursery/hash-data-using-aphash @_re_fox
+- nursery/hash-data-using-jshash @_re_fox
+- host-interaction/file-system/files/list/enumerate-files-on-windows moritz.raabe@mandiant.com anushka.virgaonkar@mandiant.com
+- nursery/check-clipboard-data anushka.virgaonkar@mandiant.com
+- nursery/clear-clipboard-data anushka.virgaonkar@mandiant.com
+- nursery/compile-dotnet-assembly anushka.virgaonkar@mandiant.com
+- nursery/create-process-via-wmi anushka.virgaonkar@mandiant.com
+- nursery/display-service-notification-message-box anushka.virgaonkar@mandiant.com
+- nursery/find-process-by-name anushka.virgaonkar@mandiant.com
+- nursery/generate-random-numbers-in-dotnet anushka.virgaonkar@mandiant.com
+- nursery/send-keystrokes anushka.virgaonkar@mandiant.com
+- nursery/send-request-in-dotnet anushka.virgaonakr@mandiant.com
+- nursery/terminate-process-by-name-in-dotnet anushka.virgaonkar@mandiant.com
+- nursery/hash-data-using-rshash @_re_fox
+- persistence/authentication-process/act-as-credential-manager-dll jakub.jozwiak@mandiant.com
+- persistence/authentication-process/act-as-password-filter-dll jakub.jozwiak@mandiant.com
+
+### Bug Fixes
+- improve handling _ prefix compile/link artifact #924 @mike-hunhoff
+- better detect OS in ELF samples #988 @williballenthin
+- display number feature zero in vverbose #1097 @mike-hunhoff
+
+### capa explorer IDA Pro plugin
+- improve file format extraction #918 @mike-hunhoff
+- remove decorators added by IDA to ELF imports #919 @mike-hunhoff
+- bug fixes for Address abstraction #1091 @mike-hunhoff
+
+### Development
+
+### Raw diffs
+- [capa v3.2.0...v4.0.0](https://github.com/mandiant/capa/compare/v3.2.0...master)
+- [capa-rules v3.2.0...v4.0.0](https://github.com/mandiant/capa-rules/compare/v3.2.0...master)
+
+## v3.2.1 (2022-06-06)
+This out-of-band release bumps the SMDA dependency version to enable installation on Python 3.10.
+
+### Bug Fixes
+
+- update SMDA dependency @mike-hunhoff #922
+
+### Raw diffs
+- [capa v3.2.0...v3.2.1](https://github.com/mandiant/capa/compare/v3.2.0...v3.2.1)
+- [capa-rules v3.2.0...v3.2.1](https://github.com/mandiant/capa-rules/compare/v3.2.0...v3.2.1)
+
+## v3.2.0 (2022-03-03)
+This release adds a new characteristic `characteristic: call $+5` enabling users to create more explicit rules. The linter now also validates ATT&CK and MBC categories. Additionally, many dependencies, including the vivisect backend, have been updated.
+
+One rule has been added and many more have been improved.
+
+Thanks for all the support, especially to @kn0wl3dge and first time contributor @uckelman-sf!
+
+### New Features
+
+- linter: validate ATT&CK/MBC categories and IDs #103 @kn0wl3dge
+- extractor: add characteristic "call $+5" feature #366 @kn0wl3dge
+
+### New Rules (1)
+
+- anti-analysis/obfuscation/obfuscated-with-advobfuscator jakub.jozwiak@mandiant.com
+
+### Bug Fixes
+
+- remove typing package as a requirement for Python 3.7+ compatibility #901 @uckelman-sf
+- elf: fix OS detection for Linux kernel modules #867 @williballenthin
+
+### Raw diffs
+- [capa v3.1.0...v3.2.0](https://github.com/mandiant/capa/compare/v3.1.0...v3.2.0)
+- [capa-rules v3.1.0...v3.2.0](https://github.com/mandiant/capa-rules/compare/v3.1.0...v3.2.0)
 
 ## v3.1.0 (2022-01-10)
 This release improves the performance of capa while also adding 23 new rules and many code quality enhancements. We profiled capa's CPU usage and optimized the way that it matches rules, such as by short circuiting when appropriate. According to our testing, the matching phase is approximately 66% faster than v3.0.3! We also added support for Python 3.10, aarch64 builds, and additional MAEC metadata in the rule headers.
@@ -1105,7 +1523,7 @@ Download a standalone binary below and checkout the readme [here on GitHub](http
   - setup: pin vivisect version @williballenthin
   - setup: bump vivisect dependency version @williballenthin
   - setup: set Python project name to `flare-capa` @williballenthin
-  - ci: run tests and linter via Github Actions @Ana06
+  - ci: run tests and linter via GitHub Actions @Ana06
   - hooks: run style checkers and hide stashed output @Ana06
   - linter: ignore period in rule filename @williballenthin
   - linter: warn on nursery rule with no changes needed @williballenthin

LICENSE.txt

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (C) 2020 Mandiant, Inc.
+   Copyright (C) 2023 Mandiant, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -2,19 +2,20 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-658-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-823-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
 capa detects capabilities in executable files.
-You run it against a PE, ELF, or shellcode file and it tells you what it thinks the program can do.
+You run it against a PE, ELF, .NET module, or shellcode file and it tells you what it thinks the program can do.
 For example, it might suggest that the file is a backdoor, is capable of installing services, or relies on HTTP to communicate.
 
 Check out:
-- the overview in our first [capa blog post](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html)
-- the major version 2.0 updates described in our [second blog post](https://www.fireeye.com/blog/threat-research/2021/07/capa-2-better-stronger-faster.html)
-- the major version 3.0 (ELF support) described in the [third blog post](https://www.fireeye.com/blog/threat-research/2021/09/elfant-in-the-room-capa-v3.html)
+- the overview in our first [capa blog post](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities)
+- the major version 2.0 updates described in our [second blog post](https://www.mandiant.com/resources/capa-2-better-stronger-faster)
+- the major version 3.0 (ELF support) described in the [third blog post](https://www.mandiant.com/resources/elfant-in-the-room-capa-v3)
+- the major version 4.0 (.NET support) described in the [fourth blog post](https://www.mandiant.com/resources/blog/capa-v4-casting-wider-net)
 
 ```
 $ capa.exe suspicious.exe
@@ -95,23 +96,32 @@ author      matthew.williams@mandiant.com
 scope       function
 att&ck      Execution::Command and Scripting Interpreter::Windows Command Shell [T1059.003]
 references  https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/ns-processthreadsapi-startupinfoa
-examples    Practical Malware Analysis Lab 14-02.exe_:0x4011C0
-function @ 0x10003A13
+function @ 0x4011C0
   and:
-    match: create a process with modified I/O handles and window @ 0x10003A13
+    match: create a process with modified I/O handles and window @ 0x4011C0
       and:
+        number: 257 = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW @ 0x4012B8
         or:
-          api: kernel32.CreateProcess @ 0x10003D6D
-        number: 0x101 @ 0x10003B03
-        or:
-          number: 0x44 @ 0x10003ADC
-        optional:
-          api: kernel32.GetStartupInfo @ 0x10003AE4
-    match: create pipe @ 0x10003A13
+          number: 68 = StartupInfo.cb (size) @ 0x401282
+        or: = API functions that accept a pointer to a STARTUPINFO structure
+          api: kernel32.CreateProcess @ 0x401343
+    match: create pipe @ 0x4011C0
       or:
-        api: kernel32.CreatePipe @ 0x10003ACB
+        api: kernel32.CreatePipe @ 0x40126F, 0x401280
+    optional:
+      match: create thread @ 0x40136A, 0x4013BA
+        or:
+          and:
+            os: windows
+            or:
+              api: kernel32.CreateThread @ 0x4013D7
+        or:
+          and:
+            os: windows
+            or:
+              api: kernel32.CreateThread @ 0x401395
     or:
-      string: cmd.exe /c  @ 0x10003AED
+      string: "cmd.exe" @ 0x4012FD
 ...
 ```
 
@@ -127,18 +137,28 @@ rule:
   meta:
     name: hash data with CRC32
     namespace: data-manipulation/checksum/crc32
-    author: moritz.raabe@mandiant.com
+    authors:
+      - moritz.raabe@mandiant.com
     scope: function
+    mbc:
+      - Data::Checksum::CRC32 [C0032.001]
     examples:
       - 2D3EDC218A90F03089CC01715A9F047F:0x403CBD
       - 7D28CB106CB54876B2A5C111724A07CD:0x402350  # RtlComputeCrc32
+      - 7EFF498DE13CC734262F87E6B3EF38AB:0x100084A6
   features:
     - or:
       - and:
         - mnemonic: shr
-        - number: 0xEDB88320
+        - or:
+          - number: 0xEDB88320
+          - bytes: 00 00 00 00 96 30 07 77 2C 61 0E EE BA 51 09 99 19 C4 6D 07 8F F4 6A 70 35 A5 63 E9 A3 95 64 9E = crc32_tab
         - number: 8
         - characteristic: nzxor
+      - and:
+        - number: 0x8320
+        - number: 0xEDB8
+        - characteristic: nzxor
       - api: RtlComputeCrc32
 ```
 

capa/engine.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,11 +8,12 @@
 
 import copy
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Mapping, Iterable
+from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator
 
 import capa.perf
 import capa.features.common
 from capa.features.common import Result, Feature
+from capa.features.address import Address
 
 if TYPE_CHECKING:
     # circular import, otherwise
@@ -26,7 +27,7 @@ if TYPE_CHECKING:
 # to collect the locations of a feature, do: `features[Number(0x10)]`
 #
 # aliased here so that the type can be documented and xref'd.
-FeatureSet = Dict[Feature, Set[int]]
+FeatureSet = Dict[Feature, Set[Address]]
 
 
 class Statement:
@@ -37,15 +38,17 @@ class Statement:
     """
 
     def __init__(self, description=None):
-        super(Statement, self).__init__()
+        super().__init__()
         self.name = self.__class__.__name__
         self.description = description
 
     def __str__(self):
+        name = self.name.lower()
+        children = ",".join(map(str, self.get_children()))
         if self.description:
-            return "%s(%s = %s)" % (self.name.lower(), ",".join(map(str, self.get_children())), self.description)
+            return f"{name}({children} = {self.description})"
         else:
-            return "%s(%s)" % (self.name.lower(), ",".join(map(str, self.get_children())))
+            return f"{name}({children})"
 
     def __repr__(self):
         return str(self)
@@ -59,21 +62,28 @@ class Statement:
         """
         raise NotImplementedError()
 
-    def get_children(self):
+    def get_children(self) -> Iterator[Union["Statement", Feature]]:
         if hasattr(self, "child"):
-            yield self.child
+            # this really confuses mypy because the property may not exist
+            # since its defined in the subclasses.
+            child = self.child  # type: ignore
+            assert isinstance(child, (Statement, Feature))
+            yield child
 
         if hasattr(self, "children"):
-            for child in getattr(self, "children"):
+            for child in self.children:
+                assert isinstance(child, (Statement, Feature))
                 yield child
 
     def replace_child(self, existing, new):
         if hasattr(self, "child"):
-            if self.child is existing:
+            # this really confuses mypy because the property may not exist
+            # since its defined in the subclasses.
+            if self.child is existing:  # type: ignore
                 self.child = new
 
         if hasattr(self, "children"):
-            children = getattr(self, "children")
+            children = self.children
             for i, child in enumerate(children):
                 if child is existing:
                     children[i] = new
@@ -89,7 +99,7 @@ class And(Statement):
     """
 
     def __init__(self, children, description=None):
-        super(And, self).__init__(description=description)
+        super().__init__(description=description)
         self.children = children
 
     def evaluate(self, ctx, short_circuit=True):
@@ -122,7 +132,7 @@ class Or(Statement):
     """
 
     def __init__(self, children, description=None):
-        super(Or, self).__init__(description=description)
+        super().__init__(description=description)
         self.children = children
 
     def evaluate(self, ctx, short_circuit=True):
@@ -149,7 +159,7 @@ class Not(Statement):
     """match only if the child evaluates to False."""
 
     def __init__(self, child, description=None):
-        super(Not, self).__init__(description=description)
+        super().__init__(description=description)
         self.child = child
 
     def evaluate(self, ctx, short_circuit=True):
@@ -171,7 +181,7 @@ class Some(Statement):
     """
 
     def __init__(self, count, children, description=None):
-        super(Some, self).__init__(description=description)
+        super().__init__(description=description)
         self.count = count
         self.children = children
 
@@ -207,7 +217,7 @@ class Range(Statement):
     """match if the child is contained in the ctx set with a count in the given range."""
 
     def __init__(self, child, min=None, max=None, description=None):
-        super(Range, self).__init__(description=description)
+        super().__init__(description=description)
         self.child = child
         self.min = min if min is not None else 0
         self.max = max if max is not None else (1 << 64 - 1)
@@ -224,9 +234,9 @@ class Range(Statement):
 
     def __str__(self):
         if self.max == (1 << 64 - 1):
-            return "range(%s, min=%d, max=infinity)" % (str(self.child), self.min)
+            return f"range({str(self.child)}, min={self.min}, max=infinity)"
         else:
-            return "range(%s, min=%d, max=%d)" % (str(self.child), self.min, self.max)
+            return f"range({str(self.child)}, min={self.min}, max={self.max})"
 
 
 class Subscope(Statement):
@@ -235,8 +245,8 @@ class Subscope(Statement):
     the engine should preprocess rules to extract subscope statements into their own rules.
     """
 
-    def __init__(self, scope, child):
-        super(Subscope, self).__init__()
+    def __init__(self, scope, child, description=None):
+        super().__init__(description=description)
         self.scope = scope
         self.child = child
 
@@ -257,10 +267,10 @@ class Subscope(Statement):
 #         inspect(match_details)
 #
 # aliased here so that the type can be documented and xref'd.
-MatchResults = Mapping[str, List[Tuple[int, Result]]]
+MatchResults = Mapping[str, List[Tuple[Address, Result]]]
 
 
-def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations: Iterable[int]):
+def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations: Iterable[Address]):
     """
     record into the given featureset that the given rule matched at the given locations.
 
@@ -277,7 +287,7 @@ def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations:
             namespace, _, _ = namespace.rpartition("/")
 
 
-def match(rules: List["capa.rules.Rule"], features: FeatureSet, va: int) -> Tuple[FeatureSet, MatchResults]:
+def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -> Tuple[FeatureSet, MatchResults]:
     """
     match the given rules against the given features,
     returning an updated set of features and the matches.
@@ -315,10 +325,10 @@ def match(rules: List["capa.rules.Rule"], features: FeatureSet, va: int) -> Tupl
             # sanity check
             assert bool(res) is True
 
-            results[rule.name].append((va, res))
+            results[rule.name].append((addr, res))
             # we need to update the current `features`
             # because subsequent iterations of this loop may use newly added features,
             # such as rule or namespace matches.
-            index_rule_matches(features, rule, [va])
+            index_rule_matches(features, rule, [addr])
 
     return (features, results)

capa/exceptions.py

@@ -0,0 +1,21 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+class UnsupportedRuntimeError(RuntimeError):
+    pass
+
+
+class UnsupportedFormatError(ValueError):
+    pass
+
+
+class UnsupportedArchError(ValueError):
+    pass
+
+
+class UnsupportedOSError(ValueError):
+    pass

capa/features/address.py

@@ -0,0 +1,121 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import abc
+
+
+class Address(abc.ABC):
+    @abc.abstractmethod
+    def __eq__(self, other):
+        ...
+
+    @abc.abstractmethod
+    def __lt__(self, other):
+        # implement < so that addresses can be sorted from low to high
+        ...
+
+    @abc.abstractmethod
+    def __hash__(self):
+        # implement hash so that addresses can be used in sets and dicts
+        ...
+
+    @abc.abstractmethod
+    def __repr__(self):
+        # implement repr to help during debugging
+        ...
+
+
+class AbsoluteVirtualAddress(int, Address):
+    """an absolute memory address"""
+
+    def __new__(cls, v):
+        assert v >= 0
+        return int.__new__(cls, v)
+
+    def __repr__(self):
+        return f"absolute(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
+
+
+class RelativeVirtualAddress(int, Address):
+    """a memory address relative to a base address"""
+
+    def __repr__(self):
+        return f"relative(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
+
+
+class FileOffsetAddress(int, Address):
+    """an address relative to the start of a file"""
+
+    def __new__(cls, v):
+        assert v >= 0
+        return int.__new__(cls, v)
+
+    def __repr__(self):
+        return f"file(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
+
+
+class DNTokenAddress(int, Address):
+    """a .NET token"""
+
+    def __new__(cls, token: int):
+        return int.__new__(cls, token)
+
+    def __repr__(self):
+        return f"token(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
+
+
+class DNTokenOffsetAddress(Address):
+    """an offset into an object specified by a .NET token"""
+
+    def __init__(self, token: int, offset: int):
+        assert offset >= 0
+        self.token = token
+        self.offset = offset
+
+    def __eq__(self, other):
+        return (self.token, self.offset) == (other.token, other.offset)
+
+    def __lt__(self, other):
+        return (self.token, self.offset) < (other.token, other.offset)
+
+    def __hash__(self):
+        return hash((self.token, self.offset))
+
+    def __repr__(self):
+        return f"token(0x{self.token:x})+(0x{self.offset:x})"
+
+    def __index__(self):
+        return self.token + self.offset
+
+
+class _NoAddress(Address):
+    def __eq__(self, other):
+        return True
+
+    def __lt__(self, other):
+        return False
+
+    def __hash__(self):
+        return hash(0)
+
+    def __repr__(self):
+        return "no address"
+
+
+NO_ADDRESS = _NoAddress()

capa/features/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -10,18 +10,11 @@ from capa.features.common import Feature
 
 
 class BasicBlock(Feature):
-    def __init__(self):
-        super(BasicBlock, self).__init__(None)
+    def __init__(self, description=None):
+        super().__init__(0, description=description)
 
     def __str__(self):
         return "basic block"
 
     def get_value_str(self):
         return ""
-
-    def freeze_serialize(self):
-        return (self.__class__.__name__, [])
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        return cls()

capa/features/common.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,10 +7,12 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import re
+import abc
 import codecs
+import typing
 import logging
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Union
+from typing import TYPE_CHECKING, Set, Dict, List, Union, Optional
 
 if TYPE_CHECKING:
     # circular import, otherwise
@@ -19,6 +21,7 @@ if TYPE_CHECKING:
 import capa.perf
 import capa.features
 import capa.features.extractors.elf
+from capa.features.address import Address
 
 logger = logging.getLogger(__name__)
 MAX_BYTES_FEATURE_SIZE = 0x100
@@ -27,6 +30,14 @@ MAX_BYTES_FEATURE_SIZE = 0x100
 THUNK_CHAIN_DEPTH_DELTA = 5
 
 
+class FeatureAccess:
+    READ = "read"
+    WRITE = "write"
+
+
+VALID_FEATURE_ACCESS = (FeatureAccess.READ, FeatureAccess.WRITE)
+
+
 def bytes_to_str(b: bytes) -> str:
     return str(codecs.encode(b, "hex").decode("utf-8"))
 
@@ -69,20 +80,13 @@ class Result:
         success: bool,
         statement: Union["capa.engine.Statement", "Feature"],
         children: List["Result"],
-        locations=None,
+        locations: Optional[Set[Address]] = None,
     ):
-        """
-        args:
-          success (bool)
-          statement (capa.engine.Statement or capa.features.Feature)
-          children (list[Result])
-          locations (iterable[VA])
-        """
-        super(Result, self).__init__()
+        super().__init__()
         self.success = success
         self.statement = statement
         self.children = children
-        self.locations = locations if locations is not None else ()
+        self.locations = locations if locations is not None else set()
 
     def __eq__(self, other):
         if isinstance(other, bool):
@@ -96,100 +100,111 @@ class Result:
         return self.success
 
 
-class Feature:
-    def __init__(self, value: Union[str, int, bytes], bitness=None, description=None):
+class Feature(abc.ABC):  # noqa: B024
+    # this is an abstract class, since we don't want anyone to instantiate it directly,
+    # but it doesn't have any abstract methods.
+
+    def __init__(
+        self,
+        value: Union[str, int, float, bytes],
+        description: Optional[str] = None,
+    ):
         """
         Args:
           value (any): the value of the feature, such as the number or string.
-          bitness (str): one of the VALID_BITNESS values, or None.
-            When None, then the feature applies to any bitness.
-            Modifies the feature name from `feature` to `feature/bitness`, like `offset/x32`.
           description (str): a human-readable description that explains the feature value.
         """
-        super(Feature, self).__init__()
-
-        if bitness is not None:
-            if bitness not in VALID_BITNESS:
-                raise ValueError("bitness '%s' must be one of %s" % (bitness, VALID_BITNESS))
-            self.name = self.__class__.__name__.lower() + "/" + bitness
-        else:
-            self.name = self.__class__.__name__.lower()
+        super().__init__()
 
+        self.name = self.__class__.__name__.lower()
         self.value = value
-        self.bitness = bitness
         self.description = description
 
     def __hash__(self):
-        return hash((self.name, self.value, self.bitness))
+        return hash((self.name, self.value))
 
     def __eq__(self, other):
-        return self.name == other.name and self.value == other.value and self.bitness == other.bitness
+        return self.name == other.name and self.value == other.value
+
+    def __lt__(self, other):
+        # implementing sorting by serializing to JSON is a huge hack.
+        # its slow, inelegant, and probably doesn't work intuitively;
+        # however, we only use it for deterministic output, so it's good enough for now.
+
+        # circular import
+        # we should fix if this wasn't already a huge hack.
+        import capa.features.freeze.features
+
+        return (
+            capa.features.freeze.features.feature_from_capa(self).json()
+            < capa.features.freeze.features.feature_from_capa(other).json()
+        )
+
+    def get_name_str(self) -> str:
+        """
+        render the name of this feature, for use by `__str__` and friends.
+        subclasses should override to customize the rendering.
+        """
+        return self.name
 
     def get_value_str(self) -> str:
         """
         render the value of this feature, for use by `__str__` and friends.
         subclasses should override to customize the rendering.
-
-        Returns: any
         """
         return str(self.value)
 
     def __str__(self):
         if self.value is not None:
             if self.description:
-                return "%s(%s = %s)" % (self.name, self.get_value_str(), self.description)
+                return f"{self.get_name_str()}({self.get_value_str()} = {self.description})"
             else:
-                return "%s(%s)" % (self.name, self.get_value_str())
+                return f"{self.get_name_str()}({self.get_value_str()})"
         else:
-            return "%s" % self.name
+            return f"{self.get_name_str()}"
 
     def __repr__(self):
         return str(self)
 
-    def evaluate(self, ctx: Dict["Feature", Set[int]], **kwargs) -> Result:
+    def evaluate(self, ctx: Dict["Feature", Set[Address]], **kwargs) -> Result:
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature." + self.name] += 1
-        return Result(self in ctx, self, [], locations=ctx.get(self, []))
-
-    def freeze_serialize(self):
-        if self.bitness is not None:
-            return (self.__class__.__name__, [self.value, {"bitness": self.bitness}])
-        else:
-            return (self.__class__.__name__, [self.value])
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        # as you can see below in code,
-        # if the last argument is a dictionary,
-        # consider it to be kwargs passed to the feature constructor.
-        if len(args) == 1:
-            return cls(*args)
-        elif isinstance(args[-1], dict):
-            kwargs = args[-1]
-            args = args[:-1]
-            return cls(*args, **kwargs)
+        return Result(self in ctx, self, [], locations=ctx.get(self, set()))
 
 
 class MatchedRule(Feature):
     def __init__(self, value: str, description=None):
-        super(MatchedRule, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "match"
 
 
 class Characteristic(Feature):
     def __init__(self, value: str, description=None):
-
-        super(Characteristic, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
 class String(Feature):
     def __init__(self, value: str, description=None):
-        super(String, self).__init__(value, description=description)
+        super().__init__(value, description=description)
+
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, str)
+        return escape_string(self.value)
+
+
+class Class(Feature):
+    def __init__(self, value: str, description=None):
+        super().__init__(value, description=description)
+
+
+class Namespace(Feature):
+    def __init__(self, value: str, description=None):
+        super().__init__(value, description=description)
 
 
 class Substring(String):
     def __init__(self, value: str, description=None):
-        super(Substring, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.value = value
 
     def evaluate(self, ctx, short_circuit=True):
@@ -198,8 +213,9 @@ class Substring(String):
 
         # mapping from string value to list of locations.
         # will unique the locations later on.
-        matches = collections.defaultdict(list)
+        matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
 
+        assert isinstance(self.value, str)
         for feature, locations in ctx.items():
             if not isinstance(feature, (String,)):
                 continue
@@ -209,32 +225,32 @@ class Substring(String):
                 raise ValueError("unexpected feature value type")
 
             if self.value in feature.value:
-                matches[feature.value].extend(locations)
+                matches[feature.value].update(locations)
                 if short_circuit:
                     # we found one matching string, thats sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 
         if matches:
-            # finalize: defaultdict -> dict
-            # which makes json serialization easier
-            matches = dict(matches)
-
             # collect all locations
             locations = set()
-            for s in matches.keys():
-                matches[s] = list(set(matches[s]))
-                locations.update(matches[s])
+            for locs in matches.values():
+                locations.update(locs)
 
             # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
             # this is because `self` may match on many strings, so we can't stuff the matched value into it.
             # instead, return a new instance that has a reference to both the substring and the matched values.
-            return Result(True, _MatchedSubstring(self, matches), [], locations=locations)
+            return Result(True, _MatchedSubstring(self, dict(matches)), [], locations=locations)
         else:
-            return Result(False, _MatchedSubstring(self, None), [])
+            return Result(False, _MatchedSubstring(self, {}), [])
+
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, str)
+        return escape_string(self.value)
 
     def __str__(self):
-        return "substring(%s)" % self.value
+        assert isinstance(self.value, str)
+        return f"substring({escape_string(self.value)})"
 
 
 class _MatchedSubstring(Substring):
@@ -245,13 +261,13 @@ class _MatchedSubstring(Substring):
     note: this type should only ever be constructed by `Substring.evaluate()`. it is not part of the public API.
     """
 
-    def __init__(self, substring: Substring, matches):
+    def __init__(self, substring: Substring, matches: Dict[str, Set[Address]]):
         """
         args:
-          substring (Substring): the substring feature that matches.
-          match (Dict[string, List[int]]|None): mapping from matching string to its locations.
+          substring: the substring feature that matches.
+          match: mapping from matching string to its locations.
         """
-        super(_MatchedSubstring, self).__init__(str(substring.value), description=substring.description)
+        super().__init__(str(substring.value), description=substring.description)
         # we want this to collide with the name of `Substring` above,
         # so that it works nicely with the renderers.
         self.name = "substring"
@@ -259,15 +275,14 @@ class _MatchedSubstring(Substring):
         self.matches = matches
 
     def __str__(self):
-        return 'substring("%s", matches = %s)' % (
-            self.value,
-            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
-        )
+        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
+        assert isinstance(self.value, str)
+        return f'substring("{self.value}", matches = {matches})'
 
 
 class Regex(String):
     def __init__(self, value: str, description=None):
-        super(Regex, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.value = value
 
         pat = self.value[len("/") : -len("/")]
@@ -277,12 +292,12 @@ class Regex(String):
             flags |= re.IGNORECASE
         try:
             self.re = re.compile(pat, flags)
-        except re.error:
+        except re.error as exc:
             if value.endswith("/i"):
                 value = value[: -len("i")]
             raise ValueError(
-                "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value
-            )
+                f"invalid regular expression: {value} it should use Python syntax, try it at https://pythex.org"
+            ) from exc
 
     def evaluate(self, ctx, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
@@ -290,7 +305,7 @@ class Regex(String):
 
         # mapping from string value to list of locations.
         # will unique the locations later on.
-        matches = collections.defaultdict(list)
+        matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
 
         for feature, locations in ctx.items():
             if not isinstance(feature, (String,)):
@@ -305,33 +320,29 @@ class Regex(String):
             # using this mode cleans is more convenient for rule authors,
             # so that they don't have to prefix/suffix their terms like: /.*foo.*/.
             if self.re.search(feature.value):
-                matches[feature.value].extend(locations)
+                matches[feature.value].update(locations)
                 if short_circuit:
                     # we found one matching string, thats sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 
         if matches:
-            # finalize: defaultdict -> dict
-            # which makes json serialization easier
-            matches = dict(matches)
-
             # collect all locations
             locations = set()
-            for s in matches.keys():
-                matches[s] = list(set(matches[s]))
-                locations.update(matches[s])
+            for locs in matches.values():
+                locations.update(locs)
 
             # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
             # this is because `self` may match on many strings, so we can't stuff the matched value into it.
             # instead, return a new instance that has a reference to both the regex and the matched values.
             # see #262.
-            return Result(True, _MatchedRegex(self, matches), [], locations=locations)
+            return Result(True, _MatchedRegex(self, dict(matches)), [], locations=locations)
         else:
-            return Result(False, _MatchedRegex(self, None), [])
+            return Result(False, _MatchedRegex(self, {}), [])
 
     def __str__(self):
-        return "regex(string =~ %s)" % self.value
+        assert isinstance(self.value, str)
+        return f"regex(string =~ {self.value})"
 
 
 class _MatchedRegex(Regex):
@@ -342,13 +353,13 @@ class _MatchedRegex(Regex):
     note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
     """
 
-    def __init__(self, regex: Regex, matches):
+    def __init__(self, regex: Regex, matches: Dict[str, Set[Address]]):
         """
         args:
-          regex (Regex): the regex feature that matches.
-          match (Dict[string, List[int]]|None): mapping from matching string to its locations.
+          regex: the regex feature that matches.
+          matches: mapping from matching string to its locations.
         """
-        super(_MatchedRegex, self).__init__(str(regex.value), description=regex.description)
+        super().__init__(str(regex.value), description=regex.description)
         # we want this to collide with the name of `Regex` above,
         # so that it works nicely with the renderers.
         self.name = "regex"
@@ -356,10 +367,9 @@ class _MatchedRegex(Regex):
         self.matches = matches
 
     def __str__(self):
-        return "regex(string =~ %s, matches = %s)" % (
-            self.value,
-            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
-        )
+        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
+        assert isinstance(self.value, str)
+        return f"regex(string =~ {self.value}, matches = {matches})"
 
 
 class StringFactory:
@@ -371,73 +381,90 @@ class StringFactory:
 
 class Bytes(Feature):
     def __init__(self, value: bytes, description=None):
-        super(Bytes, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.value = value
 
     def evaluate(self, ctx, **kwargs):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.bytes"] += 1
 
+        assert isinstance(self.value, bytes)
         for feature, locations in ctx.items():
             if not isinstance(feature, (Bytes,)):
                 continue
 
+            assert isinstance(feature.value, bytes)
             if feature.value.startswith(self.value):
                 return Result(True, self, [], locations=locations)
 
         return Result(False, self, [])
 
     def get_value_str(self):
+        assert isinstance(self.value, bytes)
         return hex_string(bytes_to_str(self.value))
 
-    def freeze_serialize(self):
-        return (self.__class__.__name__, [bytes_to_str(self.value).upper()])
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        return cls(*[codecs.decode(x, "hex") for x in args])
-
-
-# identifiers for supported bitness names that tweak a feature
-# for example, offset/x32
-BITNESS_X32 = "x32"
-BITNESS_X64 = "x64"
-VALID_BITNESS = (BITNESS_X32, BITNESS_X64)
-
 
 # other candidates here: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types
 ARCH_I386 = "i386"
 ARCH_AMD64 = "amd64"
-VALID_ARCH = (ARCH_I386, ARCH_AMD64)
+# dotnet
+ARCH_ANY = "any"
+VALID_ARCH = (ARCH_I386, ARCH_AMD64, ARCH_ANY)
 
 
 class Arch(Feature):
     def __init__(self, value: str, description=None):
-        super(Arch, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "arch"
 
 
 OS_WINDOWS = "windows"
 OS_LINUX = "linux"
 OS_MACOS = "macos"
+# dotnet
+OS_ANY = "any"
 VALID_OS = {os.value for os in capa.features.extractors.elf.OS}
-VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS})
+VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS, OS_ANY})
+# internal only, not to be used in rules
+OS_AUTO = "auto"
 
 
 class OS(Feature):
     def __init__(self, value: str, description=None):
-        super(OS, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "os"
 
+    def evaluate(self, ctx, **kwargs):
+        capa.perf.counters["evaluate.feature"] += 1
+        capa.perf.counters["evaluate.feature." + self.name] += 1
+
+        for feature, locations in ctx.items():
+            if not isinstance(feature, (OS,)):
+                continue
+
+            assert isinstance(feature.value, str)
+            if OS_ANY in (self.value, feature.value) or self.value == feature.value:
+                return Result(True, self, [], locations=locations)
+
+        return Result(False, self, [])
+
 
 FORMAT_PE = "pe"
 FORMAT_ELF = "elf"
-VALID_FORMAT = (FORMAT_PE, FORMAT_ELF)
+FORMAT_DOTNET = "dotnet"
+VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET)
+# internal only, not to be used in rules
+FORMAT_AUTO = "auto"
+FORMAT_SC32 = "sc32"
+FORMAT_SC64 = "sc64"
+FORMAT_FREEZE = "freeze"
+FORMAT_RESULT = "result"
+FORMAT_UNKNOWN = "unknown"
 
 
 class Format(Feature):
     def __init__(self, value: str, description=None):
-        super(Format, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "format"
 
 

capa/features/extractors/base_extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,23 +7,60 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import abc
-from typing import Tuple, Iterator, SupportsInt
+import dataclasses
+from typing import Any, Dict, Tuple, Union, Iterator
+from dataclasses import dataclass
 
+import capa.features.address
 from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
 
 # feature extractors may reference functions, BBs, insns by opaque handle values.
-# the only requirement of these handles are that they support `__int__`,
-# so that they can be rendered as addresses.
+# you can use the `.address` property to get and render the address of the feature.
 #
 # these handles are only consumed by routines on
 # the feature extractor from which they were created.
-#
-# int(FunctionHandle) -> function start address
-# int(BBHandle) -> BasicBlock start address
-# int(InsnHandle) -> instruction address
-FunctionHandle = SupportsInt
-BBHandle = SupportsInt
-InsnHandle = SupportsInt
+
+
+@dataclass
+class FunctionHandle:
+    """reference to a function recognized by a feature extractor.
+
+    Attributes:
+        address: the address of the function.
+        inner: extractor-specific data.
+        ctx: a context object for the extractor.
+    """
+
+    address: Address
+    inner: Any
+    ctx: Dict[str, Any] = dataclasses.field(default_factory=dict)
+
+
+@dataclass
+class BBHandle:
+    """reference to a basic block recognized by a feature extractor.
+
+    Attributes:
+        address: the address of the basic block start address.
+        inner: extractor-specific data.
+    """
+
+    address: Address
+    inner: Any
+
+
+@dataclass
+class InsnHandle:
+    """reference to a instruction recognized by a feature extractor.
+
+    Attributes:
+        address: the address of the instruction address.
+        inner: extractor-specific data.
+    """
+
+    address: Address
+    inner: Any
 
 
 class FeatureExtractor:
@@ -50,17 +87,21 @@ class FeatureExtractor:
         #  for example, the Vivisect feature extract might require the vw and/or path.
         # this base class doesn't know what to do with that info, though.
         #
-        super(FeatureExtractor, self).__init__()
+        super().__init__()
 
     @abc.abstractmethod
-    def get_base_address(self) -> int:
+    def get_base_address(self) -> Union[AbsoluteVirtualAddress, capa.features.address._NoAddress]:
         """
         fetch the preferred load address at which the sample was analyzed.
+
+        when the base address is `NO_ADDRESS`, then the loader has no concept of a preferred load address.
+        such as: shellcode, .NET modules, etc.
+        in these scenarios, RelativeVirtualAddresses aren't used.
         """
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def extract_global_features(self) -> Iterator[Tuple[Feature, int]]:
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
         """
         extract features found at every scope ("global").
 
@@ -71,12 +112,12 @@ class FeatureExtractor:
                 print('0x%x: %s', va, feature)
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def extract_file_features(self) -> Iterator[Tuple[Feature, int]]:
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
         """
         extract file-scope features.
 
@@ -87,7 +128,7 @@ class FeatureExtractor:
                 print('0x%x: %s', va, feature)
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
 
@@ -99,32 +140,33 @@ class FeatureExtractor:
         """
         raise NotImplementedError()
 
-    def is_library_function(self, va: int) -> bool:
+    def is_library_function(self, addr: Address) -> bool:
         """
         is the given address a library function?
         the backend may implement its own function matching algorithm, or none at all.
-        we accept a VA here, rather than function object, to handle addresses identified in instructions.
+        we accept an address here, rather than function object,
+         to handle addresses identified in instructions.
 
         this information is used to:
           - filter out matches in library functions (by default), and
           - recognize when to fetch symbol names for called (non-API) functions
 
         args:
-          va (int): the virtual address of a function.
+          addr (Address): the address of a function.
 
         returns:
           bool: True if the given address is the start of a library function.
         """
         return False
 
-    def get_function_name(self, va: int) -> str:
+    def get_function_name(self, addr: Address) -> str:
         """
         fetch any recognized name for the given address.
         this is only guaranteed to return a value when the given function is a recognized library function.
         we accept a VA here, rather than function object, to handle addresses identified in instructions.
 
         args:
-          va (int): the virtual address of a function.
+          addr (Address): the address of a function.
 
         returns:
           str: the function name
@@ -132,10 +174,10 @@ class FeatureExtractor:
         raises:
           KeyError: when the given function does not have a name.
         """
-        raise KeyError(va)
+        raise KeyError(addr)
 
     @abc.abstractmethod
-    def extract_function_features(self, f: FunctionHandle) -> Iterator[Tuple[Feature, int]]:
+    def extract_function_features(self, f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
         """
         extract function-scope features.
         the arguments are opaque values previously provided by `.get_functions()`, etc.
@@ -144,14 +186,14 @@ class FeatureExtractor:
 
             extractor = VivisectFeatureExtractor(vw, path)
             for function in extractor.get_functions():
-                for feature, va in extractor.extract_function_features(function):
-                    print('0x%x: %s', va, feature)
+                for feature, address in extractor.extract_function_features(function):
+                    print('0x%x: %s', address, feature)
 
         args:
           f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
 
@@ -164,7 +206,7 @@ class FeatureExtractor:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, int]]:
+    def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
         """
         extract basic block-scope features.
         the arguments are opaque values previously provided by `.get_functions()`, etc.
@@ -174,15 +216,15 @@ class FeatureExtractor:
             extractor = VivisectFeatureExtractor(vw, path)
             for function in extractor.get_functions():
                 for bb in extractor.get_basic_blocks(function):
-                    for feature, va in extractor.extract_basic_block_features(function, bb):
-                        print('0x%x: %s', va, feature)
+                    for feature, address in extractor.extract_basic_block_features(function, bb):
+                        print('0x%x: %s', address, feature)
 
         args:
           f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
           bb [BBHandle]: an opaque value previously fetched from `.get_basic_blocks()`.
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
 
@@ -195,7 +237,9 @@ class FeatureExtractor:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def extract_insn_features(self, f: FunctionHandle, bb: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, int]]:
+    def extract_insn_features(
+        self, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
         """
         extract instruction-scope features.
         the arguments are opaque values previously provided by `.get_functions()`, etc.
@@ -206,8 +250,8 @@ class FeatureExtractor:
             for function in extractor.get_functions():
                 for bb in extractor.get_basic_blocks(function):
                     for insn in extractor.get_instructions(function, bb):
-                        for feature, va in extractor.extract_insn_features(function, bb, insn):
-                            print('0x%x: %s', va, feature)
+                        for feature, address in extractor.extract_insn_features(function, bb, insn):
+                            print('0x%x: %s', address, feature)
 
         args:
           f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
@@ -215,123 +259,6 @@ class FeatureExtractor:
           insn [InsnHandle]: an opaque value previously fetched from `.get_instructions()`.
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
-
-
-class NullFeatureExtractor(FeatureExtractor):
-    """
-    An extractor that extracts some user-provided features.
-    The structure of the single parameter is demonstrated in the example below.
-
-    This is useful for testing, as we can provide expected values and see if matching works.
-    Also, this is how we represent features deserialized from a freeze file.
-
-    example::
-
-        extractor = NullFeatureExtractor({
-            'base address: 0x401000,
-            'global features': [
-                (0x0, capa.features.Arch('i386')),
-                (0x0, capa.features.OS('linux')),
-            ],
-            'file features': [
-                (0x402345, capa.features.Characteristic('embedded pe')),
-            ],
-            'functions': {
-                0x401000: {
-                    'features': [
-                        (0x401000, capa.features.Characteristic('nzxor')),
-                    ],
-                    'basic blocks': {
-                        0x401000: {
-                            'features': [
-                                (0x401000, capa.features.Characteristic('tight-loop')),
-                            ],
-                            'instructions': {
-                                0x401000: {
-                                    'features': [
-                                        (0x401000, capa.features.Characteristic('nzxor')),
-                                    ],
-                                },
-                                0x401002: ...
-                            }
-                        },
-                        0x401005: ...
-                    }
-                },
-                0x40200: ...
-            }
-        )
-    """
-
-    def __init__(self, features):
-        super(NullFeatureExtractor, self).__init__()
-        self.features = features
-
-    def get_base_address(self):
-        return self.features["base address"]
-
-    def extract_global_features(self):
-        for p in self.features.get("global features", []):
-            va, feature = p
-            yield feature, va
-
-    def extract_file_features(self):
-        for p in self.features.get("file features", []):
-            va, feature = p
-            yield feature, va
-
-    def get_functions(self):
-        for va in sorted(self.features["functions"].keys()):
-            yield va
-
-    def extract_function_features(self, f):
-        for p in self.features.get("functions", {}).get(f, {}).get("features", []):  # noqa: E127 line over-indented
-            va, feature = p
-            yield feature, va
-
-    def get_basic_blocks(self, f):
-        for va in sorted(
-            self.features.get("functions", {})  # noqa: E127 line over-indented
-            .get(f, {})
-            .get("basic blocks", {})
-            .keys()
-        ):
-            yield va
-
-    def extract_basic_block_features(self, f, bb):
-        for p in (
-            self.features.get("functions", {})  # noqa: E127 line over-indented
-            .get(f, {})
-            .get("basic blocks", {})
-            .get(bb, {})
-            .get("features", [])
-        ):
-            va, feature = p
-            yield feature, va
-
-    def get_instructions(self, f, bb):
-        for va in sorted(
-            self.features.get("functions", {})  # noqa: E127 line over-indented
-            .get(f, {})
-            .get("basic blocks", {})
-            .get(bb, {})
-            .get("instructions", {})
-            .keys()
-        ):
-            yield va
-
-    def extract_insn_features(self, f, bb, insn):
-        for p in (
-            self.features.get("functions", {})  # noqa: E127 line over-indented
-            .get(f, {})
-            .get("basic blocks", {})
-            .get(bb, {})
-            .get("instructions", {})
-            .get(insn, {})
-            .get("features", [])
-        ):
-            va, feature = p
-            yield feature, va

capa/features/extractors/binja/basicblock.py

@@ -0,0 +1,183 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import string
+import struct
+from typing import Tuple, Iterator
+
+from binaryninja import Function, Settings
+from binaryninja import BasicBlock as BinjaBasicBlock
+from binaryninja import (
+    BinaryView,
+    SymbolType,
+    RegisterValueType,
+    VariableSourceType,
+    MediumLevelILSetVar,
+    MediumLevelILOperation,
+    MediumLevelILBasicBlock,
+    MediumLevelILInstruction,
+)
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
+
+use_const_outline: bool = False
+settings: Settings = Settings()
+if settings.contains("analysis.outlining.builtins") and settings.get_bool("analysis.outlining.builtins"):
+    use_const_outline = True
+
+
+def get_printable_len_ascii(s: bytes) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    count = 0
+    for c in s:
+        if c == 0:
+            return count
+        if c < 127 and chr(c) in string.printable:
+            count += 1
+    return count
+
+
+def get_printable_len_wide(s: bytes) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    if all(c == 0x00 for c in s[1::2]):
+        return get_printable_len_ascii(s[::2])
+    return 0
+
+
+def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
+    bv: BinaryView = f.view
+
+    if il.operation != MediumLevelILOperation.MLIL_CALL:
+        return 0
+
+    target = il.dest
+    if target.operation not in [MediumLevelILOperation.MLIL_CONST, MediumLevelILOperation.MLIL_CONST_PTR]:
+        return 0
+
+    addr = target.value.value
+    sym = bv.get_symbol_at(addr)
+    if not sym or sym.type != SymbolType.LibraryFunctionSymbol:
+        return 0
+
+    if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
+        return 0
+
+    if len(il.params) < 2:
+        return 0
+
+    dest = il.params[0]
+    if dest.operation != MediumLevelILOperation.MLIL_ADDRESS_OF:
+        return 0
+
+    var = dest.src
+    if var.source_type != VariableSourceType.StackVariableSourceType:
+        return 0
+
+    src = il.params[1]
+    if src.value.type != RegisterValueType.ConstantDataAggregateValue:
+        return 0
+
+    s = f.get_constant_data(RegisterValueType.ConstantDataAggregateValue, src.value.value)
+    return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
+
+
+def get_printable_len(il: MediumLevelILSetVar) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    width = il.dest.type.width
+    value = il.src.value.value
+
+    if width == 1:
+        chars = struct.pack("<B", value & 0xFF)
+    elif width == 2:
+        chars = struct.pack("<H", value & 0xFFFF)
+    elif width == 4:
+        chars = struct.pack("<I", value & 0xFFFFFFFF)
+    elif width == 8:
+        chars = struct.pack("<Q", value & 0xFFFFFFFFFFFFFFFF)
+    else:
+        return 0
+
+    def is_printable_ascii(chars_: bytes):
+        return all(c < 127 and chr(c) in string.printable for c in chars_)
+
+    def is_printable_utf16le(chars_: bytes):
+        if all(c == 0x00 for c in chars_[1::2]):
+            return is_printable_ascii(chars_[::2])
+
+    if is_printable_ascii(chars):
+        return width
+
+    if is_printable_utf16le(chars):
+        return width // 2
+
+    return 0
+
+
+def is_mov_imm_to_stack(il: MediumLevelILInstruction) -> bool:
+    """verify instruction moves immediate onto stack"""
+    if il.operation != MediumLevelILOperation.MLIL_SET_VAR:
+        return False
+
+    if il.src.operation != MediumLevelILOperation.MLIL_CONST:
+        return False
+
+    if il.dest.source_type != VariableSourceType.StackVariableSourceType:
+        return False
+
+    return True
+
+
+def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
+    """check basic block for stackstring indicators
+
+    true if basic block contains enough moves of constant bytes to the stack
+    """
+    count = 0
+    for il in bb:
+        if use_const_outline:
+            count += get_stack_string_len(f, il)
+        else:
+            if is_mov_imm_to_stack(il):
+                count += get_printable_len(il)
+
+    if count > MIN_STACKSTRING_LEN:
+        return True
+    return False
+
+
+def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract stackstring indicators from basic block"""
+    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
+    if bb[1] is not None and bb_contains_stackstring(fh.inner, bb[1]):
+        yield Characteristic("stack string"), bbh.address
+
+
+def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract tight loop indicators from a basic block"""
+    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
+    for edge in bb[0].outgoing_edges:
+        if edge.target.start == bb[0].start:
+            yield Characteristic("tight loop"), bbh.address
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract basic block features"""
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, addr in bb_handler(fh, bbh):
+            yield feature, addr
+    yield BasicBlock(), bbh.address
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    extract_bb_stackstring,
+)

capa/features/extractors/binja/extractor.py

@@ -0,0 +1,75 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Tuple, Iterator
+
+import binaryninja as binja
+
+import capa.features.extractors.elf
+import capa.features.extractors.binja.file
+import capa.features.extractors.binja.insn
+import capa.features.extractors.binja.global_
+import capa.features.extractors.binja.function
+import capa.features.extractors.binja.basicblock
+from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+
+
+class BinjaFeatureExtractor(FeatureExtractor):
+    def __init__(self, bv: binja.BinaryView):
+        super().__init__()
+        self.bv = bv
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv))
+        self.global_features.extend(capa.features.extractors.binja.global_.extract_os(self.bv))
+        self.global_features.extend(capa.features.extractors.binja.global_.extract_arch(self.bv))
+
+    def get_base_address(self):
+        return AbsoluteVirtualAddress(self.bv.start)
+
+    def extract_global_features(self):
+        yield from self.global_features
+
+    def extract_file_features(self):
+        yield from capa.features.extractors.binja.file.extract_features(self.bv)
+
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        for f in self.bv.functions:
+            yield FunctionHandle(address=AbsoluteVirtualAddress(f.start), inner=f)
+
+    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binja.function.extract_features(fh)
+
+    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
+        f: binja.Function = fh.inner
+        # Set up a MLIL basic block dict look up to associate the disassembly basic block with its MLIL basic block
+        mlil_lookup = {}
+        for mlil_bb in f.mlil.basic_blocks:
+            mlil_lookup[mlil_bb.source_block.start] = mlil_bb
+
+        for bb in f.basic_blocks:
+            mlil_bb = mlil_lookup.get(bb.start)
+
+            yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=(bb, mlil_bb))
+
+    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binja.basicblock.extract_features(fh, bbh)
+
+    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
+        import capa.features.extractors.binja.helpers as binja_helpers
+
+        bb: Tuple[binja.BasicBlock, binja.MediumLevelILBasicBlock] = bbh.inner
+        addr = bb[0].start
+
+        for text, length in bb[0]:
+            insn = binja_helpers.DisassemblyInstruction(addr, length, text)
+            yield InsnHandle(address=AbsoluteVirtualAddress(addr), inner=insn)
+            addr += length
+
+    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
+        yield from capa.features.extractors.binja.insn.extract_features(fh, bbh, ih)

capa/features/extractors/binja/file.py

@@ -0,0 +1,167 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import struct
+from typing import Tuple, Iterator
+
+from binaryninja import Segment, BinaryView, SymbolType, SymbolBinding
+
+import capa.features.extractors.common
+import capa.features.extractors.helpers
+import capa.features.extractors.strings
+from capa.features.file import Export, Import, Section, FunctionName
+from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
+from capa.features.extractors.binja.helpers import unmangle_c_name
+
+
+def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[int, int]]:
+    """check segment for embedded PE
+
+    adapted for binja from:
+    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
+    """
+    mz_xor = [
+        (
+            capa.features.extractors.helpers.xor_static(b"MZ", i),
+            capa.features.extractors.helpers.xor_static(b"PE", i),
+            i,
+        )
+        for i in range(256)
+    ]
+
+    todo = []
+    # If this is the first segment of the binary, skip the first bytes. Otherwise, there will always be a matched
+    # PE at the start of the binaryview.
+    start = seg.start
+    if bv.view_type == "PE" and start == bv.start:
+        start += 1
+
+    for mzx, pex, i in mz_xor:
+        for off, _ in bv.find_all_data(start, seg.end, mzx):
+            todo.append((off, mzx, pex, i))
+
+    while len(todo):
+        off, mzx, pex, i = todo.pop()
+
+        # The MZ header has one field we will check e_lfanew is at 0x3c
+        e_lfanew = off + 0x3C
+
+        if seg.end < (e_lfanew + 4):
+            continue
+
+        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(bv.read(e_lfanew, 4), i))[0]
+
+        peoff = off + newoff
+        if seg.end < (peoff + 2):
+            continue
+
+        if bv.read(peoff, 2) == pex:
+            yield off, i
+
+
+def extract_file_embedded_pe(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract embedded PE features"""
+    for seg in bv.segments:
+        for ea, _ in check_segment_for_pe(bv, seg):
+            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
+
+
+def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract function exports"""
+    for sym in bv.get_symbols_of_type(SymbolType.FunctionSymbol):
+        if sym.binding in [SymbolBinding.GlobalBinding, SymbolBinding.WeakBinding]:
+            name = sym.short_name
+            yield Export(name), AbsoluteVirtualAddress(sym.address)
+            unmangled_name = unmangle_c_name(name)
+            if name != unmangled_name:
+                yield Export(unmangled_name), AbsoluteVirtualAddress(sym.address)
+
+
+def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract function imports
+
+    1. imports by ordinal:
+     - modulename.#ordinal
+
+    2. imports by name, results in two features to support importname-only
+       matching:
+     - modulename.importname
+     - importname
+    """
+    for sym in bv.get_symbols_of_type(SymbolType.ImportAddressSymbol):
+        lib_name = str(sym.namespace)
+        addr = AbsoluteVirtualAddress(sym.address)
+        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym.short_name):
+            yield Import(name), addr
+
+        ordinal = sym.ordinal
+        if ordinal != 0 and (lib_name != ""):
+            ordinal_name = f"#{ordinal}"
+            for name in capa.features.extractors.helpers.generate_symbols(lib_name, ordinal_name):
+                yield Import(name), addr
+
+
+def extract_file_section_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract section names"""
+    for name, section in bv.sections.items():
+        yield Section(name), AbsoluteVirtualAddress(section.start)
+
+
+def extract_file_strings(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract ASCII and UTF-16 LE strings"""
+    for s in bv.strings:
+        yield String(s.value), FileOffsetAddress(s.start)
+
+
+def extract_file_function_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract the names of statically-linked library functions.
+    """
+    for sym_name in bv.symbols:
+        for sym in bv.symbols[sym_name]:
+            if sym.type == SymbolType.LibraryFunctionSymbol:
+                name = sym.short_name
+                yield FunctionName(name), sym.address
+                if name.startswith("_"):
+                    # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                    # extract features for both the mangled and un-mangled representations.
+                    # e.g. `_fwrite` -> `fwrite`
+                    # see: https://stackoverflow.com/a/2628384/87207
+                    yield FunctionName(name[1:]), sym.address
+
+
+def extract_file_format(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    view_type = bv.view_type
+    if view_type in ["PE", "COFF"]:
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif view_type == "ELF":
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    elif view_type == "Raw":
+        # no file type to return when processing a binary file, but we want to continue processing
+        return
+    else:
+        raise NotImplementedError(f"unexpected file format: {view_type}")
+
+
+def extract_features(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract file features"""
+    for file_handler in FILE_HANDLERS:
+        for feature, addr in file_handler(bv):
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_strings,
+    extract_file_section_names,
+    extract_file_embedded_pe,
+    extract_file_function_names,
+    extract_file_format,
+)

capa/features/extractors/binja/find_binja_api.py

@@ -0,0 +1,35 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import subprocess
+from pathlib import Path
+
+# When the script gets executed as a standalone executable (via PyInstaller), `import binaryninja` does not work because
+# we have excluded the binaryninja module in `pyinstaller.spec`. The trick here is to call the system Python and try
+# to find out the path of the binaryninja module that has been installed.
+# Note, including the binaryninja module in the `pyintaller.spec` would not work, since the binaryninja module tries to
+# find the binaryninja core e.g., `libbinaryninjacore.dylib`, using a relative path. And this does not work when the
+# binaryninja module is extracted by the PyInstaller.
+code = r"""
+from pathlib import Path
+from importlib import util
+spec = util.find_spec('binaryninja')
+if spec is not None:
+    if len(spec.submodule_search_locations) > 0:
+            path = Path(spec.submodule_search_locations[0])
+            # encode the path with utf8 then convert to hex, make sure it can be read and restored properly
+            print(str(path.parent).encode('utf8').hex())
+"""
+
+
+def find_binja_path() -> Path:
+    raw_output = subprocess.check_output(["python", "-c", code]).decode("ascii").strip()
+    return Path(bytes.fromhex(raw_output).decode("utf8"))
+
+
+if __name__ == "__main__":
+    print(find_binja_path())

capa/features/extractors/binja/function.py

@@ -0,0 +1,68 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Tuple, Iterator
+
+from binaryninja import Function, BinaryView, LowLevelILOperation
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors import loops
+from capa.features.extractors.base_extractor import FunctionHandle
+
+
+def extract_function_calls_to(fh: FunctionHandle):
+    """extract callers to a function"""
+    func: Function = fh.inner
+
+    for caller in func.caller_sites:
+        # Everything that is a code reference to the current function is considered a caller, which actually includes
+        # many other references that are NOT a caller. For example, an instruction `push function_start` will also be
+        # considered a caller to the function
+        if caller.llil is not None and caller.llil.operation in [
+            LowLevelILOperation.LLIL_CALL,
+            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+            LowLevelILOperation.LLIL_JUMP,
+            LowLevelILOperation.LLIL_TAILCALL,
+        ]:
+            yield Characteristic("calls to"), AbsoluteVirtualAddress(caller.address)
+
+
+def extract_function_loop(fh: FunctionHandle):
+    """extract loop indicators from a function"""
+    func: Function = fh.inner
+
+    edges = []
+
+    # construct control flow graph
+    for bb in func.basic_blocks:
+        for edge in bb.outgoing_edges:
+            edges.append((bb.start, edge.target.start))
+
+    if loops.has_loop(edges):
+        yield Characteristic("loop"), fh.address
+
+
+def extract_recursive_call(fh: FunctionHandle):
+    """extract recursive function call"""
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+    if bv is None:
+        return
+
+    for ref in bv.get_code_refs(func.start):
+        if ref.function == func:
+            yield Characteristic("recursive call"), fh.address
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)

capa/features/extractors/binja/global_.py

@@ -0,0 +1,60 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import Tuple, Iterator
+
+from binaryninja import BinaryView
+
+from capa.features.common import OS, OS_MACOS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
+from capa.features.address import NO_ADDRESS, Address
+
+logger = logging.getLogger(__name__)
+
+
+def extract_os(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    name = bv.platform.name
+    if "-" in name:
+        name = name.split("-")[0]
+
+    if name == "windows":
+        yield OS(OS_WINDOWS), NO_ADDRESS
+
+    elif name == "macos":
+        yield OS(OS_MACOS), NO_ADDRESS
+
+    elif name in ["linux", "freebsd", "decree"]:
+        yield OS(name), NO_ADDRESS
+
+    else:
+        # we likely end up here:
+        #  1. handling shellcode, or
+        #  2. handling a new file format (e.g. macho)
+        #
+        # for (1) we can't do much - its shellcode and all bets are off.
+        # we could maybe accept a further CLI argument to specify the OS,
+        # but i think this would be rarely used.
+        # rules that rely on OS conditions will fail to match on shellcode.
+        #
+        # for (2), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported file format: %s, will not guess OS", name)
+        return
+
+
+def extract_arch(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    arch = bv.arch.name
+    if arch == "x86_64":
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+    elif arch == "x86":
+        yield Arch(ARCH_I386), NO_ADDRESS
+    else:
+        # we likely end up here:
+        #  1. handling a new architecture (e.g. aarch64)
+        #
+        # for (1), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported architecture: %s", arch)
+        return

capa/features/extractors/binja/helpers.py

@@ -0,0 +1,53 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import re
+from typing import List, Callable
+from dataclasses import dataclass
+
+from binaryninja import LowLevelILInstruction
+from binaryninja.architecture import InstructionTextToken
+
+
+@dataclass
+class DisassemblyInstruction:
+    address: int
+    length: int
+    text: List[InstructionTextToken]
+
+
+LLIL_VISITOR = Callable[[LowLevelILInstruction, LowLevelILInstruction, int], bool]
+
+
+def visit_llil_exprs(il: LowLevelILInstruction, func: LLIL_VISITOR):
+    # BN does not really support operand index at the disassembly level, so use the LLIL operand index as a substitute.
+    # Note, this is NOT always guaranteed to be the same as disassembly operand.
+    for i, op in enumerate(il.operands):
+        if isinstance(op, LowLevelILInstruction) and func(op, il, i):
+            visit_llil_exprs(op, func)
+
+
+def unmangle_c_name(name: str) -> str:
+    # https://learn.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170#FormatC
+    # Possible variations for BaseThreadInitThunk:
+    # @BaseThreadInitThunk@12
+    # _BaseThreadInitThunk
+    # _BaseThreadInitThunk@12
+    # It is also possible for a function to have a `Stub` appended to its name:
+    # _lstrlenWStub@4
+
+    # A small optimization to avoid running the regex too many times
+    # this still increases the unit test execution time from 170s to 200s, should be able to accelerate it
+    #
+    # TODO(xusheng): performance optimizations to improve test execution time
+    # https://github.com/mandiant/capa/issues/1610
+    if name[0] in ["@", "_"]:
+        match = re.match(r"^[@|_](.*?)(Stub)?(@\d+)?$", name)
+        if match:
+            return match.group(1)
+
+    return name

capa/features/extractors/binja/insn.py

@@ -0,0 +1,582 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Any, List, Tuple, Iterator, Optional
+
+from binaryninja import Function
+from binaryninja import BasicBlock as BinjaBasicBlock
+from binaryninja import (
+    BinaryView,
+    ILRegister,
+    SymbolType,
+    BinaryReader,
+    RegisterValueType,
+    LowLevelILOperation,
+    LowLevelILInstruction,
+)
+
+import capa.features.extractors.helpers
+from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
+from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.binja.helpers import DisassemblyInstruction, visit_llil_exprs
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+# security cookie checks may perform non-zeroing XORs, these are expected within a certain
+# byte range within the first and returning basic blocks, this helps to reduce FP features
+SECURITY_COOKIE_BYTES_DELTA = 0x40
+
+
+# check if a function is a stub function to another function/symbol. The criteria is:
+# 1. The function must only have one basic block
+# 2. The function must only make one call/jump to another address
+# If the function being checked is a stub function, returns the target address. Otherwise, return None.
+def is_stub_function(bv: BinaryView, addr: int) -> Optional[int]:
+    funcs = bv.get_functions_at(addr)
+    for func in funcs:
+        if len(func.basic_blocks) != 1:
+            continue
+
+        call_count = 0
+        call_target = None
+        for il in func.llil.instructions:
+            if il.operation in [
+                LowLevelILOperation.LLIL_CALL,
+                LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+                LowLevelILOperation.LLIL_JUMP,
+                LowLevelILOperation.LLIL_TAILCALL,
+            ]:
+                call_count += 1
+                if il.dest.value.type in [
+                    RegisterValueType.ImportedAddressValue,
+                    RegisterValueType.ConstantValue,
+                    RegisterValueType.ConstantPointerValue,
+                ]:
+                    call_target = il.dest.value.value
+
+        if call_count == 1 and call_target is not None:
+            return call_target
+
+    return None
+
+
+def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction API features
+
+    example:
+       call dword [0x00473038]
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    for llil in func.get_llils_at(ih.address):
+        if llil.operation in [
+            LowLevelILOperation.LLIL_CALL,
+            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+            LowLevelILOperation.LLIL_JUMP,
+            LowLevelILOperation.LLIL_TAILCALL,
+        ]:
+            if llil.dest.value.type not in [
+                RegisterValueType.ImportedAddressValue,
+                RegisterValueType.ConstantValue,
+                RegisterValueType.ConstantPointerValue,
+            ]:
+                continue
+            address = llil.dest.value.value
+            candidate_addrs = [address]
+            stub_addr = is_stub_function(bv, address)
+            if stub_addr is not None:
+                candidate_addrs.append(stub_addr)
+
+            for address in candidate_addrs:
+                sym = func.view.get_symbol_at(address)
+                if sym is None or sym.type not in [SymbolType.ImportAddressSymbol, SymbolType.ImportedFunctionSymbol]:
+                    continue
+
+                sym_name = sym.short_name
+
+                lib_name = ""
+                import_lib = bv.lookup_imported_object_library(sym.address)
+                if import_lib is not None:
+                    lib_name = import_lib[0].name
+                    if lib_name.endswith(".dll"):
+                        lib_name = lib_name[:-4]
+                    elif lib_name.endswith(".so"):
+                        lib_name = lib_name[:-3]
+
+                for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name):
+                    yield API(name), ih.address
+
+                if sym_name.startswith("_"):
+                    for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name[1:]):
+                        yield API(name), ih.address
+
+
+def extract_insn_number_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction number features
+    example:
+        push    3136B0h         ; dwControlCode
+    """
+    func: Function = fh.inner
+
+    results: List[Tuple[Any[Number, OperandNumber], Address]] = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation == LowLevelILOperation.LLIL_LOAD:
+            return False
+
+        if il.operation not in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            return True
+
+        for op in parent.operands:
+            if isinstance(op, ILRegister) and op.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
+                return False
+            elif isinstance(op, LowLevelILInstruction) and op.operation == LowLevelILOperation.LLIL_REG:
+                if op.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
+                    return False
+
+        raw_value = il.value.value
+        if parent.operation == LowLevelILOperation.LLIL_SUB:
+            raw_value = -raw_value
+
+        results.append((Number(raw_value), ih.address))
+        results.append((OperandNumber(index, raw_value), ih.address))
+
+        return False
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse referenced byte sequences
+    example:
+        push    offset iid_004118d4_IShellLinkA ; riid
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    candidate_addrs = set()
+
+    llil = func.get_llil_at(ih.address)
+    if llil is None or llil.operation in [LowLevelILOperation.LLIL_CALL, LowLevelILOperation.LLIL_CALL_STACK_ADJUST]:
+        return
+
+    for ref in bv.get_code_refs_from(ih.address):
+        if ref == ih.address:
+            continue
+
+        if len(bv.get_functions_containing(ref)) > 0:
+            continue
+
+        candidate_addrs.add(ref)
+
+    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            value = il.value.value
+            if value > 0:
+                candidate_addrs.add(value)
+            return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    for addr in candidate_addrs:
+        extracted_bytes = bv.read(addr, MAX_BYTES_FEATURE_SIZE)
+        if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
+            if bv.get_string_at(addr) is None:
+                # don't extract byte features for obvious strings
+                yield Bytes(extracted_bytes), ih.address
+
+
+def extract_insn_string_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction string features
+
+    example:
+        push offset aAcr     ; "ACR  > "
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    candidate_addrs = set()
+
+    # collect candidate address from code refs directly
+    for ref in bv.get_code_refs_from(ih.address):
+        if ref == ih.address:
+            continue
+
+        if len(bv.get_functions_containing(ref)) > 0:
+            continue
+
+        candidate_addrs.add(ref)
+
+    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            value = il.value.value
+            if value > 0:
+                candidate_addrs.add(value)
+            return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    # Now we have all the candidate address, check them for string or pointer to string
+    br = BinaryReader(bv)
+    for addr in candidate_addrs:
+        found = bv.get_string_at(addr)
+        if found:
+            yield String(found.value), ih.address
+
+        br.seek(addr)
+        pointer = None
+        if bv.arch.address_size == 4:
+            pointer = br.read32()
+        elif bv.arch.address_size == 8:
+            pointer = br.read64()
+
+        if pointer is not None:
+            found = bv.get_string_at(pointer)
+            if found:
+                yield String(found.value), ih.address
+
+
+def extract_insn_offset_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction structure offset features
+
+    example:
+        .text:0040112F cmp [esi+4], ebx
+    """
+    func: Function = fh.inner
+
+    results: List[Tuple[Any[Offset, OperandOffset], Address]] = []
+    address_size = func.view.arch.address_size * 8
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        #  The most common case, read/write dereference to something like `dword [eax+0x28]`
+        if il.operation in [LowLevelILOperation.LLIL_ADD, LowLevelILOperation.LLIL_SUB]:
+            left = il.left
+            right = il.right
+            # Exclude offsets based on stack/franme pointers
+            if left.operation == LowLevelILOperation.LLIL_REG and left.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
+                return True
+
+            if right.operation != LowLevelILOperation.LLIL_CONST:
+                return True
+
+            raw_value = right.value.value
+            # If this is not a dereference, then this must be an add and the offset must be in the range \
+            # [0, MAX_STRUCTURE_SIZE]. For example,
+            # add eax, 0x10,
+            # lea ebx, [eax + 1]
+            if parent.operation not in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
+                if il.operation != LowLevelILOperation.LLIL_ADD or (not 0 < raw_value < MAX_STRUCTURE_SIZE):
+                    return False
+
+            if address_size > 0:
+                # BN also encodes the constant value as two's complement, we need to restore its original value
+                value = capa.features.extractors.helpers.twos_complement(raw_value, address_size)
+            else:
+                value = raw_value
+
+            results.append((Offset(value), ih.address))
+            results.append((OperandOffset(index, value), ih.address))
+            return False
+
+        # An edge case: for code like `push dword [esi]`, we need to generate a feature for offset 0x0
+        elif il.operation in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
+            if il.operands[0].operation == LowLevelILOperation.LLIL_REG:
+                results.append((Offset(0), ih.address))
+                results.append((OperandOffset(index, 0), ih.address))
+                return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def is_nzxor_stack_cookie(f: Function, bb: BinjaBasicBlock, llil: LowLevelILInstruction) -> bool:
+    """check if nzxor exists within stack cookie delta"""
+    # TODO(xusheng): use LLIL SSA to do more accurate analysis
+    # https://github.com/mandiant/capa/issues/1609
+
+    reg_names = []
+    if llil.left.operation == LowLevelILOperation.LLIL_REG:
+        reg_names.append(llil.left.src.name)
+
+    if llil.right.operation == LowLevelILOperation.LLIL_REG:
+        reg_names.append(llil.right.src.name)
+
+    # stack cookie reg should be stack/frame pointer
+    if not any(reg in ["ebp", "esp", "rbp", "rsp", "sp"] for reg in reg_names):
+        return False
+
+    # expect security cookie init in first basic block within first bytes (instructions)
+    if len(bb.incoming_edges) == 0 and llil.address < (bb.start + SECURITY_COOKIE_BYTES_DELTA):
+        return True
+
+    # ... or within last bytes (instructions) before a return
+    if len(bb.outgoing_edges) == 0 and llil.address > (bb.end - SECURITY_COOKIE_BYTES_DELTA):
+        return True
+
+    return False
+
+
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction non-zeroing XOR instruction
+    ignore expected non-zeroing XORs, e.g. security cookies
+    """
+    func: Function = fh.inner
+
+    results = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        # If the two operands of the xor instruction are the same, the LLIL will be translated to other instructions,
+        # e.g., <llil: eax = 0>, (LLIL_SET_REG). So we do not need to check whether the two operands are the same.
+        if il.operation == LowLevelILOperation.LLIL_XOR:
+            # Exclude cases related to the stack cookie
+            if is_nzxor_stack_cookie(fh.inner, bbh.inner[0], il):
+                return False
+            results.append((Characteristic("nzxor"), ih.address))
+            return False
+        else:
+            return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_mnemonic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction mnemonic features"""
+    insn: DisassemblyInstruction = ih.inner
+    yield Mnemonic(insn.text[0].text), ih.address
+
+
+def extract_insn_obfs_call_plus_5_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse call $+5 instruction from the given instruction.
+    """
+    insn: DisassemblyInstruction = ih.inner
+    if insn.text[0].text == "call" and insn.text[2].text == "$+5" and insn.length == 5:
+        yield Characteristic("call $+5"), ih.address
+
+
+def extract_insn_peb_access_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction peb access
+
+    fs:[0x30] on x86, gs:[0x60] on x64
+    """
+    func: Function = fh.inner
+
+    results = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILOperation, index: int) -> bool:
+        if il.operation != LowLevelILOperation.LLIL_LOAD:
+            return True
+
+        src = il.src
+        if src.operation != LowLevelILOperation.LLIL_ADD:
+            return True
+
+        left = src.left
+        right = src.right
+
+        if left.operation != LowLevelILOperation.LLIL_REG:
+            return True
+
+        reg = left.src.name
+
+        if right.operation != LowLevelILOperation.LLIL_CONST:
+            return True
+
+        value = right.value.value
+        if (reg, value) not in (("fsbase", 0x30), ("gsbase", 0x60)):
+            return True
+
+        results.append((Characteristic("peb access"), ih.address))
+        return False
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_segment_access_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction fs or gs access"""
+    func: Function = fh.inner
+
+    results = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation == LowLevelILOperation.LLIL_REG:
+            reg = il.src.name
+            if reg == "fsbase":
+                results.append((Characteristic("fs access"), ih.address))
+                return False
+            elif reg == "gsbase":
+                results.append((Characteristic("gs access"), ih.address))
+                return False
+            return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_cross_section_cflow(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    if bv is None:
+        return
+
+    seg1 = bv.get_segment_at(ih.address)
+    sections1 = bv.get_sections_at(ih.address)
+    for ref in bv.get_code_refs_from(ih.address):
+        if len(bv.get_functions_at(ref)) == 0:
+            continue
+
+        seg2 = bv.get_segment_at(ref)
+        sections2 = bv.get_sections_at(ref)
+        if seg1 != seg2 or sections1 != sections2:
+            yield Characteristic("cross section flow"), ih.address
+
+
+def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract functions calls from features
+
+    most relevant at the function scope, however, its most efficient to extract at the instruction scope
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    if bv is None:
+        return
+
+    for il in func.get_llils_at(ih.address):
+        if il.operation not in [
+            LowLevelILOperation.LLIL_CALL,
+            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+            LowLevelILOperation.LLIL_TAILCALL,
+        ]:
+            continue
+
+        dest = il.dest
+        if dest.operation == LowLevelILOperation.LLIL_CONST_PTR:
+            value = dest.value.value
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
+        elif dest.operation == LowLevelILOperation.LLIL_CONST:
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value)
+        elif dest.operation == LowLevelILOperation.LLIL_LOAD:
+            indirect_src = dest.src
+            if indirect_src.operation == LowLevelILOperation.LLIL_CONST_PTR:
+                value = indirect_src.value.value
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
+            elif indirect_src.operation == LowLevelILOperation.LLIL_CONST:
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(indirect_src.value)
+        elif dest.operation == LowLevelILOperation.LLIL_REG:
+            if dest.value.type in [
+                RegisterValueType.ImportedAddressValue,
+                RegisterValueType.ConstantValue,
+                RegisterValueType.ConstantPointerValue,
+            ]:
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value.value)
+
+
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
+    does not include calls like => call ds:dword_ABD4974
+
+    most relevant at the function or basic block scope;
+    however, its most efficient to extract at the instruction scope
+    """
+    func: Function = fh.inner
+
+    llil = func.get_llil_at(ih.address)
+    if llil is None or llil.operation not in [
+        LowLevelILOperation.LLIL_CALL,
+        LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+        LowLevelILOperation.LLIL_TAILCALL,
+    ]:
+        return
+
+    if llil.dest.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+        return
+
+    if llil.dest.operation == LowLevelILOperation.LLIL_LOAD:
+        src = llil.dest.src
+        if src.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            return
+
+    yield Characteristic("indirect call"), ih.address
+
+
+def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract instruction features"""
+    for inst_handler in INSTRUCTION_HANDLERS:
+        for feature, ea in inst_handler(f, bbh, insn):
+            yield feature, ea
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_bytes_features,
+    extract_insn_string_features,
+    extract_insn_offset_features,
+    extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    extract_insn_obfs_call_plus_5_characteristic_features,
+    extract_insn_peb_access_characteristic_features,
+    extract_insn_cross_section_cflow,
+    extract_insn_segment_access_features,
+    extract_function_calls_from,
+    extract_function_indirect_call_characteristic_features,
+)

capa/features/extractors/common.py

@@ -1,34 +1,68 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import io
 import logging
 import binascii
 import contextlib
+from typing import Tuple, Iterator
 
 import pefile
 
 import capa.features
 import capa.features.extractors.elf
 import capa.features.extractors.pefile
-from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Arch, Format, String
+import capa.features.extractors.strings
+from capa.features.common import (
+    OS,
+    OS_ANY,
+    OS_AUTO,
+    ARCH_ANY,
+    FORMAT_PE,
+    FORMAT_ELF,
+    OS_WINDOWS,
+    FORMAT_FREEZE,
+    FORMAT_RESULT,
+    Arch,
+    Format,
+    String,
+    Feature,
+)
+from capa.features.freeze import is_freeze
+from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress
 
 logger = logging.getLogger(__name__)
 
+# match strings for formats
+MATCH_PE = b"MZ"
+MATCH_ELF = b"\x7fELF"
+MATCH_RESULT = b'{"meta":'
 
-def extract_file_strings(buf, **kwargs):
+
+def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[String, Address]]:
     """
     extract ASCII and UTF-16 LE strings from file
     """
     for s in capa.features.extractors.strings.extract_ascii_strings(buf):
-        yield String(s.s), s.offset
+        yield String(s.s), FileOffsetAddress(s.offset)
 
     for s in capa.features.extractors.strings.extract_unicode_strings(buf):
-        yield String(s.s), s.offset
+        yield String(s.s), FileOffsetAddress(s.offset)
 
 
-def extract_format(buf):
-    if buf.startswith(b"MZ"):
-        yield Format(FORMAT_PE), 0x0
-    elif buf.startswith(b"\x7fELF"):
-        yield Format(FORMAT_ELF), 0x0
+def extract_format(buf) -> Iterator[Tuple[Feature, Address]]:
+    if buf.startswith(MATCH_PE):
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif buf.startswith(MATCH_ELF):
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    elif is_freeze(buf):
+        yield Format(FORMAT_FREEZE), NO_ADDRESS
+    elif buf.startswith(MATCH_RESULT):
+        yield Format(FORMAT_RESULT), NO_ADDRESS
     else:
         # we likely end up here:
         #  1. handling a file format (e.g. macho)
@@ -38,11 +72,14 @@ def extract_format(buf):
         return
 
 
-def extract_arch(buf):
-    if buf.startswith(b"MZ"):
+def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
+    if buf.startswith(MATCH_PE):
         yield from capa.features.extractors.pefile.extract_file_arch(pe=pefile.PE(data=buf))
 
-    elif buf.startswith(b"\x7fELF"):
+    elif buf.startswith(MATCH_RESULT):
+        yield Arch(ARCH_ANY), NO_ADDRESS
+
+    elif buf.startswith(MATCH_ELF):
         with contextlib.closing(io.BytesIO(buf)) as f:
             arch = capa.features.extractors.elf.detect_elf_arch(f)
 
@@ -50,7 +87,7 @@ def extract_arch(buf):
             logger.debug("unsupported arch: %s", arch)
             return
 
-        yield Arch(arch), 0x0
+        yield Arch(arch), NO_ADDRESS
 
     else:
         # we likely end up here:
@@ -58,7 +95,7 @@ def extract_arch(buf):
         #  2. handling a new file format (e.g. macho)
         #
         # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the arch,
+        # we could maybe accept a further CLI argument to specify the arch,
         # but i think this would be rarely used.
         # rules that rely on arch conditions will fail to match on shellcode.
         #
@@ -67,10 +104,15 @@ def extract_arch(buf):
         return
 
 
-def extract_os(buf):
-    if buf.startswith(b"MZ"):
-        yield OS(OS_WINDOWS), 0x0
-    elif buf.startswith(b"\x7fELF"):
+def extract_os(buf, os=OS_AUTO) -> Iterator[Tuple[Feature, Address]]:
+    if os != OS_AUTO:
+        yield OS(os), NO_ADDRESS
+
+    if buf.startswith(MATCH_PE):
+        yield OS(OS_WINDOWS), NO_ADDRESS
+    elif buf.startswith(MATCH_RESULT):
+        yield OS(OS_ANY), NO_ADDRESS
+    elif buf.startswith(MATCH_ELF):
         with contextlib.closing(io.BytesIO(buf)) as f:
             os = capa.features.extractors.elf.detect_elf_os(f)
 
@@ -78,7 +120,7 @@ def extract_os(buf):
             logger.debug("unsupported os: %s", os)
             return
 
-        yield OS(os), 0x0
+        yield OS(os), NO_ADDRESS
 
     else:
         # we likely end up here:
@@ -86,8 +128,6 @@ def extract_os(buf):
         #  2. handling a new file format (e.g. macho)
         #
         # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the OS,
-        # but i think this would be rarely used.
         # rules that rely on OS conditions will fail to match on shellcode.
         #
         # for (2), this logic will need to be updated as the format is implemented.

capa/features/extractors/dnfile/extractor.py

@@ -0,0 +1,155 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+from typing import Dict, List, Tuple, Union, Iterator, Optional
+from pathlib import Path
+
+import dnfile
+from dncil.cil.opcode import OpCodes
+
+import capa.features.extractors
+import capa.features.extractors.dotnetfile
+import capa.features.extractors.dnfile.file
+import capa.features.extractors.dnfile.insn
+import capa.features.extractors.dnfile.function
+from capa.features.common import Feature
+from capa.features.address import NO_ADDRESS, Address, DNTokenAddress, DNTokenOffsetAddress
+from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.dnfile.helpers import (
+    get_dotnet_types,
+    get_dotnet_fields,
+    get_dotnet_managed_imports,
+    get_dotnet_managed_methods,
+    get_dotnet_unmanaged_imports,
+    get_dotnet_managed_method_bodies,
+)
+
+
+class DnFileFeatureExtractorCache:
+    def __init__(self, pe: dnfile.dnPE):
+        self.imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.native_imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.methods: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.fields: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.types: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+
+        for import_ in get_dotnet_managed_imports(pe):
+            self.imports[import_.token] = import_
+        for native_import in get_dotnet_unmanaged_imports(pe):
+            self.native_imports[native_import.token] = native_import
+        for method in get_dotnet_managed_methods(pe):
+            self.methods[method.token] = method
+        for field in get_dotnet_fields(pe):
+            self.fields[field.token] = field
+        for type_ in get_dotnet_types(pe):
+            self.types[type_.token] = type_
+
+    def get_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.imports.get(token)
+
+    def get_native_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.native_imports.get(token)
+
+    def get_method(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.methods.get(token)
+
+    def get_field(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.fields.get(token)
+
+    def get_type(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.types.get(token)
+
+
+class DnfileFeatureExtractor(FeatureExtractor):
+    def __init__(self, path: Path):
+        super().__init__()
+        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+
+        # pre-compute .NET token lookup tables; each .NET method has access to this cache for feature extraction
+        # most relevant at instruction scope
+        self.token_cache: DnFileFeatureExtractorCache = DnFileFeatureExtractorCache(self.pe)
+
+        # pre-compute these because we'll yield them at *every* scope.
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_format())
+        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_os(pe=self.pe))
+        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_arch(pe=self.pe))
+
+    def get_base_address(self):
+        return NO_ADDRESS
+
+    def extract_global_features(self):
+        yield from self.global_features
+
+    def extract_file_features(self):
+        yield from capa.features.extractors.dnfile.file.extract_features(self.pe)
+
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        # create a method lookup table
+        methods: Dict[Address, FunctionHandle] = {}
+        for token, method in get_dotnet_managed_method_bodies(self.pe):
+            fh: FunctionHandle = FunctionHandle(
+                address=DNTokenAddress(token),
+                inner=method,
+                ctx={"pe": self.pe, "calls_from": set(), "calls_to": set(), "cache": self.token_cache},
+            )
+
+            # method tokens should be unique
+            assert fh.address not in methods.keys()
+            methods[fh.address] = fh
+
+        # calculate unique calls to/from each method
+        for fh in methods.values():
+            for insn in fh.inner.instructions:
+                if insn.opcode not in (
+                    OpCodes.Call,
+                    OpCodes.Callvirt,
+                    OpCodes.Jmp,
+                    OpCodes.Newobj,
+                ):
+                    continue
+
+                address: DNTokenAddress = DNTokenAddress(insn.operand.value)
+
+                # record call to destination method; note: we only consider MethodDef methods for destinations
+                dest: Optional[FunctionHandle] = methods.get(address)
+                if dest is not None:
+                    dest.ctx["calls_to"].add(fh.address)
+
+                # record call from source method; note: we record all unique calls from a MethodDef method, not just
+                # those calls to other MethodDef methods e.g. calls to imported MemberRef methods
+                fh.ctx["calls_from"].add(address)
+
+        yield from methods.values()
+
+    def extract_function_features(self, fh) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.dnfile.function.extract_features(fh)
+
+    def get_basic_blocks(self, f) -> Iterator[BBHandle]:
+        # each dotnet method is considered 1 basic block
+        yield BBHandle(
+            address=f.address,
+            inner=f.inner,
+        )
+
+    def extract_basic_block_features(self, fh, bbh):
+        # we don't support basic block features
+        yield from []
+
+    def get_instructions(self, fh, bbh):
+        for insn in bbh.inner.instructions:
+            yield InsnHandle(
+                address=DNTokenOffsetAddress(bbh.address, insn.offset - (fh.inner.offset + fh.inner.header_size)),
+                inner=insn,
+            )
+
+    def extract_insn_features(self, fh, bbh, ih) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.dnfile.insn.extract_features(fh, bbh, ih)

capa/features/extractors/dnfile/file.py

@@ -0,0 +1,63 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+from typing import Tuple, Iterator
+
+import dnfile
+
+import capa.features.extractors.dotnetfile
+from capa.features.file import Import, FunctionName
+from capa.features.common import Class, Format, String, Feature, Namespace, Characteristic
+from capa.features.address import Address
+
+
+def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[Tuple[Import, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_import_names(pe=pe)
+
+
+def extract_file_format(pe: dnfile.dnPE) -> Iterator[Tuple[Format, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_format(pe=pe)
+
+
+def extract_file_function_names(pe: dnfile.dnPE) -> Iterator[Tuple[FunctionName, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_function_names(pe=pe)
+
+
+def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe)
+
+
+def extract_file_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[Tuple[Characteristic, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_mixed_mode_characteristic_features(pe=pe)
+
+
+def extract_file_namespace_features(pe: dnfile.dnPE) -> Iterator[Tuple[Namespace, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_namespace_features(pe=pe)
+
+
+def extract_file_class_features(pe: dnfile.dnPE) -> Iterator[Tuple[Class, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_class_features(pe=pe)
+
+
+def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
+    for file_handler in FILE_HANDLERS:
+        for feature, address in file_handler(pe):
+            yield feature, address
+
+
+FILE_HANDLERS = (
+    extract_file_import_names,
+    extract_file_function_names,
+    extract_file_strings,
+    extract_file_format,
+    extract_file_mixed_mode_characteristic_features,
+    extract_file_namespace_features,
+    extract_file_class_features,
+)

capa/features/extractors/dnfile/function.py

@@ -0,0 +1,50 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.extractors.base_extractor import FunctionHandle
+
+logger = logging.getLogger(__name__)
+
+
+def extract_function_calls_to(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract callers to a function"""
+    for dest in fh.ctx["calls_to"]:
+        yield Characteristic("calls to"), dest
+
+
+def extract_function_calls_from(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract callers from a function"""
+    for src in fh.ctx["calls_from"]:
+        yield Characteristic("calls from"), src
+
+
+def extract_recursive_call(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract recursive function call"""
+    if fh.address in fh.ctx["calls_to"]:
+        yield Characteristic("recursive call"), fh.address
+
+
+def extract_function_loop(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract loop indicators from a function"""
+    raise NotImplementedError()
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_calls_from, extract_recursive_call)

capa/features/extractors/dnfile/helpers.py

@@ -0,0 +1,335 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import Dict, Tuple, Union, Iterator, Optional
+
+import dnfile
+from dncil.cil.body import CilMethodBody
+from dncil.cil.error import MethodBodyFormatError
+from dncil.clr.token import Token, StringToken, InvalidToken
+from dncil.cil.body.reader import CilMethodBodyReaderBase
+
+from capa.features.common import FeatureAccess
+from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
+
+logger = logging.getLogger(__name__)
+
+
+class DnfileMethodBodyReader(CilMethodBodyReaderBase):
+    def __init__(self, pe: dnfile.dnPE, row: dnfile.mdtable.MethodDefRow):
+        self.pe: dnfile.dnPE = pe
+        self.offset: int = self.pe.get_offset_from_rva(row.Rva)
+
+    def read(self, n: int) -> bytes:
+        data: bytes = self.pe.get_data(self.pe.get_rva_from_offset(self.offset), n)
+        self.offset += n
+        return data
+
+    def tell(self) -> int:
+        return self.offset
+
+    def seek(self, offset: int) -> int:
+        self.offset = offset
+        return self.offset
+
+
+def resolve_dotnet_token(pe: dnfile.dnPE, token: Token) -> Union[dnfile.base.MDTableRow, InvalidToken, str]:
+    """map generic token to string or table row"""
+    assert pe.net is not None
+    assert pe.net.mdtables is not None
+
+    if isinstance(token, StringToken):
+        user_string: Optional[str] = read_dotnet_user_string(pe, token)
+        if user_string is None:
+            return InvalidToken(token.value)
+        return user_string
+
+    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(token.table)
+    if table is None:
+        # table index is not valid
+        return InvalidToken(token.value)
+
+    try:
+        return table.rows[token.rid - 1]
+    except IndexError:
+        # table index is valid but row index is not valid
+        return InvalidToken(token.value)
+
+
+def read_dotnet_method_body(pe: dnfile.dnPE, row: dnfile.mdtable.MethodDefRow) -> Optional[CilMethodBody]:
+    """read dotnet method body"""
+    try:
+        return CilMethodBody(DnfileMethodBodyReader(pe, row))
+    except MethodBodyFormatError as e:
+        logger.debug("failed to parse managed method body @ 0x%08x (%s)", row.Rva, e)
+        return None
+
+
+def read_dotnet_user_string(pe: dnfile.dnPE, token: StringToken) -> Optional[str]:
+    """read user string from #US stream"""
+    assert pe.net is not None
+
+    if pe.net.user_strings is None:
+        # stream may not exist (seen in obfuscated .NET)
+        logger.debug("#US stream does not exist for stream index 0x%08x", token.rid)
+        return None
+
+    try:
+        user_string: Optional[dnfile.stream.UserString] = pe.net.user_strings.get_us(token.rid)
+    except UnicodeDecodeError as e:
+        logger.debug("failed to decode #US stream index 0x%08x (%s)", token.rid, e)
+        return None
+
+    if user_string is None:
+        return None
+
+    return user_string.value
+
+
+def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get managed imports from MemberRef table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    10 - MemberRef Table
+        Each row represents an imported method
+            Class (index into the TypeRef, ModuleRef, MethodDef, TypeSpec or TypeDef tables)
+            Name (index into String heap)
+    01 - TypeRef Table
+        Each row represents an imported class, its namespace and the assembly which contains it
+            TypeName (index into String heap)
+            TypeNamespace (index into String heap)
+    """
+    for rid, member_ref in iter_dotnet_table(pe, dnfile.mdtable.MemberRef.number):
+        assert isinstance(member_ref, dnfile.mdtable.MemberRefRow)
+
+        if not isinstance(member_ref.Class.row, dnfile.mdtable.TypeRefRow):
+            # only process class imports from TypeRef table
+            continue
+
+        token: int = calculate_dotnet_token_value(dnfile.mdtable.MemberRef.number, rid)
+        access: Optional[str]
+
+        # assume .NET imports starting with get_/set_ are used to access a property
+        if member_ref.Name.startswith("get_"):
+            access = FeatureAccess.READ
+        elif member_ref.Name.startswith("set_"):
+            access = FeatureAccess.WRITE
+        else:
+            access = None
+
+        member_ref_name: str = member_ref.Name
+        if member_ref_name.startswith(("get_", "set_")):
+            # remove get_/set_ from MemberRef name
+            member_ref_name = member_ref_name[4:]
+
+        yield DnType(
+            token,
+            member_ref.Class.row.TypeName,
+            namespace=member_ref.Class.row.TypeNamespace,
+            member=member_ref_name,
+            access=access,
+        )
+
+
+def get_dotnet_methoddef_property_accessors(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
+    """get MethodDef methods used to access properties
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    24 - MethodSemantics Table
+        Links Events and Properties to specific methods. For example one Event can be associated to more methods. A property uses this table to associate get/set methods.
+            Semantics (a 2-byte bitmask of type MethodSemanticsAttributes)
+            Method (index into the MethodDef table)
+            Association (index into the Event or Property table; more precisely, a HasSemantics coded index)
+    """
+    for rid, method_semantics in iter_dotnet_table(pe, dnfile.mdtable.MethodSemantics.number):
+        assert isinstance(method_semantics, dnfile.mdtable.MethodSemanticsRow)
+
+        if method_semantics.Association.row is None:
+            logger.debug("MethodSemantics[0x%X] Association row is None", rid)
+            continue
+
+        if isinstance(method_semantics.Association.row, dnfile.mdtable.EventRow):
+            # ignore events
+            logger.debug("MethodSemantics[0x%X] ignoring Event", rid)
+            continue
+
+        if method_semantics.Method.table is None:
+            logger.debug("MethodSemantics[0x%X] Method table is None", rid)
+            continue
+
+        token: int = calculate_dotnet_token_value(
+            method_semantics.Method.table.number, method_semantics.Method.row_index
+        )
+
+        if method_semantics.Semantics.msSetter:
+            yield token, FeatureAccess.WRITE
+        elif method_semantics.Semantics.msGetter:
+            yield token, FeatureAccess.READ
+
+
+def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get managed method names from TypeDef table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    02 - TypeDef Table
+        Each row represents a class in the current assembly.
+            TypeName (index into String heap)
+            TypeNamespace (index into String heap)
+            MethodList (index into MethodDef table; it marks the first of a contiguous run of Methods owned by this Type)
+    """
+    accessor_map: Dict[int, str] = {}
+    for methoddef, methoddef_access in get_dotnet_methoddef_property_accessors(pe):
+        accessor_map[methoddef] = methoddef_access
+
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        for idx, method in enumerate(typedef.MethodList):
+            if method.table is None:
+                logger.debug("TypeDef[0x%X] MethodList[0x%X] table is None", rid, idx)
+                continue
+            if method.row is None:
+                logger.debug("TypeDef[0x%X] MethodList[0x%X] row is None", rid, idx)
+                continue
+
+            token: int = calculate_dotnet_token_value(method.table.number, method.row_index)
+            access: Optional[str] = accessor_map.get(token)
+
+            method_name: str = method.row.Name
+            if method_name.startswith(("get_", "set_")):
+                # remove get_/set_
+                method_name = method_name[4:]
+
+            yield DnType(token, typedef.TypeName, namespace=typedef.TypeNamespace, member=method_name, access=access)
+
+
+def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get fields from TypeDef table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    02 - TypeDef Table
+        Each row represents a class in the current assembly.
+            TypeName (index into String heap)
+            TypeNamespace (index into String heap)
+            FieldList (index into Field table; it marks the first of a contiguous run of Fields owned by this Type)
+    """
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        for idx, field in enumerate(typedef.FieldList):
+            if field.table is None:
+                logger.debug("TypeDef[0x%X] FieldList[0x%X] table is None", rid, idx)
+                continue
+            if field.row is None:
+                logger.debug("TypeDef[0x%X] FieldList[0x%X] row is None", rid, idx)
+                continue
+            token: int = calculate_dotnet_token_value(field.table.number, field.row_index)
+            yield DnType(token, typedef.TypeName, namespace=typedef.TypeNamespace, member=field.row.Name)
+
+
+def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
+    """get managed methods from MethodDef table"""
+    for rid, method_def in iter_dotnet_table(pe, dnfile.mdtable.MethodDef.number):
+        assert isinstance(method_def, dnfile.mdtable.MethodDefRow)
+
+        if not method_def.ImplFlags.miIL or any((method_def.Flags.mdAbstract, method_def.Flags.mdPinvokeImpl)):
+            # skip methods that do not have a method body
+            continue
+
+        body: Optional[CilMethodBody] = read_dotnet_method_body(pe, method_def)
+        if body is None:
+            logger.debug("MethodDef[0x%X] method body is None", rid)
+            continue
+
+        token: int = calculate_dotnet_token_value(dnfile.mdtable.MethodDef.number, rid)
+        yield token, body
+
+
+def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[DnUnmanagedMethod]:
+    """get unmanaged imports from ImplMap table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    28 - ImplMap Table
+        ImplMap table holds information about unmanaged methods that can be reached from managed code, using PInvoke dispatch
+            MemberForwarded (index into the Field or MethodDef table; more precisely, a MemberForwarded coded index)
+            ImportName (index into the String heap)
+            ImportScope (index into the ModuleRef table)
+    """
+    for rid, impl_map in iter_dotnet_table(pe, dnfile.mdtable.ImplMap.number):
+        assert isinstance(impl_map, dnfile.mdtable.ImplMapRow)
+
+        module: str
+        if impl_map.ImportScope.row is None:
+            logger.debug("ImplMap[0x%X] ImportScope row is None", rid)
+            module = ""
+        else:
+            module = impl_map.ImportScope.row.Name
+        method: str = impl_map.ImportName
+
+        member_forward_table: int
+        if impl_map.MemberForwarded.table is None:
+            logger.debug("ImplMap[0x%X] MemberForwarded table is None", rid)
+            continue
+        else:
+            member_forward_table = impl_map.MemberForwarded.table.number
+        member_forward_row: int = impl_map.MemberForwarded.row_index
+
+        # ECMA says "Each row of the ImplMap table associates a row in the MethodDef table (MemberForwarded) with the
+        # name of a routine (ImportName) in some unmanaged DLL (ImportScope)"; so we calculate and map the MemberForwarded
+        # MethodDef table token to help us later record native import method calls made from CIL
+        token: int = calculate_dotnet_token_value(member_forward_table, member_forward_row)
+
+        # like Kernel32.dll
+        if module and "." in module:
+            module = module.split(".")[0]
+
+        # like kernel32.CreateFileA
+        yield DnUnmanagedMethod(token, module, method)
+
+
+def get_dotnet_types(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get .NET types from TypeDef and TypeRef tables"""
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        typedef_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
+        yield DnType(typedef_token, typedef.TypeName, namespace=typedef.TypeNamespace)
+
+    for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
+        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
+
+        typeref_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
+        yield DnType(typeref_token, typeref.TypeName, namespace=typeref.TypeNamespace)
+
+
+def calculate_dotnet_token_value(table: int, rid: int) -> int:
+    return ((table & 0xFF) << Token.TABLE_SHIFT) | (rid & Token.RID_MASK)
+
+
+def is_dotnet_mixed_mode(pe: dnfile.dnPE) -> bool:
+    assert pe.net is not None
+    assert pe.net.Flags is not None
+
+    return not bool(pe.net.Flags.CLR_ILONLY)
+
+
+def iter_dotnet_table(pe: dnfile.dnPE, table_index: int) -> Iterator[Tuple[int, dnfile.base.MDTableRow]]:
+    assert pe.net is not None
+    assert pe.net.mdtables is not None
+
+    for rid, row in enumerate(pe.net.mdtables.tables.get(table_index, [])):
+        # .NET tables are 1-indexed
+        yield rid + 1, row

capa/features/extractors/dnfile/insn.py

@@ -0,0 +1,227 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Tuple, Union, Iterator, Optional
+
+if TYPE_CHECKING:
+    from capa.features.extractors.dnfile.extractor import DnFileFeatureExtractorCache
+
+import dnfile
+from dncil.clr.token import Token, StringToken, InvalidToken
+from dncil.cil.opcode import OpCodes
+
+import capa.features.extractors.helpers
+from capa.features.insn import API, Number, Property
+from capa.features.common import Class, String, Feature, Namespace, FeatureAccess, Characteristic
+from capa.features.address import Address
+from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+from capa.features.extractors.dnfile.helpers import (
+    resolve_dotnet_token,
+    read_dotnet_user_string,
+    calculate_dotnet_token_value,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def get_callee(
+    pe: dnfile.dnPE, cache: DnFileFeatureExtractorCache, token: Token
+) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+    """map .NET token to un/managed (generic) method"""
+    token_: int
+    if token.table == dnfile.mdtable.MethodSpec.number:
+        # map MethodSpec to MethodDef or MemberRef
+        row: Union[dnfile.base.MDTableRow, InvalidToken, str] = resolve_dotnet_token(pe, token)
+        assert isinstance(row, dnfile.mdtable.MethodSpecRow)
+
+        if row.Method.table is None:
+            logger.debug("MethodSpec[0x%X] Method table is None", token.rid)
+            return None
+
+        token_ = calculate_dotnet_token_value(row.Method.table.number, row.Method.row_index)
+    else:
+        token_ = token.value
+
+    callee: Optional[Union[DnType, DnUnmanagedMethod]] = cache.get_import(token_)
+    if callee is None:
+        # we must check unmanaged imports before managed methods because we map forwarded managed methods
+        # to their unmanaged imports; we prefer a forwarded managed method be mapped to its unmanaged import for analysis
+        callee = cache.get_native_import(token_)
+        if callee is None:
+            callee = cache.get_method(token_)
+    return callee
+
+
+def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction API features"""
+    if ih.inner.opcode not in (
+        OpCodes.Call,
+        OpCodes.Callvirt,
+        OpCodes.Jmp,
+        OpCodes.Newobj,
+    ):
+        return
+
+    callee: Optional[Union[DnType, DnUnmanagedMethod]] = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
+    if isinstance(callee, DnType):
+        # ignore methods used to access properties
+        if callee.access is None:
+            # like System.IO.File::Delete
+            yield API(str(callee)), ih.address
+    elif isinstance(callee, DnUnmanagedMethod):
+        # like kernel32.CreateFileA
+        for name in capa.features.extractors.helpers.generate_symbols(callee.module, callee.method):
+            yield API(name), ih.address
+
+
+def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction property features"""
+    name: Optional[str] = None
+    access: Optional[str] = None
+
+    if ih.inner.opcode in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
+        # property access via MethodDef or MemberRef
+        callee: Optional[Union[DnType, DnUnmanagedMethod]] = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
+        if isinstance(callee, DnType):
+            if callee.access is not None:
+                name = str(callee)
+                access = callee.access
+
+    elif ih.inner.opcode in (OpCodes.Ldfld, OpCodes.Ldflda, OpCodes.Ldsfld, OpCodes.Ldsflda):
+        # property read via Field
+        read_field: Optional[Union[DnType, DnUnmanagedMethod]] = fh.ctx["cache"].get_field(ih.inner.operand.value)
+        if read_field is not None:
+            name = str(read_field)
+            access = FeatureAccess.READ
+
+    elif ih.inner.opcode in (OpCodes.Stfld, OpCodes.Stsfld):
+        # property write via Field
+        write_field: Optional[Union[DnType, DnUnmanagedMethod]] = fh.ctx["cache"].get_field(ih.inner.operand.value)
+        if write_field is not None:
+            name = str(write_field)
+            access = FeatureAccess.WRITE
+
+    if name is not None:
+        if access is not None:
+            yield Property(name, access=access), ih.address
+        yield Property(name), ih.address
+
+
+def extract_insn_namespace_class_features(
+    fh: FunctionHandle, bh, ih: InsnHandle
+) -> Iterator[Tuple[Union[Namespace, Class], Address]]:
+    """parse instruction namespace and class features"""
+    type_: Optional[Union[DnType, DnUnmanagedMethod]] = None
+
+    if ih.inner.opcode in (
+        OpCodes.Call,
+        OpCodes.Callvirt,
+        OpCodes.Jmp,
+        OpCodes.Ldvirtftn,
+        OpCodes.Ldftn,
+        OpCodes.Newobj,
+    ):
+        # method call - includes managed methods (MethodDef, TypeRef) and properties (MethodSemantics, TypeRef)
+        type_ = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
+
+    elif ih.inner.opcode in (
+        OpCodes.Ldfld,
+        OpCodes.Ldflda,
+        OpCodes.Ldsfld,
+        OpCodes.Ldsflda,
+        OpCodes.Stfld,
+        OpCodes.Stsfld,
+    ):
+        # field access
+        type_ = fh.ctx["cache"].get_field(ih.inner.operand.value)
+
+    # ECMA 335 VI.C.4.10
+    elif ih.inner.opcode in (
+        OpCodes.Initobj,
+        OpCodes.Box,
+        OpCodes.Castclass,
+        OpCodes.Cpobj,
+        OpCodes.Isinst,
+        OpCodes.Ldelem,
+        OpCodes.Ldelema,
+        OpCodes.Ldobj,
+        OpCodes.Mkrefany,
+        OpCodes.Newarr,
+        OpCodes.Refanyval,
+        OpCodes.Sizeof,
+        OpCodes.Stobj,
+        OpCodes.Unbox,
+        OpCodes.Constrained,
+        OpCodes.Stelem,
+        OpCodes.Unbox_Any,
+    ):
+        # type access
+        type_ = fh.ctx["cache"].get_type(ih.inner.operand.value)
+
+    if isinstance(type_, DnType):
+        yield Class(DnType.format_name(type_.class_, namespace=type_.namespace)), ih.address
+        if type_.namespace:
+            yield Namespace(type_.namespace), ih.address
+
+
+def extract_insn_number_features(fh, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction number features"""
+    if ih.inner.is_ldc():
+        yield Number(ih.inner.get_ldc()), ih.address
+
+
+def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction string features"""
+    if not ih.inner.is_ldstr():
+        return
+
+    if not isinstance(ih.inner.operand, StringToken):
+        return
+
+    user_string: Optional[str] = read_dotnet_user_string(fh.ctx["pe"], ih.inner.operand)
+    if user_string is None:
+        return
+
+    if len(user_string) >= 4:
+        yield String(user_string), ih.address
+
+
+def extract_unmanaged_call_characteristic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Characteristic, Address]]:
+    if ih.inner.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
+        return
+
+    row: Union[str, InvalidToken, dnfile.base.MDTableRow] = resolve_dotnet_token(fh.ctx["pe"], ih.inner.operand)
+    if not isinstance(row, dnfile.mdtable.MethodDefRow):
+        return
+
+    if any((row.Flags.mdPinvokeImpl, row.ImplFlags.miUnmanaged, row.ImplFlags.miNative)):
+        yield Characteristic("unmanaged call"), ih.address
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract instruction features"""
+    for inst_handler in INSTRUCTION_HANDLERS:
+        for feature, addr in inst_handler(fh, bbh, ih):
+            assert isinstance(addr, Address)
+            yield feature, addr
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_property_features,
+    extract_insn_number_features,
+    extract_insn_string_features,
+    extract_insn_namespace_class_features,
+    extract_unmanaged_call_characteristic_features,
+)

capa/features/extractors/dnfile/types.py

@@ -0,0 +1,74 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from typing import Optional
+
+
+class DnType:
+    def __init__(self, token: int, class_: str, namespace: str = "", member: str = "", access: Optional[str] = None):
+        self.token: int = token
+        self.access: Optional[str] = access
+        self.namespace: str = namespace
+        self.class_: str = class_
+
+        if member == ".ctor":
+            member = "ctor"
+        if member == ".cctor":
+            member = "cctor"
+
+        self.member: str = member
+
+    def __hash__(self):
+        return hash((self.token, self.access, self.namespace, self.class_, self.member))
+
+    def __eq__(self, other):
+        return (
+            self.token == other.token
+            and self.access == other.access
+            and self.namespace == other.namespace
+            and self.class_ == other.class_
+            and self.member == other.member
+        )
+
+    def __str__(self):
+        return DnType.format_name(self.class_, namespace=self.namespace, member=self.member)
+
+    def __repr__(self):
+        return str(self)
+
+    @staticmethod
+    def format_name(class_: str, namespace: str = "", member: str = ""):
+        # like File::OpenRead
+        name: str = f"{class_}::{member}" if member else class_
+        if namespace:
+            # like System.IO.File::OpenRead
+            name = f"{namespace}.{name}"
+        return name
+
+
+class DnUnmanagedMethod:
+    def __init__(self, token: int, module: str, method: str):
+        self.token: int = token
+        self.module: str = module
+        self.method: str = method
+
+    def __hash__(self):
+        return hash((self.token, self.module, self.method))
+
+    def __eq__(self, other):
+        return self.token == other.token and self.module == other.module and self.method == other.method
+
+    def __str__(self):
+        return DnUnmanagedMethod.format_name(self.module, self.method)
+
+    def __repr__(self):
+        return str(self)
+
+    @staticmethod
+    def format_name(module, method):
+        return f"{module}.{method}"

capa/features/extractors/dnfile_.py

@@ -0,0 +1,158 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import Tuple, Iterator
+from pathlib import Path
+
+import dnfile
+import pefile
+
+from capa.features.common import (
+    OS,
+    OS_ANY,
+    ARCH_ANY,
+    ARCH_I386,
+    FORMAT_PE,
+    ARCH_AMD64,
+    FORMAT_DOTNET,
+    Arch,
+    Format,
+    Feature,
+)
+from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import FeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def extract_file_format(**kwargs) -> Iterator[Tuple[Feature, Address]]:
+    yield Format(FORMAT_PE), NO_ADDRESS
+    yield Format(FORMAT_DOTNET), NO_ADDRESS
+
+
+def extract_file_os(**kwargs) -> Iterator[Tuple[Feature, Address]]:
+    yield OS(OS_ANY), NO_ADDRESS
+
+
+def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Feature, Address]]:
+    # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
+    # .NET 4.5 added option: any CPU, 32-bit preferred
+    assert pe.net is not None
+    assert pe.net.Flags is not None
+
+    if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
+        yield Arch(ARCH_I386), NO_ADDRESS
+    elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+    else:
+        yield Arch(ARCH_ANY), NO_ADDRESS
+
+
+def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
+    for file_handler in FILE_HANDLERS:
+        for feature, address in file_handler(pe=pe):  # type: ignore
+            yield feature, address
+
+
+FILE_HANDLERS = (
+    # extract_file_export_names,
+    # extract_file_import_names,
+    # extract_file_section_names,
+    # extract_file_strings,
+    # extract_file_function_names,
+    extract_file_format,
+)
+
+
+def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
+    for handler in GLOBAL_HANDLERS:
+        for feature, addr in handler(pe=pe):  # type: ignore
+            yield feature, addr
+
+
+GLOBAL_HANDLERS = (
+    extract_file_os,
+    extract_file_arch,
+)
+
+
+class DnfileFeatureExtractor(FeatureExtractor):
+    def __init__(self, path: Path):
+        super().__init__()
+        self.path: Path = path
+        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+
+    def get_base_address(self) -> AbsoluteVirtualAddress:
+        return AbsoluteVirtualAddress(0x0)
+
+    def get_entry_point(self) -> int:
+        # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
+        #  True: native EP: Token
+        #  False: managed EP: RVA
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+
+        return self.pe.net.struct.EntryPointTokenOrRva
+
+    def extract_global_features(self):
+        yield from extract_global_features(self.pe)
+
+    def extract_file_features(self):
+        yield from extract_file_features(self.pe)
+
+    def is_dotnet_file(self) -> bool:
+        return bool(self.pe.net)
+
+    def is_mixed_mode(self) -> bool:
+        assert self.pe is not None
+        assert self.pe.net is not None
+        assert self.pe.net.Flags is not None
+
+        return not bool(self.pe.net.Flags.CLR_ILONLY)
+
+    def get_runtime_version(self) -> Tuple[int, int]:
+        assert self.pe is not None
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+
+        return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
+
+    def get_meta_version_string(self) -> str:
+        assert self.pe.net is not None
+        assert self.pe.net.metadata is not None
+        assert self.pe.net.metadata.struct is not None
+        assert self.pe.net.metadata.struct.Version is not None
+
+        vbuf = self.pe.net.metadata.struct.Version
+        assert isinstance(vbuf, bytes)
+
+        return vbuf.rstrip(b"\x00").decode("utf-8")
+
+    def get_functions(self):
+        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
+
+    def extract_function_features(self, f):
+        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
+
+    def get_basic_blocks(self, f):
+        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
+
+    def extract_basic_block_features(self, f, bb):
+        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
+
+    def get_instructions(self, f, bb):
+        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
+
+    def extract_insn_features(self, f, bb, insn):
+        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
+
+    def is_library_function(self, va):
+        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
+
+    def get_function_name(self, va):
+        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")

capa/features/extractors/dotnetfile.py

@@ -0,0 +1,239 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import Tuple, Iterator
+from pathlib import Path
+
+import dnfile
+import pefile
+
+import capa.features.extractors.helpers
+from capa.features.file import Import, FunctionName
+from capa.features.common import (
+    OS,
+    OS_ANY,
+    ARCH_ANY,
+    ARCH_I386,
+    FORMAT_PE,
+    ARCH_AMD64,
+    FORMAT_DOTNET,
+    Arch,
+    Class,
+    Format,
+    String,
+    Feature,
+    Namespace,
+    Characteristic,
+)
+from capa.features.address import NO_ADDRESS, Address, DNTokenAddress
+from capa.features.extractors.base_extractor import FeatureExtractor
+from capa.features.extractors.dnfile.helpers import (
+    DnType,
+    iter_dotnet_table,
+    is_dotnet_mixed_mode,
+    get_dotnet_managed_imports,
+    get_dotnet_managed_methods,
+    calculate_dotnet_token_value,
+    get_dotnet_unmanaged_imports,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def extract_file_format(**kwargs) -> Iterator[Tuple[Format, Address]]:
+    yield Format(FORMAT_PE), NO_ADDRESS
+    yield Format(FORMAT_DOTNET), NO_ADDRESS
+
+
+def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Import, Address]]:
+    for method in get_dotnet_managed_imports(pe):
+        # like System.IO.File::OpenRead
+        yield Import(str(method)), DNTokenAddress(method.token)
+
+    for imp in get_dotnet_unmanaged_imports(pe):
+        # like kernel32.CreateFileA
+        for name in capa.features.extractors.helpers.generate_symbols(imp.module, imp.method):
+            yield Import(name), DNTokenAddress(imp.token)
+
+
+def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[FunctionName, Address]]:
+    for method in get_dotnet_managed_methods(pe):
+        yield FunctionName(str(method)), DNTokenAddress(method.token)
+
+
+def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Namespace, Address]]:
+    """emit namespace features from TypeRef and TypeDef tables"""
+
+    # namespaces may be referenced multiple times, so we need to filter
+    namespaces = set()
+
+    for _, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        # emit internal .NET namespaces
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+        namespaces.add(typedef.TypeNamespace)
+
+    for _, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
+        # emit external .NET namespaces
+        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
+        namespaces.add(typeref.TypeNamespace)
+
+    # namespaces may be empty, discard
+    namespaces.discard("")
+
+    for namespace in namespaces:
+        # namespace do not have an associated token, so we yield 0x0
+        yield Namespace(namespace), NO_ADDRESS
+
+
+def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Class, Address]]:
+    """emit class features from TypeRef and TypeDef tables"""
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        # emit internal .NET classes
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        token = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
+        yield Class(DnType.format_name(typedef.TypeName, namespace=typedef.TypeNamespace)), DNTokenAddress(token)
+
+    for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
+        # emit external .NET classes
+        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
+
+        token = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
+        yield Class(DnType.format_name(typeref.TypeName, namespace=typeref.TypeNamespace)), DNTokenAddress(token)
+
+
+def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
+    yield OS(OS_ANY), NO_ADDRESS
+
+
+def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Arch, Address]]:
+    # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
+    # .NET 4.5 added option: any CPU, 32-bit preferred
+    assert pe.net is not None
+    assert pe.net.Flags is not None
+
+    if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
+        yield Arch(ARCH_I386), NO_ADDRESS
+    elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+    else:
+        yield Arch(ARCH_ANY), NO_ADDRESS
+
+
+def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[String, Address]]:
+    yield from capa.features.extractors.common.extract_file_strings(pe.__data__)
+
+
+def extract_file_mixed_mode_characteristic_features(
+    pe: dnfile.dnPE, **kwargs
+) -> Iterator[Tuple[Characteristic, Address]]:
+    if is_dotnet_mixed_mode(pe):
+        yield Characteristic("mixed mode"), NO_ADDRESS
+
+
+def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
+    for file_handler in FILE_HANDLERS:
+        for feature, addr in file_handler(pe=pe):  # type: ignore
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_file_import_names,
+    extract_file_function_names,
+    extract_file_strings,
+    extract_file_format,
+    extract_file_mixed_mode_characteristic_features,
+    extract_file_namespace_features,
+    extract_file_class_features,
+)
+
+
+def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
+    for handler in GLOBAL_HANDLERS:
+        for feature, va in handler(pe=pe):  # type: ignore
+            yield feature, va
+
+
+GLOBAL_HANDLERS = (
+    extract_file_os,
+    extract_file_arch,
+)
+
+
+class DotnetFileFeatureExtractor(FeatureExtractor):
+    def __init__(self, path: Path):
+        super().__init__()
+        self.path: Path = path
+        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+
+    def get_base_address(self):
+        return NO_ADDRESS
+
+    def get_entry_point(self) -> int:
+        # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
+        #  True: native EP: Token
+        #  False: managed EP: RVA
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+
+        return self.pe.net.struct.EntryPointTokenOrRva
+
+    def extract_global_features(self):
+        yield from extract_global_features(self.pe)
+
+    def extract_file_features(self):
+        yield from extract_file_features(self.pe)
+
+    def is_dotnet_file(self) -> bool:
+        return bool(self.pe.net)
+
+    def is_mixed_mode(self) -> bool:
+        return is_dotnet_mixed_mode(self.pe)
+
+    def get_runtime_version(self) -> Tuple[int, int]:
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+        assert self.pe.net.struct.MajorRuntimeVersion is not None
+        assert self.pe.net.struct.MinorRuntimeVersion is not None
+
+        return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
+
+    def get_meta_version_string(self) -> str:
+        assert self.pe.net is not None
+        assert self.pe.net.metadata is not None
+        assert self.pe.net.metadata.struct is not None
+        assert self.pe.net.metadata.struct.Version is not None
+
+        vbuf = self.pe.net.metadata.struct.Version
+        assert isinstance(vbuf, bytes)
+
+        return vbuf.rstrip(b"\x00").decode("utf-8")
+
+    def get_functions(self):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def extract_function_features(self, f):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def get_basic_blocks(self, f):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def extract_basic_block_features(self, f, bb):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def get_instructions(self, f, bb):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def extract_insn_features(self, f, bb, insn):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def is_library_function(self, va):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def get_function_name(self, va):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")

capa/features/extractors/elffile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,15 +7,15 @@
 # See the License for the specific language governing permissions and limitations under the License.
 import io
 import logging
-import contextlib
 from typing import Tuple, Iterator
+from pathlib import Path
 
 from elftools.elf.elffile import ELFFile, SymbolTableSection
 
 import capa.features.extractors.common
 from capa.features.file import Import, Section
 from capa.features.common import OS, FORMAT_ELF, Arch, Format, Feature
-from capa.features.extractors.elf import Arch as ElfArch
+from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
 from capa.features.extractors.base_extractor import FeatureExtractor
 
 logger = logging.getLogger(__name__)
@@ -25,29 +25,29 @@ def extract_file_import_names(elf, **kwargs):
     # see https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/scripts/readelf.py#L372
     symbol_tables = [(idx, s) for idx, s in enumerate(elf.iter_sections()) if isinstance(s, SymbolTableSection)]
 
-    for section_index, section in symbol_tables:
+    for _, section in symbol_tables:
         if not isinstance(section, SymbolTableSection):
             continue
 
         if section["sh_entsize"] == 0:
-            logger.debug("Symbol table '%s' has a sh_entsize of zero!" % (section.name))
+            logger.debug("Symbol table '%s' has a sh_entsize of zero!", section.name)
             continue
 
-        logger.debug("Symbol table '%s' contains %s entries:" % (section.name, section.num_symbols()))
+        logger.debug("Symbol table '%s' contains %s entries:", section.name, section.num_symbols())
 
-        for nsym, symbol in enumerate(section.iter_symbols()):
+        for _, symbol in enumerate(section.iter_symbols()):
             if symbol.name and symbol.entry.st_info.type == "STT_FUNC":
-                # TODO symbol address
-                # TODO symbol version info?
-                yield Import(symbol.name), 0x0
+                # TODO(williballenthin): extract symbol address
+                # https://github.com/mandiant/capa/issues/1608
+                yield Import(symbol.name), FileOffsetAddress(0x0)
 
 
 def extract_file_section_names(elf, **kwargs):
     for section in elf.iter_sections():
         if section.name:
-            yield Section(section.name), section.header.sh_addr
+            yield Section(section.name), AbsoluteVirtualAddress(section.header.sh_addr)
         elif section.is_null():
-            yield Section("NULL"), section.header.sh_addr
+            yield Section("NULL"), AbsoluteVirtualAddress(section.header.sh_addr)
 
 
 def extract_file_strings(buf, **kwargs):
@@ -58,35 +58,35 @@ def extract_file_os(elf, buf, **kwargs):
     # our current approach does not always get an OS value, e.g. for packed samples
     # for file limitation purposes, we're more lax here
     try:
-        os = next(capa.features.extractors.common.extract_os(buf))
-        yield os
+        os_tuple = next(capa.features.extractors.common.extract_os(buf))
+        yield os_tuple
     except StopIteration:
-        yield OS("unknown"), 0x0
+        yield OS("unknown"), NO_ADDRESS
 
 
 def extract_file_format(**kwargs):
-    yield Format(FORMAT_ELF), 0x0
+    yield Format(FORMAT_ELF), NO_ADDRESS
 
 
 def extract_file_arch(elf, **kwargs):
-    # TODO merge with capa.features.extractors.elf.detect_elf_arch()
     arch = elf.get_machine_arch()
     if arch == "x86":
-        yield Arch(ElfArch.I386), 0x0
+        yield Arch("i386"), NO_ADDRESS
     elif arch == "x64":
-        yield Arch(ElfArch.AMD64), 0x0
+        yield Arch("amd64"), NO_ADDRESS
     else:
         logger.warning("unsupported architecture: %s", arch)
 
 
 def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
     for file_handler in FILE_HANDLERS:
-        for feature, va in file_handler(elf=elf, buf=buf):  # type: ignore
-            yield feature, va
+        for feature, addr in file_handler(elf=elf, buf=buf):  # type: ignore
+            yield feature, addr
 
 
 FILE_HANDLERS = (
-    # TODO extract_file_export_names,
+    # TODO(williballenthin): implement extract_file_export_names
+    # https://github.com/mandiant/capa/issues/1607
     extract_file_import_names,
     extract_file_section_names,
     extract_file_strings,
@@ -97,8 +97,8 @@ FILE_HANDLERS = (
 
 def extract_global_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
     for global_handler in GLOBAL_HANDLERS:
-        for feature, va in global_handler(elf=elf, buf=buf):  # type: ignore
-            yield feature, va
+        for feature, addr in global_handler(elf=elf, buf=buf):  # type: ignore
+            yield feature, addr
 
 
 GLOBAL_HANDLERS = (
@@ -108,31 +108,28 @@ GLOBAL_HANDLERS = (
 
 
 class ElfFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
-        super(ElfFeatureExtractor, self).__init__()
-        self.path = path
-        with open(self.path, "rb") as f:
-            self.elf = ELFFile(io.BytesIO(f.read()))
+    def __init__(self, path: Path):
+        super().__init__()
+        self.path: Path = path
+        self.elf = ELFFile(io.BytesIO(path.read_bytes()))
 
     def get_base_address(self):
         # virtual address of the first segment with type LOAD
         for segment in self.elf.iter_segments():
             if segment.header.p_type == "PT_LOAD":
-                return segment.header.p_vaddr
+                return AbsoluteVirtualAddress(segment.header.p_vaddr)
 
     def extract_global_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = self.path.read_bytes()
 
-        for feature, va in extract_global_features(self.elf, buf):
-            yield feature, va
+        for feature, addr in extract_global_features(self.elf, buf):
+            yield feature, addr
 
     def extract_file_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = self.path.read_bytes()
 
-        for feature, va in extract_file_features(self.elf, buf):
-            yield feature, va
+        for feature, addr in extract_file_features(self.elf, buf):
+            yield feature, addr
 
     def get_functions(self):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
@@ -152,8 +149,8 @@ class ElfFeatureExtractor(FeatureExtractor):
     def extract_insn_features(self, f, bb, insn):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
 
-    def is_library_function(self, va):
+    def is_library_function(self, addr):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
 
-    def get_function_name(self, va):
+    def get_function_name(self, addr):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")

capa/features/extractors/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -29,8 +29,7 @@ def is_aw_function(symbol: str) -> bool:
     if symbol[-1] not in ("A", "W"):
         return False
 
-    # second to last character should be lowercase letter
-    return "a" <= symbol[-2] <= "z" or "0" <= symbol[-2] <= "9"
+    return True
 
 
 def is_ordinal(symbol: str) -> bool:
@@ -52,8 +51,11 @@ def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
       - CreateFileA
       - CreateFile
     """
+    # normalize dll name
+    dll = dll.lower()
+
     # kernel32.CreateFileA
-    yield "%s.%s" % (dll, symbol)
+    yield f"{dll}.{symbol}"
 
     if not is_ordinal(symbol):
         # CreateFileA
@@ -61,13 +63,30 @@ def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
 
     if is_aw_function(symbol):
         # kernel32.CreateFile
-        yield "%s.%s" % (dll, symbol[:-1])
+        yield f"{dll}.{symbol[:-1]}"
 
         if not is_ordinal(symbol):
             # CreateFile
             yield symbol[:-1]
 
 
+def reformat_forwarded_export_name(forwarded_name: str) -> str:
+    """
+    a forwarded export has a DLL name/path an symbol name.
+    we want the former to be lowercase, and the latter to be verbatim.
+    """
+
+    # use rpartition so we can split on separator between dll and name.
+    # the dll name can be a full path, like in the case of
+    # ef64d6d7c34250af8e21a10feb931c9b
+    # which i assume means the path can have embedded periods.
+    # so we don't want the first period, we want the last.
+    forwarded_dll, _, forwarded_symbol = forwarded_name.rpartition(".")
+    forwarded_dll = forwarded_dll.lower()
+
+    return f"{forwarded_dll}.{forwarded_symbol}"
+
+
 def all_zeros(bytez: bytes) -> bool:
     return all(b == 0 for b in builtins.bytes(bytez))
 
@@ -110,7 +129,6 @@ def carve_pe(pbytes: bytes, offset: int = 0) -> Iterator[Tuple[int, int]]:
     todo = [(off, mzx, pex, key) for (off, mzx, pex, key) in todo if off != -1]
 
     while len(todo):
-
         off, mzx, pex, key = todo.pop()
 
         # The MZ header has one field we will check

capa/features/extractors/ida/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,22 +8,21 @@
 
 import string
 import struct
+from typing import Tuple, Iterator
 
 import idaapi
 
 import capa.features.extractors.ida.helpers
-from capa.features.common import Characteristic
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
 from capa.features.basicblock import BasicBlock
 from capa.features.extractors.ida import helpers
 from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
 
 
-def get_printable_len(op):
-    """Return string length if all operand bytes are ascii or utf16-le printable
-
-    args:
-        op (IDA op_t)
-    """
+def get_printable_len(op: idaapi.op_t) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
     op_val = capa.features.extractors.ida.helpers.mask_op_val(op)
 
     if op.dtype == idaapi.dt_byte:
@@ -35,14 +34,14 @@ def get_printable_len(op):
     elif op.dtype == idaapi.dt_qword:
         chars = struct.pack("<Q", op_val)
     else:
-        raise ValueError("Unhandled operand data type 0x%x." % op.dtype)
+        raise ValueError(f"Unhandled operand data type 0x{op.dtype:x}.")
 
-    def is_printable_ascii(chars):
-        return all(c < 127 and chr(c) in string.printable for c in chars)
+    def is_printable_ascii(chars_: bytes):
+        return all(c < 127 and chr(c) in string.printable for c in chars_)
 
-    def is_printable_utf16le(chars):
-        if all(c == 0x00 for c in chars[1::2]):
-            return is_printable_ascii(chars[::2])
+    def is_printable_utf16le(chars_: bytes):
+        if all(c == 0x00 for c in chars_[1::2]):
+            return is_printable_ascii(chars_[::2])
 
     if is_printable_ascii(chars):
         return idaapi.get_dtype_size(op.dtype)
@@ -53,12 +52,8 @@ def get_printable_len(op):
     return 0
 
 
-def is_mov_imm_to_stack(insn):
-    """verify instruction moves immediate onto stack
-
-    args:
-        insn (IDA insn_t)
-    """
+def is_mov_imm_to_stack(insn: idaapi.insn_t) -> bool:
+    """verify instruction moves immediate onto stack"""
     if insn.Op2.type != idaapi.o_imm:
         return False
 
@@ -71,14 +66,10 @@ def is_mov_imm_to_stack(insn):
     return True
 
 
-def bb_contains_stackstring(f, bb):
+def bb_contains_stackstring(f: idaapi.func_t, bb: idaapi.BasicBlock) -> bool:
     """check basic block for stackstring indicators
 
     true if basic block contains enough moves of constant bytes to the stack
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
     """
     count = 0
     for insn in capa.features.extractors.ida.helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
@@ -89,57 +80,27 @@ def bb_contains_stackstring(f, bb):
     return False
 
 
-def extract_bb_stackstring(f, bb):
-    """extract stackstring indicators from basic block
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-    """
-    if bb_contains_stackstring(f, bb):
-        yield Characteristic("stack string"), bb.start_ea
+def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract stackstring indicators from basic block"""
+    if bb_contains_stackstring(fh.inner, bbh.inner):
+        yield Characteristic("stack string"), bbh.address
 
 
-def extract_bb_tight_loop(f, bb):
-    """extract tight loop indicators from a basic block
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-    """
-    if capa.features.extractors.ida.helpers.is_basic_block_tight_loop(bb):
-        yield Characteristic("tight loop"), bb.start_ea
+def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract tight loop indicators from a basic block"""
+    if capa.features.extractors.ida.helpers.is_basic_block_tight_loop(bbh.inner):
+        yield Characteristic("tight loop"), bbh.address
 
 
-def extract_features(f, bb):
-    """extract basic block features
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-    """
+def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract basic block features"""
     for bb_handler in BASIC_BLOCK_HANDLERS:
-        for (feature, ea) in bb_handler(f, bb):
-            yield feature, ea
-    yield BasicBlock(), bb.start_ea
+        for feature, addr in bb_handler(fh, bbh):
+            yield feature, addr
+    yield BasicBlock(), bbh.address
 
 
 BASIC_BLOCK_HANDLERS = (
     extract_bb_tight_loop,
     extract_bb_stackstring,
 )
-
-
-def main():
-    features = []
-    for f in helpers.get_functions(skip_thunks=True, skip_libs=True):
-        for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
-            features.extend(list(extract_features(f, bb)))
-
-    import pprint
-
-    pprint.pprint(features)
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ida/extractor.py

@@ -1,10 +1,12 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Tuple, Iterator
+
 import idaapi
 
 import capa.ida.helpers
@@ -14,57 +16,21 @@ import capa.features.extractors.ida.insn
 import capa.features.extractors.ida.global_
 import capa.features.extractors.ida.function
 import capa.features.extractors.ida.basicblock
-from capa.features.extractors.base_extractor import FeatureExtractor
-
-
-class FunctionHandle:
-    """this acts like an idaapi.func_t but with __int__()"""
-
-    def __init__(self, inner):
-        self._inner = inner
-
-    def __int__(self):
-        return self.start_ea
-
-    def __getattr__(self, name):
-        return getattr(self._inner, name)
-
-
-class BasicBlockHandle:
-    """this acts like an idaapi.BasicBlock but with __int__()"""
-
-    def __init__(self, inner):
-        self._inner = inner
-
-    def __int__(self):
-        return self.start_ea
-
-    def __getattr__(self, name):
-        return getattr(self._inner, name)
-
-
-class InstructionHandle:
-    """this acts like an idaapi.insn_t but with __int__()"""
-
-    def __init__(self, inner):
-        self._inner = inner
-
-    def __int__(self):
-        return self.ea
-
-    def __getattr__(self, name):
-        return getattr(self._inner, name)
+from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
 
 
 class IdaFeatureExtractor(FeatureExtractor):
     def __init__(self):
-        super(IdaFeatureExtractor, self).__init__()
-        self.global_features = []
+        super().__init__()
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.ida.file.extract_file_format())
         self.global_features.extend(capa.features.extractors.ida.global_.extract_os())
         self.global_features.extend(capa.features.extractors.ida.global_.extract_arch())
 
     def get_base_address(self):
-        return idaapi.get_imagebase()
+        return AbsoluteVirtualAddress(idaapi.get_imagebase())
 
     def extract_global_features(self):
         yield from self.global_features
@@ -72,41 +38,34 @@ class IdaFeatureExtractor(FeatureExtractor):
     def extract_file_features(self):
         yield from capa.features.extractors.ida.file.extract_features()
 
-    def get_functions(self):
+    def get_functions(self) -> Iterator[FunctionHandle]:
         import capa.features.extractors.ida.helpers as ida_helpers
 
-        # data structure shared across functions yielded here.
-        # useful for caching analysis relevant across a single workspace.
-        ctx = {}
-
         # ignore library functions and thunk functions as identified by IDA
-        for f in ida_helpers.get_functions(skip_thunks=True, skip_libs=True):
-            setattr(f, "ctx", ctx)
-            yield FunctionHandle(f)
+        yield from ida_helpers.get_functions(skip_thunks=True, skip_libs=True)
 
     @staticmethod
-    def get_function(ea):
+    def get_function(ea: int) -> FunctionHandle:
         f = idaapi.get_func(ea)
-        setattr(f, "ctx", {})
-        return FunctionHandle(f)
+        return FunctionHandle(address=AbsoluteVirtualAddress(f.start_ea), inner=f)
 
-    def extract_function_features(self, f):
-        yield from capa.features.extractors.ida.function.extract_features(f)
+    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.ida.function.extract_features(fh)
 
-    def get_basic_blocks(self, f):
+    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
         import capa.features.extractors.ida.helpers as ida_helpers
 
-        for bb in ida_helpers.get_function_blocks(f):
-            yield BasicBlockHandle(bb)
+        for bb in ida_helpers.get_function_blocks(fh.inner):
+            yield BBHandle(address=AbsoluteVirtualAddress(bb.start_ea), inner=bb)
 
-    def extract_basic_block_features(self, f, bb):
-        yield from capa.features.extractors.ida.basicblock.extract_features(f, bb)
+    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.ida.basicblock.extract_features(fh, bbh)
 
-    def get_instructions(self, f, bb):
+    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
         import capa.features.extractors.ida.helpers as ida_helpers
 
-        for insn in ida_helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
-            yield InstructionHandle(insn)
+        for insn in ida_helpers.get_instructions_in_range(bbh.inner.start_ea, bbh.inner.end_ea):
+            yield InsnHandle(address=AbsoluteVirtualAddress(insn.ea), inner=insn)
 
-    def extract_insn_features(self, f, bb, insn):
-        yield from capa.features.extractors.ida.insn.extract_features(f, bb, insn)
+    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
+        yield from capa.features.extractors.ida.insn.extract_features(fh, bbh, ih)

capa/features/extractors/ida/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,27 +7,29 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import struct
+from typing import Tuple, Iterator
 
 import idc
 import idaapi
 import idautils
-import ida_loader
+import ida_entry
 
+import capa.features.extractors.common
 import capa.features.extractors.helpers
 import capa.features.extractors.strings
 import capa.features.extractors.ida.helpers
 from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Format, String, Characteristic
+from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
+
+MAX_OFFSET_PE_AFTER_MZ = 0x200
 
 
-def check_segment_for_pe(seg):
+def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
     """check segment for embedded PE
 
     adapted for IDA from:
-    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
-
-    args:
-        seg (IDA segment_t)
+    https://github.com/vivisect/vivisect/blob/91e8419a861f49779f18316f155311967e696836/PE/carve.py#L25
     """
     seg_max = seg.end_ea
     mz_xor = [
@@ -40,14 +42,15 @@ def check_segment_for_pe(seg):
     ]
 
     todo = []
-    for (mzx, pex, i) in mz_xor:
+    for mzx, pex, i in mz_xor:
+        # find all segment offsets containing XOR'd "MZ" bytes
         for off in capa.features.extractors.ida.helpers.find_byte_sequence(seg.start_ea, seg.end_ea, mzx):
             todo.append((off, mzx, pex, i))
 
     while len(todo):
         off, mzx, pex, i = todo.pop()
 
-        # The MZ header has one field we will check e_lfanew is at 0x3c
+        # MZ header has one field we will check e_lfanew is at 0x3c
         e_lfanew = off + 0x3C
 
         if seg_max < (e_lfanew + 4):
@@ -55,18 +58,19 @@ def check_segment_for_pe(seg):
 
         newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(idc.get_bytes(e_lfanew, 4), i))[0]
 
+        # assume XOR'd "PE" bytes exist within threshold
+        if newoff > MAX_OFFSET_PE_AFTER_MZ:
+            continue
+
         peoff = off + newoff
         if seg_max < (peoff + 2):
             continue
 
         if idc.get_bytes(peoff, 2) == pex:
-            yield (off, i)
-
-        for nextres in capa.features.extractors.ida.helpers.find_byte_sequence(off + 1, seg.end_ea, mzx):
-            todo.append((nextres, mzx, pex, i))
+            yield off, i
 
 
-def extract_file_embedded_pe():
+def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
     """extract embedded PE features
 
     IDA must load resource sections for this to be complete
@@ -74,17 +78,23 @@ def extract_file_embedded_pe():
         - Check 'Load resource sections' when opening binary in IDA manually
     """
     for seg in capa.features.extractors.ida.helpers.get_segments(skip_header_segments=True):
-        for (ea, _) in check_segment_for_pe(seg):
-            yield Characteristic("embedded pe"), ea
+        for ea, _ in check_segment_for_pe(seg):
+            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
 
 
-def extract_file_export_names():
+def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
     """extract function exports"""
-    for (_, _, ea, name) in idautils.Entries():
-        yield Export(name), ea
+    for _, ordinal, ea, name in idautils.Entries():
+        forwarded_name = ida_entry.get_entry_forwarder(ordinal)
+        if forwarded_name is None:
+            yield Export(name), AbsoluteVirtualAddress(ea)
+        else:
+            forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
+            yield Export(forwarded_name), AbsoluteVirtualAddress(ea)
+            yield Characteristic("forwarded export"), AbsoluteVirtualAddress(ea)
 
 
-def extract_file_import_names():
+def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
     """extract function imports
 
     1. imports by ordinal:
@@ -95,28 +105,32 @@ def extract_file_import_names():
      - modulename.importname
      - importname
     """
-    for (ea, info) in capa.features.extractors.ida.helpers.get_file_imports().items():
+    for ea, info in capa.features.extractors.ida.helpers.get_file_imports().items():
+        addr = AbsoluteVirtualAddress(ea)
         if info[1] and info[2]:
             # e.g. in mimikatz: ('cabinet', 'FCIAddFile', 11L)
             # extract by name here and by ordinal below
             for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1]):
-                yield Import(name), ea
+                yield Import(name), addr
             dll = info[0]
-            symbol = "#%d" % (info[2])
+            symbol = f"#{info[2]}"
         elif info[1]:
             dll = info[0]
             symbol = info[1]
         elif info[2]:
             dll = info[0]
-            symbol = "#%d" % (info[2])
+            symbol = f"#{info[2]}"
         else:
             continue
 
         for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-            yield Import(name), ea
+            yield Import(name), addr
+
+    for ea, info in capa.features.extractors.ida.helpers.get_file_externs().items():
+        yield Import(info[1]), AbsoluteVirtualAddress(ea)
 
 
-def extract_file_section_names():
+def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
     """extract section names
 
     IDA must load resource sections for this to be complete
@@ -124,10 +138,10 @@ def extract_file_section_names():
         - Check 'Load resource sections' when opening binary in IDA manually
     """
     for seg in capa.features.extractors.ida.helpers.get_segments(skip_header_segments=True):
-        yield Section(idaapi.get_segm_name(seg)), seg.start_ea
+        yield Section(idaapi.get_segm_name(seg)), AbsoluteVirtualAddress(seg.start_ea)
 
 
-def extract_file_strings():
+def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
     """extract ASCII and UTF-16 LE strings
 
     IDA must load resource sections for this to be complete
@@ -137,41 +151,50 @@ def extract_file_strings():
     for seg in capa.features.extractors.ida.helpers.get_segments():
         seg_buff = capa.features.extractors.ida.helpers.get_segment_buffer(seg)
 
+        # differing to common string extractor factor in segment offset here
         for s in capa.features.extractors.strings.extract_ascii_strings(seg_buff):
-            yield String(s.s), (seg.start_ea + s.offset)
+            yield String(s.s), FileOffsetAddress(seg.start_ea + s.offset)
 
         for s in capa.features.extractors.strings.extract_unicode_strings(seg_buff):
-            yield String(s.s), (seg.start_ea + s.offset)
+            yield String(s.s), FileOffsetAddress(seg.start_ea + s.offset)
 
 
-def extract_file_function_names():
+def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
     """
     extract the names of statically-linked library functions.
     """
     for ea in idautils.Functions():
+        addr = AbsoluteVirtualAddress(ea)
         if idaapi.get_func(ea).flags & idaapi.FUNC_LIB:
             name = idaapi.get_name(ea)
-            yield FunctionName(name), ea
+            yield FunctionName(name), addr
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                yield FunctionName(name[1:]), addr
 
 
-def extract_file_format():
-    format_name = ida_loader.get_file_type_name()
+def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
+    file_info = idaapi.get_inf_structure()
 
-    if "PE" in format_name:
-        yield Format(FORMAT_PE), 0x0
-    elif "ELF64" in format_name:
-        yield Format(FORMAT_ELF), 0x0
-    elif "ELF32" in format_name:
-        yield Format(FORMAT_ELF), 0x0
+    if file_info.filetype in (idaapi.f_PE, idaapi.f_COFF):
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif file_info.filetype == idaapi.f_ELF:
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    elif file_info.filetype == idaapi.f_BIN:
+        # no file type to return when processing a binary file, but we want to continue processing
+        return
     else:
-        raise NotImplementedError("file format: %s", format_name)
+        raise NotImplementedError(f"unexpected file format: {file_info.filetype}")
 
 
-def extract_features():
+def extract_features() -> Iterator[Tuple[Feature, Address]]:
     """extract file features"""
     for file_handler in FILE_HANDLERS:
-        for feature, va in file_handler():
-            yield feature, va
+        for feature, addr in file_handler():
+            yield feature, addr
 
 
 FILE_HANDLERS = (
@@ -183,14 +206,3 @@ FILE_HANDLERS = (
     extract_file_function_names,
     extract_file_format,
 )
-
-
-def main():
-    """ """
-    import pprint
-
-    pprint.pprint(list(extract_features()))
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ida/function.py

@@ -1,35 +1,31 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+from typing import Tuple, Iterator
 
 import idaapi
 import idautils
 
 import capa.features.extractors.ida.helpers
-from capa.features.common import Characteristic
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
 from capa.features.extractors import loops
+from capa.features.extractors.base_extractor import FunctionHandle
 
 
-def extract_function_calls_to(f):
-    """extract callers to a function
-
-    args:
-        f (IDA func_t)
-    """
-    for ea in idautils.CodeRefsTo(f.start_ea, True):
-        yield Characteristic("calls to"), ea
+def extract_function_calls_to(fh: FunctionHandle):
+    """extract callers to a function"""
+    for ea in idautils.CodeRefsTo(fh.inner.start_ea, True):
+        yield Characteristic("calls to"), AbsoluteVirtualAddress(ea)
 
 
-def extract_function_loop(f):
-    """extract loop indicators from a function
-
-    args:
-        f (IDA func_t)
-    """
+def extract_function_loop(fh: FunctionHandle):
+    """extract loop indicators from a function"""
+    f: idaapi.func_t = fh.inner
     edges = []
 
     # construct control flow graph
@@ -38,43 +34,19 @@ def extract_function_loop(f):
             edges.append((bb.start_ea, succ.start_ea))
 
     if loops.has_loop(edges):
-        yield Characteristic("loop"), f.start_ea
+        yield Characteristic("loop"), fh.address
 
 
-def extract_recursive_call(f):
-    """extract recursive function call
-
-    args:
-        f (IDA func_t)
-    """
-    if capa.features.extractors.ida.helpers.is_function_recursive(f):
-        yield Characteristic("recursive call"), f.start_ea
+def extract_recursive_call(fh: FunctionHandle):
+    """extract recursive function call"""
+    if capa.features.extractors.ida.helpers.is_function_recursive(fh.inner):
+        yield Characteristic("recursive call"), fh.address
 
 
-def extract_features(f):
-    """extract function features
-
-    arg:
-        f (IDA func_t)
-    """
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
     for func_handler in FUNCTION_HANDLERS:
-        for (feature, ea) in func_handler(f):
-            yield feature, ea
+        for feature, addr in func_handler(fh):
+            yield feature, addr
 
 
 FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
-
-
-def main():
-    """ """
-    features = []
-    for f in capa.features.extractors.ida.get_functions(skip_thunks=True, skip_libs=True):
-        features.extend(list(extract_features(f)))
-
-    import pprint
-
-    pprint.pprint(features)
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ida/global_.py

@@ -1,27 +1,36 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import logging
 import contextlib
+from typing import Tuple, Iterator
 
 import idaapi
 import ida_loader
 
 import capa.ida.helpers
 import capa.features.extractors.elf
-from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch
+from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
+from capa.features.address import NO_ADDRESS, Address
 
 logger = logging.getLogger(__name__)
 
 
-def extract_os():
-    format_name = ida_loader.get_file_type_name()
+def extract_os() -> Iterator[Tuple[Feature, Address]]:
+    format_name: str = ida_loader.get_file_type_name()
 
     if "PE" in format_name:
-        yield OS(OS_WINDOWS), 0x0
+        yield OS(OS_WINDOWS), NO_ADDRESS
 
     elif "ELF" in format_name:
         with contextlib.closing(capa.ida.helpers.IDAIO()) as f:
             os = capa.features.extractors.elf.detect_elf_os(f)
 
-        yield OS(os), 0x0
+        yield OS(os), NO_ADDRESS
 
     else:
         # we likely end up here:
@@ -29,7 +38,7 @@ def extract_os():
         #  2. handling a new file format (e.g. macho)
         #
         # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the OS,
+        # we could maybe accept a further CLI argument to specify the OS,
         # but i think this would be rarely used.
         # rules that rely on OS conditions will fail to match on shellcode.
         #
@@ -38,12 +47,12 @@ def extract_os():
         return
 
 
-def extract_arch():
-    info = idaapi.get_inf_structure()
+def extract_arch() -> Iterator[Tuple[Feature, Address]]:
+    info: idaapi.idainfo = idaapi.get_inf_structure()
     if info.procname == "metapc" and info.is_64bit():
-        yield Arch(ARCH_AMD64), 0x0
+        yield Arch(ARCH_AMD64), NO_ADDRESS
     elif info.procname == "metapc" and info.is_32bit():
-        yield Arch(ARCH_I386), 0x0
+        yield Arch(ARCH_I386), NO_ADDRESS
     elif info.procname == "metapc":
         logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel")
         return

capa/features/extractors/ida/helpers.py

@@ -1,18 +1,24 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import functools
+from typing import Any, Dict, Tuple, Iterator, Optional
 
 import idc
 import idaapi
 import idautils
 import ida_bytes
+import ida_segment
+
+from capa.features.address import AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import FunctionHandle
 
 
-def find_byte_sequence(start, end, seq):
+def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
     """yield all ea of a given byte sequence
 
     args:
@@ -20,32 +26,33 @@ def find_byte_sequence(start, end, seq):
         end: max virtual address
         seq: bytes to search e.g. b"\x01\x03"
     """
-    seq = " ".join(["%02x" % b for b in seq])
+    seqstr = " ".join([f"{b:02x}" for b in seq])
     while True:
-        ea = idaapi.find_binary(start, end, seq, 0, idaapi.SEARCH_DOWN)
+        # TODO(mike-hunhoff): find_binary is deprecated. Please use ida_bytes.bin_search() instead.
+        # https://github.com/mandiant/capa/issues/1606
+        ea = idaapi.find_binary(start, end, seqstr, 0, idaapi.SEARCH_DOWN)
         if ea == idaapi.BADADDR:
             break
         start = ea + 1
         yield ea
 
 
-def get_functions(start=None, end=None, skip_thunks=False, skip_libs=False):
+def get_functions(
+    start: Optional[int] = None, end: Optional[int] = None, skip_thunks: bool = False, skip_libs: bool = False
+) -> Iterator[FunctionHandle]:
     """get functions, range optional
 
     args:
         start: min virtual address
         end: max virtual address
-
-    ret:
-        yield func_t*
     """
     for ea in idautils.Functions(start=start, end=end):
         f = idaapi.get_func(ea)
         if not (skip_thunks and (f.flags & idaapi.FUNC_THUNK) or skip_libs and (f.flags & idaapi.FUNC_LIB)):
-            yield f
+            yield FunctionHandle(address=AbsoluteVirtualAddress(ea), inner=f)
 
 
-def get_segments(skip_header_segments=False):
+def get_segments(skip_header_segments=False) -> Iterator[idaapi.segment_t]:
     """get list of segments (sections) in the binary image
 
     args:
@@ -57,7 +64,7 @@ def get_segments(skip_header_segments=False):
             yield seg
 
 
-def get_segment_buffer(seg):
+def get_segment_buffer(seg: idaapi.segment_t) -> bytes:
     """return bytes stored in a given segment
 
     decrease buffer size until IDA is able to read bytes from the segment
@@ -75,9 +82,22 @@ def get_segment_buffer(seg):
     return buff if buff else b""
 
 
-def get_file_imports():
+def inspect_import(imports, library, ea, function, ordinal):
+    if function and function.startswith("__imp_"):
+        # handle mangled PE imports
+        function = function[len("__imp_") :]
+
+    if function and "@@" in function:
+        # handle mangled ELF imports, like "fopen@@glibc_2.2.5"
+        function, _, _ = function.partition("@@")
+
+    imports[ea] = (library.lower(), function, ordinal)
+    return True
+
+
+def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
     """get file imports"""
-    imports = {}
+    imports: Dict[int, Tuple[str, str, int]] = {}
 
     for idx in range(idaapi.get_import_module_qty()):
         library = idaapi.get_import_module_name(idx)
@@ -85,26 +105,38 @@ def get_file_imports():
         if not library:
             continue
 
-        def inspect_import(ea, function, ordinal):
-            if function and function.startswith("__imp_"):
-                # handle mangled names starting
-                function = function[len("__imp_") :]
-            imports[ea] = (library.lower(), function, ordinal)
-            return True
+        # IDA uses section names for the library of ELF imports, like ".dynsym".
+        # These are not useful to us, we may need to expand this list over time
+        # TODO(williballenthin): find all section names used by IDA
+        # https://github.com/mandiant/capa/issues/1419
+        if library == ".dynsym":
+            library = ""
 
-        idaapi.enum_import_names(idx, inspect_import)
+        cb = functools.partial(inspect_import, imports, library)
+        idaapi.enum_import_names(idx, cb)
 
     return imports
 
 
-def get_instructions_in_range(start, end):
+def get_file_externs() -> Dict[int, Tuple[str, str, int]]:
+    externs = {}
+
+    for seg in get_segments(skip_header_segments=True):
+        if seg.type != ida_segment.SEG_XTRN:
+            continue
+
+        for ea in idautils.Functions(seg.start_ea, seg.end_ea):
+            externs[ea] = ("", idaapi.get_func_name(ea), -1)
+
+    return externs
+
+
+def get_instructions_in_range(start: int, end: int) -> Iterator[idaapi.insn_t]:
     """yield instructions in range
 
     args:
         start: virtual address (inclusive)
         end: virtual address (exclusive)
-    yield:
-        (insn_t*)
     """
     for head in idautils.Heads(start, end):
         insn = idautils.DecodeInstruction(head)
@@ -112,7 +144,7 @@ def get_instructions_in_range(start, end):
             yield insn
 
 
-def is_operand_equal(op1, op2):
+def is_operand_equal(op1: idaapi.op_t, op2: idaapi.op_t) -> bool:
     """compare two IDA op_t"""
     if op1.flags != op2.flags:
         return False
@@ -138,7 +170,7 @@ def is_operand_equal(op1, op2):
     return True
 
 
-def is_basic_block_equal(bb1, bb2):
+def is_basic_block_equal(bb1: idaapi.BasicBlock, bb2: idaapi.BasicBlock) -> bool:
     """compare two IDA BasicBlock"""
     if bb1.start_ea != bb2.start_ea:
         return False
@@ -152,12 +184,12 @@ def is_basic_block_equal(bb1, bb2):
     return True
 
 
-def basic_block_size(bb):
+def basic_block_size(bb: idaapi.BasicBlock) -> int:
     """calculate size of basic block"""
     return bb.end_ea - bb.start_ea
 
 
-def read_bytes_at(ea, count):
+def read_bytes_at(ea: int, count: int) -> bytes:
     """ """
     # check if byte has a value, see get_wide_byte doc
     if not idc.is_loaded(ea):
@@ -170,10 +202,10 @@ def read_bytes_at(ea, count):
         return idc.get_bytes(ea, count)
 
 
-def find_string_at(ea, min=4):
+def find_string_at(ea: int, min_: int = 4) -> str:
     """check if ASCII string exists at a given virtual address"""
     found = idaapi.get_strlit_contents(ea, -1, idaapi.STRTYPE_C)
-    if found and len(found) > min:
+    if found and len(found) >= min_:
         try:
             found = found.decode("ascii")
             # hacky check for IDA bug; get_strlit_contents also reads Unicode as
@@ -187,7 +219,7 @@ def find_string_at(ea, min=4):
     return ""
 
 
-def get_op_phrase_info(op):
+def get_op_phrase_info(op: idaapi.op_t) -> Dict:
     """parse phrase features from operand
 
     Pretty much dup of sark's implementation:
@@ -197,7 +229,8 @@ def get_op_phrase_info(op):
         return {}
 
     scale = 1 << ((op.specflag2 & 0xC0) >> 6)
-    offset = op.addr
+    # IDA ea_t may be 32- or 64-bit; we assume displacement can only be 32-bit
+    offset = op.addr & 0xFFFFFFFF
 
     if op.specflag1 == 0:
         index = None
@@ -224,46 +257,44 @@ def get_op_phrase_info(op):
     return {"base": base, "index": index, "scale": scale, "offset": offset}
 
 
-def is_op_write(insn, op):
+def is_op_write(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
     """Check if an operand is written to (destination operand)"""
     return idaapi.has_cf_chg(insn.get_canon_feature(), op.n)
 
 
-def is_op_read(insn, op):
+def is_op_read(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
     """Check if an operand is read from (source operand)"""
     return idaapi.has_cf_use(insn.get_canon_feature(), op.n)
 
 
-def is_op_offset(insn, op):
+def is_op_offset(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
     """Check is an operand has been marked as an offset (by auto-analysis or manually)"""
     flags = idaapi.get_flags(insn.ea)
     return ida_bytes.is_off(flags, op.n)
 
 
-def is_sp_modified(insn):
+def is_sp_modified(insn: idaapi.insn_t) -> bool:
     """determine if instruction modifies SP, ESP, RSP"""
-    for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
-        if op.reg == idautils.procregs.sp.reg and is_op_write(insn, op):
-            # register is stack and written
-            return True
-    return False
+    return any(
+        op.reg == idautils.procregs.sp.reg and is_op_write(insn, op)
+        for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,))
+    )
 
 
-def is_bp_modified(insn):
+def is_bp_modified(insn: idaapi.insn_t) -> bool:
     """check if instruction modifies BP, EBP, RBP"""
-    for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
-        if op.reg == idautils.procregs.bp.reg and is_op_write(insn, op):
-            # register is base and written
-            return True
-    return False
+    return any(
+        op.reg == idautils.procregs.bp.reg and is_op_write(insn, op)
+        for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,))
+    )
 
 
-def is_frame_register(reg):
+def is_frame_register(reg: int) -> bool:
     """check if register is sp or bp"""
     return reg in (idautils.procregs.sp.reg, idautils.procregs.bp.reg)
 
 
-def get_insn_ops(insn, target_ops=()):
+def get_insn_ops(insn: idaapi.insn_t, target_ops: Optional[Tuple[Any]] = None) -> idaapi.op_t:
     """yield op_t for instruction, filter on type if specified"""
     for op in insn.ops:
         if op.type == idaapi.o_void:
@@ -274,12 +305,12 @@ def get_insn_ops(insn, target_ops=()):
         yield op
 
 
-def is_op_stack_var(ea, index):
+def is_op_stack_var(ea: int, index: int) -> bool:
     """check if operand is a stack variable"""
     return idaapi.is_stkvar(idaapi.get_flags(ea), index)
 
 
-def mask_op_val(op):
+def mask_op_val(op: idaapi.op_t) -> int:
     """mask value by data type
 
     necessary due to a bug in AMD64
@@ -299,26 +330,15 @@ def mask_op_val(op):
     return masks.get(op.dtype, op.value) & op.value
 
 
-def is_function_recursive(f):
-    """check if function is recursive
-
-    args:
-        f (IDA func_t)
-    """
-    for ref in idautils.CodeRefsTo(f.start_ea, True):
-        if f.contains(ref):
-            return True
-    return False
+def is_function_recursive(f: idaapi.func_t) -> bool:
+    """check if function is recursive"""
+    return any(f.contains(ref) for ref in idautils.CodeRefsTo(f.start_ea, True))
 
 
-def is_basic_block_tight_loop(bb):
+def is_basic_block_tight_loop(bb: idaapi.BasicBlock) -> bool:
     """check basic block loops to self
 
     true if last instruction in basic block branches to basic block start
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
     """
     bb_end = idc.prev_head(bb.end_ea)
     if bb.start_ea < bb_end:
@@ -328,7 +348,7 @@ def is_basic_block_tight_loop(bb):
     return False
 
 
-def find_data_reference_from_insn(insn, max_depth=10):
+def find_data_reference_from_insn(insn: idaapi.insn_t, max_depth: int = 10) -> int:
     """search for data reference from instruction, return address of instruction if no reference exists"""
     depth = 0
     ea = insn.ea
@@ -358,19 +378,17 @@ def find_data_reference_from_insn(insn, max_depth=10):
     return ea
 
 
-def get_function_blocks(f):
-    """yield basic blocks contained in specified function
-
-    args:
-        f (IDA func_t)
-    yield:
-        block (IDA BasicBlock)
-    """
+def get_function_blocks(f: idaapi.func_t) -> Iterator[idaapi.BasicBlock]:
+    """yield basic blocks contained in specified function"""
     # leverage idaapi.FC_NOEXT flag to ignore useless external blocks referenced by the function
-    for block in idaapi.FlowChart(f, flags=(idaapi.FC_PREDS | idaapi.FC_NOEXT)):
-        yield block
+    yield from idaapi.FlowChart(f, flags=(idaapi.FC_PREDS | idaapi.FC_NOEXT))
 
 
-def is_basic_block_return(bb):
+def is_basic_block_return(bb: idaapi.BasicBlock) -> bool:
     """check if basic block is return block"""
     return bb.type == idaapi.fcb_ret
+
+
+def has_sib(oper: idaapi.op_t) -> bool:
+    # via: https://reverseengineering.stackexchange.com/a/14300
+    return oper.specflag1 == 1

capa/features/extractors/ida/insn.py

@@ -1,10 +1,11 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+from typing import Any, Dict, Tuple, Iterator
 
 import idc
 import idaapi
@@ -12,47 +13,29 @@ import idautils
 
 import capa.features.extractors.helpers
 import capa.features.extractors.ida.helpers
-from capa.features.insn import API, Number, Offset, Mnemonic
-from capa.features.common import (
-    BITNESS_X32,
-    BITNESS_X64,
-    MAX_BYTES_FEATURE_SIZE,
-    THUNK_CHAIN_DEPTH_DELTA,
-    Bytes,
-    String,
-    Characteristic,
-)
+from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
+from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
 
 # security cookie checks may perform non-zeroing XORs, these are expected within a certain
 # byte range within the first and returning basic blocks, this helps to reduce FP features
 SECURITY_COOKIE_BYTES_DELTA = 0x40
 
 
-def get_bitness(ctx):
-    """
-    fetch the BITNESS_* constant for the currently open workspace.
-
-    via Tamir Bahar/@tmr232
-    https://reverseengineering.stackexchange.com/a/11398/17194
-    """
-    if "bitness" not in ctx:
-        info = idaapi.get_inf_structure()
-        if info.is_64bit():
-            ctx["bitness"] = BITNESS_X64
-        elif info.is_32bit():
-            ctx["bitness"] = BITNESS_X32
-        else:
-            raise ValueError("unexpected bitness")
-    return ctx["bitness"]
-
-
-def get_imports(ctx):
+def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
     if "imports_cache" not in ctx:
         ctx["imports_cache"] = capa.features.extractors.ida.helpers.get_file_imports()
     return ctx["imports_cache"]
 
 
-def check_for_api_call(ctx, insn):
+def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    if "externs_cache" not in ctx:
+        ctx["externs_cache"] = capa.features.extractors.ida.helpers.get_file_externs()
+    return ctx["externs_cache"]
+
+
+def check_for_api_call(insn: idaapi.insn_t, funcs: Dict[int, Any]) -> Iterator[Any]:
     """check instruction for API call"""
     info = ()
     ref = insn.ea
@@ -69,7 +52,7 @@ def check_for_api_call(ctx, insn):
             except IndexError:
                 break
 
-        info = get_imports(ctx).get(ref, ())
+        info = funcs.get(ref, ())
         if info:
             break
 
@@ -78,27 +61,31 @@ def check_for_api_call(ctx, insn):
             break
 
     if info:
-        yield "%s.%s" % (info[0], info[1])
+        yield info
 
 
-def extract_insn_api_features(f, bb, insn):
-    """parse instruction API features
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
+def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction API features
 
     example:
-        call dword [0x00473038]
+       call dword [0x00473038]
     """
-    if not insn.get_canon_mnem() in ("call", "jmp"):
+    insn: idaapi.insn_t = ih.inner
+
+    if insn.get_canon_mnem() not in ("call", "jmp"):
         return
 
-    for api in check_for_api_call(f.ctx, insn):
-        dll, _, symbol = api.rpartition(".")
-        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-            yield API(name), insn.ea
+    # check calls to imported functions
+    for api in check_for_api_call(insn, get_imports(fh.ctx)):
+        # tuple (<module>, <function>, <ordinal>)
+        for name in capa.features.extractors.helpers.generate_symbols(api[0], api[1]):
+            yield API(name), ih.address
+
+    # check calls to extern functions
+    for api in check_for_api_call(insn, get_externs(fh.ctx)):
+        # tuple (<module>, <function>, <ordinal>)
+        yield API(api[1]), ih.address
 
     # extract IDA/FLIRT recognized API functions
     targets = tuple(idautils.CodeRefsFrom(insn.ea, False))
@@ -113,20 +100,25 @@ def extract_insn_api_features(f, bb, insn):
 
     if target_func.flags & idaapi.FUNC_LIB:
         name = idaapi.get_name(target_func.start_ea)
-        yield API(name), insn.ea
+        yield API(name), ih.address
+        if name.startswith("_"):
+            # some linkers may prefix linked routines with a `_` to avoid name collisions.
+            # extract features for both the mangled and un-mangled representations.
+            # e.g. `_fwrite` -> `fwrite`
+            # see: https://stackoverflow.com/a/2628384/87207
+            yield API(name[1:]), ih.address
 
 
-def extract_insn_number_features(f, bb, insn):
-    """parse instruction number features
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
-
+def extract_insn_number_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction number features
     example:
         push    3136B0h         ; dwControlCode
     """
+    insn: idaapi.insn_t = ih.inner
+
     if idaapi.is_ret_insn(insn):
         # skip things like:
         #   .text:0042250E retn 8
@@ -137,7 +129,11 @@ def extract_insn_number_features(f, bb, insn):
         #   .text:00401145 add esp, 0Ch
         return
 
-    for op in capa.features.extractors.ida.helpers.get_insn_ops(insn, target_ops=(idaapi.o_imm, idaapi.o_mem)):
+    for i, op in enumerate(insn.ops):
+        if op.type == idaapi.o_void:
+            break
+        if op.type not in (idaapi.o_imm, idaapi.o_mem):
+            continue
         # skip things like:
         #   .text:00401100 shr eax, offset loc_C
         if capa.features.extractors.ida.helpers.is_op_offset(insn, op):
@@ -148,21 +144,27 @@ def extract_insn_number_features(f, bb, insn):
         else:
             const = op.addr
 
-        yield Number(const), insn.ea
-        yield Number(const, bitness=get_bitness(f.ctx)), insn.ea
+        yield Number(const), ih.address
+        yield OperandNumber(i, const), ih.address
+
+        if insn.itype == idaapi.NN_add and 0 < const < MAX_STRUCTURE_SIZE and op.type == idaapi.o_imm:
+            # for pattern like:
+            #
+            #     add eax, 0x10
+            #
+            # assume 0x10 is also an offset (imagine eax is a pointer).
+            yield Offset(const), ih.address
+            yield OperandOffset(i, const), ih.address
 
 
-def extract_insn_bytes_features(f, bb, insn):
-    """parse referenced byte sequences
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
-
+def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse referenced byte sequences
     example:
         push    offset iid_004118d4_IShellLinkA ; riid
     """
+    insn: idaapi.insn_t = ih.inner
+
     if idaapi.is_call_insn(insn):
         return
 
@@ -170,43 +172,54 @@ def extract_insn_bytes_features(f, bb, insn):
     if ref != insn.ea:
         extracted_bytes = capa.features.extractors.ida.helpers.read_bytes_at(ref, MAX_BYTES_FEATURE_SIZE)
         if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-            yield Bytes(extracted_bytes), insn.ea
+            if not capa.features.extractors.ida.helpers.find_string_at(ref):
+                # don't extract byte features for obvious strings
+                yield Bytes(extracted_bytes), ih.address
 
 
-def extract_insn_string_features(f, bb, insn):
-    """parse instruction string features
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
+def extract_insn_string_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction string features
 
     example:
         push offset aAcr     ; "ACR  > "
     """
+    insn: idaapi.insn_t = ih.inner
+
     ref = capa.features.extractors.ida.helpers.find_data_reference_from_insn(insn)
     if ref != insn.ea:
         found = capa.features.extractors.ida.helpers.find_string_at(ref)
         if found:
-            yield String(found), insn.ea
+            yield String(found), ih.address
 
 
-def extract_insn_offset_features(f, bb, insn):
-    """parse instruction structure offset features
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
+def extract_insn_offset_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction structure offset features
 
     example:
         .text:0040112F cmp [esi+4], ebx
     """
-    for op in capa.features.extractors.ida.helpers.get_insn_ops(insn, target_ops=(idaapi.o_phrase, idaapi.o_displ)):
+    insn: idaapi.insn_t = ih.inner
+
+    for i, op in enumerate(insn.ops):
+        if op.type == idaapi.o_void:
+            break
+        if op.type not in (idaapi.o_phrase, idaapi.o_displ):
+            continue
         if capa.features.extractors.ida.helpers.is_op_stack_var(insn.ea, op.n):
             continue
+
         p_info = capa.features.extractors.ida.helpers.get_op_phrase_info(op)
-        op_off = p_info.get("offset", 0)
+
+        op_off = p_info.get("offset")
+        if op_off is None:
+            continue
+
         if idaapi.is_mapped(op_off):
             # Ignore:
             #   mov esi, dword_1005B148[esi]
@@ -217,12 +230,32 @@ def extract_insn_offset_features(f, bb, insn):
         # https://stackoverflow.com/questions/31853189/x86-64-assembly-why-displacement-not-64-bits
         op_off = capa.features.extractors.helpers.twos_complement(op_off, 32)
 
-        yield Offset(op_off), insn.ea
-        yield Offset(op_off, bitness=get_bitness(f.ctx)), insn.ea
+        yield Offset(op_off), ih.address
+        yield OperandOffset(i, op_off), ih.address
+
+        if (
+            insn.itype == idaapi.NN_lea
+            and i == 1
+            # o_displ is used for both:
+            #   [eax+1]
+            #   [eax+ebx+2]
+            and op.type == idaapi.o_displ
+            # but the SIB is only present for [eax+ebx+2]
+            # which we don't want
+            and not capa.features.extractors.ida.helpers.has_sib(op)
+        ):
+            # for pattern like:
+            #
+            #     lea eax, [ebx + 1]
+            #
+            # assume 1 is also an offset (imagine ebx is a zero register).
+            yield Number(op_off), ih.address
+            yield OperandNumber(i, op_off), ih.address
 
 
-def contains_stack_cookie_keywords(s):
-    """check if string contains stack cookie keywords
+def contains_stack_cookie_keywords(s: str) -> bool:
+    """
+    check if string contains stack cookie keywords
 
     Examples:
         xor     ecx, ebp ; StackCookie
@@ -236,7 +269,7 @@ def contains_stack_cookie_keywords(s):
     return any(keyword in s for keyword in ("stack", "security"))
 
 
-def bb_stack_cookie_registers(bb):
+def bb_stack_cookie_registers(bb: idaapi.BasicBlock) -> Iterator[int]:
     """scan basic block for stack cookie operations
 
     yield registers ids that may have been used for stack cookie operations
@@ -270,7 +303,7 @@ def bb_stack_cookie_registers(bb):
                     yield op.reg
 
 
-def is_nzxor_stack_cookie_delta(f, bb, insn):
+def is_nzxor_stack_cookie_delta(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.insn_t) -> bool:
     """check if nzxor exists within stack cookie delta"""
     # security cookie check should use SP or BP
     if not capa.features.extractors.ida.helpers.is_frame_register(insn.Op2.reg):
@@ -293,7 +326,7 @@ def is_nzxor_stack_cookie_delta(f, bb, insn):
     return False
 
 
-def is_nzxor_stack_cookie(f, bb, insn):
+def is_nzxor_stack_cookie(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.insn_t) -> bool:
     """check if nzxor is related to stack cookie"""
     if contains_stack_cookie_keywords(idaapi.get_cmt(insn.ea, False)):
         # Example:
@@ -310,37 +343,49 @@ def is_nzxor_stack_cookie(f, bb, insn):
     return False
 
 
-def extract_insn_nzxor_characteristic_features(f, bb, insn):
-    """parse instruction non-zeroing XOR instruction
-
-    ignore expected non-zeroing XORs, e.g. security cookies
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
     """
+    parse instruction non-zeroing XOR instruction
+    ignore expected non-zeroing XORs, e.g. security cookies
+    """
+    insn: idaapi.insn_t = ih.inner
+
     if insn.itype not in (idaapi.NN_xor, idaapi.NN_xorpd, idaapi.NN_xorps, idaapi.NN_pxor):
         return
     if capa.features.extractors.ida.helpers.is_operand_equal(insn.Op1, insn.Op2):
         return
-    if is_nzxor_stack_cookie(f, bb, insn):
+    if is_nzxor_stack_cookie(fh.inner, bbh.inner, insn):
         return
-    yield Characteristic("nzxor"), insn.ea
+    yield Characteristic("nzxor"), ih.address
 
 
-def extract_insn_mnemonic_features(f, bb, insn):
-    """parse instruction mnemonic features
+def extract_insn_mnemonic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction mnemonic features"""
+    yield Mnemonic(idc.print_insn_mnem(ih.inner.ea)), ih.address
 
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
+
+def extract_insn_obfs_call_plus_5_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
     """
-    yield Mnemonic(idc.print_insn_mnem(insn.ea)), insn.ea
+    parse call $+5 instruction from the given instruction.
+    """
+    insn: idaapi.insn_t = ih.inner
+
+    if not idaapi.is_call_insn(insn):
+        return
+
+    if insn.ea + 5 == idc.get_operand_value(insn.ea, 0):
+        yield Characteristic("call $+5"), ih.address
 
 
-def extract_insn_peb_access_characteristic_features(f, bb, insn):
+def extract_insn_peb_access_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
     """parse instruction peb access
 
     fs:[0x30] on x86, gs:[0x60] on x64
@@ -348,51 +393,61 @@ def extract_insn_peb_access_characteristic_features(f, bb, insn):
     TODO:
         IDA should be able to do this..
     """
+    insn: idaapi.insn_t = ih.inner
+
     if insn.itype not in (idaapi.NN_push, idaapi.NN_mov):
         return
 
-    if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
+    if all(op.type != idaapi.o_mem for op in insn.ops):
         # try to optimize for only memory references
         return
 
     disasm = idc.GetDisasm(insn.ea)
 
     if " fs:30h" in disasm or " gs:60h" in disasm:
-        # TODO: replace above with proper IDA
-        yield Characteristic("peb access"), insn.ea
+        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
+        # scanning the disassembly text is a hack.
+        # https://github.com/mandiant/capa/issues/1605
+        yield Characteristic("peb access"), ih.address
 
 
-def extract_insn_segment_access_features(f, bb, insn):
+def extract_insn_segment_access_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
     """parse instruction fs or gs access
 
     TODO:
         IDA should be able to do this...
     """
-    if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
+    insn: idaapi.insn_t = ih.inner
+
+    if all(op.type != idaapi.o_mem for op in insn.ops):
         # try to optimize for only memory references
         return
 
     disasm = idc.GetDisasm(insn.ea)
 
     if " fs:" in disasm:
-        # TODO: replace above with proper IDA
-        yield Characteristic("fs access"), insn.ea
+        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
+        # scanning the disassembly text is a hack.
+        # https://github.com/mandiant/capa/issues/1605
+        yield Characteristic("fs access"), ih.address
 
     if " gs:" in disasm:
-        # TODO: replace above with proper IDA
-        yield Characteristic("gs access"), insn.ea
+        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
+        # scanning the disassembly text is a hack.
+        # https://github.com/mandiant/capa/issues/1605
+        yield Characteristic("gs access"), ih.address
 
 
-def extract_insn_cross_section_cflow(f, bb, insn):
-    """inspect the instruction for a CALL or JMP that crosses section boundaries
+def extract_insn_cross_section_cflow(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
+    insn: idaapi.insn_t = ih.inner
 
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
-    """
     for ref in idautils.CodeRefsFrom(insn.ea, False):
-        if ref in get_imports(f.ctx).keys():
+        if ref in get_imports(fh.ctx):
             # ignore API calls
             continue
         if not idaapi.getseg(ref):
@@ -400,50 +455,40 @@ def extract_insn_cross_section_cflow(f, bb, insn):
             continue
         if idaapi.getseg(ref) == idaapi.getseg(insn.ea):
             continue
-        yield Characteristic("cross section flow"), insn.ea
+        yield Characteristic("cross section flow"), ih.address
 
 
-def extract_function_calls_from(f, bb, insn):
+def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """extract functions calls from features
 
     most relevant at the function scope, however, its most efficient to extract at the instruction scope
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
     """
+    insn: idaapi.insn_t = ih.inner
+
     if idaapi.is_call_insn(insn):
         for ref in idautils.CodeRefsFrom(insn.ea, False):
-            yield Characteristic("calls from"), ref
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(ref)
 
 
-def extract_function_indirect_call_characteristic_features(f, bb, insn):
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
     """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
     does not include calls like => call ds:dword_ABD4974
 
     most relevant at the function or basic block scope;
     however, its most efficient to extract at the instruction scope
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
     """
+    insn: idaapi.insn_t = ih.inner
+
     if idaapi.is_call_insn(insn) and idc.get_operand_type(insn.ea, 0) in (idc.o_reg, idc.o_phrase, idc.o_displ):
-        yield Characteristic("indirect call"), insn.ea
+        yield Characteristic("indirect call"), ih.address
 
 
-def extract_features(f, bb, insn):
-    """extract instruction features
-
-    args:
-        f (IDA func_t)
-        bb (IDA BasicBlock)
-        insn (IDA insn_t)
-    """
+def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract instruction features"""
     for inst_handler in INSTRUCTION_HANDLERS:
-        for (feature, ea) in inst_handler(f, bb, insn):
+        for feature, ea in inst_handler(f, bbh, insn):
             yield feature, ea
 
 
@@ -455,26 +500,10 @@ INSTRUCTION_HANDLERS = (
     extract_insn_offset_features,
     extract_insn_nzxor_characteristic_features,
     extract_insn_mnemonic_features,
+    extract_insn_obfs_call_plus_5_characteristic_features,
     extract_insn_peb_access_characteristic_features,
     extract_insn_cross_section_cflow,
     extract_insn_segment_access_features,
     extract_function_calls_from,
     extract_function_indirect_call_characteristic_features,
 )
-
-
-def main():
-    """ """
-    features = []
-    for f in capa.features.extractors.ida.helpers.get_functions(skip_thunks=True, skip_libs=True):
-        for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
-            for insn in capa.features.extractors.ida.helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
-                features.extend(list(extract_features(f, bb, insn)))
-
-    import pprint
-
-    pprint.pprint(features)
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/loops.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/null.py

@@ -0,0 +1,79 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Dict, List, Tuple
+from dataclasses import dataclass
+
+from capa.features.common import Feature
+from capa.features.address import NO_ADDRESS, Address
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+
+
+@dataclass
+class InstructionFeatures:
+    features: List[Tuple[Address, Feature]]
+
+
+@dataclass
+class BasicBlockFeatures:
+    features: List[Tuple[Address, Feature]]
+    instructions: Dict[Address, InstructionFeatures]
+
+
+@dataclass
+class FunctionFeatures:
+    features: List[Tuple[Address, Feature]]
+    basic_blocks: Dict[Address, BasicBlockFeatures]
+
+
+@dataclass
+class NullFeatureExtractor(FeatureExtractor):
+    """
+    An extractor that extracts some user-provided features.
+
+    This is useful for testing, as we can provide expected values and see if matching works.
+    """
+
+    base_address: Address
+    global_features: List[Feature]
+    file_features: List[Tuple[Address, Feature]]
+    functions: Dict[Address, FunctionFeatures]
+
+    def get_base_address(self):
+        return self.base_address
+
+    def extract_global_features(self):
+        for feature in self.global_features:
+            yield feature, NO_ADDRESS
+
+    def extract_file_features(self):
+        for address, feature in self.file_features:
+            yield feature, address
+
+    def get_functions(self):
+        for address in sorted(self.functions.keys()):
+            yield FunctionHandle(address, None)
+
+    def extract_function_features(self, f):
+        for address, feature in self.functions[f.address].features:
+            yield feature, address
+
+    def get_basic_blocks(self, f):
+        for address in sorted(self.functions[f.address].basic_blocks.keys()):
+            yield BBHandle(address, None)
+
+    def extract_basic_block_features(self, f, bb):
+        for address, feature in self.functions[f.address].basic_blocks[bb.address].features:
+            yield feature, address
+
+    def get_instructions(self, f, bb):
+        for address in sorted(self.functions[f.address].basic_blocks[bb.address].instructions.keys()):
+            yield InsnHandle(address, None)
+
+    def extract_insn_features(self, f, bb, insn):
+        for address, feature in self.functions[f.address].basic_blocks[bb.address].instructions[insn.address].features:
+            yield feature, address

capa/features/extractors/pefile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,6 +7,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import logging
+from pathlib import Path
 
 import pefile
 
@@ -17,6 +18,7 @@ import capa.features.extractors.helpers
 import capa.features.extractors.strings
 from capa.features.file import Export, Import, Section
 from capa.features.common import OS, ARCH_I386, FORMAT_PE, ARCH_AMD64, OS_WINDOWS, Arch, Format, Characteristic
+from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
 from capa.features.extractors.base_extractor import FeatureExtractor
 
 logger = logging.getLogger(__name__)
@@ -24,7 +26,7 @@ logger = logging.getLogger(__name__)
 
 def extract_file_embedded_pe(buf, **kwargs):
     for offset, _ in capa.features.extractors.helpers.carve_pe(buf, 1):
-        yield Characteristic("embedded pe"), offset
+        yield Characteristic("embedded pe"), FileOffsetAddress(offset)
 
 
 def extract_file_export_names(pe, **kwargs):
@@ -38,8 +40,20 @@ def extract_file_export_names(pe, **kwargs):
                 name = export.name.partition(b"\x00")[0].decode("ascii")
             except UnicodeDecodeError:
                 continue
-            va = base_address + export.address
-            yield Export(name), va
+
+            if export.forwarder is None:
+                va = base_address + export.address
+                yield Export(name), AbsoluteVirtualAddress(va)
+
+            else:
+                try:
+                    forwarded_name = export.forwarder.partition(b"\x00")[0].decode("ascii")
+                except UnicodeDecodeError:
+                    continue
+                forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
+                va = base_address + export.address
+                yield Export(forwarded_name), AbsoluteVirtualAddress(va)
+                yield Characteristic("forwarded export"), AbsoluteVirtualAddress(va)
 
 
 def extract_file_import_names(pe, **kwargs):
@@ -63,7 +77,7 @@ def extract_file_import_names(pe, **kwargs):
 
             for imp in dll.imports:
                 if imp.import_by_ordinal:
-                    impname = "#%s" % imp.ordinal
+                    impname = f"#{imp.ordinal}"
                 else:
                     try:
                         impname = imp.name.partition(b"\x00")[0].decode("ascii")
@@ -71,7 +85,7 @@ def extract_file_import_names(pe, **kwargs):
                         continue
 
                 for name in capa.features.extractors.helpers.generate_symbols(modname, impname):
-                    yield Import(name), imp.address
+                    yield Import(name), AbsoluteVirtualAddress(imp.address)
 
 
 def extract_file_section_names(pe, **kwargs):
@@ -83,7 +97,7 @@ def extract_file_section_names(pe, **kwargs):
         except UnicodeDecodeError:
             continue
 
-        yield Section(name), base_address + section.VirtualAddress
+        yield Section(name), AbsoluteVirtualAddress(base_address + section.VirtualAddress)
 
 
 def extract_file_strings(buf, **kwargs):
@@ -103,18 +117,18 @@ def extract_file_function_names(**kwargs):
 def extract_file_os(**kwargs):
     # assuming PE -> Windows
     # though i suppose they're also used by UEFI
-    yield OS(OS_WINDOWS), 0x0
+    yield OS(OS_WINDOWS), NO_ADDRESS
 
 
 def extract_file_format(**kwargs):
-    yield Format(FORMAT_PE), 0x0
+    yield Format(FORMAT_PE), NO_ADDRESS
 
 
 def extract_file_arch(pe, **kwargs):
     if pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_I386"]:
-        yield Arch(ARCH_I386), 0x0
+        yield Arch(ARCH_I386), NO_ADDRESS
     elif pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_AMD64"]:
-        yield Arch(ARCH_AMD64), 0x0
+        yield Arch(ARCH_AMD64), NO_ADDRESS
     else:
         logger.warning("unsupported architecture: %s", pefile.MACHINE_TYPE[pe.FILE_HEADER.Machine])
 
@@ -132,7 +146,8 @@ def extract_file_features(pe, buf):
     """
 
     for file_handler in FILE_HANDLERS:
-        for feature, va in file_handler(pe=pe, buf=buf):
+        # file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
+        for feature, va in file_handler(pe=pe, buf=buf):  # type: ignore
             yield feature, va
 
 
@@ -159,7 +174,8 @@ def extract_global_features(pe, buf):
       Tuple[Feature, VA]: a feature and its location.
     """
     for handler in GLOBAL_HANDLERS:
-        for feature, va in handler(pe=pe, buf=buf):
+        # file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
+        for feature, va in handler(pe=pe, buf=buf):  # type: ignore
             yield feature, va
 
 
@@ -170,23 +186,21 @@ GLOBAL_HANDLERS = (
 
 
 class PefileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
-        super(PefileFeatureExtractor, self).__init__()
-        self.path = path
-        self.pe = pefile.PE(path)
+    def __init__(self, path: Path):
+        super().__init__()
+        self.path: Path = path
+        self.pe = pefile.PE(str(path))
 
     def get_base_address(self):
-        return self.pe.OPTIONAL_HEADER.ImageBase
+        return AbsoluteVirtualAddress(self.pe.OPTIONAL_HEADER.ImageBase)
 
     def extract_global_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = Path(self.path).read_bytes()
 
         yield from extract_global_features(self.pe, buf)
 
     def extract_file_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = Path(self.path).read_bytes()
 
         yield from extract_file_features(self.pe, buf)
 

capa/features/extractors/smda/basicblock.py

@@ -1,130 +0,0 @@
-import string
-import struct
-
-from capa.features.common import Characteristic
-from capa.features.basicblock import BasicBlock
-from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-
-
-def _bb_has_tight_loop(f, bb):
-    """
-    parse tight loops, true if last instruction in basic block branches to bb start
-    """
-    return bb.offset in f.blockrefs[bb.offset] if bb.offset in f.blockrefs else False
-
-
-def extract_bb_tight_loop(f, bb):
-    """check basic block for tight loop indicators"""
-    if _bb_has_tight_loop(f, bb):
-        yield Characteristic("tight loop"), bb.offset
-
-
-def _bb_has_stackstring(f, bb):
-    """
-    extract potential stackstring creation, using the following heuristics:
-      - basic block contains enough moves of constant bytes to the stack
-    """
-    count = 0
-    for instr in bb.getInstructions():
-        if is_mov_imm_to_stack(instr):
-            count += get_printable_len(instr.getDetailed())
-        if count > MIN_STACKSTRING_LEN:
-            return True
-    return False
-
-
-def get_operands(smda_ins):
-    return [o.strip() for o in smda_ins.operands.split(",")]
-
-
-def extract_stackstring(f, bb):
-    """check basic block for stackstring indicators"""
-    if _bb_has_stackstring(f, bb):
-        yield Characteristic("stack string"), bb.offset
-
-
-def is_mov_imm_to_stack(smda_ins):
-    """
-    Return if instruction moves immediate onto stack
-    """
-    if not smda_ins.mnemonic.startswith("mov"):
-        return False
-
-    try:
-        dst, src = get_operands(smda_ins)
-    except ValueError:
-        # not two operands
-        return False
-
-    try:
-        int(src, 16)
-    except ValueError:
-        return False
-
-    if not any(regname in dst for regname in ["ebp", "rbp", "esp", "rsp"]):
-        return False
-
-    return True
-
-
-def is_printable_ascii(chars):
-    return all(c < 127 and chr(c) in string.printable for c in chars)
-
-
-def is_printable_utf16le(chars):
-    if all(c == 0x00 for c in chars[1::2]):
-        return is_printable_ascii(chars[::2])
-
-
-def get_printable_len(instr):
-    """
-    Return string length if all operand bytes are ascii or utf16-le printable
-
-    Works on a capstone instruction
-    """
-    # should have exactly two operands for mov immediate
-    if len(instr.operands) != 2:
-        return 0
-
-    op_value = instr.operands[1].value.imm
-
-    if instr.imm_size == 1:
-        chars = struct.pack("<B", op_value & 0xFF)
-    elif instr.imm_size == 2:
-        chars = struct.pack("<H", op_value & 0xFFFF)
-    elif instr.imm_size == 4:
-        chars = struct.pack("<I", op_value & 0xFFFFFFFF)
-    elif instr.imm_size == 8:
-        chars = struct.pack("<Q", op_value & 0xFFFFFFFFFFFFFFFF)
-    else:
-        raise ValueError("Unhandled operand data type 0x%x." % instr.imm_size)
-
-    if is_printable_ascii(chars):
-        return instr.imm_size
-    if is_printable_utf16le(chars):
-        return instr.imm_size // 2
-
-    return 0
-
-
-def extract_features(f, bb):
-    """
-    extract features from the given basic block.
-
-    args:
-      f (smda.common.SmdaFunction): the function from which to extract features
-      bb (smda.common.SmdaBasicBlock): the basic block to process.
-
-    yields:
-      Tuple[Feature, int]: the features and their location found in this basic block.
-    """
-    yield BasicBlock(), bb.offset
-    for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, va in bb_handler(f, bb):
-            yield feature, va
-
-
-BASIC_BLOCK_HANDLERS = (
-    extract_bb_tight_loop,
-    extract_stackstring,
-)

capa/features/extractors/smda/extractor.py

@@ -1,53 +0,0 @@
-from smda.common.SmdaReport import SmdaReport
-
-import capa.features.extractors.common
-import capa.features.extractors.smda.file
-import capa.features.extractors.smda.insn
-import capa.features.extractors.smda.global_
-import capa.features.extractors.smda.function
-import capa.features.extractors.smda.basicblock
-from capa.features.extractors.base_extractor import FeatureExtractor
-
-
-class SmdaFeatureExtractor(FeatureExtractor):
-    def __init__(self, smda_report: SmdaReport, path):
-        super(SmdaFeatureExtractor, self).__init__()
-        self.smda_report = smda_report
-        self.path = path
-        with open(self.path, "rb") as f:
-            self.buf = f.read()
-
-        # pre-compute these because we'll yield them at *every* scope.
-        self.global_features = []
-        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf))
-        self.global_features.extend(capa.features.extractors.smda.global_.extract_arch(self.smda_report))
-
-    def get_base_address(self):
-        return self.smda_report.base_addr
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.smda.file.extract_features(self.smda_report, self.buf)
-
-    def get_functions(self):
-        for function in self.smda_report.getFunctions():
-            yield function
-
-    def extract_function_features(self, f):
-        yield from capa.features.extractors.smda.function.extract_features(f)
-
-    def get_basic_blocks(self, f):
-        for bb in f.getBlocks():
-            yield bb
-
-    def extract_basic_block_features(self, f, bb):
-        yield from capa.features.extractors.smda.basicblock.extract_features(f, bb)
-
-    def get_instructions(self, f, bb):
-        for smda_ins in bb.getInstructions():
-            yield smda_ins
-
-    def extract_insn_features(self, f, bb, insn):
-        yield from capa.features.extractors.smda.insn.extract_features(f, bb, insn)

capa/features/extractors/smda/file.py

@@ -1,102 +0,0 @@
-# if we have SMDA we definitely have lief
-import lief
-
-import capa.features.extractors.common
-import capa.features.extractors.helpers
-import capa.features.extractors.strings
-from capa.features.file import Export, Import, Section
-from capa.features.common import String, Characteristic
-
-
-def extract_file_embedded_pe(buf, **kwargs):
-    for offset, _ in capa.features.extractors.helpers.carve_pe(buf, 1):
-        yield Characteristic("embedded pe"), offset
-
-
-def extract_file_export_names(buf, **kwargs):
-    lief_binary = lief.parse(buf)
-
-    if lief_binary is not None:
-        for function in lief_binary.exported_functions:
-            yield Export(function.name), function.address
-
-
-def extract_file_import_names(smda_report, buf):
-    # extract import table info via LIEF
-    lief_binary = lief.parse(buf)
-    if not isinstance(lief_binary, lief.PE.Binary):
-        return
-    for imported_library in lief_binary.imports:
-        library_name = imported_library.name.lower()
-        library_name = library_name[:-4] if library_name.endswith(".dll") else library_name
-        for func in imported_library.entries:
-            va = func.iat_address + smda_report.base_addr
-            if func.name:
-                for name in capa.features.extractors.helpers.generate_symbols(library_name, func.name):
-                    yield Import(name), va
-            elif func.is_ordinal:
-                for name in capa.features.extractors.helpers.generate_symbols(library_name, "#%s" % func.ordinal):
-                    yield Import(name), va
-
-
-def extract_file_section_names(buf, **kwargs):
-    lief_binary = lief.parse(buf)
-    if not isinstance(lief_binary, lief.PE.Binary):
-        return
-    if lief_binary and lief_binary.sections:
-        base_address = lief_binary.optional_header.imagebase
-        for section in lief_binary.sections:
-            yield Section(section.name), base_address + section.virtual_address
-
-
-def extract_file_strings(buf, **kwargs):
-    """
-    extract ASCII and UTF-16 LE strings from file
-    """
-    for s in capa.features.extractors.strings.extract_ascii_strings(buf):
-        yield String(s.s), s.offset
-
-    for s in capa.features.extractors.strings.extract_unicode_strings(buf):
-        yield String(s.s), s.offset
-
-
-def extract_file_function_names(smda_report, **kwargs):
-    """
-    extract the names of statically-linked library functions.
-    """
-    if False:
-        # using a `yield` here to force this to be a generator, not function.
-        yield NotImplementedError("SMDA doesn't have library matching")
-    return
-
-
-def extract_file_format(buf, **kwargs):
-    yield from capa.features.extractors.common.extract_format(buf)
-
-
-def extract_features(smda_report, buf):
-    """
-    extract file features from given workspace
-
-    args:
-      smda_report (smda.common.SmdaReport): a SmdaReport
-      buf: the raw bytes of the sample
-
-    yields:
-      Tuple[Feature, VA]: a feature and its location.
-    """
-
-    for file_handler in FILE_HANDLERS:
-        for feature, va in file_handler(smda_report=smda_report, buf=buf):
-            yield feature, va
-
-
-FILE_HANDLERS = (
-    extract_file_embedded_pe,
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_section_names,
-    extract_file_strings,
-    extract_file_function_names,
-    extract_file_format,
-)

capa/features/extractors/smda/function.py

@@ -1,38 +0,0 @@
-from capa.features.common import Characteristic
-from capa.features.extractors import loops
-
-
-def extract_function_calls_to(f):
-    for inref in f.inrefs:
-        yield Characteristic("calls to"), inref
-
-
-def extract_function_loop(f):
-    """
-    parse if a function has a loop
-    """
-    edges = []
-    for bb_from, bb_tos in f.blockrefs.items():
-        for bb_to in bb_tos:
-            edges.append((bb_from, bb_to))
-
-    if edges and loops.has_loop(edges):
-        yield Characteristic("loop"), f.offset
-
-
-def extract_features(f):
-    """
-    extract features from the given function.
-
-    args:
-      f (smda.common.SmdaFunction): the function from which to extract features
-
-    yields:
-      Tuple[Feature, int]: the features and their location found in this function.
-    """
-    for func_handler in FUNCTION_HANDLERS:
-        for feature, va in func_handler(f):
-            yield feature, va
-
-
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)

capa/features/extractors/smda/global_.py

@@ -1,20 +0,0 @@
-import logging
-
-from capa.features.common import ARCH_I386, ARCH_AMD64, Arch
-
-logger = logging.getLogger(__name__)
-
-
-def extract_arch(smda_report):
-    if smda_report.architecture == "intel":
-        if smda_report.bitness == 32:
-            yield Arch(ARCH_I386), 0x0
-        elif smda_report.bitness == 64:
-            yield Arch(ARCH_AMD64), 0x0
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", smda_report.architecture)
-        return

capa/features/extractors/smda/insn.py

@@ -1,397 +0,0 @@
-import re
-import string
-import struct
-
-from smda.common.SmdaReport import SmdaReport
-
-import capa.features.extractors.helpers
-from capa.features.insn import API, Number, Offset, Mnemonic
-from capa.features.common import (
-    BITNESS_X32,
-    BITNESS_X64,
-    MAX_BYTES_FEATURE_SIZE,
-    THUNK_CHAIN_DEPTH_DELTA,
-    Bytes,
-    String,
-    Characteristic,
-)
-
-# security cookie checks may perform non-zeroing XORs, these are expected within a certain
-# byte range within the first and returning basic blocks, this helps to reduce FP features
-SECURITY_COOKIE_BYTES_DELTA = 0x40
-PATTERN_HEXNUM = re.compile(r"[+\-] (?P<num>0x[a-fA-F0-9]+)")
-PATTERN_SINGLENUM = re.compile(r"[+\-] (?P<num>[0-9])")
-
-
-def get_bitness(smda_report):
-    if smda_report.architecture == "intel":
-        if smda_report.bitness == 32:
-            return BITNESS_X32
-        elif smda_report.bitness == 64:
-            return BITNESS_X64
-    else:
-        raise NotImplementedError
-
-
-def extract_insn_api_features(f, bb, insn):
-    """parse API features from the given instruction."""
-    if insn.offset in f.apirefs:
-        api_entry = f.apirefs[insn.offset]
-        # reformat
-        dll_name, api_name = api_entry.split("!")
-        dll_name = dll_name.split(".")[0]
-        dll_name = dll_name.lower()
-        for name in capa.features.extractors.helpers.generate_symbols(dll_name, api_name):
-            yield API(name), insn.offset
-    elif insn.offset in f.outrefs:
-        current_function = f
-        current_instruction = insn
-        for index in range(THUNK_CHAIN_DEPTH_DELTA):
-            if current_function and len(current_function.outrefs[current_instruction.offset]) == 1:
-                target = current_function.outrefs[current_instruction.offset][0]
-                referenced_function = current_function.smda_report.getFunction(target)
-                if referenced_function:
-                    # TODO SMDA: implement this function for both jmp and call, checking if function has 1 instruction which refs an API
-                    if referenced_function.isApiThunk():
-                        api_entry = (
-                            referenced_function.apirefs[target] if target in referenced_function.apirefs else None
-                        )
-                        if api_entry:
-                            # reformat
-                            dll_name, api_name = api_entry.split("!")
-                            dll_name = dll_name.split(".")[0]
-                            dll_name = dll_name.lower()
-                            for name in capa.features.extractors.helpers.generate_symbols(dll_name, api_name):
-                                yield API(name), insn.offset
-                    elif referenced_function.num_instructions == 1 and referenced_function.num_outrefs == 1:
-                        current_function = referenced_function
-                        current_instruction = [i for i in referenced_function.getInstructions()][0]
-                else:
-                    return
-
-
-def extract_insn_number_features(f, bb, insn):
-    """parse number features from the given instruction."""
-    # example:
-    #
-    #     push    3136B0h         ; dwControlCode
-    operands = [o.strip() for o in insn.operands.split(",")]
-    if insn.mnemonic == "add" and operands[0] in ["esp", "rsp"]:
-        # skip things like:
-        #
-        #    .text:00401140                 call    sub_407E2B
-        #    .text:00401145                 add     esp, 0Ch
-        return
-    for operand in operands:
-        try:
-            # The result of bitwise operations is calculated as though carried out
-            # in two’s complement with an infinite number of sign bits
-            value = int(operand, 16) & ((1 << f.smda_report.bitness) - 1)
-
-            yield Number(value), insn.offset
-            yield Number(value, bitness=get_bitness(f.smda_report)), insn.offset
-        except:
-            continue
-
-
-def read_bytes(smda_report, va, num_bytes=None):
-    """
-    read up to MAX_BYTES_FEATURE_SIZE from the given address.
-    """
-
-    rva = va - smda_report.base_addr
-    if smda_report.buffer is None:
-        raise ValueError("buffer is empty")
-    buffer_end = len(smda_report.buffer)
-    max_bytes = num_bytes if num_bytes is not None else MAX_BYTES_FEATURE_SIZE
-    if rva + max_bytes > buffer_end:
-        return smda_report.buffer[rva:]
-    else:
-        return smda_report.buffer[rva : rva + max_bytes]
-
-
-def derefs(smda_report, p):
-    """
-    recursively follow the given pointer, yielding the valid memory addresses along the way.
-    useful when you may have a pointer to string, or pointer to pointer to string, etc.
-
-    this is a "do what i mean" type of helper function.
-
-    based on the implementation in viv/insn.py
-    """
-    depth = 0
-    while True:
-        if not smda_report.isAddrWithinMemoryImage(p):
-            return
-        yield p
-
-        bytes_ = read_bytes(smda_report, p, num_bytes=4)
-        val = struct.unpack("I", bytes_)[0]
-
-        # sanity: pointer points to self
-        if val == p:
-            return
-
-        # sanity: avoid chains of pointers that are unreasonably deep
-        depth += 1
-        if depth > 10:
-            return
-
-        p = val
-
-
-def extract_insn_bytes_features(f, bb, insn):
-    """
-    parse byte sequence features from the given instruction.
-    example:
-        #     push    offset iid_004118d4_IShellLinkA ; riid
-    """
-    for data_ref in insn.getDataRefs():
-        for v in derefs(f.smda_report, data_ref):
-            bytes_read = read_bytes(f.smda_report, v)
-            if bytes_read is None:
-                continue
-            if capa.features.extractors.helpers.all_zeros(bytes_read):
-                continue
-
-            yield Bytes(bytes_read), insn.offset
-
-
-def detect_ascii_len(smda_report, offset):
-    if smda_report.buffer is None:
-        return 0
-    ascii_len = 0
-    rva = offset - smda_report.base_addr
-    char = smda_report.buffer[rva]
-    while char < 127 and chr(char) in string.printable:
-        ascii_len += 1
-        rva += 1
-        char = smda_report.buffer[rva]
-    if char == 0:
-        return ascii_len
-    return 0
-
-
-def detect_unicode_len(smda_report, offset):
-    if smda_report.buffer is None:
-        return 0
-    unicode_len = 0
-    rva = offset - smda_report.base_addr
-    char = smda_report.buffer[rva]
-    second_char = smda_report.buffer[rva + 1]
-    while char < 127 and chr(char) in string.printable and second_char == 0:
-        unicode_len += 2
-        rva += 2
-        char = smda_report.buffer[rva]
-        second_char = smda_report.buffer[rva + 1]
-    if char == 0 and second_char == 0:
-        return unicode_len
-    return 0
-
-
-def read_string(smda_report, offset):
-    alen = detect_ascii_len(smda_report, offset)
-    if alen > 1:
-        return read_bytes(smda_report, offset, alen).decode("utf-8")
-    ulen = detect_unicode_len(smda_report, offset)
-    if ulen > 2:
-        return read_bytes(smda_report, offset, ulen).decode("utf-16")
-
-
-def extract_insn_string_features(f, bb, insn):
-    """parse string features from the given instruction."""
-    # example:
-    #
-    #     push    offset aAcr     ; "ACR  > "
-    for data_ref in insn.getDataRefs():
-        for v in derefs(f.smda_report, data_ref):
-            string_read = read_string(f.smda_report, v)
-            if string_read:
-                yield String(string_read.rstrip("\x00")), insn.offset
-
-
-def extract_insn_offset_features(f, bb, insn):
-    """parse structure offset features from the given instruction."""
-    # examples:
-    #
-    #     mov eax, [esi + 4]
-    #     mov eax, [esi + ecx + 16384]
-    operands = [o.strip() for o in insn.operands.split(",")]
-    for operand in operands:
-        if not "ptr" in operand:
-            continue
-        if "esp" in operand or "ebp" in operand or "rbp" in operand:
-            continue
-        number = 0
-        number_hex = re.search(PATTERN_HEXNUM, operand)
-        number_int = re.search(PATTERN_SINGLENUM, operand)
-        if number_hex:
-            number = int(number_hex.group("num"), 16)
-            number = -1 * number if number_hex.group().startswith("-") else number
-        elif number_int:
-            number = int(number_int.group("num"))
-            number = -1 * number if number_int.group().startswith("-") else number
-        yield Offset(number), insn.offset
-        yield Offset(number, bitness=get_bitness(f.smda_report)), insn.offset
-
-
-def is_security_cookie(f, bb, insn):
-    """
-    check if an instruction is related to security cookie checks
-    """
-    # security cookie check should use SP or BP
-    operands = [o.strip() for o in insn.operands.split(",")]
-    if operands[1] not in ["esp", "ebp", "rsp", "rbp"]:
-        return False
-    for index, block in enumerate(f.getBlocks()):
-        # expect security cookie init in first basic block within first bytes (instructions)
-        block_instructions = [i for i in block.getInstructions()]
-        if index == 0 and insn.offset < (block_instructions[0].offset + SECURITY_COOKIE_BYTES_DELTA):
-            return True
-        # ... or within last bytes (instructions) before a return
-        if block_instructions[-1].mnemonic.startswith("ret") and insn.offset > (
-            block_instructions[-1].offset - SECURITY_COOKIE_BYTES_DELTA
-        ):
-            return True
-    return False
-
-
-def extract_insn_nzxor_characteristic_features(f, bb, insn):
-    """
-    parse non-zeroing XOR instruction from the given instruction.
-    ignore expected non-zeroing XORs, e.g. security cookies.
-    """
-
-    if insn.mnemonic not in ("xor", "xorpd", "xorps", "pxor"):
-        return
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    if operands[0] == operands[1]:
-        return
-
-    if is_security_cookie(f, bb, insn):
-        return
-
-    yield Characteristic("nzxor"), insn.offset
-
-
-def extract_insn_mnemonic_features(f, bb, insn):
-    """parse mnemonic features from the given instruction."""
-    yield Mnemonic(insn.mnemonic), insn.offset
-
-
-def extract_insn_peb_access_characteristic_features(f, bb, insn):
-    """
-    parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
-    """
-
-    if insn.mnemonic not in ["push", "mov"]:
-        return
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    for operand in operands:
-        if "fs:" in operand and "0x30" in operand:
-            yield Characteristic("peb access"), insn.offset
-        elif "gs:" in operand and "0x60" in operand:
-            yield Characteristic("peb access"), insn.offset
-
-
-def extract_insn_segment_access_features(f, bb, insn):
-    """parse the instruction for access to fs or gs"""
-    operands = [o.strip() for o in insn.operands.split(",")]
-    for operand in operands:
-        if "fs:" in operand:
-            yield Characteristic("fs access"), insn.offset
-        elif "gs:" in operand:
-            yield Characteristic("gs access"), insn.offset
-
-
-def extract_insn_cross_section_cflow(f, bb, insn):
-    """
-    inspect the instruction for a CALL or JMP that crosses section boundaries.
-    """
-    if insn.mnemonic in ["call", "jmp"]:
-        if insn.offset in f.apirefs:
-            return
-
-        smda_report = insn.smda_function.smda_report
-        if insn.offset in f.outrefs:
-            for target in f.outrefs[insn.offset]:
-                if smda_report.getSection(insn.offset) != smda_report.getSection(target):
-                    yield Characteristic("cross section flow"), insn.offset
-        elif insn.operands.startswith("0x"):
-            target = int(insn.operands, 16)
-            if smda_report.getSection(insn.offset) != smda_report.getSection(target):
-                yield Characteristic("cross section flow"), insn.offset
-
-
-# this is a feature that's most relevant at the function scope,
-# however, its most efficient to extract at the instruction scope.
-def extract_function_calls_from(f, bb, insn):
-    if insn.mnemonic != "call":
-        return
-
-    if insn.offset in f.outrefs:
-        for outref in f.outrefs[insn.offset]:
-            yield Characteristic("calls from"), outref
-
-            if outref == f.offset:
-                # if we found a jump target and it's the function address
-                # mark as recursive
-                yield Characteristic("recursive call"), outref
-    if insn.offset in f.apirefs:
-        yield Characteristic("calls from"), insn.offset
-
-
-# this is a feature that's most relevant at the function or basic block scope,
-# however, its most efficient to extract at the instruction scope.
-def extract_function_indirect_call_characteristic_features(f, bb, insn):
-    """
-    extract indirect function call characteristic (e.g., call eax or call dword ptr [edx+4])
-    does not include calls like => call ds:dword_ABD4974
-    """
-    if insn.mnemonic != "call":
-        return
-    if insn.operands.startswith("0x"):
-        return False
-    if "qword ptr" in insn.operands and "rip" in insn.operands:
-        return False
-    if insn.operands.startswith("dword ptr [0x"):
-        return False
-    # call edx
-    # call dword ptr [eax+50h]
-    # call qword ptr [rsp+78h]
-    yield Characteristic("indirect call"), insn.offset
-
-
-def extract_features(f, bb, insn):
-    """
-    extract features from the given insn.
-
-    args:
-      f (smda.common.SmdaFunction): the function to process.
-      bb (smda.common.SmdaBasicBlock): the basic block to process.
-      insn (smda.common.SmdaInstruction): the instruction to process.
-
-    yields:
-      Tuple[Feature, int]: the features and their location found in this insn.
-    """
-    for insn_handler in INSTRUCTION_HANDLERS:
-        for feature, va in insn_handler(f, bb, insn):
-            yield feature, va
-
-
-INSTRUCTION_HANDLERS = (
-    extract_insn_api_features,
-    extract_insn_number_features,
-    extract_insn_string_features,
-    extract_insn_bytes_features,
-    extract_insn_offset_features,
-    extract_insn_nzxor_characteristic_features,
-    extract_insn_mnemonic_features,
-    extract_insn_peb_access_characteristic_features,
-    extract_insn_cross_section_cflow,
-    extract_insn_segment_access_features,
-    extract_function_calls_from,
-    extract_function_indirect_call_characteristic_features,
-)

capa/features/extractors/strings.py

@@ -1,6 +1,6 @@
 # strings code from FLOSS, https://github.com/mandiant/flare-floss
 #
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -9,6 +9,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import re
+import contextlib
 from collections import namedtuple
 
 ASCII_BYTE = r" !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t".encode(
@@ -81,24 +82,5 @@ def extract_unicode_strings(buf, n=4):
         reg = b"((?:[%s]\x00){%d,})" % (ASCII_BYTE, n)
         r = re.compile(reg)
     for match in r.finditer(buf):
-        try:
+        with contextlib.suppress(UnicodeDecodeError):
             yield String(match.group().decode("utf-16"), match.start())
-        except UnicodeDecodeError:
-            pass
-
-
-def main():
-    import sys
-
-    with open(sys.argv[1], "rb") as f:
-        b = f.read()
-
-    for s in extract_ascii_strings(b):
-        print("0x{:x}: {:s}".format(s.offset, s.s))
-
-    for s in extract_unicode_strings(b):
-        print("0x{:x}: {:s}".format(s.offset, s.s))
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/viv/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,27 +8,30 @@
 
 import string
 import struct
+from typing import Tuple, Iterator
 
 import envi
 import envi.archs.i386.disasm
 
-from capa.features.common import Characteristic
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
 from capa.features.basicblock import BasicBlock
 from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
 
 
-def interface_extract_basic_block_XXX(f, bb):
+def interface_extract_basic_block_XXX(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     parse features from the given basic block.
 
     args:
-      f (viv_utils.Function): the function to process.
-      bb (viv_utils.BasicBlock): the basic block to process.
+      f: the function to process.
+      bb: the basic block to process.
 
     yields:
-      (Feature, int): the feature and the address at which its found.
+      (Feature, Address): the feature and the address at which its found.
     """
-    yield NotImplementedError("feature"), NotImplementedError("virtual address")
+    raise NotImplementedError
 
 
 def _bb_has_tight_loop(f, bb):
@@ -44,10 +47,10 @@ def _bb_has_tight_loop(f, bb):
     return False
 
 
-def extract_bb_tight_loop(f, bb):
+def extract_bb_tight_loop(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
     """check basic block for tight loop indicators"""
-    if _bb_has_tight_loop(f, bb):
-        yield Characteristic("tight loop"), bb.va
+    if _bb_has_tight_loop(f, bb.inner):
+        yield Characteristic("tight loop"), bb.address
 
 
 def _bb_has_stackstring(f, bb):
@@ -67,10 +70,10 @@ def _bb_has_stackstring(f, bb):
     return False
 
 
-def extract_stackstring(f, bb):
+def extract_stackstring(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
     """check basic block for stackstring indicators"""
-    if _bb_has_stackstring(f, bb):
-        yield Characteristic("stack string"), bb.va
+    if _bb_has_stackstring(f, bb.inner):
+        yield Characteristic("stack string"), bb.address
 
 
 def is_mov_imm_to_stack(instr: envi.archs.i386.disasm.i386Opcode) -> bool:
@@ -89,7 +92,6 @@ def is_mov_imm_to_stack(instr: envi.archs.i386.disasm.i386Opcode) -> bool:
     if not src.isImmed():
         return False
 
-    # TODO what about 64-bit operands?
     if not isinstance(dst, envi.archs.i386.disasm.i386SibOper) and not isinstance(
         dst, envi.archs.i386.disasm.i386RegMemOper
     ):
@@ -118,7 +120,7 @@ def get_printable_len(oper: envi.archs.i386.disasm.i386ImmOper) -> int:
     elif oper.tsize == 8:
         chars = struct.pack("<Q", oper.imm)
     else:
-        raise ValueError("unexpected oper.tsize: %d" % (oper.tsize))
+        raise ValueError(f"unexpected oper.tsize: {oper.tsize}")
 
     if is_printable_ascii(chars):
         return oper.tsize
@@ -143,7 +145,7 @@ def is_printable_utf16le(chars: bytes) -> bool:
     return False
 
 
-def extract_features(f, bb):
+def extract_features(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     extract features from the given basic block.
 
@@ -154,10 +156,10 @@ def extract_features(f, bb):
     yields:
       Tuple[Feature, int]: the features and their location found in this basic block.
     """
-    yield BasicBlock(), bb.va
+    yield BasicBlock(), AbsoluteVirtualAddress(bb.inner.va)
     for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, va in bb_handler(f, bb):
-            yield feature, va
+        for feature, addr in bb_handler(f, bb):
+            yield feature, addr
 
 
 BASIC_BLOCK_HANDLERS = (

capa/features/extractors/viv/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,6 +6,8 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import logging
+from typing import Any, Dict, List, Tuple, Iterator
+from pathlib import Path
 
 import viv_utils
 import viv_utils.flirt
@@ -16,40 +18,29 @@ import capa.features.extractors.viv.insn
 import capa.features.extractors.viv.global_
 import capa.features.extractors.viv.function
 import capa.features.extractors.viv.basicblock
-from capa.features.extractors.base_extractor import FeatureExtractor
+from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
 
 logger = logging.getLogger(__name__)
 
 
-class InstructionHandle:
-    """this acts like a vivisect.Opcode but with an __int__() method"""
-
-    def __init__(self, inner):
-        self._inner = inner
-
-    def __int__(self):
-        return self.va
-
-    def __getattr__(self, name):
-        return getattr(self._inner, name)
-
-
 class VivisectFeatureExtractor(FeatureExtractor):
-    def __init__(self, vw, path):
-        super(VivisectFeatureExtractor, self).__init__()
+    def __init__(self, vw, path: Path, os):
+        super().__init__()
         self.vw = vw
         self.path = path
-        with open(self.path, "rb") as f:
-            self.buf = f.read()
+        self.buf = path.read_bytes()
 
         # pre-compute these because we'll yield them at *every* scope.
-        self.global_features = []
-        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf))
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.viv.file.extract_file_format(self.buf))
+        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf, os))
         self.global_features.extend(capa.features.extractors.viv.global_.extract_arch(self.vw))
 
     def get_base_address(self):
         # assume there is only one file loaded into the vw
-        return list(self.vw.filemeta.values())[0]["imagebase"]
+        return AbsoluteVirtualAddress(list(self.vw.filemeta.values())[0]["imagebase"])
 
     def extract_global_features(self):
         yield from self.global_features
@@ -57,28 +48,36 @@ class VivisectFeatureExtractor(FeatureExtractor):
     def extract_file_features(self):
         yield from capa.features.extractors.viv.file.extract_features(self.vw, self.buf)
 
-    def get_functions(self):
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        cache: Dict[str, Any] = {}
         for va in sorted(self.vw.getFunctions()):
-            yield viv_utils.Function(self.vw, va)
+            yield FunctionHandle(
+                address=AbsoluteVirtualAddress(va), inner=viv_utils.Function(self.vw, va), ctx={"cache": cache}
+            )
 
-    def extract_function_features(self, f):
-        yield from capa.features.extractors.viv.function.extract_features(f)
+    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.viv.function.extract_features(fh)
 
-    def get_basic_blocks(self, f):
-        return f.basic_blocks
+    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
+        f: viv_utils.Function = fh.inner
+        for bb in f.basic_blocks:
+            yield BBHandle(address=AbsoluteVirtualAddress(bb.va), inner=bb)
 
-    def extract_basic_block_features(self, f, bb):
-        yield from capa.features.extractors.viv.basicblock.extract_features(f, bb)
+    def extract_basic_block_features(self, fh: FunctionHandle, bbh) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.viv.basicblock.extract_features(fh, bbh)
 
-    def get_instructions(self, f, bb):
+    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
+        bb: viv_utils.BasicBlock = bbh.inner
         for insn in bb.instructions:
-            yield InstructionHandle(insn)
+            yield InsnHandle(address=AbsoluteVirtualAddress(insn.va), inner=insn)
 
-    def extract_insn_features(self, f, bb, insn):
-        yield from capa.features.extractors.viv.insn.extract_features(f, bb, insn)
+    def extract_insn_features(
+        self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.viv.insn.extract_features(fh, bbh, ih)
 
-    def is_library_function(self, va):
-        return viv_utils.flirt.is_library_function(self.vw, va)
+    def is_library_function(self, addr):
+        return viv_utils.flirt.is_library_function(self.vw, addr)
 
-    def get_function_name(self, va):
-        return viv_utils.get_function_name(self.vw, va)
+    def get_function_name(self, addr):
+        return viv_utils.get_function_name(self.vw, addr)

capa/features/extractors/viv/file.py

@@ -1,12 +1,14 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+from typing import Tuple, Iterator
 
 import PE.carve as pe_carve  # vivisect PE
+import vivisect
 import viv_utils
 import viv_utils.flirt
 
@@ -15,20 +17,46 @@ import capa.features.extractors.common
 import capa.features.extractors.helpers
 import capa.features.extractors.strings
 from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import String, Characteristic
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, FileOffsetAddress, AbsoluteVirtualAddress
 
 
-def extract_file_embedded_pe(buf, **kwargs):
+def extract_file_embedded_pe(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     for offset, _ in pe_carve.carve(buf, 1):
-        yield Characteristic("embedded pe"), offset
+        yield Characteristic("embedded pe"), FileOffsetAddress(offset)
 
 
-def extract_file_export_names(vw, **kwargs):
+def get_first_vw_filename(vw: vivisect.VivWorkspace):
+    # vivisect associates metadata with each file that its loaded into the workspace.
+    # capa only loads a single file into each workspace.
+    # so to access the metadata for the file in question, we can just take the first one.
+    # otherwise, we'd have to pass around the module name of the file we're analyzing,
+    # which is a pain.
+    #
+    # so this is a simplifying assumption.
+    return next(iter(vw.filemeta.keys()))
+
+
+def extract_file_export_names(vw: vivisect.VivWorkspace, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     for va, _, name, _ in vw.getExports():
-        yield Export(name), va
+        yield Export(name), AbsoluteVirtualAddress(va)
+
+    if vw.getMeta("Format") == "pe":
+        pe = vw.parsedbin
+        baseaddr = pe.IMAGE_NT_HEADERS.OptionalHeader.ImageBase
+        for rva, _, forwarded_name in vw.getFileMeta(get_first_vw_filename(vw), "forwarders"):
+            try:
+                forwarded_name = forwarded_name.partition(b"\x00")[0].decode("ascii")
+            except UnicodeDecodeError:
+                continue
+
+            forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
+            va = baseaddr + rva
+            yield Export(forwarded_name), AbsoluteVirtualAddress(va)
+            yield Characteristic("forwarded export"), AbsoluteVirtualAddress(va)
 
 
-def extract_file_import_names(vw, **kwargs):
+def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     """
     extract imported function names
     1. imports by ordinal:
@@ -42,10 +70,11 @@ def extract_file_import_names(vw, **kwargs):
         modname, impname = tinfo.split(".", 1)
         if is_viv_ord_impname(impname):
             # replace ord prefix with #
-            impname = "#%s" % impname[len("ord") :]
+            impname = "#" + impname[len("ord") :]
 
+        addr = AbsoluteVirtualAddress(va)
         for name in capa.features.extractors.helpers.generate_symbols(modname, impname):
-            yield Import(name), va
+            yield Import(name), addr
 
 
 def is_viv_ord_impname(impname: str) -> bool:
@@ -62,30 +91,37 @@ def is_viv_ord_impname(impname: str) -> bool:
         return True
 
 
-def extract_file_section_names(vw, **kwargs):
+def extract_file_section_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     for va, _, segname, _ in vw.getSegments():
-        yield Section(segname), va
+        yield Section(segname), AbsoluteVirtualAddress(va)
 
 
-def extract_file_strings(buf, **kwargs):
+def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     yield from capa.features.extractors.common.extract_file_strings(buf)
 
 
-def extract_file_function_names(vw, **kwargs):
+def extract_file_function_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     """
     extract the names of statically-linked library functions.
     """
     for va in sorted(vw.getFunctions()):
+        addr = AbsoluteVirtualAddress(va)
         if viv_utils.flirt.is_library_function(vw, va):
             name = viv_utils.get_function_name(vw, va)
-            yield FunctionName(name), va
+            yield FunctionName(name), addr
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                yield FunctionName(name[1:]), addr
 
 
-def extract_file_format(buf, **kwargs):
+def extract_file_format(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     yield from capa.features.extractors.common.extract_format(buf)
 
 
-def extract_features(vw, buf: bytes):
+def extract_features(vw, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
     """
     extract file features from given workspace
 
@@ -94,12 +130,12 @@ def extract_features(vw, buf: bytes):
       buf: the raw input file bytes
 
     yields:
-      Tuple[Feature, VA]: a feature and its location.
+      Tuple[Feature, Address]: a feature and its location.
     """
 
     for file_handler in FILE_HANDLERS:
-        for feature, va in file_handler(vw=vw, buf=buf):  # type: ignore
-            yield feature, va
+        for feature, addr in file_handler(vw=vw, buf=buf):  # type: ignore
+            yield feature, addr
 
 
 FILE_HANDLERS = (

capa/features/extractors/viv/function.py

@@ -1,45 +1,80 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+from typing import Tuple, Iterator
 
 import envi
+import viv_utils
 import vivisect.const
 
-from capa.features.common import Characteristic
+from capa.features.file import FunctionName
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
 from capa.features.extractors import loops
+from capa.features.extractors.elf import SymTab
+from capa.features.extractors.base_extractor import FunctionHandle
 
 
-def interface_extract_function_XXX(f):
+def interface_extract_function_XXX(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     parse features from the given function.
 
     args:
-      f (viv_utils.Function): the function to process.
+      f: the function to process.
 
     yields:
-      (Feature, int): the feature and the address at which its found.
+      (Feature, Address): the feature and the address at which its found.
     """
-    yield NotImplementedError("feature"), NotImplementedError("virtual address")
+    raise NotImplementedError
 
 
-def extract_function_calls_to(f):
+def extract_function_symtab_names(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    if fh.inner.vw.metadata["Format"] == "elf":
+        # the file's symbol table gets added to the metadata of the vivisect workspace.
+        # this is in order to eliminate the computational overhead of refetching symtab each time.
+        if "symtab" not in fh.ctx["cache"]:
+            try:
+                fh.ctx["cache"]["symtab"] = SymTab.from_Elf(fh.inner.vw.parsedbin)
+            except Exception:
+                fh.ctx["cache"]["symtab"] = None
+
+        symtab = fh.ctx["cache"]["symtab"]
+        if symtab:
+            for symbol in symtab.get_symbols():
+                sym_name = symtab.get_name(symbol)
+                sym_value = symbol.value
+                sym_info = symbol.info
+
+                STT_FUNC = 0x2
+                if sym_value == fh.address and sym_info & STT_FUNC != 0:
+                    yield FunctionName(sym_name), fh.address
+
+
+def extract_function_calls_to(fhandle: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    f: viv_utils.Function = fhandle.inner
     for src, _, _, _ in f.vw.getXrefsTo(f.va, rtype=vivisect.const.REF_CODE):
-        yield Characteristic("calls to"), src
+        yield Characteristic("calls to"), AbsoluteVirtualAddress(src)
 
 
-def extract_function_loop(f):
+def extract_function_loop(fhandle: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     parse if a function has a loop
     """
+    f: viv_utils.Function = fhandle.inner
+
     edges = []
 
     for bb in f.basic_blocks:
         if len(bb.instructions) > 0:
             for bva, bflags in bb.instructions[-1].getBranches():
+                if bva is None:
+                    # vivisect may be unable to recover the call target, e.g. on dynamic calls like `call esi`
+                    # for this bva is None, and we don't want to add it for loop detection, ref: vivisect#574
+                    continue
                 # vivisect does not set branch flags for non-conditional jmp so add explicit check
                 if (
                     bflags & envi.BR_COND
@@ -50,22 +85,26 @@ def extract_function_loop(f):
                     edges.append((bb.va, bva))
 
     if edges and loops.has_loop(edges):
-        yield Characteristic("loop"), f.va
+        yield Characteristic("loop"), fhandle.address
 
 
-def extract_features(f):
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     extract features from the given function.
 
     args:
-      f (viv_utils.Function): the function from which to extract features
+      fh: the function handle from which to extract features
 
     yields:
       Tuple[Feature, int]: the features and their location found in this function.
     """
     for func_handler in FUNCTION_HANDLERS:
-        for feature, va in func_handler(f):
-            yield feature, va
+        for feature, addr in func_handler(fh):
+            yield feature, addr
 
 
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)
+FUNCTION_HANDLERS = (
+    extract_function_symtab_names,
+    extract_function_calls_to,
+    extract_function_loop,
+)

capa/features/extractors/viv/global_.py

@@ -1,19 +1,26 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import logging
+from typing import Tuple, Iterator
 
-import envi.archs.i386
-import envi.archs.amd64
-
-from capa.features.common import ARCH_I386, ARCH_AMD64, Arch
+from capa.features.common import ARCH_I386, ARCH_AMD64, Arch, Feature
+from capa.features.address import NO_ADDRESS, Address
 
 logger = logging.getLogger(__name__)
 
 
-def extract_arch(vw):
-    if isinstance(vw.arch, envi.archs.amd64.Amd64Module):
-        yield Arch(ARCH_AMD64), 0x0
+def extract_arch(vw) -> Iterator[Tuple[Feature, Address]]:
+    arch = vw.getMeta("Architecture")
+    if arch == "amd64":
+        yield Arch(ARCH_AMD64), NO_ADDRESS
 
-    elif isinstance(vw.arch, envi.archs.i386.i386Module):
-        yield Arch(ARCH_I386), 0x0
+    elif arch == "i386":
+        yield Arch(ARCH_I386), NO_ADDRESS
 
     else:
         # we likely end up here:

capa/features/extractors/viv/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/viv/indirect_calls.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,7 +7,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import collections
-from typing import TYPE_CHECKING, Set, List, Deque, Tuple, Union, Optional
+from typing import Set, List, Deque, Tuple, Optional
 
 import envi
 import vivisect.const
@@ -15,9 +15,6 @@ import envi.archs.i386.disasm
 import envi.archs.amd64.disasm
 from vivisect import VivWorkspace
 
-if TYPE_CHECKING:
-    from capa.features.extractors.viv.extractor import InstructionHandle
-
 # pull out consts for lookup performance
 i386RegOper = envi.archs.i386.disasm.i386RegOper
 i386ImmOper = envi.archs.i386.disasm.i386ImmOper
@@ -45,7 +42,7 @@ def get_previous_instructions(vw: VivWorkspace, va: int) -> List[int]:
     ret = []
 
     # find the immediate prior instruction.
-    # ensure that it fallsthrough to this one.
+    # ensure that it falls through to this one.
     loc = vw.getPrevLocation(va, adjacent=True)
     if loc is not None:
         ploc = vw.getPrevLocation(va, adjacent=True)
@@ -62,7 +59,7 @@ def get_previous_instructions(vw: VivWorkspace, va: int) -> List[int]:
     #
     # from vivisect.const:
     # xref: (XR_FROM, XR_TO, XR_RTYPE, XR_RFLAG)
-    for (xfrom, _, _, xflag) in vw.getXrefsTo(va, REF_CODE):
+    for xfrom, _, _, xflag in vw.getXrefsTo(va, REF_CODE):
         if (xflag & FAR_BRANCH_MASK) != 0:
             continue
         ret.append(xfrom)
@@ -74,7 +71,7 @@ class NotFoundError(Exception):
     pass
 
 
-def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int, None]]:
+def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Optional[int]]:
     """
     scan backwards from the given address looking for assignments to the given register.
     if a constant, return that value.
@@ -90,8 +87,8 @@ def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int
     raises:
       NotFoundError: when the definition cannot be found.
     """
-    q = collections.deque()  # type: Deque[int]
-    seen = set([])  # type: Set[int]
+    q: Deque[int] = collections.deque()
+    seen: Set[int] = set()
 
     q.extend(get_previous_instructions(vw, va))
     while q:
@@ -135,16 +132,14 @@ def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int
     raise NotFoundError()
 
 
-def is_indirect_call(vw: VivWorkspace, va: int, insn: Optional["InstructionHandle"] = None) -> bool:
+def is_indirect_call(vw: VivWorkspace, va: int, insn: envi.Opcode) -> bool:
     if insn is None:
         insn = vw.parseOpcode(va)
 
     return insn.mnem in ("call", "jmp") and isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper)
 
 
-def resolve_indirect_call(
-    vw: VivWorkspace, va: int, insn: Optional["InstructionHandle"] = None
-) -> Tuple[int, Optional[int]]:
+def resolve_indirect_call(vw: VivWorkspace, va: int, insn: envi.Opcode) -> Tuple[int, Optional[int]]:
     """
     inspect the given indirect call instruction and attempt to resolve the target address.
 

capa/features/extractors/viv/insn.py

@@ -1,10 +1,12 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Tuple, Callable, Iterator
+
 import envi
 import envi.exc
 import viv_utils
@@ -17,16 +19,11 @@ import envi.archs.amd64.disasm
 
 import capa.features.extractors.helpers
 import capa.features.extractors.viv.helpers
-from capa.features.insn import API, Number, Offset, Mnemonic
-from capa.features.common import (
-    BITNESS_X32,
-    BITNESS_X64,
-    MAX_BYTES_FEATURE_SIZE,
-    THUNK_CHAIN_DEPTH_DELTA,
-    Bytes,
-    String,
-    Characteristic,
-)
+from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
+from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.elf import SymTab
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
 from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_indirect_call
 
 # security cookie checks may perform non-zeroing XORs, these are expected within a certain
@@ -34,27 +31,21 @@ from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_i
 SECURITY_COOKIE_BYTES_DELTA = 0x40
 
 
-def get_bitness(vw):
-    bitness = vw.getMeta("Architecture")
-    if bitness == "i386":
-        return BITNESS_X32
-    elif bitness == "amd64":
-        return BITNESS_X64
-
-
-def interface_extract_instruction_XXX(f, bb, insn):
+def interface_extract_instruction_XXX(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
     """
     parse features from the given instruction.
 
     args:
-      f (viv_utils.Function): the function to process.
-      bb (viv_utils.BasicBlock): the basic block to process.
-      insn (vivisect...Instruction): the instruction to process.
+      fh: the function handle to process.
+      bbh: the basic block handle to process.
+      ih: the instruction handle to process.
 
     yields:
-      (Feature, int): the feature and the address at which its found.
+      (Feature, Address): the feature and the address at which its found.
     """
-    yield NotImplementedError("feature"), NotImplementedError("virtual address")
+    raise NotImplementedError
 
 
 def get_imports(vw):
@@ -74,12 +65,15 @@ def get_imports(vw):
         return imports
 
 
-def extract_insn_api_features(f, bb, insn):
-    """parse API features from the given instruction."""
+def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse API features from the given instruction.
 
-    # example:
-    #
-    #    call dword [0x00473038]
+    example:
+       call dword [0x00473038]
+    """
+    insn: envi.Opcode = ih.inner
+    f: viv_utils.Function = fh.inner
     if insn.mnem not in ("call", "jmp"):
         return
 
@@ -96,7 +90,7 @@ def extract_insn_api_features(f, bb, insn):
         if target in imports:
             dll, symbol = imports[target]
             for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-                yield API(name), insn.va
+                yield API(name), ih.address
 
     # call via thunk on x86,
     # see 9324d1a8ae37a36ae560c37448c9705a at 0x407985
@@ -116,16 +110,42 @@ def extract_insn_api_features(f, bb, insn):
         if not target:
             return
 
+        if f.vw.metadata["Format"] == "elf":
+            if "symtab" not in fh.ctx["cache"]:
+                # the symbol table gets stored as a function's attribute in order to avoid running
+                # this code everytime the call is made, thus preventing the computational overhead.
+                try:
+                    fh.ctx["cache"]["symtab"] = SymTab.from_Elf(f.vw.parsedbin)
+                except Exception:
+                    fh.ctx["cache"]["symtab"] = None
+
+            symtab = fh.ctx["cache"]["symtab"]
+            if symtab:
+                for symbol in symtab.get_symbols():
+                    sym_name = symtab.get_name(symbol)
+                    sym_value = symbol.value
+                    sym_info = symbol.info
+
+                    STT_FUNC = 0x2
+                    if sym_value == target and sym_info & STT_FUNC != 0:
+                        yield API(sym_name), ih.address
+
         if viv_utils.flirt.is_library_function(f.vw, target):
             name = viv_utils.get_function_name(f.vw, target)
-            yield API(name), insn.va
+            yield API(name), ih.address
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                yield API(name[1:]), ih.address
             return
 
         for _ in range(THUNK_CHAIN_DEPTH_DELTA):
             if target in imports:
                 dll, symbol = imports[target]
                 for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-                    yield API(name), insn.va
+                    yield API(name), ih.address
 
             # if jump leads to an ENDBRANCH instruction, skip it
             if f.vw.getByteDef(target)[1].startswith(b"\xf3\x0f\x1e"):
@@ -145,7 +165,7 @@ def extract_insn_api_features(f, bb, insn):
         if target in imports:
             dll, symbol = imports[target]
             for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-                yield API(name), insn.va
+                yield API(name), ih.address
 
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper):
         try:
@@ -162,38 +182,7 @@ def extract_insn_api_features(f, bb, insn):
         if target in imports:
             dll, symbol = imports[target]
             for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-                yield API(name), insn.va
-
-
-def extract_insn_number_features(f, bb, insn):
-    """parse number features from the given instruction."""
-    # example:
-    #
-    #     push    3136B0h         ; dwControlCode
-    for oper in insn.opers:
-        # this is for both x32 and x64
-        if not isinstance(oper, (envi.archs.i386.disasm.i386ImmOper, envi.archs.i386.disasm.i386ImmMemOper)):
-            continue
-
-        if isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
-            v = oper.getOperValue(oper)
-        else:
-            v = oper.getOperAddr(oper)
-
-        if f.vw.probeMemory(v, 1, envi.memory.MM_READ):
-            # this is a valid address
-            # assume its not also a constant.
-            continue
-
-        if insn.mnem == "add" and insn.opers[0].isReg() and insn.opers[0].reg == envi.archs.i386.regs.REG_ESP:
-            # skip things like:
-            #
-            #    .text:00401140                 call    sub_407E2B
-            #    .text:00401145                 add     esp, 0Ch
-            return
-
-        yield Number(v), insn.va
-        yield Number(v, bitness=get_bitness(f.vw)), insn.va
+                yield API(name), ih.address
 
 
 def derefs(vw, p):
@@ -207,8 +196,13 @@ def derefs(vw, p):
     while True:
         if not vw.isValidPointer(p):
             return
+
         yield p
 
+        if vw.isProbablyString(p) or vw.isProbablyUnicode(p):
+            # don't deref strings that coincidentally are pointers
+            return
+
         try:
             next = vw.readMemoryPtr(p)
         except Exception:
@@ -266,12 +260,15 @@ def read_bytes(vw, va: int) -> bytes:
         raise
 
 
-def extract_insn_bytes_features(f, bb, insn):
+def extract_insn_bytes_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     parse byte sequence features from the given instruction.
     example:
         #     push    offset iid_004118d4_IShellLinkA ; riid
     """
+    insn: envi.Opcode = ih.inner
+    f: viv_utils.Function = fh.inner
+
     if insn.mnem == "call":
         return
 
@@ -291,16 +288,20 @@ def extract_insn_bytes_features(f, bb, insn):
         else:
             continue
 
-        for v in derefs(f.vw, v):
+        for vv in derefs(f.vw, v):
             try:
-                buf = read_bytes(f.vw, v)
+                buf = read_bytes(f.vw, vv)
             except envi.exc.SegmentationViolation:
                 continue
 
             if capa.features.extractors.helpers.all_zeros(buf):
                 continue
 
-            yield Bytes(buf), insn.va
+            if f.vw.isProbablyString(vv) or f.vw.isProbablyUnicode(vv):
+                # don't extract byte features for obvious strings
+                continue
+
+            yield Bytes(buf), ih.address
 
 
 def read_string(vw, offset: int) -> str:
@@ -310,7 +311,12 @@ def read_string(vw, offset: int) -> str:
         pass
     else:
         if alen > 0:
-            return read_memory(vw, offset, alen).decode("utf-8")
+            buf = read_memory(vw, offset, alen)
+            if b"\x00" in buf:
+                # account for bug #1271.
+                # remove when vivisect is fixed.
+                buf = buf.partition(b"\x00")[0]
+            return buf.decode("utf-8")
 
     try:
         ulen = vw.detectUnicode(offset)
@@ -329,80 +335,13 @@ def read_string(vw, offset: int) -> str:
                 # vivisect seems to mis-detect the end unicode strings
                 # off by two, too short
                 ulen += 2
-            return read_memory(vw, offset, ulen).decode("utf-16")
+            # partition to account for bug #1271.
+            # remove when vivisect is fixed.
+            return read_memory(vw, offset, ulen).decode("utf-16").partition("\x00")[0]
 
     raise ValueError("not a string", offset)
 
 
-def extract_insn_string_features(f, bb, insn):
-    """parse string features from the given instruction."""
-    # example:
-    #
-    #     push    offset aAcr     ; "ACR  > "
-
-    for oper in insn.opers:
-        if isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
-            v = oper.getOperValue(oper)
-        elif isinstance(oper, envi.archs.i386.disasm.i386ImmMemOper):
-            # like 0x10056CB4 in `lea eax, dword [0x10056CB4]`
-            v = oper.imm
-        elif isinstance(oper, envi.archs.i386.disasm.i386SibOper):
-            # like 0x401000 in `mov eax, 0x401000[2 * ebx]`
-            v = oper.imm
-        elif isinstance(oper, envi.archs.amd64.disasm.Amd64RipRelOper):
-            v = oper.getOperAddr(insn)
-        else:
-            continue
-
-        for v in derefs(f.vw, v):
-            try:
-                s = read_string(f.vw, v)
-            except ValueError:
-                continue
-            else:
-                yield String(s.rstrip("\x00")), insn.va
-
-
-def extract_insn_offset_features(f, bb, insn):
-    """parse structure offset features from the given instruction."""
-    # example:
-    #
-    #     .text:0040112F    cmp     [esi+4], ebx
-    for oper in insn.opers:
-
-        # this is for both x32 and x64
-        # like [esi + 4]
-        #       reg   ^
-        #             disp
-        if isinstance(oper, envi.archs.i386.disasm.i386RegMemOper):
-            if oper.reg == envi.archs.i386.regs.REG_ESP:
-                continue
-
-            if oper.reg == envi.archs.i386.regs.REG_EBP:
-                continue
-
-            # TODO: do x64 support for real.
-            if oper.reg == envi.archs.amd64.regs.REG_RBP:
-                continue
-
-            # viv already decodes offsets as signed
-            v = oper.disp
-
-            yield Offset(v), insn.va
-            yield Offset(v, bitness=get_bitness(f.vw)), insn.va
-
-        # like: [esi + ecx + 16384]
-        #        reg   ^     ^
-        #              index ^
-        #                    disp
-        elif isinstance(oper, envi.archs.i386.disasm.i386SibOper):
-            # viv already decodes offsets as signed
-            v = oper.disp
-
-            yield Offset(v), insn.va
-            yield Offset(v, bitness=get_bitness(f.vw)), insn.va
-
-
 def is_security_cookie(f, bb, insn) -> bool:
     """
     check if an instruction is related to security cookie checks
@@ -412,7 +351,6 @@ def is_security_cookie(f, bb, insn) -> bool:
     if oper.isReg() and oper.reg not in [
         envi.archs.i386.regs.REG_ESP,
         envi.archs.i386.regs.REG_EBP,
-        # TODO: do x64 support for real.
         envi.archs.amd64.regs.REG_RBP,
         envi.archs.amd64.regs.REG_RSP,
     ]:
@@ -431,11 +369,17 @@ def is_security_cookie(f, bb, insn) -> bool:
     return False
 
 
-def extract_insn_nzxor_characteristic_features(f, bb, insn):
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle, bbhandle: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
     """
     parse non-zeroing XOR instruction from the given instruction.
     ignore expected non-zeroing XORs, e.g. security cookies.
     """
+    insn: envi.Opcode = ih.inner
+    bb: viv_utils.BasicBlock = bbhandle.inner
+    f: viv_utils.Function = fh.inner
+
     if insn.mnem not in ("xor", "xorpd", "xorps", "pxor"):
         return
 
@@ -445,19 +389,37 @@ def extract_insn_nzxor_characteristic_features(f, bb, insn):
     if is_security_cookie(f, bb, insn):
         return
 
-    yield Characteristic("nzxor"), insn.va
+    yield Characteristic("nzxor"), ih.address
 
 
-def extract_insn_mnemonic_features(f, bb, insn):
+def extract_insn_mnemonic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """parse mnemonic features from the given instruction."""
-    yield Mnemonic(insn.mnem), insn.va
+    yield Mnemonic(ih.inner.mnem), ih.address
 
 
-def extract_insn_peb_access_characteristic_features(f, bb, insn):
+def extract_insn_obfs_call_plus_5_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse call $+5 instruction from the given instruction.
+    """
+    insn: envi.Opcode = ih.inner
+
+    if insn.mnem != "call":
+        return
+
+    if isinstance(insn.opers[0], envi.archs.i386.disasm.i386PcRelOper):
+        if insn.va + 5 == insn.opers[0].getOperValue(insn):
+            yield Characteristic("call $+5"), ih.address
+
+    if isinstance(insn.opers[0], (envi.archs.i386.disasm.i386ImmMemOper, envi.archs.amd64.disasm.Amd64RipRelOper)):
+        if insn.va + 5 == insn.opers[0].getOperAddr(insn):
+            yield Characteristic("call $+5"), ih.address
+
+
+def extract_insn_peb_access_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
     """
-    # TODO handle where fs/gs are loaded into a register or onto the stack and used later
+    insn: envi.Opcode = ih.inner
 
     if insn.mnem not in ["push", "mov"]:
         return
@@ -476,7 +438,7 @@ def extract_insn_peb_access_characteristic_features(f, bb, insn):
             if (isinstance(oper, envi.archs.i386.disasm.i386RegMemOper) and oper.disp == 0x30) or (
                 isinstance(oper, envi.archs.i386.disasm.i386ImmMemOper) and oper.imm == 0x30
             ):
-                yield Characteristic("peb access"), insn.va
+                yield Characteristic("peb access"), ih.address
     elif "gs" in prefix:
         for oper in insn.opers:
             if (
@@ -484,20 +446,22 @@ def extract_insn_peb_access_characteristic_features(f, bb, insn):
                 or (isinstance(oper, envi.archs.amd64.disasm.i386SibOper) and oper.imm == 0x60)
                 or (isinstance(oper, envi.archs.amd64.disasm.i386ImmMemOper) and oper.imm == 0x60)
             ):
-                yield Characteristic("peb access"), insn.va
+                yield Characteristic("peb access"), ih.address
     else:
         pass
 
 
-def extract_insn_segment_access_features(f, bb, insn):
+def extract_insn_segment_access_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """parse the instruction for access to fs or gs"""
+    insn: envi.Opcode = ih.inner
+
     prefix = insn.getPrefixName()
 
     if prefix == "fs":
-        yield Characteristic("fs access"), insn.va
+        yield Characteristic("fs access"), ih.address
 
     if prefix == "gs":
-        yield Characteristic("gs access"), insn.va
+        yield Characteristic("gs access"), ih.address
 
 
 def get_section(vw, va: int):
@@ -508,10 +472,13 @@ def get_section(vw, va: int):
     raise KeyError(va)
 
 
-def extract_insn_cross_section_cflow(f, bb, insn):
+def extract_insn_cross_section_cflow(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     inspect the instruction for a CALL or JMP that crosses section boundaries.
     """
+    insn: envi.Opcode = ih.inner
+    f: viv_utils.Function = fh.inner
+
     for va, flags in insn.getBranches():
         if va is None:
             # va may be none for dynamic branches that haven't been resolved, such as `jmp eax`.
@@ -538,7 +505,7 @@ def extract_insn_cross_section_cflow(f, bb, insn):
                     continue
 
             if get_section(f.vw, insn.va) != get_section(f.vw, va):
-                yield Characteristic("cross section flow"), insn.va
+                yield Characteristic("cross section flow"), ih.address
 
         except KeyError:
             continue
@@ -546,7 +513,10 @@ def extract_insn_cross_section_cflow(f, bb, insn):
 
 # this is a feature that's most relevant at the function scope,
 # however, its most efficient to extract at the instruction scope.
-def extract_function_calls_from(f, bb, insn):
+def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    insn: envi.Opcode = ih.inner
+    f: viv_utils.Function = fh.inner
+
     if insn.mnem != "call":
         return
 
@@ -556,7 +526,8 @@ def extract_function_calls_from(f, bb, insn):
     if isinstance(insn.opers[0], envi.archs.i386.disasm.i386ImmMemOper):
         oper = insn.opers[0]
         target = oper.getOperAddr(insn)
-        yield Characteristic("calls from"), target
+        if target >= 0:
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
 
     # call via thunk on x86,
     # see 9324d1a8ae37a36ae560c37448c9705a at 0x407985
@@ -565,43 +536,192 @@ def extract_function_calls_from(f, bb, insn):
     # see Lab21-01.exe_:0x140001178
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386PcRelOper):
         target = insn.opers[0].getOperValue(insn)
-        yield Characteristic("calls from"), target
+        if target >= 0:
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
 
     # call via IAT, x64
     elif isinstance(insn.opers[0], envi.archs.amd64.disasm.Amd64RipRelOper):
         op = insn.opers[0]
         target = op.getOperAddr(insn)
-        yield Characteristic("calls from"), target
+        if target >= 0:
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
 
     if target and target == f.va:
         # if we found a jump target and it's the function address
         # mark as recursive
-        yield Characteristic("recursive call"), target
+        yield Characteristic("recursive call"), AbsoluteVirtualAddress(target)
 
 
 # this is a feature that's most relevant at the function or basic block scope,
 # however, its most efficient to extract at the instruction scope.
-def extract_function_indirect_call_characteristic_features(f, bb, insn):
+def extract_function_indirect_call_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
     """
     extract indirect function call characteristic (e.g., call eax or call dword ptr [edx+4])
     does not include calls like => call ds:dword_ABD4974
     """
+    insn: envi.Opcode = ih.inner
+
     if insn.mnem != "call":
         return
 
     # Checks below work for x86 and x64
     if isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper):
         # call edx
-        yield Characteristic("indirect call"), insn.va
+        yield Characteristic("indirect call"), ih.address
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegMemOper):
         # call dword ptr [eax+50h]
-        yield Characteristic("indirect call"), insn.va
+        yield Characteristic("indirect call"), ih.address
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386SibOper):
         # call qword ptr [rsp+78h]
-        yield Characteristic("indirect call"), insn.va
+        yield Characteristic("indirect call"), ih.address
 
 
-def extract_features(f, bb, insn):
+def extract_op_number_features(
+    fh: FunctionHandle, bb, ih: InsnHandle, i, oper: envi.Operand
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse number features from the given operand.
+
+    example:
+        push    3136B0h         ; dwControlCode
+    """
+    insn: envi.Opcode = ih.inner
+    f: viv_utils.Function = fh.inner
+
+    # this is for both x32 and x64
+    if not isinstance(oper, (envi.archs.i386.disasm.i386ImmOper, envi.archs.i386.disasm.i386ImmMemOper)):
+        return
+
+    if isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
+        v = oper.getOperValue(oper)
+    else:
+        v = oper.getOperAddr(oper)
+
+    if f.vw.probeMemory(v, 1, envi.memory.MM_READ):
+        # this is a valid address
+        # assume its not also a constant.
+        return
+
+    if insn.mnem == "add" and insn.opers[0].isReg() and insn.opers[0].reg == envi.archs.i386.regs.REG_ESP:
+        # skip things like:
+        #
+        #    .text:00401140                 call    sub_407E2B
+        #    .text:00401145                 add     esp, 0Ch
+        return
+
+    yield Number(v), ih.address
+    yield OperandNumber(i, v), ih.address
+
+    if insn.mnem == "add" and 0 < v < MAX_STRUCTURE_SIZE and isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
+        # for pattern like:
+        #
+        #     add eax, 0x10
+        #
+        # assume 0x10 is also an offset (imagine eax is a pointer).
+        yield Offset(v), ih.address
+        yield OperandOffset(i, v), ih.address
+
+
+def extract_op_offset_features(
+    fh: FunctionHandle, bb, ih: InsnHandle, i, oper: envi.Operand
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse structure offset features from the given operand."""
+    # example:
+    #
+    #     .text:0040112F    cmp     [esi+4], ebx
+    insn: envi.Opcode = ih.inner
+    f: viv_utils.Function = fh.inner
+
+    # this is for both x32 and x64
+    # like [esi + 4]
+    #       reg   ^
+    #             disp
+    if isinstance(oper, envi.archs.i386.disasm.i386RegMemOper):
+        if oper.reg == envi.archs.i386.regs.REG_ESP:
+            return
+
+        if oper.reg == envi.archs.i386.regs.REG_EBP:
+            return
+
+        if oper.reg == envi.archs.amd64.regs.REG_RBP:
+            return
+
+        # viv already decodes offsets as signed
+        v = oper.disp
+
+        yield Offset(v), ih.address
+        yield OperandOffset(i, v), ih.address
+
+        if insn.mnem == "lea" and i == 1 and not f.vw.probeMemory(v, 1, envi.memory.MM_READ):
+            # for pattern like:
+            #
+            #     lea eax, [ebx + 1]
+            #
+            # assume 1 is also an offset (imagine ebx is a zero register).
+            yield Number(v), ih.address
+            yield OperandNumber(i, v), ih.address
+
+    # like: [esi + ecx + 16384]
+    #        reg   ^     ^
+    #              index ^
+    #                    disp
+    elif isinstance(oper, envi.archs.i386.disasm.i386SibOper):
+        # viv already decodes offsets as signed
+        v = oper.disp
+
+        yield Offset(v), ih.address
+        yield OperandOffset(i, v), ih.address
+
+
+def extract_op_string_features(
+    fh: FunctionHandle, bb, ih: InsnHandle, i, oper: envi.Operand
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse string features from the given operand."""
+    # example:
+    #
+    #     push    offset aAcr     ; "ACR  > "
+    insn: envi.Opcode = ih.inner
+    f: viv_utils.Function = fh.inner
+
+    if isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
+        v = oper.getOperValue(oper)
+    elif isinstance(oper, envi.archs.i386.disasm.i386ImmMemOper):
+        # like 0x10056CB4 in `lea eax, dword [0x10056CB4]`
+        v = oper.imm
+    elif isinstance(oper, envi.archs.i386.disasm.i386SibOper):
+        # like 0x401000 in `mov eax, 0x401000[2 * ebx]`
+        v = oper.imm
+    elif isinstance(oper, envi.archs.amd64.disasm.Amd64RipRelOper):
+        v = oper.getOperAddr(insn)
+    else:
+        return
+
+    for vv in derefs(f.vw, v):
+        try:
+            s = read_string(f.vw, vv).rstrip("\x00")
+        except ValueError:
+            continue
+        else:
+            if len(s) >= 4:
+                yield String(s), ih.address
+
+
+def extract_operand_features(f: FunctionHandle, bb, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    for i, oper in enumerate(insn.inner.opers):
+        for op_handler in OPERAND_HANDLERS:
+            for feature, addr in op_handler(f, bb, insn, i, oper):
+                yield feature, addr
+
+
+OPERAND_HANDLERS: List[
+    Callable[[FunctionHandle, BBHandle, InsnHandle, int, envi.Operand], Iterator[Tuple[Feature, Address]]]
+] = [
+    extract_op_number_features,
+    extract_op_offset_features,
+    extract_op_string_features,
+]
+
+
+def extract_features(f, bb, insn) -> Iterator[Tuple[Feature, Address]]:
     """
     extract features from the given insn.
 
@@ -611,24 +731,23 @@ def extract_features(f, bb, insn):
       insn (vivisect...Instruction): the instruction to process.
 
     yields:
-      Tuple[Feature, int]: the features and their location found in this insn.
+      Tuple[Feature, Address]: the features and their location found in this insn.
     """
     for insn_handler in INSTRUCTION_HANDLERS:
-        for feature, va in insn_handler(f, bb, insn):
-            yield feature, va
+        for feature, addr in insn_handler(f, bb, insn):
+            yield feature, addr
 
 
-INSTRUCTION_HANDLERS = (
+INSTRUCTION_HANDLERS: List[Callable[[FunctionHandle, BBHandle, InsnHandle], Iterator[Tuple[Feature, Address]]]] = [
     extract_insn_api_features,
-    extract_insn_number_features,
-    extract_insn_string_features,
     extract_insn_bytes_features,
-    extract_insn_offset_features,
     extract_insn_nzxor_characteristic_features,
     extract_insn_mnemonic_features,
+    extract_insn_obfs_call_plus_5_characteristic_features,
     extract_insn_peb_access_characteristic_features,
     extract_insn_cross_section_cflow,
     extract_insn_segment_access_features,
     extract_function_calls_from,
     extract_function_indirect_call_characteristic_features,
-)
+    extract_operand_features,
+]

capa/features/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -12,19 +12,19 @@ from capa.features.common import Feature
 class Export(Feature):
     def __init__(self, value: str, description=None):
         # value is export name
-        super(Export, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
 class Import(Feature):
     def __init__(self, value: str, description=None):
         # value is import name
-        super(Import, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
 class Section(Feature):
     def __init__(self, value: str, description=None):
         # value is section name
-        super(Section, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
 class FunctionName(Feature):
@@ -32,7 +32,7 @@ class FunctionName(Feature):
 
     def __init__(self, name: str, description=None):
         # value is function name
-        super(FunctionName, self).__init__(name, description=description)
+        super().__init__(name, description=description)
         # override the name property set by `capa.features.Feature`
         # that would be `functionname` (note missing dash)
         self.name = "function-name"

capa/features/freeze.py

@@ -1,279 +0,0 @@
-"""
-capa freeze file format: `| capa0000 | + zlib(utf-8(json(...)))`
-
-json format:
-
-    {
-      'version': 1,
-      'base address': int(base address),
-      'functions': {
-        int(function va): {
-          int(basic block va): [int(instruction va), ...]
-          ...
-        },
-        ...
-      },
-      'scopes': {
-        'global': [
-          (str(name), [any(arg), ...], int(va), ()),
-          ...
-        },
-        'file': [
-          (str(name), [any(arg), ...], int(va), ()),
-          ...
-        },
-        'function': [
-          (str(name), [any(arg), ...], int(va), (int(function va), )),
-          ...
-        ],
-        'basic block': [
-          (str(name), [any(arg), ...], int(va), (int(function va),
-                                                 int(basic block va))),
-          ...
-        ],
-        'instruction': [
-          (str(name), [any(arg), ...], int(va), (int(function va),
-                                                 int(basic block va),
-                                                 int(instruction va))),
-          ...
-        ],
-      }
-    }
-
-Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
-You may obtain a copy of the License at: [package root]/LICENSE.txt
-Unless required by applicable law or agreed to in writing, software distributed under the License
- is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and limitations under the License.
-"""
-import json
-import zlib
-import logging
-
-import capa.features.file
-import capa.features.insn
-import capa.features.common
-import capa.features.basicblock
-import capa.features.extractors.base_extractor
-from capa.helpers import hex
-
-logger = logging.getLogger(__name__)
-
-
-def serialize_feature(feature):
-    return feature.freeze_serialize()
-
-
-KNOWN_FEATURES = {F.__name__: F for F in capa.features.common.Feature.__subclasses__()}
-
-
-def deserialize_feature(doc):
-    F = KNOWN_FEATURES[doc[0]]
-    return F.freeze_deserialize(doc[1])
-
-
-def dumps(extractor):
-    """
-    serialize the given extractor to a string
-
-    args:
-      extractor: capa.features.extractors.base_extractor.FeatureExtractor:
-
-    returns:
-      str: the serialized features.
-    """
-    ret = {
-        "version": 1,
-        "base address": extractor.get_base_address(),
-        "functions": {},
-        "scopes": {
-            "global": [],
-            "file": [],
-            "function": [],
-            "basic block": [],
-            "instruction": [],
-        },
-    }
-    for feature, va in extractor.extract_global_features():
-        ret["scopes"]["global"].append(serialize_feature(feature) + (hex(va), ()))
-
-    for feature, va in extractor.extract_file_features():
-        ret["scopes"]["file"].append(serialize_feature(feature) + (hex(va), ()))
-
-    for f in extractor.get_functions():
-        ret["functions"][hex(f)] = {}
-
-        for feature, va in extractor.extract_function_features(f):
-            ret["scopes"]["function"].append(serialize_feature(feature) + (hex(va), (hex(f),)))
-
-        for bb in extractor.get_basic_blocks(f):
-            ret["functions"][hex(f)][hex(bb)] = []
-
-            for feature, va in extractor.extract_basic_block_features(f, bb):
-                ret["scopes"]["basic block"].append(
-                    serialize_feature(feature)
-                    + (
-                        hex(va),
-                        (
-                            hex(f),
-                            hex(bb),
-                        ),
-                    )
-                )
-
-            for insnva, insn in sorted(
-                [(int(insn), insn) for insn in extractor.get_instructions(f, bb)], key=lambda p: p[0]
-            ):
-                ret["functions"][hex(f)][hex(bb)].append(hex(insnva))
-
-                for feature, va in extractor.extract_insn_features(f, bb, insn):
-                    ret["scopes"]["instruction"].append(
-                        serialize_feature(feature)
-                        + (
-                            hex(va),
-                            (
-                                hex(f),
-                                hex(bb),
-                                hex(insnva),
-                            ),
-                        )
-                    )
-    return json.dumps(ret)
-
-
-def loads(s):
-    """deserialize a set of features (as a NullFeatureExtractor) from a string."""
-    doc = json.loads(s)
-
-    if doc.get("version") != 1:
-        raise ValueError("unsupported freeze format version: %d" % (doc.get("version")))
-
-    features = {
-        "base address": doc.get("base address"),
-        "global features": [],
-        "file features": [],
-        "functions": {},
-    }
-
-    for fva, function in doc.get("functions", {}).items():
-        fva = int(fva, 0x10)
-        features["functions"][fva] = {
-            "features": [],
-            "basic blocks": {},
-        }
-
-        for bbva, bb in function.items():
-            bbva = int(bbva, 0x10)
-            features["functions"][fva]["basic blocks"][bbva] = {
-                "features": [],
-                "instructions": {},
-            }
-
-            for insnva in bb:
-                insnva = int(insnva, 0x10)
-                features["functions"][fva]["basic blocks"][bbva]["instructions"][insnva] = {
-                    "features": [],
-                }
-
-    # in the following blocks, each entry looks like:
-    #
-    #     ('MatchedRule', ('foo', ), '0x401000', ('0x401000', ))
-    #      ^^^^^^^^^^^^^  ^^^^^^^^^  ^^^^^^^^^^  ^^^^^^^^^^^^^^
-    #      feature name   args       addr         func/bb/insn
-    for feature in doc.get("scopes", {}).get("global", []):
-        va, loc = feature[2:]
-        va = int(va, 0x10)
-        feature = deserialize_feature(feature[:2])
-        features["global features"].append((va, feature))
-
-    for feature in doc.get("scopes", {}).get("file", []):
-        va, loc = feature[2:]
-        va = int(va, 0x10)
-        feature = deserialize_feature(feature[:2])
-        features["file features"].append((va, feature))
-
-    for feature in doc.get("scopes", {}).get("function", []):
-        # fetch the pair like:
-        #
-        #     ('0x401000', ('0x401000', ))
-        #      ^^^^^^^^^^  ^^^^^^^^^^^^^^
-        #      addr         func/bb/insn
-        va, loc = feature[2:]
-        va = int(va, 0x10)
-        loc = [int(lo, 0x10) for lo in loc]
-
-        # decode the feature from the pair like:
-        #
-        #     ('MatchedRule', ('foo', ))
-        #      ^^^^^^^^^^^^^  ^^^^^^^^^
-        #      feature name   args
-        feature = deserialize_feature(feature[:2])
-        features["functions"][loc[0]]["features"].append((va, feature))
-
-    for feature in doc.get("scopes", {}).get("basic block", []):
-        va, loc = feature[2:]
-        va = int(va, 0x10)
-        loc = [int(lo, 0x10) for lo in loc]
-        feature = deserialize_feature(feature[:2])
-        features["functions"][loc[0]]["basic blocks"][loc[1]]["features"].append((va, feature))
-
-    for feature in doc.get("scopes", {}).get("instruction", []):
-        va, loc = feature[2:]
-        va = int(va, 0x10)
-        loc = [int(lo, 0x10) for lo in loc]
-        feature = deserialize_feature(feature[:2])
-        features["functions"][loc[0]]["basic blocks"][loc[1]]["instructions"][loc[2]]["features"].append((va, feature))
-
-    return capa.features.extractors.base_extractor.NullFeatureExtractor(features)
-
-
-MAGIC = "capa0000".encode("ascii")
-
-
-def dump(extractor):
-    """serialize the given extractor to a byte array."""
-    return MAGIC + zlib.compress(dumps(extractor).encode("utf-8"))
-
-
-def is_freeze(buf: bytes) -> bool:
-    return buf[: len(MAGIC)] == MAGIC
-
-
-def load(buf):
-    """deserialize a set of features (as a NullFeatureExtractor) from a byte array."""
-    if not is_freeze(buf):
-        raise ValueError("missing magic header")
-    return loads(zlib.decompress(buf[len(MAGIC) :]).decode("utf-8"))
-
-
-def main(argv=None):
-    import sys
-    import argparse
-
-    import capa.main
-
-    if argv is None:
-        argv = sys.argv[1:]
-
-    parser = argparse.ArgumentParser(description="save capa features to a file")
-    capa.main.install_common_args(parser, {"sample", "format", "backend", "signatures"})
-    parser.add_argument("output", type=str, help="Path to output file")
-    args = parser.parse_args(args=argv)
-    capa.main.handle_common_args(args)
-
-    sigpaths = capa.main.get_signatures(args.signatures)
-
-    extractor = capa.main.get_extractor(args.sample, args.format, args.backend, sigpaths, False)
-
-    with open(args.output, "wb") as f:
-        f.write(dump(extractor))
-
-    return 0
-
-
-if __name__ == "__main__":
-    import sys
-
-    sys.exit(main())

capa/features/freeze/__init__.py

@@ -0,0 +1,410 @@
+"""
+capa freeze file format: `| capa0000 | + zlib(utf-8(json(...)))`
+
+Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+You may obtain a copy of the License at: [package root]/LICENSE.txt
+Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+"""
+import zlib
+import logging
+from enum import Enum
+from typing import List, Tuple, Union
+
+from pydantic import Field, BaseModel
+
+import capa.helpers
+import capa.version
+import capa.features.file
+import capa.features.insn
+import capa.features.common
+import capa.features.address
+import capa.features.basicblock
+import capa.features.extractors.base_extractor
+from capa.helpers import assert_never
+from capa.features.freeze.features import Feature, feature_from_capa
+
+logger = logging.getLogger(__name__)
+
+
+class HashableModel(BaseModel):
+    class Config:
+        frozen = True
+
+
+class AddressType(str, Enum):
+    ABSOLUTE = "absolute"
+    RELATIVE = "relative"
+    FILE = "file"
+    DN_TOKEN = "dn token"
+    DN_TOKEN_OFFSET = "dn token offset"
+    NO_ADDRESS = "no address"
+
+
+class Address(HashableModel):
+    type: AddressType
+    value: Union[int, Tuple[int, int], None]
+
+    @classmethod
+    def from_capa(cls, a: capa.features.address.Address) -> "Address":
+        if isinstance(a, capa.features.address.AbsoluteVirtualAddress):
+            return cls(type=AddressType.ABSOLUTE, value=int(a))
+
+        elif isinstance(a, capa.features.address.RelativeVirtualAddress):
+            return cls(type=AddressType.RELATIVE, value=int(a))
+
+        elif isinstance(a, capa.features.address.FileOffsetAddress):
+            return cls(type=AddressType.FILE, value=int(a))
+
+        elif isinstance(a, capa.features.address.DNTokenAddress):
+            return cls(type=AddressType.DN_TOKEN, value=int(a))
+
+        elif isinstance(a, capa.features.address.DNTokenOffsetAddress):
+            return cls(type=AddressType.DN_TOKEN_OFFSET, value=(a.token, a.offset))
+
+        elif a == capa.features.address.NO_ADDRESS or isinstance(a, capa.features.address._NoAddress):
+            return cls(type=AddressType.NO_ADDRESS, value=None)
+
+        elif isinstance(a, capa.features.address.Address) and not issubclass(type(a), capa.features.address.Address):
+            raise ValueError("don't use an Address instance directly")
+
+        elif isinstance(a, capa.features.address.Address):
+            raise ValueError("don't use an Address instance directly")
+
+        else:
+            assert_never(a)
+
+    def to_capa(self) -> capa.features.address.Address:
+        if self.type is AddressType.ABSOLUTE:
+            assert isinstance(self.value, int)
+            return capa.features.address.AbsoluteVirtualAddress(self.value)
+
+        elif self.type is AddressType.RELATIVE:
+            assert isinstance(self.value, int)
+            return capa.features.address.RelativeVirtualAddress(self.value)
+
+        elif self.type is AddressType.FILE:
+            assert isinstance(self.value, int)
+            return capa.features.address.FileOffsetAddress(self.value)
+
+        elif self.type is AddressType.DN_TOKEN:
+            assert isinstance(self.value, int)
+            return capa.features.address.DNTokenAddress(self.value)
+
+        elif self.type is AddressType.DN_TOKEN_OFFSET:
+            assert isinstance(self.value, tuple)
+            token, offset = self.value
+            assert isinstance(token, int)
+            assert isinstance(offset, int)
+            return capa.features.address.DNTokenOffsetAddress(token, offset)
+
+        elif self.type is AddressType.NO_ADDRESS:
+            return capa.features.address.NO_ADDRESS
+
+        else:
+            assert_never(self.type)
+
+    def __lt__(self, other: "Address") -> bool:
+        if self.type != other.type:
+            return self.type < other.type
+
+        if self.type is AddressType.NO_ADDRESS:
+            return True
+
+        else:
+            assert self.type == other.type
+            # mypy doesn't realize we've proven that either
+            # both are ints, or both are tuples of ints.
+            # and both of these are comparable.
+            return self.value < other.value  # type: ignore
+
+
+class GlobalFeature(HashableModel):
+    feature: Feature
+
+
+class FileFeature(HashableModel):
+    address: Address
+    feature: Feature
+
+
+class FunctionFeature(HashableModel):
+    """
+    args:
+        function: the address of the function to which this feature belongs.
+        address: the address at which this feature is found.
+
+    function != address because, e.g., the feature may be found *within* the scope (function).
+    versus right at its starting address.
+    """
+
+    function: Address
+    address: Address
+    feature: Feature
+
+
+class BasicBlockFeature(HashableModel):
+    """
+    args:
+        basic_block: the address of the basic block to which this feature belongs.
+        address: the address at which this feature is found.
+
+    basic_block != address because, e.g., the feature may be found *within* the scope (basic block).
+    versus right at its starting address.
+    """
+
+    basic_block: Address = Field(alias="basic block")
+    address: Address
+    feature: Feature
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class InstructionFeature(HashableModel):
+    """
+    args:
+        instruction: the address of the instruction to which this feature belongs.
+        address: the address at which this feature is found.
+
+    instruction != address because, e.g., the feature may be found *within* the scope (basic block),
+    versus right at its starting address.
+    """
+
+    instruction: Address
+    address: Address
+    feature: Feature
+
+
+class InstructionFeatures(BaseModel):
+    address: Address
+    features: Tuple[InstructionFeature, ...]
+
+
+class BasicBlockFeatures(BaseModel):
+    address: Address
+    features: Tuple[BasicBlockFeature, ...]
+    instructions: Tuple[InstructionFeatures, ...]
+
+
+class FunctionFeatures(BaseModel):
+    address: Address
+    features: Tuple[FunctionFeature, ...]
+    basic_blocks: Tuple[BasicBlockFeatures, ...] = Field(alias="basic blocks")
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class Features(BaseModel):
+    global_: Tuple[GlobalFeature, ...] = Field(alias="global")
+    file: Tuple[FileFeature, ...]
+    functions: Tuple[FunctionFeatures, ...]
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class Extractor(BaseModel):
+    name: str
+    version: str = capa.version.__version__
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+class Freeze(BaseModel):
+    version: int = 2
+    base_address: Address = Field(alias="base address")
+    extractor: Extractor
+    features: Features
+
+    class Config:
+        allow_population_by_field_name = True
+
+
+def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> str:
+    """
+    serialize the given extractor to a string
+    """
+
+    global_features: List[GlobalFeature] = []
+    for feature, _ in extractor.extract_global_features():
+        global_features.append(
+            GlobalFeature(
+                feature=feature_from_capa(feature),
+            )
+        )
+
+    file_features: List[FileFeature] = []
+    for feature, address in extractor.extract_file_features():
+        file_features.append(
+            FileFeature(
+                feature=feature_from_capa(feature),
+                address=Address.from_capa(address),
+            )
+        )
+
+    function_features: List[FunctionFeatures] = []
+    for f in extractor.get_functions():
+        faddr = Address.from_capa(f.address)
+        ffeatures = [
+            FunctionFeature(
+                function=faddr,
+                address=Address.from_capa(addr),
+                feature=feature_from_capa(feature),
+            )
+            for feature, addr in extractor.extract_function_features(f)
+        ]
+
+        basic_blocks = []
+        for bb in extractor.get_basic_blocks(f):
+            bbaddr = Address.from_capa(bb.address)
+            bbfeatures = [
+                BasicBlockFeature(
+                    basic_block=bbaddr,
+                    address=Address.from_capa(addr),
+                    feature=feature_from_capa(feature),
+                )  # type: ignore
+                # Mypy is unable to recognise `basic_block` as a argument due to alias
+                for feature, addr in extractor.extract_basic_block_features(f, bb)
+            ]
+
+            instructions = []
+            for insn in extractor.get_instructions(f, bb):
+                iaddr = Address.from_capa(insn.address)
+                ifeatures = [
+                    InstructionFeature(
+                        instruction=iaddr,
+                        address=Address.from_capa(addr),
+                        feature=feature_from_capa(feature),
+                    )
+                    for feature, addr in extractor.extract_insn_features(f, bb, insn)
+                ]
+
+                instructions.append(
+                    InstructionFeatures(
+                        address=iaddr,
+                        features=tuple(ifeatures),
+                    )
+                )
+
+            basic_blocks.append(
+                BasicBlockFeatures(
+                    address=bbaddr,
+                    features=tuple(bbfeatures),
+                    instructions=tuple(instructions),
+                )
+            )
+
+        function_features.append(
+            FunctionFeatures(
+                address=faddr,
+                features=tuple(ffeatures),
+                basic_blocks=basic_blocks,
+            )  # type: ignore
+            # Mypy is unable to recognise `basic_blocks` as a argument due to alias
+        )
+
+    features = Features(
+        global_=global_features,
+        file=tuple(file_features),
+        functions=tuple(function_features),
+    )  # type: ignore
+    # Mypy is unable to recognise `global_` as a argument due to alias
+
+    freeze = Freeze(
+        version=2,
+        base_address=Address.from_capa(extractor.get_base_address()),
+        extractor=Extractor(name=extractor.__class__.__name__),
+        features=features,
+    )  # type: ignore
+    # Mypy is unable to recognise `base_address` as a argument due to alias
+
+    return freeze.json()
+
+
+def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
+    """deserialize a set of features (as a NullFeatureExtractor) from a string."""
+    import capa.features.extractors.null as null
+
+    freeze = Freeze.parse_raw(s)
+    if freeze.version != 2:
+        raise ValueError(f"unsupported freeze format version: {freeze.version}")
+
+    return null.NullFeatureExtractor(
+        base_address=freeze.base_address.to_capa(),
+        global_features=[f.feature.to_capa() for f in freeze.features.global_],
+        file_features=[(f.address.to_capa(), f.feature.to_capa()) for f in freeze.features.file],
+        functions={
+            f.address.to_capa(): null.FunctionFeatures(
+                features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in f.features],
+                basic_blocks={
+                    bb.address.to_capa(): null.BasicBlockFeatures(
+                        features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in bb.features],
+                        instructions={
+                            i.address.to_capa(): null.InstructionFeatures(
+                                features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in i.features]
+                            )
+                            for i in bb.instructions
+                        },
+                    )
+                    for bb in f.basic_blocks
+                },
+            )
+            for f in freeze.features.functions
+        },
+    )
+
+
+MAGIC = "capa0000".encode("ascii")
+
+
+def dump(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> bytes:
+    """serialize the given extractor to a byte array."""
+    return MAGIC + zlib.compress(dumps(extractor).encode("utf-8"))
+
+
+def is_freeze(buf: bytes) -> bool:
+    return buf[: len(MAGIC)] == MAGIC
+
+
+def load(buf: bytes) -> capa.features.extractors.base_extractor.FeatureExtractor:
+    """deserialize a set of features (as a NullFeatureExtractor) from a byte array."""
+    if not is_freeze(buf):
+        raise ValueError("missing magic header")
+    return loads(zlib.decompress(buf[len(MAGIC) :]).decode("utf-8"))
+
+
+def main(argv=None):
+    import sys
+    import argparse
+    from pathlib import Path
+
+    import capa.main
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(description="save capa features to a file")
+    capa.main.install_common_args(parser, {"sample", "format", "backend", "os", "signatures"})
+    parser.add_argument("output", type=str, help="Path to output file")
+    args = parser.parse_args(args=argv)
+    capa.main.handle_common_args(args)
+
+    sigpaths = capa.main.get_signatures(args.signatures)
+
+    extractor = capa.main.get_extractor(args.sample, args.format, args.os, args.backend, sigpaths, False)
+
+    Path(args.output).write_bytes(dump(extractor))
+
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(main())

capa/features/freeze/features.py

@@ -0,0 +1,378 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import binascii
+from typing import Union, Optional
+
+from pydantic import Field, BaseModel
+
+import capa.features.file
+import capa.features.insn
+import capa.features.common
+import capa.features.basicblock
+
+
+class FeatureModel(BaseModel):
+    class Config:
+        frozen = True
+        allow_population_by_field_name = True
+
+    def to_capa(self) -> capa.features.common.Feature:
+        if isinstance(self, OSFeature):
+            return capa.features.common.OS(self.os, description=self.description)
+
+        elif isinstance(self, ArchFeature):
+            return capa.features.common.Arch(self.arch, description=self.description)
+
+        elif isinstance(self, FormatFeature):
+            return capa.features.common.Format(self.format, description=self.description)
+
+        elif isinstance(self, MatchFeature):
+            return capa.features.common.MatchedRule(self.match, description=self.description)
+
+        elif isinstance(
+            self,
+            CharacteristicFeature,
+        ):
+            return capa.features.common.Characteristic(self.characteristic, description=self.description)
+
+        elif isinstance(self, ExportFeature):
+            return capa.features.file.Export(self.export, description=self.description)
+
+        elif isinstance(self, ImportFeature):
+            return capa.features.file.Import(self.import_, description=self.description)
+
+        elif isinstance(self, SectionFeature):
+            return capa.features.file.Section(self.section, description=self.description)
+
+        elif isinstance(self, FunctionNameFeature):
+            return capa.features.file.FunctionName(self.function_name, description=self.description)
+
+        elif isinstance(self, SubstringFeature):
+            return capa.features.common.Substring(self.substring, description=self.description)
+
+        elif isinstance(self, RegexFeature):
+            return capa.features.common.Regex(self.regex, description=self.description)
+
+        elif isinstance(self, StringFeature):
+            return capa.features.common.String(self.string, description=self.description)
+
+        elif isinstance(self, ClassFeature):
+            return capa.features.common.Class(self.class_, description=self.description)
+
+        elif isinstance(self, NamespaceFeature):
+            return capa.features.common.Namespace(self.namespace, description=self.description)
+
+        elif isinstance(self, BasicBlockFeature):
+            return capa.features.basicblock.BasicBlock(description=self.description)
+
+        elif isinstance(self, APIFeature):
+            return capa.features.insn.API(self.api, description=self.description)
+
+        elif isinstance(self, PropertyFeature):
+            return capa.features.insn.Property(self.property, access=self.access, description=self.description)
+
+        elif isinstance(self, NumberFeature):
+            return capa.features.insn.Number(self.number, description=self.description)
+
+        elif isinstance(self, BytesFeature):
+            return capa.features.common.Bytes(binascii.unhexlify(self.bytes), description=self.description)
+
+        elif isinstance(self, OffsetFeature):
+            return capa.features.insn.Offset(self.offset, description=self.description)
+
+        elif isinstance(self, MnemonicFeature):
+            return capa.features.insn.Mnemonic(self.mnemonic, description=self.description)
+
+        elif isinstance(self, OperandNumberFeature):
+            return capa.features.insn.OperandNumber(
+                self.index,
+                self.operand_number,
+                description=self.description,
+            )
+
+        elif isinstance(self, OperandOffsetFeature):
+            return capa.features.insn.OperandOffset(
+                self.index,
+                self.operand_offset,
+                description=self.description,
+            )
+
+        else:
+            raise NotImplementedError(f"Feature.to_capa({type(self)}) not implemented")
+
+
+def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
+    if isinstance(f, capa.features.common.OS):
+        assert isinstance(f.value, str)
+        return OSFeature(os=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Arch):
+        assert isinstance(f.value, str)
+        return ArchFeature(arch=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Format):
+        assert isinstance(f.value, str)
+        return FormatFeature(format=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.MatchedRule):
+        assert isinstance(f.value, str)
+        return MatchFeature(match=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Characteristic):
+        assert isinstance(f.value, str)
+        return CharacteristicFeature(characteristic=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.file.Export):
+        assert isinstance(f.value, str)
+        return ExportFeature(export=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.file.Import):
+        assert isinstance(f.value, str)
+        return ImportFeature(import_=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `import_` as a argument due to alias
+
+    elif isinstance(f, capa.features.file.Section):
+        assert isinstance(f.value, str)
+        return SectionFeature(section=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.file.FunctionName):
+        assert isinstance(f.value, str)
+        return FunctionNameFeature(function_name=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `function_name` as a argument due to alias
+
+    # must come before check for String due to inheritance
+    elif isinstance(f, capa.features.common.Substring):
+        assert isinstance(f.value, str)
+        return SubstringFeature(substring=f.value, description=f.description)
+
+    # must come before check for String due to inheritance
+    elif isinstance(f, capa.features.common.Regex):
+        assert isinstance(f.value, str)
+        return RegexFeature(regex=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.String):
+        assert isinstance(f.value, str)
+        return StringFeature(string=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Class):
+        assert isinstance(f.value, str)
+        return ClassFeature(class_=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `class_` as a argument due to alias
+
+    elif isinstance(f, capa.features.common.Namespace):
+        assert isinstance(f.value, str)
+        return NamespaceFeature(namespace=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.basicblock.BasicBlock):
+        return BasicBlockFeature(description=f.description)
+
+    elif isinstance(f, capa.features.insn.API):
+        assert isinstance(f.value, str)
+        return APIFeature(api=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.insn.Property):
+        assert isinstance(f.value, str)
+        return PropertyFeature(property=f.value, access=f.access, description=f.description)
+
+    elif isinstance(f, capa.features.insn.Number):
+        assert isinstance(f.value, (int, float))
+        return NumberFeature(number=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.common.Bytes):
+        buf = f.value
+        assert isinstance(buf, bytes)
+        return BytesFeature(bytes=binascii.hexlify(buf).decode("ascii"), description=f.description)
+
+    elif isinstance(f, capa.features.insn.Offset):
+        assert isinstance(f.value, int)
+        return OffsetFeature(offset=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.insn.Mnemonic):
+        assert isinstance(f.value, str)
+        return MnemonicFeature(mnemonic=f.value, description=f.description)
+
+    elif isinstance(f, capa.features.insn.OperandNumber):
+        assert isinstance(f.value, int)
+        return OperandNumberFeature(index=f.index, operand_number=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `operand_number` as a argument due to alias
+
+    elif isinstance(f, capa.features.insn.OperandOffset):
+        assert isinstance(f.value, int)
+        return OperandOffsetFeature(index=f.index, operand_offset=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `operand_offset` as a argument due to alias
+
+    else:
+        raise NotImplementedError(f"feature_from_capa({type(f)}) not implemented")
+
+
+class OSFeature(FeatureModel):
+    type: str = "os"
+    os: str
+    description: Optional[str]
+
+
+class ArchFeature(FeatureModel):
+    type: str = "arch"
+    arch: str
+    description: Optional[str]
+
+
+class FormatFeature(FeatureModel):
+    type: str = "format"
+    format: str
+    description: Optional[str]
+
+
+class MatchFeature(FeatureModel):
+    type: str = "match"
+    match: str
+    description: Optional[str]
+
+
+class CharacteristicFeature(FeatureModel):
+    type: str = "characteristic"
+    characteristic: str
+    description: Optional[str]
+
+
+class ExportFeature(FeatureModel):
+    type: str = "export"
+    export: str
+    description: Optional[str]
+
+
+class ImportFeature(FeatureModel):
+    type: str = "import"
+    import_: str = Field(alias="import")
+    description: Optional[str]
+
+
+class SectionFeature(FeatureModel):
+    type: str = "section"
+    section: str
+    description: Optional[str]
+
+
+class FunctionNameFeature(FeatureModel):
+    type: str = "function name"
+    function_name: str = Field(alias="function name")
+    description: Optional[str]
+
+
+class SubstringFeature(FeatureModel):
+    type: str = "substring"
+    substring: str
+    description: Optional[str]
+
+
+class RegexFeature(FeatureModel):
+    type: str = "regex"
+    regex: str
+    description: Optional[str]
+
+
+class StringFeature(FeatureModel):
+    type: str = "string"
+    string: str
+    description: Optional[str]
+
+
+class ClassFeature(FeatureModel):
+    type: str = "class"
+    class_: str = Field(alias="class")
+    description: Optional[str]
+
+
+class NamespaceFeature(FeatureModel):
+    type: str = "namespace"
+    namespace: str
+    description: Optional[str]
+
+
+class BasicBlockFeature(FeatureModel):
+    type: str = "basic block"
+    description: Optional[str]
+
+
+class APIFeature(FeatureModel):
+    type: str = "api"
+    api: str
+    description: Optional[str]
+
+
+class PropertyFeature(FeatureModel):
+    type: str = "property"
+    access: Optional[str]
+    property: str
+    description: Optional[str]
+
+
+class NumberFeature(FeatureModel):
+    type: str = "number"
+    number: Union[int, float]
+    description: Optional[str]
+
+
+class BytesFeature(FeatureModel):
+    type: str = "bytes"
+    bytes: str
+    description: Optional[str]
+
+
+class OffsetFeature(FeatureModel):
+    type: str = "offset"
+    offset: int
+    description: Optional[str]
+
+
+class MnemonicFeature(FeatureModel):
+    type: str = "mnemonic"
+    mnemonic: str
+    description: Optional[str]
+
+
+class OperandNumberFeature(FeatureModel):
+    type: str = "operand number"
+    index: int
+    operand_number: int = Field(alias="operand number")
+    description: Optional[str]
+
+
+class OperandOffsetFeature(FeatureModel):
+    type: str = "operand offset"
+    index: int
+    operand_offset: int = Field(alias="operand offset")
+    description: Optional[str]
+
+
+Feature = Union[
+    OSFeature,
+    ArchFeature,
+    FormatFeature,
+    MatchFeature,
+    CharacteristicFeature,
+    ExportFeature,
+    ImportFeature,
+    SectionFeature,
+    FunctionNameFeature,
+    SubstringFeature,
+    RegexFeature,
+    StringFeature,
+    ClassFeature,
+    NamespaceFeature,
+    APIFeature,
+    PropertyFeature,
+    NumberFeature,
+    BytesFeature,
+    OffsetFeature,
+    MnemonicFeature,
+    OperandNumberFeature,
+    OperandOffsetFeature,
+    # Note! this must be last, see #1161
+    BasicBlockFeature,
+]

capa/features/insn.py

@@ -1,41 +1,169 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import abc
+from typing import Union, Optional
 
-import capa.render.utils
-from capa.features.common import Feature
+import capa.helpers
+from capa.features.common import VALID_FEATURE_ACCESS, Feature
+
+
+def hex(n: int) -> str:
+    """render the given number using upper case hex, like: 0x123ABC"""
+    if n < 0:
+        return f"-0x{(-n):X}"
+    else:
+        return f"0x{(n):X}"
 
 
 class API(Feature):
     def __init__(self, name: str, description=None):
-        # Downcase library name if given
-        if "." in name:
-            modname, _, impname = name.rpartition(".")
-            name = modname.lower() + "." + impname
+        super().__init__(name, description=description)
 
-        super(API, self).__init__(name, description=description)
+
+class _AccessFeature(Feature, abc.ABC):
+    # superclass: don't use directly
+    def __init__(self, value: str, access: Optional[str] = None, description: Optional[str] = None):
+        super().__init__(value, description=description)
+        if access is not None:
+            if access not in VALID_FEATURE_ACCESS:
+                raise ValueError(f"{self.name} access type {access} not valid")
+        self.access = access
+
+    def __hash__(self):
+        return hash((self.name, self.value, self.access))
+
+    def __eq__(self, other):
+        return super().__eq__(other) and self.access == other.access
+
+    def get_name_str(self) -> str:
+        if self.access is not None:
+            return f"{self.name}/{self.access}"
+        return self.name
+
+
+class Property(_AccessFeature):
+    def __init__(self, value: str, access: Optional[str] = None, description=None):
+        super().__init__(value, access=access, description=description)
 
 
 class Number(Feature):
-    def __init__(self, value: int, bitness=None, description=None):
-        super(Number, self).__init__(value, bitness=bitness, description=description)
+    def __init__(self, value: Union[int, float], description=None):
+        """
+        args:
+          value (int or float): positive or negative integer, or floating point number.
+
+        the range of the value is:
+          - if positive, the range of u64
+          - if negative, the range of i64
+          - if floating, the range and precision of double
+        """
+        super().__init__(value, description=description)
 
     def get_value_str(self):
-        return capa.render.utils.hex(self.value)
+        if isinstance(self.value, int):
+            return capa.helpers.hex(self.value)
+        elif isinstance(self.value, float):
+            return str(self.value)
+        else:
+            raise ValueError(f"invalid value type {type(self.value)}")
+
+
+# max recognized structure size (and therefore, offset size)
+MAX_STRUCTURE_SIZE = 0x10000
 
 
 class Offset(Feature):
-    def __init__(self, value: int, bitness=None, description=None):
-        super(Offset, self).__init__(value, bitness=bitness, description=description)
+    def __init__(self, value: int, description=None):
+        """
+        args:
+          value (int): the offset, which can be positive or negative.
+
+        the range of the value is:
+          - if positive, the range of u64
+          - if negative, the range of i64
+        """
+        super().__init__(value, description=description)
 
     def get_value_str(self):
-        return capa.render.utils.hex(self.value)
+        assert isinstance(self.value, int)
+        return hex(self.value)
 
 
 class Mnemonic(Feature):
     def __init__(self, value: str, description=None):
-        super(Mnemonic, self).__init__(value, description=description)
+        super().__init__(value, description=description)
+
+
+# max number of operands to consider for a given instruction.
+# since we only support Intel and .NET, we can assume this is 3
+# which covers cases up to e.g. "vinserti128 ymm0,ymm0,ymm5,1"
+MAX_OPERAND_COUNT = 4
+MAX_OPERAND_INDEX = MAX_OPERAND_COUNT - 1
+
+
+class _Operand(Feature, abc.ABC):
+    # superclass: don't use directly
+    # subclasses should set self.name and provide the value string formatter
+    def __init__(self, index: int, value: Union[int, float], description=None):
+        super().__init__(value, description=description)
+        self.index = index
+
+    def __hash__(self):
+        return hash((self.name, self.value))
+
+    def __eq__(self, other):
+        return super().__eq__(other) and self.index == other.index
+
+
+class OperandNumber(_Operand):
+    # cached names so we don't do extra string formatting every ctor
+    NAMES = [f"operand[{i}].number" for i in range(MAX_OPERAND_COUNT)]
+
+    # operand[i].number: 0x12
+    def __init__(self, index: int, value: Union[int, float], description=None):
+        """
+        args:
+          value (int or float): positive or negative integer, or floating point number.
+
+        the range of the value is:
+          - if positive, the range of u64
+          - if negative, the range of i64
+          - if floating, the range and precision of double
+        """
+        super().__init__(index, value, description=description)
+        self.name = self.NAMES[index]
+
+    def get_value_str(self) -> str:
+        if isinstance(self.value, int):
+            return capa.helpers.hex(self.value)
+        elif isinstance(self.value, float):
+            return str(self.value)
+        else:
+            raise ValueError("invalid value type")
+
+
+class OperandOffset(_Operand):
+    # cached names so we don't do extra string formatting every ctor
+    NAMES = [f"operand[{i}].offset" for i in range(MAX_OPERAND_COUNT)]
+
+    # operand[i].offset: 0x12
+    def __init__(self, index: int, value: int, description=None):
+        """
+        args:
+          value (int): the offset, which can be positive or negative.
+
+        the range of the value is:
+          - if positive, the range of u64
+          - if negative, the range of i64
+        """
+        super().__init__(index, value, description=description)
+        self.name = self.NAMES[index]
+
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, int)
+        return hex(self.value)

capa/helpers.py

@@ -1,37 +1,159 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-
-import os
+import inspect
+import logging
+import contextlib
+import importlib.util
 from typing import NoReturn
+from pathlib import Path
 
-_hex = hex
+import tqdm
+
+from capa.exceptions import UnsupportedFormatError
+from capa.features.common import FORMAT_PE, FORMAT_SC32, FORMAT_SC64, FORMAT_DOTNET, FORMAT_UNKNOWN, Format
+
+EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32")
+EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64")
+EXTENSIONS_ELF = "elf_"
+
+logger = logging.getLogger("capa")
 
 
-def hex(i):
-    return _hex(int(i))
+def hex(n: int) -> str:
+    """render the given number using upper case hex, like: 0x123ABC"""
+    if n < 0:
+        return f"-0x{(-n):X}"
+    else:
+        return f"0x{(n):X}"
 
 
-def get_file_taste(sample_path: str) -> bytes:
-    if not os.path.exists(sample_path):
-        raise IOError("sample path %s does not exist or cannot be accessed" % sample_path)
-    with open(sample_path, "rb") as f:
-        taste = f.read(8)
+def get_file_taste(sample_path: Path) -> bytes:
+    if not sample_path.exists():
+        raise IOError(f"sample path {sample_path} does not exist or cannot be accessed")
+    taste = sample_path.open("rb").read(8)
     return taste
 
 
 def is_runtime_ida():
+    return importlib.util.find_spec("idc") is not None
+
+
+def assert_never(value) -> NoReturn:
+    # careful: python -O will remove this assertion.
+    # but this is only used for type checking, so it's ok.
+    assert False, f"Unhandled value: {value} ({type(value).__name__})"  # noqa: B011
+
+
+def get_format_from_extension(sample: Path) -> str:
+    if sample.name.endswith(EXTENSIONS_SHELLCODE_32):
+        return FORMAT_SC32
+    elif sample.name.endswith(EXTENSIONS_SHELLCODE_64):
+        return FORMAT_SC64
+    return FORMAT_UNKNOWN
+
+
+def get_auto_format(path: Path) -> str:
+    format_ = get_format(path)
+    if format_ == FORMAT_UNKNOWN:
+        format_ = get_format_from_extension(path)
+    if format_ == FORMAT_UNKNOWN:
+        raise UnsupportedFormatError()
+    return format_
+
+
+def get_format(sample: Path) -> str:
+    # imported locally to avoid import cycle
+    from capa.features.extractors.common import extract_format
+    from capa.features.extractors.dnfile_ import DnfileFeatureExtractor
+
+    buf = sample.read_bytes()
+
+    for feature, _ in extract_format(buf):
+        if feature == Format(FORMAT_PE):
+            dnfile_extractor = DnfileFeatureExtractor(sample)
+            if dnfile_extractor.is_dotnet_file():
+                feature = Format(FORMAT_DOTNET)
+
+        assert isinstance(feature.value, str)
+        return feature.value
+
+    return FORMAT_UNKNOWN
+
+
+@contextlib.contextmanager
+def redirecting_print_to_tqdm(disable_progress):
+    """
+    tqdm (progress bar) expects to have fairly tight control over console output.
+    so calls to `print()` will break the progress bar and make things look bad.
+    so, this context manager temporarily replaces the `print` implementation
+    with one that is compatible with tqdm.
+    via: https://stackoverflow.com/a/42424890/87207
+    """
+    old_print = print  # noqa: T202 [reserved word print used]
+
+    def new_print(*args, **kwargs):
+        # If tqdm.tqdm.write raises error, use builtin print
+        if disable_progress:
+            old_print(*args, **kwargs)
+        else:
+            try:
+                tqdm.tqdm.write(*args, **kwargs)
+            except Exception:
+                old_print(*args, **kwargs)
+
     try:
-        import idc
-    except ImportError:
-        return False
-    else:
-        return True
+        # Globally replace print with new_print.
+        # Verified this works manually on Python 3.11:
+        #     >>> import inspect
+        #     >>> inspect.builtins
+        #     <module 'builtins' (built-in)>
+        inspect.builtins.print = new_print  # type: ignore
+        yield
+    finally:
+        inspect.builtins.print = old_print  # type: ignore
 
 
-def assert_never(value: NoReturn) -> NoReturn:
-    assert False, f"Unhandled value: {value} ({type(value).__name__})"
+def log_unsupported_format_error():
+    logger.error("-" * 80)
+    logger.error(" Input file does not appear to be a PE or ELF file.")
+    logger.error(" ")
+    logger.error(
+        " capa currently only supports analyzing PE and ELF files (or shellcode, when using --format sc32|sc64)."
+    )
+    logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
+    logger.error("-" * 80)
+
+
+def log_unsupported_os_error():
+    logger.error("-" * 80)
+    logger.error(" Input file does not appear to target a supported OS.")
+    logger.error(" ")
+    logger.error(
+        " capa currently only supports analyzing executables for some operating systems (including Windows and Linux)."
+    )
+    logger.error("-" * 80)
+
+
+def log_unsupported_arch_error():
+    logger.error("-" * 80)
+    logger.error(" Input file does not appear to target a supported architecture.")
+    logger.error(" ")
+    logger.error(" capa currently only supports analyzing x86 (32- and 64-bit).")
+    logger.error("-" * 80)
+
+
+def log_unsupported_runtime_error():
+    logger.error("-" * 80)
+    logger.error(" Unsupported runtime or Python interpreter.")
+    logger.error(" ")
+    logger.error(" capa supports running under Python 3.8 and higher.")
+    logger.error(" ")
+    logger.error(
+        " If you're seeing this message on the command line, please ensure you're running a supported Python version."
+    )
+    logger.error("-" * 80)

capa/ida/helpers.py

@@ -1,23 +1,31 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-
+import json
 import logging
 import datetime
+import contextlib
+from typing import List, Optional
+from pathlib import Path
 
 import idc
 import idaapi
 import idautils
 import ida_bytes
 import ida_loader
+from netnode import netnode
 
 import capa
 import capa.version
+import capa.render.utils as rutils
 import capa.features.common
+import capa.features.freeze
+import capa.render.result_document as rdoc
+from capa.features.address import AbsoluteVirtualAddress
 
 logger = logging.getLogger("capa")
 
@@ -26,23 +34,29 @@ SUPPORTED_FILE_TYPES = (
     idaapi.f_PE,
     idaapi.f_ELF,
     idaapi.f_BIN,
+    idaapi.f_COFF,
     # idaapi.f_MACHO,
 )
 
 # arch type as returned by idainfo.procname
 SUPPORTED_ARCH_TYPES = ("metapc",)
 
+CAPA_NETNODE = f"$ com.mandiant.capa.v{capa.version.__version__}"
+NETNODE_RESULTS = "results"
+NETNODE_RULES_CACHE_ID = "rules-cache-id"
+
 
 def inform_user_ida_ui(message):
-    idaapi.info("%s. Please refer to IDA Output window for more information." % message)
+    # this isn't a logger, this is IDA's logging facility
+    idaapi.info(f"{message}. Please refer to IDA Output window for more information.")  # noqa: G004
 
 
 def is_supported_ida_version():
     version = float(idaapi.get_kernel_version())
-    if version < 7.4 or version >= 8:
+    if version < 7.4 or version >= 9:
         warning_msg = "This plugin does not support your IDA Pro version"
         logger.warning(warning_msg)
-        logger.warning("Your IDA Pro version is: %s. Supported versions are: IDA >= 7.4 and IDA < 8.0." % version)
+        logger.warning("Your IDA Pro version is: %s. Supported versions are: IDA >= 7.4 and IDA < 9.0.", version)
         return False
     return True
 
@@ -107,33 +121,57 @@ def get_file_sha256():
     return sha256
 
 
-def collect_metadata():
+def collect_metadata(rules: List[Path]):
     """ """
     md5 = get_file_md5()
     sha256 = get_file_sha256()
 
-    return {
-        "timestamp": datetime.datetime.now().isoformat(),
-        # "argv" is not relevant here
-        "sample": {
-            "md5": md5,
-            "sha1": "",  # not easily accessible
-            "sha256": sha256,
-            "path": idaapi.get_input_file_path(),
-        },
-        "analysis": {
-            "format": idaapi.get_file_type_name(),
-            "extractor": "ida",
-            "base_address": idaapi.get_imagebase(),
-            "layout": {
+    info: idaapi.idainfo = idaapi.get_inf_structure()
+    if info.procname == "metapc" and info.is_64bit():
+        arch = "x86_64"
+    elif info.procname == "metapc" and info.is_32bit():
+        arch = "x86"
+    else:
+        arch = "unknown arch"
+
+    format_name: str = ida_loader.get_file_type_name()
+    if "PE" in format_name:
+        os = "windows"
+    elif "ELF" in format_name:
+        with contextlib.closing(capa.ida.helpers.IDAIO()) as f:
+            os = capa.features.extractors.elf.detect_elf_os(f)
+    else:
+        os = "unknown os"
+
+    return rdoc.Metadata(
+        timestamp=datetime.datetime.now(),
+        version=capa.version.__version__,
+        argv=(),
+        sample=rdoc.Sample(
+            md5=md5,
+            sha1="",  # not easily accessible
+            sha256=sha256,
+            path=idaapi.get_input_file_path(),
+        ),
+        analysis=rdoc.Analysis(
+            format=idaapi.get_file_type_name(),
+            arch=arch,
+            os=os,
+            extractor="ida",
+            rules=tuple(r.resolve().absolute().as_posix() for r in rules),
+            base_address=capa.features.freeze.Address.from_capa(idaapi.get_imagebase()),
+            layout=rdoc.Layout(
+                functions=(),
                 # this is updated after capabilities have been collected.
                 # will look like:
                 #
                 # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
-            },
-        },
-        "version": capa.version.__version__,
-    }
+            ),
+            # ignore these for now - not used by IDA plugin.
+            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
+            library_functions=(),
+        ),
+    )
 
 
 class IDAIO:
@@ -143,7 +181,7 @@ class IDAIO:
     """
 
     def __init__(self):
-        super(IDAIO, self).__init__()
+        super().__init__()
         self.offset = 0
 
     def seek(self, offset, whence=0):
@@ -153,11 +191,63 @@ class IDAIO:
     def read(self, size):
         ea = ida_loader.get_fileregion_ea(self.offset)
         if ea == idc.BADADDR:
-            # best guess, such as if file is mapped at address 0x0.
-            ea = self.offset
+            logger.debug("cannot read 0x%x bytes at 0x%x (ea: BADADDR)", size, self.offset)
+            return b""
 
         logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, ea)
-        return ida_bytes.get_bytes(ea, size)
+
+        # get_bytes returns None on error, for consistency with read always return bytes
+        return ida_bytes.get_bytes(ea, size) or b""
 
     def close(self):
         return
+
+
+def save_cached_results(resdoc):
+    logger.debug("saving cached capa results to netnode '%s'", CAPA_NETNODE)
+    n = netnode.Netnode(CAPA_NETNODE)
+    n[NETNODE_RESULTS] = resdoc.json()
+
+
+def idb_contains_cached_results() -> bool:
+    try:
+        n = netnode.Netnode(CAPA_NETNODE)
+        return bool(n.get(NETNODE_RESULTS))
+    except netnode.NetnodeCorruptError as e:
+        logger.exception(str(e))
+        return False
+
+
+def load_and_verify_cached_results() -> Optional[rdoc.ResultDocument]:
+    """verifies that cached results have valid (mapped) addresses for the current database"""
+    logger.debug("loading cached capa results from netnode '%s'", CAPA_NETNODE)
+
+    n = netnode.Netnode(CAPA_NETNODE)
+    doc = rdoc.ResultDocument.parse_obj(json.loads(n[NETNODE_RESULTS]))
+
+    for rule in rutils.capability_rules(doc):
+        for location_, _ in rule.matches:
+            location = location_.to_capa()
+            if isinstance(location, AbsoluteVirtualAddress):
+                ea = int(location)
+                if not idaapi.is_mapped(ea):
+                    logger.error("cached address %s is not a valid location in this database", hex(ea))
+                    return None
+    return doc
+
+
+def save_rules_cache_id(ruleset_id):
+    logger.debug("saving ruleset ID to netnode '%s'", CAPA_NETNODE)
+    n = netnode.Netnode(CAPA_NETNODE)
+    n[NETNODE_RULES_CACHE_ID] = ruleset_id
+
+
+def load_rules_cache_id():
+    n = netnode.Netnode(CAPA_NETNODE)
+    return n[NETNODE_RULES_CACHE_ID]
+
+
+def delete_cached_results():
+    logger.debug("deleting cached capa data")
+    n = netnode.Netnode(CAPA_NETNODE)
+    del n[NETNODE_RESULTS]

capa/ida/plugin/README.md

@@ -1,8 +1,8 @@
 ![capa explorer](../../../.github/capa-explorer-logo.png)
 
 capa explorer is an IDAPython plugin that integrates the FLARE team's open-source framework, capa, with IDA Pro. capa is a framework that uses a well-defined collection of rules to 
-identify capabilities in a program. You can run capa against a PE file or shellcode and it tells you what it thinks the program can do. For example, it might suggest that 
-the program is a backdoor, can install services, or relies on HTTP to communicate. capa explorer runs capa directly against your IDA Pro database (IDB) without requiring access
+identify capabilities in a program. You can run capa against a PE file, ELF file, or shellcode and it tells you what it thinks the program can do. For example, it might suggest that 
+the program is a backdoor, can install services, or relies on HTTP to communicate. capa explorer runs capa analysis on your IDA Pro database (IDB) without needing access
 to the original binary file. Once a database has been analyzed, capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted from your IDB.
 
 We love using capa explorer during malware analysis because it teaches us what parts of a program suggest a behavior. As we click on rows, capa explorer jumps directly 
@@ -21,74 +21,43 @@ We can use capa explorer to navigate our Disassembly view directly to the suspec
 Using the `Rule Information` and `Details` columns capa explorer shows us that the suspect function matched `self delete via COMSPEC environment variable` because it contains capa rule matches for `create process`, `get COMSPEC environment variable`,
 and `query environment variable`, references to the strings `COMSPEC`, ` > nul`, and `/c del `, and calls to the Windows API functions `GetEnvironmentVariableA` and `ShellExecuteEx`.
 
-capa explorer also helps you build new capa rules. To start select the `Rule Generator` tab, navigate to a function in your Disassembly view,
+capa explorer also helps you build and test new capa rules. To start, select the `Rule Generator` tab, navigate to a function in your Disassembly view,
 and click `Analyze`. capa explorer will extract features from the function and display them in the `Features` pane. You can add features listed in this pane to the `Editor` pane
 by either double-clicking a feature or using multi-select + right-click to add multiple features at once. The `Preview` and `Editor` panes help edit your rule. Use the `Preview` pane
-to modify the rule text directly and the `Editor` pane to construct and rearrange your hierarchy of statements and features. When you finish a rule you can save it directly to a file by clicking `Save`.
+to modify rule text directly and the `Editor` pane to construct and rearrange your hierarchy of statements and features. When you finish a rule you can save it directly to a file by clicking `Save`.
 
 ![](../../../doc/img/rulegen_expanded.png)
 
-For more information on the FLARE team's open-source framework, capa, check out the overview in our first [blog](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html).
+For more information on the FLARE team's open-source framework, capa, check out the overview in our first [blog](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities).
 
 ## Getting Started
 
-### Requirements
-
-capa explorer supports Python versions >= 3.6.x and the following IDA Pro versions:
-
-* IDA 7.4
-* IDA 7.5
-* IDA 7.6 (caveat below)
-* IDA 7.7
-
-capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.6.x). Based on our testing the following matrix shows the Python versions supported
-by each supported IDA version:
-
-| | IDA 7.4 | IDA 7.5 | IDA 7.6 |
-| --- | --- | --- | --- |
-| Python 3.6.x | Yes | Yes | Yes |
-| Python 3.7.x | Yes | Yes | Yes |
-| Python 3.8.x | Partial (see below) | Yes | Yes |
-| Python 3.9.x | No | Partial (see below) | Yes |
-
-To use capa explorer with IDA 7.4 and Python 3.8.x you must follow the instructions provided by hex-rays [here](https://hex-rays.com/blog/ida-7-4-and-python-3-8/).
-
-To use capa explorer with IDA 7.5 and Python 3.9.x you must follow the instructions provided by hex-rays [here](https://hex-rays.com/blog/python-3-9-support-for-ida-7-5/).
-
-If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues).
-
-#### IDA 7.6 caveat: IDA 7.6sp1 or patch required
-
-As described [here](https://www.hex-rays.com/blog/ida-7-6-empty-qtreeview-qtreewidget/):
-
-> A rather nasty issue evaded our testing and found its way into IDA 7.6: using the PyQt5 modules that are shipped with IDA, QTreeView (or QTreeWidget) instances will always fail to display contents.
-
-Therefore, in order to use capa under IDA 7.6 you need the [Service Pack 1 for IDA 7.6](https://www.hex-rays.com/products/ida/news/7_6sp1). Alternatively, you can download and install the fix corresponding to your IDA installation, replacing the original QtWidgets DLL with the one contained in the .zip file (links to Hex-Rays):
-
-
-  - Windows: [pyqt5_qtwidgets_win](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_win.zip)
-  - Linux: [pyqt5_qtwidgets_linux](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_linux.zip)
-  - MacOS (Intel): [pyqt5_qtwidgets_mac_x64](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_mac_x64.zip)
-  - MacOS (AppleSilicon): [pyqt5_qtwidgets_mac_arm](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_mac_arm.zip)
-
-
-### Supported File Types
-
-capa explorer is limited to the file types supported by capa, which include:
-
-* Windows x86 (32- and 64-bit) PE and ELF files
-* Windows x86 (32- and 64-bit) shellcode
-
 ### Installation
 
 You can install capa explorer using the following steps:
 
-1. Install capa and its dependencies from PyPI for the Python interpreter used by your IDA installation:
+1. Install capa and its dependencies from PyPI using the Python interpreter configured for your IDA installation:
     ```
     $ pip install flare-capa
     ```
-3. Download the [standard collection of capa rules](https://github.com/mandiant/capa-rules) (capa explorer needs capa rules to analyze a database)
-4. Copy [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory
+2. Download and extract the [official capa rules](https://github.com/mandiant/capa-rules/releases) that match the version of capa you have installed
+   1. Use the following command to view the version of capa you have installed:
+   ```commandline
+    $ pip show flare-capa
+    OR
+    $ capa --version
+    ```
+3. Copy [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory
+   - find your plugin directories via `idaapi.get_ida_subdirs("plugins")` or see this [Hex-Rays blog](https://hex-rays.com/blog/igors-tip-of-the-week-103-sharing-plugins-between-ida-installs/)
+   - common paths are `%APPDATA%\Hex-Rays\IDA Pro\plugins` (Windows) or `$HOME/.idapro/plugins` on Linux/Mac
+
+### Supported File Types
+
+capa explorer is limited to the file types supported by capa, which include:
+
+* Windows x86 (32- and 64-bit) PE files
+* Windows x86 (32- and 64-bit) shellcode
+* ELF files on various operating systems
 
 ### Usage
 
@@ -98,19 +67,20 @@ You can install capa explorer using the following steps:
 3. Select the `Program Analysis` tab
 4. Click the `Analyze` button
 
-When running capa explorer for the first time you are prompted to select a file directory containing capa rules. The plugin conveniently
-remembers your selection for future runs; you can change this selection and other default settings by clicking `Settings`. We recommend 
-downloading and using the [standard collection of capa rules](https://github.com/mandiant/capa-rules) when getting started with the plugin.
+The first time you run capa explorer you will be asked to specify a local directory containing capa rules to use for analysis. We recommend downloading and extracting the [official capa rules](https://github.com/mandiant/capa-rules/releases) that match 
+the version of capa you have installed (see installation instructions above for more details). capa explorer remembers your selection for future analysis which you
+can update using the `Settings` button.
 
 #### Tips for Program Analysis
 
 * Start analysis by clicking the `Analyze` button
+  * capa explorer caches results to the database and reuses them across IDA sessions 
 * Reset the plugin user interface and remove highlighting from your Disassembly view by clicking the `Reset` button
-* Change your capa rules directory and other default settings by clicking `Settings`
+* Change your local capa rules directory, auto analysis settings, and other default settings by clicking the `Settings` button
 * Hover your cursor over a rule match to view the source content of the rule
 * Double-click the `Address` column to navigate your Disassembly view to the address of the associated feature
 * Double-click a result in the `Rule Information` column to expand its children
-* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in your Dissasembly view
+* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in your Disassembly view
 
 #### Tips for Rule Generator
 
@@ -123,6 +93,22 @@ downloading and using the [standard collection of capa rules](https://github.com
 * Directly edit rule text and metadata fields using the `Preview` pane
 * Change the default rule author and default rule scope displayed in the `Preview` pane by clicking `Settings`
 
+### Requirements
+
+capa explorer supports Python versions >= 3.8.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:
+
+* IDA 7.4
+* IDA 7.5
+* IDA 7.6 Service Pack 1
+* IDA 7.7
+* IDA 8.0
+* IDA 8.1
+* IDA 8.2
+
+capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.8.x).
+
+If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues).
+
 ## Development
 
 capa explorer is packaged with capa so you will need to install capa locally for development. You can install capa locally by following the steps outlined in `Method 3: Inspecting the capa source code` of the [capa 

capa/ida/plugin/__init__.py

@@ -1,11 +1,10 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-
 import logging
 
 import idaapi
@@ -18,7 +17,6 @@ logger = logging.getLogger(__name__)
 
 
 class CapaExplorerPlugin(idaapi.plugin_t):
-
     # Mandatory definitions
     PLUGIN_NAME = "FLARE capa explorer"
     PLUGIN_VERSION = "1.0.0"
@@ -40,6 +38,12 @@ class CapaExplorerPlugin(idaapi.plugin_t):
         """called when IDA is loading the plugin"""
         logging.basicConfig(level=logging.INFO)
 
+        # do not load plugin unless hosted in idaq (IDA Qt)
+        if not idaapi.is_idaq():
+            # note: it does not appear that IDA calls "init" by default when hosted in idat; we keep this
+            # check here for good measure
+            return idaapi.PLUGIN_SKIP
+
         import capa.ida.helpers
 
         # do not load plugin if IDA version/file type not supported
@@ -63,7 +67,16 @@ class CapaExplorerPlugin(idaapi.plugin_t):
           arg (int): bitflag. Setting LSB enables automatic analysis upon
           loading. The other bits are currently undefined. See `form.Options`.
         """
-        self.form = CapaExplorerForm(self.PLUGIN_NAME, arg)
+        if not self.form:
+            self.form = CapaExplorerForm(self.PLUGIN_NAME, arg)
+        else:
+            widget = idaapi.find_widget(self.form.form_title)
+            if widget:
+                idaapi.activate_widget(widget, True)
+            else:
+                self.form.Show()
+                self.form.load_capa_results(False, True)
+
         return True
 
 
@@ -86,7 +99,7 @@ class CapaExplorerPlugin(idaapi.plugin_t):
 # so we need to register a callback that's invoked from the main thread after the plugin is registered.
 #
 # after a lot of guess-and-check, we can use `UI_Hooks.updated_actions` to
-#  receive notications after IDA has created an action for each plugin.
+#  receive notifications after IDA has created an action for each plugin.
 # so, create this hook, wait for capa plugin to load, set the icon, and unhook.
 
 
@@ -94,7 +107,7 @@ class OnUpdatedActionsHook(ida_kernwin.UI_Hooks):
     """register a callback to be invoked each time the UI actions are updated"""
 
     def __init__(self, cb):
-        super(OnUpdatedActionsHook, self).__init__()
+        super().__init__()
         self.cb = cb
 
     def updated_actions(self):

capa/ida/plugin/cache.py

@@ -0,0 +1,229 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import itertools
+import collections
+from typing import Set, Dict, Tuple, Union, Optional
+
+import capa.engine
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.features.address import NO_ADDRESS, Address
+from capa.ida.plugin.extractor import CapaExplorerFeatureExtractor
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+
+class CapaRuleGenFeatureCacheNode:
+    def __init__(
+        self,
+        inner: Optional[Union[FunctionHandle, BBHandle, InsnHandle]],
+        parent: Optional[CapaRuleGenFeatureCacheNode],
+    ):
+        self.inner: Optional[Union[FunctionHandle, BBHandle, InsnHandle]] = inner
+        self.address = NO_ADDRESS if self.inner is None else self.inner.address
+        self.parent: Optional[CapaRuleGenFeatureCacheNode] = parent
+
+        if self.parent is not None:
+            self.parent.children.add(self)
+
+        self.features: FeatureSet = collections.defaultdict(set)
+        self.children: Set[CapaRuleGenFeatureCacheNode] = set()
+
+    def __hash__(self):
+        # TODO(mike-hunhoff): confirm this is unique enough
+        # https://github.com/mandiant/capa/issues/1604
+        return hash((self.address,))
+
+    def __eq__(self, other):
+        if not isinstance(other, type(self)):
+            return NotImplemented
+        # TODO(mike-hunhoff): confirm this is unique enough
+        # https://github.com/mandiant/capa/issues/1604
+        return self.address == other.address
+
+
+class CapaRuleGenFeatureCache:
+    def __init__(self, extractor: CapaExplorerFeatureExtractor):
+        self.extractor = extractor
+        self.global_features: FeatureSet = collections.defaultdict(set)
+
+        self.file_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(None, None)
+        self.func_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
+        self.bb_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
+        self.insn_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
+
+        self._find_global_features()
+        self._find_file_features()
+
+    def _find_global_features(self):
+        for feature, addr in self.extractor.extract_global_features():
+            # not all global features may have virtual addresses.
+            # if not, then at least ensure the feature shows up in the index.
+            # the set of addresses will still be empty.
+            if addr is not None:
+                self.global_features[feature].add(addr)
+            else:
+                if feature not in self.global_features:
+                    self.global_features[feature] = set()
+
+    def _find_file_features(self):
+        # not all file features may have virtual addresses.
+        # if not, then at least ensure the feature shows up in the index.
+        # the set of addresses will still be empty.
+        for feature, addr in self.extractor.extract_file_features():
+            if addr is not None:
+                self.file_node.features[feature].add(addr)
+            else:
+                if feature not in self.file_node.features:
+                    self.file_node.features[feature] = set()
+
+    def _find_function_and_below_features(self, fh: FunctionHandle):
+        f_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(fh, self.file_node)
+
+        # extract basic block and below features
+        for bbh in self.extractor.get_basic_blocks(fh):
+            bb_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(bbh, f_node)
+
+            # extract instruction features
+            for ih in self.extractor.get_instructions(fh, bbh):
+                inode: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(ih, bb_node)
+
+                for feature, addr in self.extractor.extract_insn_features(fh, bbh, ih):
+                    inode.features[feature].add(addr)
+
+                self.insn_nodes[inode.address] = inode
+
+            # extract basic block features
+            for feature, addr in self.extractor.extract_basic_block_features(fh, bbh):
+                bb_node.features[feature].add(addr)
+
+            # store basic block features in cache and function parent
+            self.bb_nodes[bb_node.address] = bb_node
+
+        # extract function features
+        for feature, addr in self.extractor.extract_function_features(fh):
+            f_node.features[feature].add(addr)
+
+        self.func_nodes[f_node.address] = f_node
+
+    def _find_instruction_capabilities(
+        self, ruleset: RuleSet, insn: CapaRuleGenFeatureCacheNode
+    ) -> Tuple[FeatureSet, MatchResults]:
+        features: FeatureSet = collections.defaultdict(set)
+
+        for feature, locs in itertools.chain(insn.features.items(), self.global_features.items()):
+            features[feature].update(locs)
+
+        _, matches = ruleset.match(Scope.INSTRUCTION, features, insn.address)
+        for name, result in matches.items():
+            rule = ruleset[name]
+            for addr, _ in result:
+                capa.engine.index_rule_matches(features, rule, [addr])
+
+        return features, matches
+
+    def _find_basic_block_capabilities(
+        self, ruleset: RuleSet, bb: CapaRuleGenFeatureCacheNode
+    ) -> Tuple[FeatureSet, MatchResults, MatchResults]:
+        features: FeatureSet = collections.defaultdict(set)
+        insn_matches: MatchResults = collections.defaultdict(list)
+
+        for insn in bb.children:
+            ifeatures, imatches = self._find_instruction_capabilities(ruleset, insn)
+            for feature, locs in ifeatures.items():
+                features[feature].update(locs)
+            for name, result in imatches.items():
+                insn_matches[name].extend(result)
+
+        for feature, locs in itertools.chain(bb.features.items(), self.global_features.items()):
+            features[feature].update(locs)
+
+        _, matches = ruleset.match(Scope.BASIC_BLOCK, features, bb.address)
+        for name, result in matches.items():
+            rule = ruleset[name]
+            for loc, _ in result:
+                capa.engine.index_rule_matches(features, rule, [loc])
+
+        return features, matches, insn_matches
+
+    def find_code_capabilities(
+        self, ruleset: RuleSet, fh: FunctionHandle
+    ) -> Tuple[FeatureSet, MatchResults, MatchResults, MatchResults]:
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self._get_cached_func_node(fh)
+        if f_node is None:
+            return {}, {}, {}, {}
+
+        insn_matches: MatchResults = collections.defaultdict(list)
+        bb_matches: MatchResults = collections.defaultdict(list)
+        function_features: FeatureSet = collections.defaultdict(set)
+
+        for bb in f_node.children:
+            features, bmatches, imatches = self._find_basic_block_capabilities(ruleset, bb)
+            for feature, locs in features.items():
+                function_features[feature].update(locs)
+            for name, result in bmatches.items():
+                bb_matches[name].extend(result)
+            for name, result in imatches.items():
+                insn_matches[name].extend(result)
+
+        for feature, locs in itertools.chain(f_node.features.items(), self.global_features.items()):
+            function_features[feature].update(locs)
+
+        _, function_matches = ruleset.match(Scope.FUNCTION, function_features, f_node.address)
+        return function_features, function_matches, bb_matches, insn_matches
+
+    def find_file_capabilities(self, ruleset: RuleSet) -> Tuple[FeatureSet, MatchResults]:
+        features: FeatureSet = collections.defaultdict(set)
+
+        for func_node in self.file_node.children:
+            assert func_node.inner is not None
+            assert isinstance(func_node.inner, FunctionHandle)
+
+            func_features, _, _, _ = self.find_code_capabilities(ruleset, func_node.inner)
+            for feature, locs in func_features.items():
+                features[feature].update(locs)
+
+        for feature, locs in itertools.chain(self.file_node.features.items(), self.global_features.items()):
+            features[feature].update(locs)
+
+        _, matches = ruleset.match(Scope.FILE, features, NO_ADDRESS)
+        return features, matches
+
+    def _get_cached_func_node(self, fh: FunctionHandle) -> Optional[CapaRuleGenFeatureCacheNode]:
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self.func_nodes.get(fh.address)
+        if f_node is None:
+            # function is not in our cache, do extraction now
+            self._find_function_and_below_features(fh)
+            f_node = self.func_nodes.get(fh.address)
+        return f_node
+
+    def get_all_function_features(self, fh: FunctionHandle) -> FeatureSet:
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self._get_cached_func_node(fh)
+        if f_node is None:
+            return {}
+
+        all_function_features: FeatureSet = collections.defaultdict(set)
+        all_function_features.update(f_node.features)
+
+        for bb_node in f_node.children:
+            for i_node in bb_node.children:
+                for feature, locs in i_node.features.items():
+                    all_function_features[feature].update(locs)
+            for feature, locs in bb_node.features.items():
+                all_function_features[feature].update(locs)
+
+        # include global features just once
+        for feature, locs in self.global_features.items():
+            all_function_features[feature].update(locs)
+
+        return all_function_features
+
+    def get_all_file_features(self):
+        yield from itertools.chain(self.file_node.features.items(), self.global_features.items())

capa/ida/plugin/capa_explorer.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/error.py

@@ -0,0 +1,13 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+
+class UserCancelledError(Exception):
+    """throw exception when user cancels action"""
+
+    pass

capa/ida/plugin/extractor.py

@@ -0,0 +1,44 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import ida_kernwin
+from PyQt5 import QtCore
+
+from capa.ida.plugin.error import UserCancelledError
+from capa.features.extractors.ida.extractor import IdaFeatureExtractor
+from capa.features.extractors.base_extractor import FunctionHandle
+
+
+class CapaExplorerProgressIndicator(QtCore.QObject):
+    """implement progress signal, used during feature extraction"""
+
+    progress = QtCore.pyqtSignal(str)
+
+    def update(self, text):
+        """emit progress update
+
+        check if user cancelled action, raise exception for parent function to catch
+        """
+        if ida_kernwin.user_cancelled():
+            raise UserCancelledError("user cancelled")
+        self.progress.emit(f"extracting features from {text}")
+
+
+class CapaExplorerFeatureExtractor(IdaFeatureExtractor):
+    """subclass the IdaFeatureExtractor
+
+    track progress during feature extraction, also allow user to cancel feature extraction
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.indicator = CapaExplorerProgressIndicator()
+
+    def extract_function_features(self, fh: FunctionHandle):
+        self.indicator.update(f"function at {hex(fh.inner.start_ea)}")
+        return super().extract_function_features(fh)

capa/ida/plugin/hooks.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -16,7 +16,7 @@ class CapaExplorerIdaHooks(idaapi.UI_Hooks):
         @param screen_ea_changed_hook: function hook for IDA screen ea changed
         @param action_hooks: dict of IDA action handles
         """
-        super(CapaExplorerIdaHooks, self).__init__()
+        super().__init__()
 
         self.screen_ea_changed_hook = screen_ea_changed_hook
         self.process_action_hooks = action_hooks
@@ -30,7 +30,7 @@ class CapaExplorerIdaHooks(idaapi.UI_Hooks):
 
         @retval must be 0
         """
-        self.process_action_handle = self.process_action_hooks.get(name, None)
+        self.process_action_handle = self.process_action_hooks.get(name)
 
         if self.process_action_handle:
             self.process_action_handle(self.process_action_meta)

capa/ida/plugin/item.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,12 +7,14 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import codecs
+from typing import List, Iterator, Optional
 
 import idc
 import idaapi
 from PyQt5 import QtCore
 
 import capa.ida.helpers
+from capa.features.address import Address, FileOffsetAddress, AbsoluteVirtualAddress
 
 
 def info_to_name(display):
@@ -26,19 +28,19 @@ def info_to_name(display):
         return ""
 
 
-def location_to_hex(location):
-    """convert location to hex for display"""
-    return "%08X" % location
+def ea_to_hex(ea):
+    """convert effective address (ea) to hex for display"""
+    return f"{hex(ea)}"
 
 
 class CapaExplorerDataItem:
     """store data for CapaExplorerDataModel"""
 
-    def __init__(self, parent, data, can_check=True):
+    def __init__(self, parent: Optional["CapaExplorerDataItem"], data: List[str], can_check=True):
         """initialize item"""
         self.pred = parent
         self._data = data
-        self.children = []
+        self._children: List["CapaExplorerDataItem"] = []
         self._checked = False
         self._can_check = can_check
 
@@ -76,29 +78,29 @@ class CapaExplorerDataItem:
         """get item is checked"""
         return self._checked
 
-    def appendChild(self, item):
+    def appendChild(self, item: "CapaExplorerDataItem"):
         """add a new child to specified item
 
         @param item: CapaExplorerDataItem
         """
-        self.children.append(item)
+        self._children.append(item)
 
-    def child(self, row):
+    def child(self, row: int) -> "CapaExplorerDataItem":
         """get child row
 
         @param row: row number
         """
-        return self.children[row]
+        return self._children[row]
 
-    def childCount(self):
+    def childCount(self) -> int:
         """get child count"""
-        return len(self.children)
+        return len(self._children)
 
-    def columnCount(self):
+    def columnCount(self) -> int:
         """get column count"""
         return len(self._data)
 
-    def data(self, column):
+    def data(self, column: int) -> Optional[str]:
         """get data at column
 
         @param: column number
@@ -108,17 +110,17 @@ class CapaExplorerDataItem:
         except IndexError:
             return None
 
-    def parent(self):
+    def parent(self) -> Optional["CapaExplorerDataItem"]:
         """get parent"""
         return self.pred
 
-    def row(self):
+    def row(self) -> int:
         """get row location"""
         if self.pred:
-            return self.pred.children.index(self)
+            return self.pred._children.index(self)
         return 0
 
-    def setData(self, column, value):
+    def setData(self, column: int, value: str):
         """set data in column
 
         @param column: column number
@@ -126,14 +128,13 @@ class CapaExplorerDataItem:
         """
         self._data[column] = value
 
-    def children(self):
+    def children(self) -> Iterator["CapaExplorerDataItem"]:
         """yield children"""
-        for child in self.children:
-            yield child
+        yield from self._children
 
     def removeChildren(self):
         """remove children"""
-        del self.children[:]
+        del self._children[:]
 
     def __str__(self):
         """get string representation of columns
@@ -148,7 +149,7 @@ class CapaExplorerDataItem:
         return self._data[0]
 
     @property
-    def location(self):
+    def location(self) -> Optional[int]:
         """return data stored in location column"""
         try:
             # address stored as str, convert to int before return
@@ -167,7 +168,9 @@ class CapaExplorerRuleItem(CapaExplorerDataItem):
 
     fmt = "%s (%d matches)"
 
-    def __init__(self, parent, name, namespace, count, source, can_check=True):
+    def __init__(
+        self, parent: CapaExplorerDataItem, name: str, namespace: str, count: int, source: str, can_check=True
+    ):
         """initialize item
 
         @param parent: parent node
@@ -177,7 +180,7 @@ class CapaExplorerRuleItem(CapaExplorerDataItem):
         @param source: rule source (tooltip)
         """
         display = self.fmt % (name, count) if count > 1 else name
-        super(CapaExplorerRuleItem, self).__init__(parent, [display, "", namespace], can_check)
+        super().__init__(parent, [display, "", namespace], can_check)
         self._source = source
 
     @property
@@ -189,14 +192,14 @@ class CapaExplorerRuleItem(CapaExplorerDataItem):
 class CapaExplorerRuleMatchItem(CapaExplorerDataItem):
     """store data for rule match"""
 
-    def __init__(self, parent, display, source=""):
+    def __init__(self, parent: CapaExplorerDataItem, display: str, source=""):
         """initialize item
 
         @param parent: parent node
         @param display: text to display in UI
         @param source: rule match source to display (tooltip)
         """
-        super(CapaExplorerRuleMatchItem, self).__init__(parent, [display, "", ""])
+        super().__init__(parent, [display, "", ""])
         self._source = source
 
     @property
@@ -210,20 +213,20 @@ class CapaExplorerFunctionItem(CapaExplorerDataItem):
 
     fmt = "function(%s)"
 
-    def __init__(self, parent, location, can_check=True):
+    def __init__(self, parent: CapaExplorerDataItem, location: Address, can_check=True):
         """initialize item
 
         @param parent: parent node
         @param location: virtual address of function as seen by IDA
         """
-        super(CapaExplorerFunctionItem, self).__init__(
-            parent, [self.fmt % idaapi.get_name(location), location_to_hex(location), ""], can_check
-        )
+        assert isinstance(location, AbsoluteVirtualAddress)
+        ea = int(location)
+        super().__init__(parent, [self.fmt % idaapi.get_name(ea), ea_to_hex(ea), ""], can_check)
 
     @property
     def info(self):
         """return function name"""
-        info = super(CapaExplorerFunctionItem, self).info
+        info = super().info
         display = info_to_name(info)
         return display if display else info
 
@@ -243,13 +246,13 @@ class CapaExplorerSubscopeItem(CapaExplorerDataItem):
 
     fmt = "subscope(%s)"
 
-    def __init__(self, parent, scope):
+    def __init__(self, parent: CapaExplorerDataItem, scope):
         """initialize item
 
         @param parent: parent node
         @param scope: subscope name
         """
-        super(CapaExplorerSubscopeItem, self).__init__(parent, [self.fmt % scope, "", ""])
+        super().__init__(parent, [self.fmt % scope, "", ""])
 
 
 class CapaExplorerBlockItem(CapaExplorerDataItem):
@@ -257,19 +260,29 @@ class CapaExplorerBlockItem(CapaExplorerDataItem):
 
     fmt = "basic block(loc_%08X)"
 
-    def __init__(self, parent, location):
+    def __init__(self, parent: CapaExplorerDataItem, location: Address):
         """initialize item
 
         @param parent: parent node
         @param location: virtual address of basic block as seen by IDA
         """
-        super(CapaExplorerBlockItem, self).__init__(parent, [self.fmt % location, location_to_hex(location), ""])
+        assert isinstance(location, AbsoluteVirtualAddress)
+        ea = int(location)
+        super().__init__(parent, [self.fmt % ea, ea_to_hex(ea), ""])
+
+
+class CapaExplorerInstructionItem(CapaExplorerBlockItem):
+    """store data for instruction match"""
+
+    fmt = "instruction(loc_%08X)"
 
 
 class CapaExplorerDefaultItem(CapaExplorerDataItem):
     """store data for default match e.g. statement (and, or)"""
 
-    def __init__(self, parent, display, details="", location=None):
+    def __init__(
+        self, parent: CapaExplorerDataItem, display: str, details: str = "", location: Optional[Address] = None
+    ):
         """initialize item
 
         @param parent: parent node
@@ -277,14 +290,20 @@ class CapaExplorerDefaultItem(CapaExplorerDataItem):
         @param details: text to display in details section of UI
         @param location: virtual address as seen by IDA
         """
-        location = location_to_hex(location) if location else ""
-        super(CapaExplorerDefaultItem, self).__init__(parent, [display, location, details])
+        ea = None
+        if location:
+            assert isinstance(location, AbsoluteVirtualAddress)
+            ea = int(location)
+
+        super().__init__(parent, [display, ea_to_hex(ea) if ea is not None else "", details])
 
 
 class CapaExplorerFeatureItem(CapaExplorerDataItem):
     """store data for feature match"""
 
-    def __init__(self, parent, display, location="", details=""):
+    def __init__(
+        self, parent: CapaExplorerDataItem, display: str, location: Optional[Address] = None, details: str = ""
+    ):
         """initialize item
 
         @param parent: parent node
@@ -292,14 +311,18 @@ class CapaExplorerFeatureItem(CapaExplorerDataItem):
         @param details: text to display in details section of UI
         @param location: virtual address as seen by IDA
         """
-        location = location_to_hex(location) if location else ""
-        super(CapaExplorerFeatureItem, self).__init__(parent, [display, location, details])
+        if location:
+            assert isinstance(location, (AbsoluteVirtualAddress, FileOffsetAddress))
+            ea = int(location)
+            super().__init__(parent, [display, ea_to_hex(ea), details])
+        else:
+            super().__init__(parent, [display, "", details])
 
 
 class CapaExplorerInstructionViewItem(CapaExplorerFeatureItem):
     """store data for instruction match"""
 
-    def __init__(self, parent, display, location):
+    def __init__(self, parent: CapaExplorerDataItem, display: str, location: Address):
         """initialize item
 
         details section shows disassembly view for match
@@ -308,15 +331,17 @@ class CapaExplorerInstructionViewItem(CapaExplorerFeatureItem):
         @param display: text to display in UI
         @param location: virtual address as seen by IDA
         """
-        details = capa.ida.helpers.get_disasm_line(location)
-        super(CapaExplorerInstructionViewItem, self).__init__(parent, display, location=location, details=details)
-        self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)
+        assert isinstance(location, AbsoluteVirtualAddress)
+        ea = int(location)
+        details = capa.ida.helpers.get_disasm_line(ea)
+        super().__init__(parent, display, location=location, details=details)
+        self.ida_highlight = idc.get_color(ea, idc.CIC_ITEM)
 
 
 class CapaExplorerByteViewItem(CapaExplorerFeatureItem):
     """store data for byte match"""
 
-    def __init__(self, parent, display, location):
+    def __init__(self, parent: CapaExplorerDataItem, display: str, location: Address):
         """initialize item
 
         details section shows byte preview for match
@@ -325,26 +350,32 @@ class CapaExplorerByteViewItem(CapaExplorerFeatureItem):
         @param display: text to display in UI
         @param location: virtual address as seen by IDA
         """
-        byte_snap = idaapi.get_bytes(location, 32)
+        assert isinstance(location, (AbsoluteVirtualAddress, FileOffsetAddress))
+        ea = int(location)
+
+        byte_snap = idaapi.get_bytes(ea, 32)
 
         details = ""
         if byte_snap:
             byte_snap = codecs.encode(byte_snap, "hex").upper()
             details = " ".join([byte_snap[i : i + 2].decode() for i in range(0, len(byte_snap), 2)])
 
-        super(CapaExplorerByteViewItem, self).__init__(parent, display, location=location, details=details)
-        self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)
+        super().__init__(parent, display, location=location, details=details)
+        self.ida_highlight = idc.get_color(ea, idc.CIC_ITEM)
 
 
 class CapaExplorerStringViewItem(CapaExplorerFeatureItem):
     """store data for string match"""
 
-    def __init__(self, parent, display, location, value):
+    def __init__(self, parent: CapaExplorerDataItem, display: str, location: Address, value: str):
         """initialize item
 
         @param parent: parent node
         @param display: text to display in UI
         @param location: virtual address as seen by IDA
         """
-        super(CapaExplorerStringViewItem, self).__init__(parent, display, location=location, details=value)
-        self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)
+        assert isinstance(location, (AbsoluteVirtualAddress, FileOffsetAddress))
+        ea = int(location)
+
+        super().__init__(parent, display, location=location, details=value)
+        self.ida_highlight = idc.get_color(ea, idc.CIC_ITEM)

capa/ida/plugin/model.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,7 +6,8 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from collections import deque, defaultdict
+from typing import Set, Dict, List, Tuple, Optional
+from collections import deque
 
 import idc
 import idaapi
@@ -16,6 +17,9 @@ import capa.rules
 import capa.ida.helpers
 import capa.render.utils as rutils
 import capa.features.common
+import capa.features.freeze as frz
+import capa.render.result_document as rd
+import capa.features.freeze.features as frzf
 from capa.ida.plugin.item import (
     CapaExplorerDataItem,
     CapaExplorerRuleItem,
@@ -27,8 +31,10 @@ from capa.ida.plugin.item import (
     CapaExplorerSubscopeItem,
     CapaExplorerRuleMatchItem,
     CapaExplorerStringViewItem,
+    CapaExplorerInstructionItem,
     CapaExplorerInstructionViewItem,
 )
+from capa.features.address import Address, AbsoluteVirtualAddress
 
 # default highlight color used in IDA window
 DEFAULT_HIGHLIGHT = 0xE6C700
@@ -45,7 +51,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
     def __init__(self, parent=None):
         """initialize model"""
-        super(CapaExplorerDataModel, self).__init__(parent)
+        super().__init__(parent)
         # root node does not have parent, contains header columns
         self.root_node = CapaExplorerDataItem(None, ["Rule Information", "Address", "Details"])
 
@@ -137,6 +143,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                     CapaExplorerFunctionItem,
                     CapaExplorerFeatureItem,
                     CapaExplorerSubscopeItem,
+                    CapaExplorerInstructionItem,
                 ),
             )
             and column == CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION
@@ -342,7 +349,14 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
         return item.childCount()
 
-    def render_capa_doc_statement_node(self, parent, statement, locations, doc):
+    def render_capa_doc_statement_node(
+        self,
+        parent: CapaExplorerDataItem,
+        match: rd.Match,
+        statement: rd.Statement,
+        locations: List[Address],
+        doc: rd.ResultDocument,
+    ):
         """render capa statement read from doc
 
         @param parent: parent to which new child is assigned
@@ -350,132 +364,156 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         @param locations: locations of children (applies to range only?)
         @param doc: result doc
         """
-        if statement["type"] in ("and", "or", "optional"):
-            display = statement["type"]
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
-            return CapaExplorerDefaultItem(parent, display)
-        elif statement["type"] == "not":
-            # TODO: do we display 'not'
+
+        if isinstance(statement, rd.CompoundStatement):
+            if statement.type != rd.CompoundStatementType.NOT:
+                display = statement.type
+                if statement.description:
+                    display += f" ({statement.description})"
+                return CapaExplorerDefaultItem(parent, display)
+        elif isinstance(statement, rd.CompoundStatement) and statement.type == rd.CompoundStatementType.NOT:
+            # TODO(mike-hunhoff): verify that we can display NOT statements
+            # https://github.com/mandiant/capa/issues/1602
             pass
-        elif statement["type"] == "some":
-            display = "%d or more" % statement["count"]
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
+        elif isinstance(statement, rd.SomeStatement):
+            display = f"{statement.count} or more"
+            if statement.description:
+                display += f" ({statement.description})"
             return CapaExplorerDefaultItem(parent, display)
-        elif statement["type"] == "range":
+        elif isinstance(statement, rd.RangeStatement):
             # `range` is a weird node, its almost a hybrid of statement + feature.
             # it is a specific feature repeated multiple times.
             # there's no additional logic in the feature part, just the existence of a feature.
             # so, we have to inline some of the feature rendering here.
-            display = "count(%s): " % self.capa_doc_feature_to_display(statement["child"])
+            display = f"count({self.capa_doc_feature_to_display(statement.child)}): "
 
-            if statement["max"] == statement["min"]:
-                display += "%d" % (statement["min"])
-            elif statement["min"] == 0:
-                display += "%d or fewer" % (statement["max"])
-            elif statement["max"] == (1 << 64 - 1):
-                display += "%d or more" % (statement["min"])
+            if statement.max == statement.min:
+                display += f"{statement.min}"
+            elif statement.min == 0:
+                display += f"{statement.max} or fewer"
+            elif statement.max == (1 << 64 - 1):
+                display += f"{statement.min} or more"
             else:
-                display += "between %d and %d" % (statement["min"], statement["max"])
+                display += f"between {statement.min} and {statement.max}"
 
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
+            if statement.description:
+                display += f" ({statement.description})"
 
             parent2 = CapaExplorerFeatureItem(parent, display=display)
 
             for location in locations:
                 # for each location render child node for range statement
-                self.render_capa_doc_feature(parent2, statement["child"], location, doc)
+                self.render_capa_doc_feature(parent2, match, statement.child, location, doc)
 
             return parent2
-        elif statement["type"] == "subscope":
-            display = statement[statement["type"]]
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
+        elif isinstance(statement, rd.SubscopeStatement):
+            display = str(statement.scope)
+            if statement.description:
+                display += f" ({statement.description})"
             return CapaExplorerSubscopeItem(parent, display)
         else:
             raise RuntimeError("unexpected match statement type: " + str(statement))
 
-    def render_capa_doc_match(self, parent, match, doc):
+    def render_capa_doc_match(self, parent: CapaExplorerDataItem, match: rd.Match, doc: rd.ResultDocument):
         """render capa match read from doc
 
         @param parent: parent node to which new child is assigned
         @param match: match read from doc
         @param doc: result doc
         """
-        if not match["success"]:
-            # TODO: display failed branches at some point? Help with debugging rules?
+        if not match.success:
+            # TODO(mike-hunhoff): display failed branches at some point? Help with debugging rules?
+            # https://github.com/mandiant/capa/issues/1601
             return
 
         # optional statement with no successful children is empty
-        if match["node"].get("statement", {}).get("type") == "optional" and not any(
-            map(lambda m: m["success"], match["children"])
-        ):
-            return
+        if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
+            if not any(m.success for m in match.children):
+                return
 
-        if match["node"]["type"] == "statement":
+        if isinstance(match.node, rd.StatementNode):
             parent2 = self.render_capa_doc_statement_node(
-                parent, match["node"]["statement"], match.get("locations", []), doc
+                parent, match, match.node.statement, [addr.to_capa() for addr in match.locations], doc
             )
-        elif match["node"]["type"] == "feature":
+        elif isinstance(match.node, rd.FeatureNode):
             parent2 = self.render_capa_doc_feature_node(
-                parent, match["node"]["feature"], match.get("locations", []), doc
+                parent, match, match.node.feature, [addr.to_capa() for addr in match.locations], doc
             )
         else:
-            raise RuntimeError("unexpected node type: " + str(match["node"]["type"]))
+            raise RuntimeError("unexpected node type: " + str(match.node.type))
 
-        for child in match.get("children", []):
+        for child in match.children:
             self.render_capa_doc_match(parent2, child, doc)
 
-    def render_capa_doc_by_function(self, doc):
-        """ """
-        matches_by_function = {}
+    def render_capa_doc_by_function(self, doc: rd.ResultDocument):
+        """render rule matches by function meaning each rule match is nested under function where it was found"""
+        matches_by_function: Dict[AbsoluteVirtualAddress, Tuple[CapaExplorerFunctionItem, Set[str]]] = {}
         for rule in rutils.capability_rules(doc):
-            for ea in rule["matches"].keys():
-                ea = capa.ida.helpers.get_func_start_ea(ea)
-                if ea is None:
-                    # file scope, skip rendering in this mode
+            match_eas: List[int] = []
+
+            # initial pass of rule matches
+            for addr_, _ in rule.matches:
+                addr: Address = addr_.to_capa()
+                if isinstance(addr, AbsoluteVirtualAddress):
+                    match_eas.append(int(addr))
+
+            for ea in match_eas:
+                func_ea: Optional[int] = capa.ida.helpers.get_func_start_ea(ea)
+                if func_ea is None:
+                    # rule match address is not located in a defined function
                     continue
-                if not matches_by_function.get(ea, ()):
-                    # new function root
-                    matches_by_function[ea] = (CapaExplorerFunctionItem(self.root_node, ea, can_check=False), [])
-                function_root, match_cache = matches_by_function[ea]
-                if rule["meta"]["name"] in match_cache:
-                    # rule match already rendered for this function root, skip it
+
+                func_address: AbsoluteVirtualAddress = AbsoluteVirtualAddress(func_ea)
+                if not matches_by_function.get(func_address, ()):
+                    # create a new function root to nest its rule matches; Note: we must use the address of the
+                    # function here so everything is displayed properly
+                    matches_by_function[func_address] = (
+                        CapaExplorerFunctionItem(self.root_node, func_address, can_check=False),
+                        set(),
+                    )
+
+                func_root, func_match_cache = matches_by_function[func_address]
+                if rule.meta.name in func_match_cache:
+                    # only nest each rule once, so if found, skip
                     continue
-                match_cache.append(rule["meta"]["name"])
+
+                # add matched rule to its function cache; create a new rule node whose parent is the matched
+                # function node
+                func_match_cache.add(rule.meta.name)
                 CapaExplorerRuleItem(
-                    function_root,
-                    rule["meta"]["name"],
-                    rule["meta"].get("namespace"),
-                    len(rule["matches"]),
-                    rule["source"],
+                    func_root,
+                    rule.meta.name,
+                    rule.meta.namespace or "",
+                    len([ea for ea in match_eas if capa.ida.helpers.get_func_start_ea(ea) == func_ea]),
+                    rule.source,
                     can_check=False,
                 )
 
-    def render_capa_doc_by_program(self, doc):
+    def render_capa_doc_by_program(self, doc: rd.ResultDocument):
         """ """
         for rule in rutils.capability_rules(doc):
-            rule_name = rule["meta"]["name"]
-            rule_namespace = rule["meta"].get("namespace")
-            parent = CapaExplorerRuleItem(
-                self.root_node, rule_name, rule_namespace, len(rule["matches"]), rule["source"]
-            )
+            rule_name = rule.meta.name
+            rule_namespace = rule.meta.namespace or ""
+            parent = CapaExplorerRuleItem(self.root_node, rule_name, rule_namespace, len(rule.matches), rule.source)
 
-            for (location, match) in doc["rules"][rule["meta"]["name"]]["matches"].items():
-                if rule["meta"]["scope"] == capa.rules.FILE_SCOPE:
+            for location_, match in rule.matches:
+                location = location_.to_capa()
+
+                parent2: CapaExplorerDataItem
+                if rule.meta.scope == capa.rules.FILE_SCOPE:
                     parent2 = parent
-                elif rule["meta"]["scope"] == capa.rules.FUNCTION_SCOPE:
+                elif rule.meta.scope == capa.rules.FUNCTION_SCOPE:
                     parent2 = CapaExplorerFunctionItem(parent, location)
-                elif rule["meta"]["scope"] == capa.rules.BASIC_BLOCK_SCOPE:
+                elif rule.meta.scope == capa.rules.BASIC_BLOCK_SCOPE:
                     parent2 = CapaExplorerBlockItem(parent, location)
+                elif rule.meta.scope == capa.rules.INSTRUCTION_SCOPE:
+                    parent2 = CapaExplorerInstructionItem(parent, location)
                 else:
-                    raise RuntimeError("unexpected rule scope: " + str(rule["meta"]["scope"]))
+                    raise RuntimeError("unexpected rule scope: " + str(rule.meta.scope))
 
                 self.render_capa_doc_match(parent2, match, doc)
 
-    def render_capa_doc(self, doc, by_function):
+    def render_capa_doc(self, doc: rd.ResultDocument, by_function: bool):
         """render capa features specified in doc
 
         @param doc: capa result doc
@@ -491,27 +529,44 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         # inform model changes have ended
         self.endResetModel()
 
-    def capa_doc_feature_to_display(self, feature):
+    def capa_doc_feature_to_display(self, feature: frzf.Feature):
         """convert capa doc feature type string to display string for ui
 
         @param feature: capa feature read from doc
         """
-        key = feature["type"]
-        value = feature[feature["type"]]
-        if value:
-            if key == "string":
-                value = '"%s"' % capa.features.common.escape_string(value)
-            if feature.get("description", ""):
-                return "%s(%s = %s)" % (key, value, feature["description"])
-            else:
-                return "%s(%s)" % (key, value)
-        else:
-            return "%s" % key
+        key = feature.type
+        value = feature.dict(by_alias=True).get(feature.type)
 
-    def render_capa_doc_feature_node(self, parent, feature, locations, doc):
+        if value:
+            if isinstance(feature, frzf.StringFeature):
+                value = f'"{capa.features.common.escape_string(value)}"'
+
+            if isinstance(feature, frzf.PropertyFeature) and feature.access is not None:
+                key = f"property/{feature.access}"
+            elif isinstance(feature, frzf.OperandNumberFeature):
+                key = f"operand[{feature.index}].number"
+            elif isinstance(feature, frzf.OperandOffsetFeature):
+                key = f"operand[{feature.index}].offset"
+
+            if feature.description:
+                return f"{key}({value} = {feature.description})"
+            else:
+                return f"{key}({value})"
+        else:
+            return f"{key}"
+
+    def render_capa_doc_feature_node(
+        self,
+        parent: CapaExplorerDataItem,
+        match: rd.Match,
+        feature: frzf.Feature,
+        locations: List[Address],
+        doc: rd.ResultDocument,
+    ):
         """process capa doc feature node
 
         @param parent: parent node to which child is assigned
+        @param match: match information
         @param feature: capa doc feature node
         @param locations: locations identified for feature
         @param doc: capa doc
@@ -522,6 +577,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             # only one location for feature so no need to nest children
             parent2 = self.render_capa_doc_feature(
                 parent,
+                match,
                 feature,
                 next(iter(locations)),
                 doc,
@@ -532,81 +588,114 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             parent2 = CapaExplorerFeatureItem(parent, display)
 
             for location in sorted(locations):
-                self.render_capa_doc_feature(parent2, feature, location, doc)
+                self.render_capa_doc_feature(parent2, match, feature, location, doc)
 
         return parent2
 
-    def render_capa_doc_feature(self, parent, feature, location, doc, display="-"):
+    def render_capa_doc_feature(
+        self,
+        parent: CapaExplorerDataItem,
+        match: rd.Match,
+        feature: frzf.Feature,
+        location: Address,
+        doc: rd.ResultDocument,
+        display="-",
+    ):
         """render capa feature read from doc
 
         @param parent: parent node to which new child is assigned
+        @param match: match information
         @param feature: feature read from doc
         @param doc: capa feature doc
         @param location: address of feature
         @param display: text to display in plugin UI
         """
+
         # special handling for characteristic pending type
-        if feature["type"] == "characteristic":
-            if feature[feature["type"]] in ("embedded pe",):
+        if isinstance(feature, frzf.CharacteristicFeature):
+            characteristic = feature.characteristic
+            if characteristic in ("embedded pe",):
                 return CapaExplorerByteViewItem(parent, display, location)
 
-            if feature[feature["type"]] in ("loop", "recursive call", "tight loop"):
+            if characteristic in ("loop", "recursive call", "tight loop"):
                 return CapaExplorerFeatureItem(parent, display=display)
 
             # default to instruction view for all other characteristics
             return CapaExplorerInstructionViewItem(parent, display, location)
 
-        if feature["type"] == "match":
+        elif isinstance(feature, frzf.MatchFeature):
             # display content of rule for all rule matches
-            return CapaExplorerRuleMatchItem(
-                parent, display, source=doc["rules"].get(feature[feature["type"]], {}).get("source", "")
-            )
+            matched_rule_source = ""
 
-        if feature["type"] in ("regex", "substring"):
-            for s, locations in feature["matches"].items():
-                if location in locations:
-                    return CapaExplorerStringViewItem(
-                        parent, display, location, '"' + capa.features.common.escape_string(s) + '"'
-                    )
+            # check if match is a matched rule
+            matched_rule = doc.rules.get(feature.match)
+            if matched_rule is not None:
+                matched_rule_source = matched_rule.source
+
+            return CapaExplorerRuleMatchItem(parent, display, source=matched_rule_source)
+
+        elif isinstance(feature, (frzf.RegexFeature, frzf.SubstringFeature)):
+            for capture, addrs in sorted(match.captures.items()):
+                for addr in addrs:
+                    assert isinstance(addr, frz.Address)
+                    if location == addr.value:
+                        return CapaExplorerStringViewItem(
+                            parent, display, location, '"' + capa.features.common.escape_string(capture) + '"'
+                        )
 
             # programming error: the given location should always be found in the regex matches
             raise ValueError("regex match at location not found")
 
-        if feature["type"] == "basicblock":
+        elif isinstance(feature, frzf.BasicBlockFeature):
             return CapaExplorerBlockItem(parent, location)
 
-        if feature["type"] in (
-            "bytes",
-            "api",
-            "mnemonic",
-            "number",
-            "offset",
-            "number/x32",
-            "number/x64",
-            "offset/x32",
-            "offset/x64",
+        elif isinstance(
+            feature,
+            (
+                frzf.BytesFeature,
+                frzf.APIFeature,
+                frzf.MnemonicFeature,
+                frzf.NumberFeature,
+                frzf.OffsetFeature,
+                frzf.OperandNumberFeature,
+                frzf.OperandOffsetFeature,
+            ),
         ):
             # display instruction preview
             return CapaExplorerInstructionViewItem(parent, display, location)
 
-        if feature["type"] in ("section",):
+        elif isinstance(feature, frzf.SectionFeature):
             # display byte preview
             return CapaExplorerByteViewItem(parent, display, location)
 
-        if feature["type"] in ("string",):
+        elif isinstance(feature, frzf.StringFeature):
             # display string preview
             return CapaExplorerStringViewItem(
-                parent, display, location, '"%s"' % capa.features.common.escape_string(feature[feature["type"]])
+                parent, display, location, f'"{capa.features.common.escape_string(feature.string)}"'
             )
 
-        if feature["type"] in ("import", "export", "function-name"):
+        elif isinstance(
+            feature,
+            (
+                frzf.ImportFeature,
+                frzf.ExportFeature,
+                frzf.FunctionNameFeature,
+            ),
+        ):
             # display no preview
             return CapaExplorerFeatureItem(parent, location=location, display=display)
 
-        if feature["type"] in ("arch", "os", "format"):
+        elif isinstance(
+            feature,
+            (
+                frzf.ArchFeature,
+                frzf.OSFeature,
+                frzf.FormatFeature,
+            ),
+        ):
             return CapaExplorerFeatureItem(parent, display=display)
 
-        raise RuntimeError("unexpected feature type: " + str(feature["type"]))
+        raise RuntimeError("unexpected feature type: " + str(feature.type))
 
     def update_function_name(self, old_name, new_name):
         """update all instances of old function name with new function name

capa/ida/plugin/proxy.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -22,7 +22,7 @@ class CapaExplorerRangeProxyModel(QtCore.QSortFilterProxyModel):
 
     def __init__(self, parent=None):
         """initialize proxy filter"""
-        super(CapaExplorerRangeProxyModel, self).__init__(parent)
+        super().__init__(parent)
         self.min_ea = None
         self.max_ea = None
 
@@ -92,7 +92,7 @@ class CapaExplorerRangeProxyModel(QtCore.QSortFilterProxyModel):
         @param parent: QModelIndex of parent
         """
         # filter not set
-        if self.min_ea is None and self.max_ea is None:
+        if self.min_ea is None or self.max_ea is None:
             return True
 
         index = self.sourceModel().index(row, 0, parent)
@@ -145,7 +145,7 @@ class CapaExplorerSearchProxyModel(QtCore.QSortFilterProxyModel):
 
     def __init__(self, parent=None):
         """ """
-        super(CapaExplorerSearchProxyModel, self).__init__(parent)
+        super().__init__(parent)
         self.query = ""
         self.setFilterKeyColumn(-1)  # all columns
 

capa/ida/plugin/view.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,6 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import re
+from typing import Dict, Optional
 from collections import Counter
 
 import idc
@@ -18,6 +19,7 @@ import capa.ida.helpers
 import capa.features.common
 import capa.features.basicblock
 from capa.ida.plugin.item import CapaExplorerFunctionItem
+from capa.features.address import AbsoluteVirtualAddress, _NoAddress
 from capa.ida.plugin.model import CapaExplorerDataModel
 
 MAX_SECTION_SIZE = 750
@@ -57,12 +59,12 @@ def parse_yaml_line(feature):
         if m:
             # reconstruct count without description
             feature, value, description, count = m.groups()
-            feature = "- count(%s(%s)): %s" % (feature, value, count)
+            feature = f"- count({feature}({value})): {count}"
     elif not feature.startswith("#"):
         feature, _, comment = feature.partition("#")
         feature, _, description = feature.partition("=")
 
-    return map(lambda o: o.strip(), (feature, description, comment))
+    return (o.strip() for o in (feature, description, comment))
 
 
 def parse_node_for_feature(feature, description, comment, depth):
@@ -71,18 +73,18 @@ def parse_node_for_feature(feature, description, comment, depth):
     display = ""
 
     if feature.startswith("#"):
-        display += "%s%s\n" % (" " * depth, feature)
+        display += f"{' '*depth}{feature}\n"
     elif description:
-        if feature.startswith(("- and", "- or", "- optional", "- basic block", "- not")):
-            display += "%s%s" % (" " * depth, feature)
+        if feature.startswith(("- and", "- or", "- optional", "- basic block", "- not", "- instruction:")):
+            display += f"{' '*depth}{feature}\n"
             if comment:
-                display += " # %s" % comment
-            display += "\n%s- description: %s\n" % (" " * (depth + 2), description)
+                display += f" # {comment}"
+            display += f"\n{' '*(depth+2)}- description: {description}\n"
         elif feature.startswith("- string"):
-            display += "%s%s" % (" " * depth, feature)
+            display += f"{' '*depth}{feature}\n"
             if comment:
-                display += " # %s" % comment
-            display += "\n%sdescription: %s\n" % (" " * (depth + 2), description)
+                display += f" # {comment}"
+            display += f"\n{' '*(depth+2)}description: {description}\n"
         elif feature.startswith("- count"):
             # count is weird, we need to format description based on feature type, so we parse with regex
             # assume format - count(<feature_name>(<feature_value>)): <count>
@@ -90,28 +92,22 @@ def parse_node_for_feature(feature, description, comment, depth):
             if m:
                 name, value, count = m.groups()
                 if name in ("string",):
-                    display += "%s%s" % (" " * depth, feature)
+                    display += f"{' '*depth}{feature}"
                     if comment:
-                        display += " # %s" % comment
-                    display += "\n%sdescription: %s\n" % (" " * (depth + 2), description)
+                        display += f" # {comment}"
+                    display += f"\n{' '*(depth+2)}description: {description}\n"
                 else:
-                    display += "%s- count(%s(%s = %s)): %s" % (
-                        " " * depth,
-                        name,
-                        value,
-                        description,
-                        count,
-                    )
+                    display += f"{' '*depth}- count({name}({value} = {description})): {count}"
                     if comment:
-                        display += " # %s\n" % comment
+                        display += f" # {comment}\n"
         else:
-            display += "%s%s = %s" % (" " * depth, feature, description)
+            display += f"{' '*depth}{feature} = {description}"
             if comment:
-                display += " # %s\n" % comment
+                display += f" # {comment}\n"
     else:
-        display += "%s%s" % (" " * depth, feature)
+        display += f"{' '*depth}{feature}"
         if comment:
-            display += " # %s\n" % comment
+            display += f" # {comment}\n"
 
     return display if display.endswith("\n") else display + "\n"
 
@@ -172,17 +168,17 @@ def resize_columns_to_content(header):
         header.resizeSection(0, MAX_SECTION_SIZE)
 
 
-class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
-
+class CapaExplorerRulegenPreview(QtWidgets.QTextEdit):
     INDENT = " " * 2
 
     def __init__(self, parent=None):
         """ """
-        super(CapaExplorerRulgenPreview, self).__init__(parent)
+        super().__init__(parent)
 
         self.setFont(QtGui.QFont("Courier", weight=QtGui.QFont.Bold))
         self.setLineWrapMode(QtWidgets.QTextEdit.NoWrap)
         self.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
+        self.setAcceptRichText(False)
 
     def reset_view(self):
         """ """
@@ -196,13 +192,15 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
             "  meta:",
             "    name: <insert_name>",
             "    namespace: <insert_namespace>",
-            "    author: %s" % author,
-            "    scope: %s" % scope,
-            "    references: <insert_references>",
+            "    authors:",
+            f"      - {author}",
+            f"    scope: {scope}",
+            "    references:",
+            "      - <insert_references>",
             "    examples:",
-            "      - %s:0x%X" % (capa.ida.helpers.get_file_md5().upper(), ea)
+            f"      - {capa.ida.helpers.get_file_md5().upper()}:{hex(ea)}"
             if ea
-            else "      - %s" % (capa.ida.helpers.get_file_md5().upper()),
+            else f"      - {capa.ida.helpers.get_file_md5().upper()}",
             "  features:",
         ]
         self.setText("\n".join(metadata_default))
@@ -251,7 +249,7 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
                     lines_modified = 0
                     first_modified = False
                     change = []
-                    for (lineno, line) in enumerate(plain[start_lineno : end_lineno + 1]):
+                    for lineno, line in enumerate(plain[start_lineno : end_lineno + 1]):
                         if line.startswith(self.INDENT):
                             if lineno == 0:
                                 # keep track if first line is modified, so we can properly display
@@ -282,7 +280,7 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
                 self.set_selection(select_start_ppos, select_end_ppos, len(self.toPlainText()))
                 self.verticalScrollBar().setSliderPosition(scroll_ppos)
         else:
-            super(CapaExplorerRulgenPreview, self).keyPressEvent(e)
+            super().keyPressEvent(e)
 
     def count_previous_lines_from_block(self, block):
         """calculate number of lines preceding block"""
@@ -302,13 +300,12 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
         self.setTextCursor(cursor)
 
 
-class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
-
+class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
     updated = QtCore.pyqtSignal()
 
     def __init__(self, preview, parent=None):
         """ """
-        super(CapaExplorerRulgenEditor, self).__init__(parent)
+        super().__init__(parent)
 
         self.preview = preview
 
@@ -372,18 +369,18 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
 
     def dragMoveEvent(self, e):
         """ """
-        super(CapaExplorerRulgenEditor, self).dragMoveEvent(e)
+        super().dragMoveEvent(e)
 
     def dragEventEnter(self, e):
         """ """
-        super(CapaExplorerRulgenEditor, self).dragEventEnter(e)
+        super().dragEventEnter(e)
 
     def dropEvent(self, e):
         """ """
         if not self.indexAt(e.pos()).isValid():
             return
 
-        super(CapaExplorerRulgenEditor, self).dropEvent(e)
+        super().dropEvent(e)
 
         self.update_preview()
         expand_tree(self.invisibleRootItem())
@@ -425,6 +422,10 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
             # add default child expression when nesting under basic block
             new_parent.setExpanded(True)
             new_parent = self.new_expression_node(new_parent, ("- or:", ""))
+        elif "instruction" in action.data()[0]:
+            # add default child expression when nesting under instruction
+            new_parent.setExpanded(True)
+            new_parent = self.new_expression_node(new_parent, ("- or:", ""))
 
         for o in self.get_features(selected=True):
             # take child from its parent by index, add to new parent
@@ -437,7 +438,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         expression, o = action.data()
         if "basic block" in expression and "basic block" not in o.text(
-            CapaExplorerRulgenEditor.get_column_feature_index()
+            CapaExplorerRulegenEditor.get_column_feature_index()
         ):
             # current expression is "basic block", and not changing to "basic block" expression
             children = o.takeChildren()
@@ -445,7 +446,17 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
             for child in children:
                 new_parent.addChild(child)
             new_parent.setExpanded(True)
-        o.setText(CapaExplorerRulgenEditor.get_column_feature_index(), expression)
+        elif "instruction" in expression and "instruction" not in o.text(
+            CapaExplorerRulegenEditor.get_column_feature_index()
+        ):
+            # current expression is "instruction", and not changing to "instruction" expression
+            children = o.takeChildren()
+            new_parent = self.new_expression_node(o, ("- or:", ""))
+            for child in children:
+                new_parent.addChild(child)
+            new_parent.setExpanded(True)
+
+        o.setText(CapaExplorerRulegenEditor.get_column_feature_index(), expression)
 
     def slot_clear_all(self, action):
         """ """
@@ -456,7 +467,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         if not self.indexAt(pos).isValid():
             # user selected invalid index
             self.load_custom_context_menu_invalid_index(pos)
-        elif self.itemAt(pos).capa_type == CapaExplorerRulgenEditor.get_node_type_expression():
+        elif self.itemAt(pos).capa_type == CapaExplorerRulegenEditor.get_node_type_expression():
             # user selected expression node
             self.load_custom_context_menu_expression(pos)
         else:
@@ -468,8 +479,8 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
     def slot_item_double_clicked(self, o, column):
         """ """
         if column in (
-            CapaExplorerRulgenEditor.get_column_comment_index(),
-            CapaExplorerRulgenEditor.get_column_description_index(),
+            CapaExplorerRulegenEditor.get_column_comment_index(),
+            CapaExplorerRulegenEditor.get_column_description_index(),
         ):
             o.setFlags(o.flags() | QtCore.Qt.ItemIsEditable)
             self.editItem(o, column)
@@ -488,12 +499,13 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
             rule_text += "\n  features:\n"
 
         for o in iterate_tree(self):
-            feature, description, comment = map(lambda o: o.strip(), tuple(o.text(i) for i in range(3)))
+            feature, description, comment = (o.strip() for o in tuple(o.text(i) for i in range(3)))
             rule_text += parse_node_for_feature(feature, description, comment, calc_item_depth(o))
 
-        # FIXME we avoid circular update by disabling signals when updating
+        # TODO(mike-hunhoff): we avoid circular update by disabling signals when updating
         # the preview. Preferably we would refactor the code to avoid this
-        # in the first place
+        # in the first place.
+        # https://github.com/mandiant/capa/issues/1600
         self.preview.blockSignals(True)
         self.preview.setPlainText(rule_text)
         self.preview.blockSignals(False)
@@ -518,11 +530,12 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
             ("not", ("- not:",), self.slot_nest_features),
             ("optional", ("- optional:",), self.slot_nest_features),
             ("basic block", ("- basic block:",), self.slot_nest_features),
+            ("instruction", ("- instruction:",), self.slot_nest_features),
         )
 
         # build submenu with modify actions
         sub_menu = build_context_menu(self.parent(), sub_actions)
-        sub_menu.setTitle("Nest feature%s" % ("" if len(tuple(self.get_features(selected=True))) == 1 else "s"))
+        sub_menu.setTitle(f"Nest feature{'' if len(tuple(self.get_features(selected=True))) == 1 else 's'}")
 
         # build main menu with submenu + main actions
         menu = build_context_menu(self.parent(), (sub_menu,) + actions)
@@ -539,6 +552,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
             ("not", ("- not:", self.itemAt(pos)), self.slot_edit_expression),
             ("optional", ("- optional:", self.itemAt(pos)), self.slot_edit_expression),
             ("basic block", ("- basic block:", self.itemAt(pos)), self.slot_edit_expression),
+            ("instruction", ("- instruction:", self.itemAt(pos)), self.slot_edit_expression),
         )
 
         # build submenu with modify actions
@@ -555,7 +569,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         font = QtGui.QFont()
         font.setBold(True)
 
-        o.setFont(CapaExplorerRulgenEditor.get_column_feature_index(), font)
+        o.setFont(CapaExplorerRulegenEditor.get_column_feature_index(), font)
 
     def style_feature_node(self, o):
         """ """
@@ -566,8 +580,8 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         font.setWeight(QtGui.QFont.Medium)
         brush.setColor(QtGui.QColor(*COLOR_GREEN_RGB))
 
-        o.setFont(CapaExplorerRulgenEditor.get_column_feature_index(), font)
-        o.setForeground(CapaExplorerRulgenEditor.get_column_feature_index(), brush)
+        o.setFont(CapaExplorerRulegenEditor.get_column_feature_index(), font)
+        o.setForeground(CapaExplorerRulegenEditor.get_column_feature_index(), brush)
 
     def style_comment_node(self, o):
         """ """
@@ -575,22 +589,22 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         font.setBold(True)
         font.setFamily("Courier")
 
-        o.setFont(CapaExplorerRulgenEditor.get_column_feature_index(), font)
+        o.setFont(CapaExplorerRulegenEditor.get_column_feature_index(), font)
 
     def set_expression_node(self, o):
         """ """
-        setattr(o, "capa_type", CapaExplorerRulgenEditor.get_node_type_expression())
+        setattr(o, "capa_type", CapaExplorerRulegenEditor.get_node_type_expression())
         self.style_expression_node(o)
 
     def set_feature_node(self, o):
         """ """
-        setattr(o, "capa_type", CapaExplorerRulgenEditor.get_node_type_feature())
+        setattr(o, "capa_type", CapaExplorerRulegenEditor.get_node_type_feature())
         o.setFlags(o.flags() & ~QtCore.Qt.ItemIsDropEnabled)
         self.style_feature_node(o)
 
     def set_comment_node(self, o):
         """ """
-        setattr(o, "capa_type", CapaExplorerRulgenEditor.get_node_type_comment())
+        setattr(o, "capa_type", CapaExplorerRulegenEditor.get_node_type_comment())
         o.setFlags(o.flags() & ~QtCore.Qt.ItemIsDropEnabled)
 
         self.style_comment_node(o)
@@ -599,7 +613,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         o = QtWidgets.QTreeWidgetItem(parent)
         self.set_expression_node(o)
-        for (i, v) in enumerate(values):
+        for i, v in enumerate(values):
             o.setText(i, v)
         return o
 
@@ -607,7 +621,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         o = QtWidgets.QTreeWidgetItem(parent)
         self.set_feature_node(o)
-        for (i, v) in enumerate(values):
+        for i, v in enumerate(values):
             o.setText(i, v)
         return o
 
@@ -615,7 +629,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         o = QtWidgets.QTreeWidgetItem(parent)
         self.set_comment_node(o)
-        for (i, v) in enumerate(values):
+        for i, v in enumerate(values):
             o.setText(i, v)
         return o
 
@@ -634,23 +648,23 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         counted = list(zip(Counter(features).keys(), Counter(features).values()))
 
         # single features
-        for (k, v) in filter(lambda t: t[1] == 1, counted):
+        for k, _ in filter(lambda t: t[1] == 1, counted):
             if isinstance(k, (capa.features.common.String,)):
-                value = '"%s"' % capa.features.common.escape_string(k.get_value_str())
+                value = f'"{capa.features.common.escape_string(k.get_value_str())}"'
             else:
                 value = k.get_value_str()
-            self.new_feature_node(top_node, ("- %s: %s" % (k.name.lower(), value), ""))
+            self.new_feature_node(top_node, (f"- {k.name.lower()}: {value}", ""))
 
         # n > 1 features
-        for (k, v) in filter(lambda t: t[1] > 1, counted):
+        for k, v in filter(lambda t: t[1] > 1, counted):
             if k.value:
                 if isinstance(k, (capa.features.common.String,)):
-                    value = '"%s"' % capa.features.common.escape_string(k.get_value_str())
+                    value = f'"{capa.features.common.escape_string(k.get_value_str())}"'
                 else:
                     value = k.get_value_str()
-                display = "- count(%s(%s)): %d" % (k.name.lower(), value, v)
+                display = f"- count({k.name.lower()}({value})): {v}"
             else:
-                display = "- count(%s): %d" % (k.name.lower(), v)
+                display = f"- count({k.name.lower()}): {v}"
             self.new_feature_node(top_node, (display, ""))
 
         self.update_preview()
@@ -670,10 +684,12 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
 
             # we don't add a new node for description; either set description column of parent's last child
             # or the parent itself
-            if parent.childCount():
-                parent.child(parent.childCount() - 1).setText(1, feature.lstrip("description:").lstrip())
-            else:
-                parent.setText(1, feature.lstrip("description:").lstrip())
+            if feature.startswith("description:"):
+                description = feature[len("description:") :].lstrip()
+                if parent.childCount():
+                    parent.child(parent.childCount() - 1).setText(1, description)
+                else:
+                    parent.setText(1, description)
             return None
         elif feature.startswith("- description:"):
             if not parent:
@@ -681,22 +697,23 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
                 return None
 
             # we don't add a new node for description; set the description column of the parent instead
-            parent.setText(1, feature.lstrip("- description:").lstrip())
+            description = feature[len("- description:") :].lstrip()
+            parent.setText(1, description)
             return None
 
         node = QtWidgets.QTreeWidgetItem(parent)
 
         # set node text to data parsed from feature
-        for (idx, text) in enumerate((feature, comment, description)):
+        for idx, text in enumerate((feature, comment, description)):
             node.setText(idx, text)
 
         # we need to set our own type so we can control the GUI accordingly
-        if feature.startswith(("- and:", "- or:", "- not:", "- basic block:", "- optional:")):
-            setattr(node, "capa_type", CapaExplorerRulgenEditor.get_node_type_expression())
+        if feature.startswith(("- and:", "- or:", "- not:", "- basic block:", "- instruction:", "- optional:")):
+            setattr(node, "capa_type", CapaExplorerRulegenEditor.get_node_type_expression())
         elif feature.startswith("#"):
-            setattr(node, "capa_type", CapaExplorerRulgenEditor.get_node_type_comment())
+            setattr(node, "capa_type", CapaExplorerRulegenEditor.get_node_type_comment())
         else:
-            setattr(node, "capa_type", CapaExplorerRulgenEditor.get_node_type_feature())
+            setattr(node, "capa_type", CapaExplorerRulegenEditor.get_node_type_feature())
 
         # format the node based on its type
         (self.set_expression_node, self.set_feature_node, self.set_comment_node)[node.capa_type](node)
@@ -758,7 +775,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         """ """
         for feature in filter(
             lambda o: o.capa_type
-            in (CapaExplorerRulgenEditor.get_node_type_feature(), CapaExplorerRulgenEditor.get_node_type_comment()),
+            in (CapaExplorerRulegenEditor.get_node_type_feature(), CapaExplorerRulegenEditor.get_node_type_comment()),
             tuple(iterate_tree(self)),
         ):
             if feature in ignore:
@@ -770,7 +787,7 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
     def get_expressions(self, selected=False, ignore=()):
         """ """
         for expression in filter(
-            lambda o: o.capa_type == CapaExplorerRulgenEditor.get_node_type_expression(), tuple(iterate_tree(self))
+            lambda o: o.capa_type == CapaExplorerRulegenEditor.get_node_type_expression(), tuple(iterate_tree(self))
         ):
             if expression in ignore:
                 continue
@@ -782,12 +799,12 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
 class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
     def __init__(self, editor, parent=None):
         """ """
-        super(CapaExplorerRulegenFeatures, self).__init__(parent)
+        super().__init__(parent)
 
         self.parent_items = {}
         self.editor = editor
 
-        self.setHeaderLabels(["Feature", "Virtual Address"])
+        self.setHeaderLabels(["Feature", "Address"])
         self.setStyleSheet("QTreeView::item {padding-right: 15 px;padding-bottom: 2 px;}")
 
         # configure view columns to auto-resize
@@ -862,7 +879,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
             if isinstance(self.selectedItems()[0].data(0, 0x100), capa.features.common.Bytes):
                 actions.append(("Add n bytes...", (), self.slot_add_n_bytes_feature))
         else:
-            action_add_features_fmt = "Add %d features" % selected_items_count
+            action_add_features_fmt = f"Add {selected_items_count} features"
 
         actions.append((action_add_features_fmt, (), self.slot_add_selected_features))
 
@@ -979,7 +996,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
         o = QtWidgets.QTreeWidgetItem(parent)
 
         self.set_parent_node(o)
-        for (i, v) in enumerate(data):
+        for i, v in enumerate(data):
             o.setText(i, v)
         if feature:
             o.setData(0, 0x100, feature)
@@ -991,14 +1008,14 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
         o = QtWidgets.QTreeWidgetItem(parent)
 
         self.set_leaf_node(o)
-        for (i, v) in enumerate(data):
+        for i, v in enumerate(data):
             o.setText(i, v)
         if feature:
             o.setData(0, 0x100, feature)
 
         return o
 
-    def load_features(self, file_features, func_features={}):
+    def load_features(self, file_features, func_features: Optional[Dict] = None):
         """ """
         self.parse_features_for_tree(self.new_parent_node(self, ("File Scope",)), file_features)
         if func_features:
@@ -1010,17 +1027,20 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
         self.parent_items = {}
 
         def format_address(e):
-            return "%X" % e if e else ""
+            if isinstance(e, AbsoluteVirtualAddress):
+                return f"{hex(int(e))}"
+            else:
+                return ""
 
         def format_feature(feature):
             """ """
             name = feature.name.lower()
             value = feature.get_value_str()
             if isinstance(feature, (capa.features.common.String,)):
-                value = '"%s"' % capa.features.common.escape_string(value)
-            return "%s(%s)" % (name, value)
+                value = f'"{capa.features.common.escape_string(value)}"'
+            return f"{name}({value})"
 
-        for (feature, eas) in sorted(features.items(), key=lambda k: sorted(k[1])):
+        for feature, addrs in sorted(features.items(), key=lambda k: sorted(k[1])):
             if isinstance(feature, capa.features.basicblock.BasicBlock):
                 # filter basic blocks for now, we may want to add these back in some time
                 # in the future
@@ -1032,7 +1052,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
 
             # level 1
             if feature not in self.parent_items:
-                if len(eas) > 1:
+                if len(addrs) > 1:
                     self.parent_items[feature] = self.new_parent_node(
                         self.parent_items[type(feature)], (format_feature(feature),), feature=feature
                     )
@@ -1042,18 +1062,18 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
                     )
 
             # level n > 1
-            if len(eas) > 1:
-                for ea in sorted(eas):
+            if len(addrs) > 1:
+                for addr in sorted(addrs):
                     self.new_leaf_node(
-                        self.parent_items[feature], (format_feature(feature), format_address(ea)), feature=feature
+                        self.parent_items[feature], (format_feature(feature), format_address(addr)), feature=feature
                     )
             else:
-                if eas:
-                    ea = eas.pop()
+                if addrs:
+                    addr = addrs.pop()
                 else:
                     # some features may not have an address e.g. "format"
-                    ea = ""
-                for (i, v) in enumerate((format_feature(feature), format_address(ea))):
+                    addr = _NoAddress()
+                for i, v in enumerate((format_feature(feature), format_address(addr))):
                     self.parent_items[feature].setText(i, v)
                 self.parent_items[feature].setData(0, 0x100, feature)
 
@@ -1069,7 +1089,7 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
 
     def __init__(self, model, parent=None):
         """initialize view"""
-        super(CapaExplorerQtreeView, self).__init__(parent)
+        super().__init__(parent)
 
         self.setModel(model)
 
@@ -1204,8 +1224,7 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
             yield self.new_action(*action)
 
         # add default actions
-        for action in self.load_default_context_menu_actions(data):
-            yield action
+        yield from self.load_default_context_menu_actions(data)
 
     def load_default_context_menu(self, pos, item, model_index):
         """create default custom context menu

capa/optimizer.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import logging
 
 import capa.engine as ceng
@@ -22,7 +29,7 @@ def get_node_cost(node):
         # substring and regex features require a full scan of each string
         # which we anticipate is more expensive then a hash lookup feature (e.g. mnemonic or count).
         #
-        # TODO: compute the average cost of these feature relative to hash feature
+        # fun research: compute the average cost of these feature relative to hash feature
         # and adjust the factor accordingly.
         return 2
 
@@ -47,7 +54,7 @@ def optimize_statement(statement):
 
     if isinstance(statement, (ceng.And, ceng.Or, ceng.Some)):
         # has .children
-        statement.children = sorted(statement.children, key=lambda n: get_node_cost(n))
+        statement.children = sorted(statement.children, key=get_node_cost)
         return
     elif isinstance(statement, (ceng.Not, ceng.Range)):
         # has .child

capa/perf.py

@@ -1,8 +1,15 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import typing
 import collections
-from typing import Dict
 
 # this structure is unstable and may change before the next major release.
-counters: Dict[str, int] = collections.Counter()
+counters: typing.Counter[str] = collections.Counter()
 
 
 def reset():

capa/render/default.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -11,7 +11,8 @@ import collections
 import tabulate
 
 import capa.render.utils as rutils
-import capa.render.result_document
+import capa.render.result_document as rd
+import capa.features.freeze.features as frzf
 from capa.rules import RuleSet
 from capa.engine import MatchResults
 from capa.render.utils import StringIO
@@ -27,50 +28,49 @@ def width(s: str, character_count: int) -> str:
         return s
 
 
-def render_meta(doc, ostream: StringIO):
+def render_meta(doc: rd.ResultDocument, ostream: StringIO):
     rows = [
-        (width("md5", 22), width(doc["meta"]["sample"]["md5"], 82)),
-        ("sha1", doc["meta"]["sample"]["sha1"]),
-        ("sha256", doc["meta"]["sample"]["sha256"]),
-        ("os", doc["meta"]["analysis"]["os"]),
-        ("format", doc["meta"]["analysis"]["format"]),
-        ("arch", doc["meta"]["analysis"]["arch"]),
-        ("path", doc["meta"]["sample"]["path"]),
+        (width("md5", 22), width(doc.meta.sample.md5, 82)),
+        ("sha1", doc.meta.sample.sha1),
+        ("sha256", doc.meta.sample.sha256),
+        ("os", doc.meta.analysis.os),
+        ("format", doc.meta.analysis.format),
+        ("arch", doc.meta.analysis.arch),
+        ("path", doc.meta.sample.path),
     ]
 
-    ostream.write(tabulate.tabulate(rows, tablefmt="psql"))
+    ostream.write(tabulate.tabulate(rows, tablefmt="mixed_outline"))
     ostream.write("\n")
 
 
-def find_subrule_matches(doc):
+def find_subrule_matches(doc: rd.ResultDocument):
     """
     collect the rule names that have been matched as a subrule match.
     this way we can avoid displaying entries for things that are too specific.
     """
-    matches = set([])
+    matches = set()
 
-    def rec(node):
-        if not node["success"]:
+    def rec(match: rd.Match):
+        if not match.success:
             # there's probably a bug here for rules that do `not: match: ...`
             # but we don't have any examples of this yet
             return
 
-        elif node["node"]["type"] == "statement":
-            for child in node["children"]:
+        elif isinstance(match.node, rd.StatementNode):
+            for child in match.children:
                 rec(child)
 
-        elif node["node"]["type"] == "feature":
-            if node["node"]["feature"]["type"] == "match":
-                matches.add(node["node"]["feature"]["match"])
+        elif isinstance(match.node, rd.FeatureNode) and isinstance(match.node.feature, frzf.MatchFeature):
+            matches.add(match.node.feature.match)
 
     for rule in rutils.capability_rules(doc):
-        for node in rule["matches"].values():
-            rec(node)
+        for _, match in rule.matches:
+            rec(match)
 
     return matches
 
 
-def render_capabilities(doc, ostream: StringIO):
+def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
     """
     example::
 
@@ -86,29 +86,29 @@ def render_capabilities(doc, ostream: StringIO):
 
     rows = []
     for rule in rutils.capability_rules(doc):
-        if rule["meta"]["name"] in subrule_matches:
+        if rule.meta.name in subrule_matches:
             # rules that are also matched by other rules should not get rendered by default.
             # this cuts down on the amount of output while giving approx the same detail.
             # see #224
             continue
 
-        count = len(rule["matches"])
+        count = len(rule.matches)
         if count == 1:
-            capability = rutils.bold(rule["meta"]["name"])
+            capability = rutils.bold(rule.meta.name)
         else:
-            capability = "%s (%d matches)" % (rutils.bold(rule["meta"]["name"]), count)
-        rows.append((capability, rule["meta"]["namespace"]))
+            capability = f"{rutils.bold(rule.meta.name)} ({count} matches)"
+        rows.append((capability, rule.meta.namespace))
 
     if rows:
         ostream.write(
-            tabulate.tabulate(rows, headers=[width("CAPABILITY", 50), width("NAMESPACE", 50)], tablefmt="psql")
+            tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline")
         )
         ostream.write("\n")
     else:
         ostream.writeln(rutils.bold("no capabilities found"))
 
 
-def render_attack(doc, ostream: StringIO):
+def render_attack(doc: rd.ResultDocument, ostream: StringIO):
     """
     example::
 
@@ -126,20 +126,17 @@ def render_attack(doc, ostream: StringIO):
     """
     tactics = collections.defaultdict(set)
     for rule in rutils.capability_rules(doc):
-        if not rule["meta"].get("att&ck"):
-            continue
-
-        for attack in rule["meta"]["att&ck"]:
-            tactics[attack["tactic"]].add((attack["technique"], attack.get("subtechnique"), attack["id"]))
+        for attack in rule.meta.attack:
+            tactics[attack.tactic].add((attack.technique, attack.subtechnique, attack.id))
 
     rows = []
     for tactic, techniques in sorted(tactics.items()):
         inner_rows = []
-        for (technique, subtechnique, id) in sorted(techniques):
-            if subtechnique is None:
-                inner_rows.append("%s %s" % (rutils.bold(technique), id))
+        for technique, subtechnique, id in sorted(techniques):
+            if not subtechnique:
+                inner_rows.append(f"{rutils.bold(technique)} {id}")
             else:
-                inner_rows.append("%s::%s %s" % (rutils.bold(technique), subtechnique, id))
+                inner_rows.append(f"{rutils.bold(technique)}::{subtechnique} {id}")
         rows.append(
             (
                 rutils.bold(tactic.upper()),
@@ -150,13 +147,13 @@ def render_attack(doc, ostream: StringIO):
     if rows:
         ostream.write(
             tabulate.tabulate(
-                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="psql"
+                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="mixed_grid"
             )
         )
         ostream.write("\n")
 
 
-def render_mbc(doc, ostream: StringIO):
+def render_mbc(doc: rd.ResultDocument, ostream: StringIO):
     """
     example::
 
@@ -172,20 +169,17 @@ def render_mbc(doc, ostream: StringIO):
     """
     objectives = collections.defaultdict(set)
     for rule in rutils.capability_rules(doc):
-        if not rule["meta"].get("mbc"):
-            continue
-
-        for mbc in rule["meta"]["mbc"]:
-            objectives[mbc["objective"]].add((mbc["behavior"], mbc.get("method"), mbc["id"]))
+        for mbc in rule.meta.mbc:
+            objectives[mbc.objective].add((mbc.behavior, mbc.method, mbc.id))
 
     rows = []
     for objective, behaviors in sorted(objectives.items()):
         inner_rows = []
-        for (behavior, method, id) in sorted(behaviors):
-            if method is None:
-                inner_rows.append("%s [%s]" % (rutils.bold(behavior), id))
+        for behavior, method, id in sorted(behaviors):
+            if not method:
+                inner_rows.append(f"{rutils.bold(behavior)} [{id}]")
             else:
-                inner_rows.append("%s::%s [%s]" % (rutils.bold(behavior), method, id))
+                inner_rows.append(f"{rutils.bold(behavior)}::{method} [{id}]")
         rows.append(
             (
                 rutils.bold(objective.upper()),
@@ -195,12 +189,14 @@ def render_mbc(doc, ostream: StringIO):
 
     if rows:
         ostream.write(
-            tabulate.tabulate(rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="psql")
+            tabulate.tabulate(
+                rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="mixed_grid"
+            )
         )
         ostream.write("\n")
 
 
-def render_default(doc):
+def render_default(doc: rd.ResultDocument):
     ostream = rutils.StringIO()
 
     render_meta(doc, ostream)
@@ -215,5 +211,5 @@ def render_default(doc):
 
 
 def render(meta, rules: RuleSet, capabilities: MatchResults) -> str:
-    doc = capa.render.result_document.convert_capabilities_to_result_document(meta, rules, capabilities)
+    doc = rd.ResultDocument.from_capa(meta, rules, capabilities)
     return render_default(doc)

capa/render/json.py

@@ -1,33 +1,14 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import json
-
-import capa.render.result_document
+import capa.render.result_document as rd
 from capa.rules import RuleSet
 from capa.engine import MatchResults
 
 
-class CapaJsonObjectEncoder(json.JSONEncoder):
-    """JSON encoder that emits Python sets as sorted lists"""
-
-    def default(self, obj):
-        if isinstance(obj, (list, dict, int, float, bool, type(None))) or isinstance(obj, str):
-            return json.JSONEncoder.default(self, obj)
-        elif isinstance(obj, set):
-            return list(sorted(obj))
-        else:
-            # probably will TypeError
-            return json.JSONEncoder.default(self, obj)
-
-
 def render(meta, rules: RuleSet, capabilities: MatchResults) -> str:
-    return json.dumps(
-        capa.render.result_document.convert_capabilities_to_result_document(meta, rules, capabilities),
-        cls=CapaJsonObjectEncoder,
-        sort_keys=True,
-    )
+    return rd.ResultDocument.from_capa(meta, rules, capabilities).json(exclude_none=True)

capa/render/proto/__init__.py

@@ -0,0 +1,737 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+"""
+Convert capa results to protobuf format.
+The functionality here is similar to the various *from_capa functions, e.g. ResultDocument.from_capa() or
+feature_from_capa.
+
+For few classes we can rely on the proto json parser (e.g. RuleMetadata).
+
+For most classes (e.g. RuleMatches) conversion is tricky, because we use natively unsupported types (e.g. tuples),
+several classes with unions, and more complex layouts. So, it's more straight forward to convert explicitly vs.
+massaging the data so the protobuf json parser works.
+
+Of note, the 3 in `syntax = "proto3"` has nothing to do with the 2 in capa_pb2.py;
+see details in https://github.com/grpc/grpc/issues/15444#issuecomment-396442980.
+
+First compile the protobuf to generate an API file and a mypy stub file
+$ protoc.exe --python_out=. --mypy_out=. <path_to_proto> (e.g. capa/render/proto/capa.proto)
+
+Alternatively, --pyi_out=. can be used to generate a Python Interface file that supports development
+"""
+import datetime
+from typing import Any, Dict, Union
+
+import google.protobuf.json_format
+
+import capa.rules
+import capa.features.freeze as frz
+import capa.render.proto.capa_pb2 as capa_pb2
+import capa.render.result_document as rd
+import capa.features.freeze.features as frzf
+from capa.helpers import assert_never
+from capa.features.freeze import AddressType
+
+
+def dict_tuple_to_list_values(d: Dict) -> Dict:
+    o = {}
+    for k, v in d.items():
+        if isinstance(v, tuple):
+            o[k] = list(v)
+        else:
+            o[k] = v
+    return o
+
+
+def int_to_pb2(v: int) -> capa_pb2.Integer:
+    if v < -2_147_483_648:
+        raise ValueError(f"value underflow: {v}")
+    if v > 0xFFFFFFFFFFFFFFFF:
+        raise ValueError(f"value overflow: {v}")
+
+    if v < 0:
+        return capa_pb2.Integer(i=v)
+    else:
+        return capa_pb2.Integer(u=v)
+
+
+def number_to_pb2(v: Union[int, float]) -> capa_pb2.Number:
+    if isinstance(v, float):
+        return capa_pb2.Number(f=v)
+    elif isinstance(v, int):
+        i = int_to_pb2(v)
+        if v < 0:
+            return capa_pb2.Number(i=i.i)
+        else:
+            return capa_pb2.Number(u=i.u)
+    else:
+        assert_never(v)
+
+
+def addr_to_pb2(addr: frz.Address) -> capa_pb2.Address:
+    if addr.type is AddressType.ABSOLUTE:
+        assert isinstance(addr.value, int)
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_ABSOLUTE, v=int_to_pb2(addr.value))
+
+    elif addr.type is AddressType.RELATIVE:
+        assert isinstance(addr.value, int)
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_RELATIVE, v=int_to_pb2(addr.value))
+
+    elif addr.type is AddressType.FILE:
+        assert isinstance(addr.value, int)
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_FILE, v=int_to_pb2(addr.value))
+
+    elif addr.type is AddressType.DN_TOKEN:
+        assert isinstance(addr.value, int)
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_DN_TOKEN, v=int_to_pb2(addr.value))
+
+    elif addr.type is AddressType.DN_TOKEN_OFFSET:
+        assert isinstance(addr.value, tuple)
+        token, offset = addr.value
+        assert isinstance(token, int)
+        assert isinstance(offset, int)
+        return capa_pb2.Address(
+            type=capa_pb2.AddressType.ADDRESSTYPE_DN_TOKEN_OFFSET,
+            token_offset=capa_pb2.Token_Offset(token=int_to_pb2(token), offset=offset),
+        )
+
+    elif addr.type is AddressType.NO_ADDRESS:
+        # value == None, so only set type
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_NO_ADDRESS)
+
+    else:
+        assert_never(addr)
+
+
+def scope_to_pb2(scope: capa.rules.Scope) -> capa_pb2.Scope.ValueType:
+    if scope == capa.rules.Scope.FILE:
+        return capa_pb2.Scope.SCOPE_FILE
+    elif scope == capa.rules.Scope.FUNCTION:
+        return capa_pb2.Scope.SCOPE_FUNCTION
+    elif scope == capa.rules.Scope.BASIC_BLOCK:
+        return capa_pb2.Scope.SCOPE_BASIC_BLOCK
+    elif scope == capa.rules.Scope.INSTRUCTION:
+        return capa_pb2.Scope.SCOPE_INSTRUCTION
+    else:
+        assert_never(scope)
+
+
+def metadata_to_pb2(meta: rd.Metadata) -> capa_pb2.Metadata:
+    return capa_pb2.Metadata(
+        timestamp=str(meta.timestamp),
+        version=meta.version,
+        argv=meta.argv,
+        sample=google.protobuf.json_format.ParseDict(meta.sample.dict(), capa_pb2.Sample()),
+        analysis=capa_pb2.Analysis(
+            format=meta.analysis.format,
+            arch=meta.analysis.arch,
+            os=meta.analysis.os,
+            extractor=meta.analysis.extractor,
+            rules=list(meta.analysis.rules),
+            base_address=addr_to_pb2(meta.analysis.base_address),
+            layout=capa_pb2.Layout(
+                functions=[
+                    capa_pb2.FunctionLayout(
+                        address=addr_to_pb2(f.address),
+                        matched_basic_blocks=[
+                            capa_pb2.BasicBlockLayout(address=addr_to_pb2(bb.address)) for bb in f.matched_basic_blocks
+                        ],
+                    )
+                    for f in meta.analysis.layout.functions
+                ]
+            ),
+            feature_counts=capa_pb2.FeatureCounts(
+                file=meta.analysis.feature_counts.file,
+                functions=[
+                    capa_pb2.FunctionFeatureCount(address=addr_to_pb2(f.address), count=f.count)
+                    for f in meta.analysis.feature_counts.functions
+                ],
+            ),
+            library_functions=[
+                capa_pb2.LibraryFunction(address=addr_to_pb2(lf.address), name=lf.name)
+                for lf in meta.analysis.library_functions
+            ],
+        ),
+    )
+
+
+def statement_to_pb2(statement: rd.Statement) -> capa_pb2.StatementNode:
+    if isinstance(statement, rd.RangeStatement):
+        return capa_pb2.StatementNode(
+            range=capa_pb2.RangeStatement(
+                type="range",
+                description=statement.description,
+                min=statement.min,
+                max=statement.max,
+                child=feature_to_pb2(statement.child),
+            ),
+            type="statement",
+        )
+
+    elif isinstance(statement, rd.SomeStatement):
+        return capa_pb2.StatementNode(
+            some=capa_pb2.SomeStatement(type=statement.type, description=statement.description, count=statement.count),
+            type="statement",
+        )
+
+    elif isinstance(statement, rd.SubscopeStatement):
+        return capa_pb2.StatementNode(
+            subscope=capa_pb2.SubscopeStatement(
+                type=statement.type,
+                description=statement.description,
+                scope=scope_to_pb2(statement.scope),
+            ),
+            type="statement",
+        )
+
+    elif isinstance(statement, rd.CompoundStatement):
+        return capa_pb2.StatementNode(
+            compound=capa_pb2.CompoundStatement(type=statement.type, description=statement.description),
+            type="statement",
+        )
+
+    else:
+        assert_never(statement)
+
+
+def feature_to_pb2(f: frzf.Feature) -> capa_pb2.FeatureNode:
+    if isinstance(f, frzf.OSFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", os=capa_pb2.OSFeature(type=f.type, os=f.os, description=f.description)
+        )
+
+    elif isinstance(f, frzf.ArchFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", arch=capa_pb2.ArchFeature(type=f.type, arch=f.arch, description=f.description)
+        )
+
+    elif isinstance(f, frzf.FormatFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", format=capa_pb2.FormatFeature(type=f.type, format=f.format, description=f.description)
+        )
+
+    elif isinstance(f, frzf.MatchFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            match=capa_pb2.MatchFeature(
+                type=f.type,
+                match=f.match,
+                description=f.description,
+            ),
+        )
+
+    elif isinstance(f, frzf.CharacteristicFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            characteristic=capa_pb2.CharacteristicFeature(
+                type=f.type, characteristic=f.characteristic, description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.ExportFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", export=capa_pb2.ExportFeature(type=f.type, export=f.export, description=f.description)
+        )
+
+    elif isinstance(f, frzf.ImportFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", import_=capa_pb2.ImportFeature(type=f.type, import_=f.import_, description=f.description)
+        )
+
+    elif isinstance(f, frzf.SectionFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", section=capa_pb2.SectionFeature(type=f.type, section=f.section, description=f.description)
+        )
+
+    elif isinstance(f, frzf.FunctionNameFeature):
+        return capa_pb2.FeatureNode(
+            type="function name",
+            function_name=capa_pb2.FunctionNameFeature(
+                type=f.type, function_name=f.function_name, description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.SubstringFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            substring=capa_pb2.SubstringFeature(type=f.type, substring=f.substring, description=f.description),
+        )
+
+    elif isinstance(f, frzf.RegexFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", regex=capa_pb2.RegexFeature(type=f.type, regex=f.regex, description=f.description)
+        )
+
+    elif isinstance(f, frzf.StringFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            string=capa_pb2.StringFeature(
+                type=f.type,
+                string=f.string,
+                description=f.description,
+            ),
+        )
+
+    elif isinstance(f, frzf.ClassFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", class_=capa_pb2.ClassFeature(type=f.type, class_=f.class_, description=f.description)
+        )
+
+    elif isinstance(f, frzf.NamespaceFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            namespace=capa_pb2.NamespaceFeature(type=f.type, namespace=f.namespace, description=f.description),
+        )
+
+    elif isinstance(f, frzf.APIFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", api=capa_pb2.APIFeature(type=f.type, api=f.api, description=f.description)
+        )
+
+    elif isinstance(f, frzf.PropertyFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            property_=capa_pb2.PropertyFeature(
+                type=f.type, access=f.access, property_=f.property, description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.NumberFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            number=capa_pb2.NumberFeature(type=f.type, number=number_to_pb2(f.number), description=f.description),
+        )
+
+    elif isinstance(f, frzf.BytesFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", bytes=capa_pb2.BytesFeature(type=f.type, bytes=f.bytes, description=f.description)
+        )
+
+    elif isinstance(f, frzf.OffsetFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            offset=capa_pb2.OffsetFeature(type=f.type, offset=int_to_pb2(f.offset), description=f.description),
+        )
+
+    elif isinstance(f, frzf.MnemonicFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            mnemonic=capa_pb2.MnemonicFeature(type=f.type, mnemonic=f.mnemonic, description=f.description),
+        )
+
+    elif isinstance(f, frzf.OperandNumberFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            operand_number=capa_pb2.OperandNumberFeature(
+                type=f.type, index=f.index, operand_number=int_to_pb2(f.operand_number), description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.OperandOffsetFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            operand_offset=capa_pb2.OperandOffsetFeature(
+                type=f.type, index=f.index, operand_offset=int_to_pb2(f.operand_offset), description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.BasicBlockFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", basic_block=capa_pb2.BasicBlockFeature(type=f.type, description=f.description)
+        )
+
+    else:
+        assert_never(f)
+
+
+def node_to_pb2(node: rd.Node) -> Union[capa_pb2.FeatureNode, capa_pb2.StatementNode]:
+    if isinstance(node, rd.StatementNode):
+        return statement_to_pb2(node.statement)
+
+    elif isinstance(node, rd.FeatureNode):
+        return feature_to_pb2(node.feature)
+
+    else:
+        assert_never(node)
+
+
+def match_to_pb2(match: rd.Match) -> capa_pb2.Match:
+    node = node_to_pb2(match.node)
+    children = list(map(match_to_pb2, match.children))
+    locations = list(map(addr_to_pb2, match.locations))
+
+    if isinstance(node, capa_pb2.StatementNode):
+        return capa_pb2.Match(
+            success=match.success,
+            statement=node,
+            children=children,
+            locations=locations,
+            captures={},
+        )
+
+    elif isinstance(node, capa_pb2.FeatureNode):
+        return capa_pb2.Match(
+            success=match.success,
+            feature=node,
+            children=children,
+            locations=locations,
+            captures={
+                capture: capa_pb2.Addresses(address=list(map(addr_to_pb2, locs)))
+                for capture, locs in match.captures.items()
+            },
+        )
+
+    else:
+        assert_never(match)
+
+
+def rule_metadata_to_pb2(rule_metadata: rd.RuleMetadata) -> capa_pb2.RuleMetadata:
+    # after manual type conversions to the RuleMetadata, we can rely on the protobuf json parser
+    # conversions include tuple -> list and rd.Enum -> proto.enum
+    meta = dict_tuple_to_list_values(rule_metadata.dict())
+    meta["scope"] = scope_to_pb2(meta["scope"])
+    meta["attack"] = list(map(dict_tuple_to_list_values, meta.get("attack", [])))
+    meta["mbc"] = list(map(dict_tuple_to_list_values, meta.get("mbc", [])))
+
+    return google.protobuf.json_format.ParseDict(meta, capa_pb2.RuleMetadata())
+
+
+def doc_to_pb2(doc: rd.ResultDocument) -> capa_pb2.ResultDocument:
+    rule_matches: Dict[str, capa_pb2.RuleMatches] = {}
+    for rule_name, matches in doc.rules.items():
+        m = capa_pb2.RuleMatches(
+            meta=rule_metadata_to_pb2(matches.meta),
+            source=matches.source,
+            matches=[
+                capa_pb2.Pair_Address_Match(address=addr_to_pb2(addr), match=match_to_pb2(match))
+                for addr, match in matches.matches
+            ],
+        )
+        rule_matches[rule_name] = m
+
+    r = capa_pb2.ResultDocument(meta=metadata_to_pb2(doc.meta), rules=rule_matches)
+
+    return r
+
+
+def int_from_pb2(v: capa_pb2.Integer) -> int:
+    type = v.WhichOneof("value")
+    if type == "u":
+        return v.u
+    elif type == "i":
+        return v.i
+    else:
+        assert_never(type)
+
+
+def number_from_pb2(v: capa_pb2.Number) -> Union[int, float]:
+    type = v.WhichOneof("value")
+    if type == "u":
+        return v.u
+    elif type == "i":
+        return v.i
+    elif type == "f":
+        return v.f
+    else:
+        assert_never(type)
+
+
+def addr_from_pb2(addr: capa_pb2.Address) -> frz.Address:
+    if addr.type == capa_pb2.AddressType.ADDRESSTYPE_ABSOLUTE:
+        return frz.Address(type=frz.AddressType.ABSOLUTE, value=int_from_pb2(addr.v))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_RELATIVE:
+        return frz.Address(type=frz.AddressType.RELATIVE, value=int_from_pb2(addr.v))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_FILE:
+        return frz.Address(type=frz.AddressType.FILE, value=int_from_pb2(addr.v))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_DN_TOKEN:
+        return frz.Address(type=frz.AddressType.DN_TOKEN, value=int_from_pb2(addr.v))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_DN_TOKEN_OFFSET:
+        token = int_from_pb2(addr.token_offset.token)
+        offset = addr.token_offset.offset
+        return frz.Address(type=frz.AddressType.DN_TOKEN_OFFSET, value=(token, offset))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_NO_ADDRESS:
+        return frz.Address(type=frz.AddressType.NO_ADDRESS, value=None)
+
+    else:
+        assert_never(addr)
+
+
+def scope_from_pb2(scope: capa_pb2.Scope.ValueType) -> capa.rules.Scope:
+    if scope == capa_pb2.Scope.SCOPE_FILE:
+        return capa.rules.Scope.FILE
+    elif scope == capa_pb2.Scope.SCOPE_FUNCTION:
+        return capa.rules.Scope.FUNCTION
+    elif scope == capa_pb2.Scope.SCOPE_BASIC_BLOCK:
+        return capa.rules.Scope.BASIC_BLOCK
+    elif scope == capa_pb2.Scope.SCOPE_INSTRUCTION:
+        return capa.rules.Scope.INSTRUCTION
+    else:
+        assert_never(scope)
+
+
+def metadata_from_pb2(meta: capa_pb2.Metadata) -> rd.Metadata:
+    return rd.Metadata(
+        timestamp=datetime.datetime.fromisoformat(meta.timestamp),
+        version=meta.version,
+        argv=tuple(meta.argv) if meta.argv else None,
+        sample=rd.Sample(
+            md5=meta.sample.md5,
+            sha1=meta.sample.sha1,
+            sha256=meta.sample.sha256,
+            path=meta.sample.path,
+        ),
+        analysis=rd.Analysis(
+            format=meta.analysis.format,
+            arch=meta.analysis.arch,
+            os=meta.analysis.os,
+            extractor=meta.analysis.extractor,
+            rules=tuple(meta.analysis.rules),
+            base_address=addr_from_pb2(meta.analysis.base_address),
+            layout=rd.Layout(
+                functions=tuple(
+                    [
+                        rd.FunctionLayout(
+                            address=addr_from_pb2(f.address),
+                            matched_basic_blocks=tuple(
+                                [
+                                    rd.BasicBlockLayout(address=addr_from_pb2(bb.address))
+                                    for bb in f.matched_basic_blocks
+                                ]
+                            ),
+                        )
+                        for f in meta.analysis.layout.functions
+                    ]
+                )
+            ),
+            feature_counts=rd.FeatureCounts(
+                file=meta.analysis.feature_counts.file,
+                functions=tuple(
+                    [
+                        rd.FunctionFeatureCount(address=addr_from_pb2(f.address), count=f.count)
+                        for f in meta.analysis.feature_counts.functions
+                    ]
+                ),
+            ),
+            library_functions=tuple(
+                [
+                    rd.LibraryFunction(address=addr_from_pb2(lf.address), name=lf.name)
+                    for lf in meta.analysis.library_functions
+                ]
+            ),
+        ),
+    )
+
+
+def statement_from_pb2(statement: capa_pb2.StatementNode) -> rd.Statement:
+    type_ = statement.WhichOneof("statement")
+
+    if type_ == "range":
+        return rd.RangeStatement(
+            min=statement.range.min,
+            max=statement.range.max,
+            child=feature_from_pb2(statement.range.child),
+            description=statement.range.description or None,
+        )
+    elif type_ == "some":
+        return rd.SomeStatement(
+            count=statement.some.count,
+            description=statement.some.description or None,
+        )
+    elif type_ == "subscope":
+        return rd.SubscopeStatement(
+            scope=scope_from_pb2(statement.subscope.scope),
+            description=statement.subscope.description or None,
+        )
+    elif type_ == "compound":
+        return rd.CompoundStatement(
+            type=statement.compound.type,
+            description=statement.compound.description or None,
+        )
+    else:
+        assert_never(type_)
+
+
+def feature_from_pb2(f: capa_pb2.FeatureNode) -> frzf.Feature:
+    type_ = f.WhichOneof("feature")
+
+    # mypy gets angry below because ff may have a different type in each branch,
+    # even though we don't use ff outside each branch.
+    # so we just let mypy know that ff might be any type to silence that warning.
+    # upstream issue: https://github.com/python/mypy/issues/6233
+    ff: Any
+
+    if type_ == "os":
+        ff = f.os
+        return frzf.OSFeature(os=ff.os, description=ff.description or None)
+    elif type_ == "arch":
+        ff = f.arch
+        return frzf.ArchFeature(arch=ff.arch, description=ff.description or None)
+    elif type_ == "format":
+        ff = f.format
+        return frzf.FormatFeature(format=ff.format, description=ff.description or None)
+    elif type_ == "match":
+        ff = f.match
+        return frzf.MatchFeature(match=ff.match, description=ff.description or None)
+    elif type_ == "characteristic":
+        ff = f.characteristic
+        return frzf.CharacteristicFeature(characteristic=ff.characteristic, description=ff.description or None)
+    elif type_ == "export":
+        ff = f.export
+        return frzf.ExportFeature(export=ff.export, description=ff.description or None)
+    elif type_ == "import_":
+        ff = f.import_
+        return frzf.ImportFeature(import_=ff.import_, description=ff.description or None)  # type: ignore
+        # Mypy is unable to recognize `import_` as an argument
+    elif type_ == "section":
+        ff = f.section
+        return frzf.SectionFeature(section=ff.section, description=ff.description or None)
+    elif type_ == "function_name":
+        ff = f.function_name
+        return frzf.FunctionNameFeature(function_name=ff.function_name, description=ff.description or None)  # type: ignore
+    elif type_ == "substring":
+        ff = f.substring
+        return frzf.SubstringFeature(substring=ff.substring, description=ff.description or None)
+    elif type_ == "regex":
+        ff = f.regex
+        return frzf.RegexFeature(regex=ff.regex, description=ff.description or None)
+    elif type_ == "string":
+        ff = f.string
+        return frzf.StringFeature(string=ff.string, description=ff.description or None)
+    elif type_ == "class_":
+        ff = f.class_
+        return frzf.ClassFeature(class_=ff.class_, description=ff.description or None)  # type: ignore
+        # Mypy is unable to recognize `class_` as an argument due to aliasing
+    elif type_ == "namespace":
+        ff = f.namespace
+        return frzf.NamespaceFeature(namespace=ff.namespace, description=ff.description or None)
+    elif type_ == "api":
+        ff = f.api
+        return frzf.APIFeature(api=ff.api, description=ff.description or None)
+    elif type_ == "property_":
+        ff = f.property_
+        return frzf.PropertyFeature(property=ff.property_, access=ff.access or None, description=ff.description or None)
+    elif type_ == "number":
+        ff = f.number
+        return frzf.NumberFeature(number=number_from_pb2(ff.number), description=ff.description or None)
+    elif type_ == "bytes":
+        ff = f.bytes
+        return frzf.BytesFeature(bytes=ff.bytes, description=ff.description or None)
+    elif type_ == "offset":
+        ff = f.offset
+        return frzf.OffsetFeature(offset=int_from_pb2(ff.offset), description=ff.description or None)
+    elif type_ == "mnemonic":
+        ff = f.mnemonic
+        return frzf.MnemonicFeature(mnemonic=ff.mnemonic, description=ff.description or None)
+    elif type_ == "operand_number":
+        ff = f.operand_number
+        return frzf.OperandNumberFeature(
+            index=ff.index, operand_number=number_from_pb2(ff.operand_number), description=ff.description or None
+        )  # type: ignore
+    elif type_ == "operand_offset":
+        ff = f.operand_offset
+        return frzf.OperandOffsetFeature(
+            index=ff.index, operand_offset=int_from_pb2(ff.operand_offset), description=ff.description or None
+        )  # type: ignore
+        # Mypy is unable to recognize `operand_offset` as an argument due to aliasing
+    elif type_ == "basic_block":
+        ff = f.basic_block
+        return frzf.BasicBlockFeature(description=ff.description or None)
+    else:
+        assert_never(type_)
+
+
+def match_from_pb2(match: capa_pb2.Match) -> rd.Match:
+    children = list(map(match_from_pb2, match.children))
+    locations = list(map(addr_from_pb2, match.locations))
+
+    node_type = match.WhichOneof("node")
+    if node_type == "statement":
+        return rd.Match(
+            success=match.success,
+            node=rd.StatementNode(statement=statement_from_pb2(match.statement)),
+            children=tuple(children),
+            locations=tuple(locations),
+            captures={},
+        )
+    elif node_type == "feature":
+        return rd.Match(
+            success=match.success,
+            node=rd.FeatureNode(feature=feature_from_pb2(match.feature)),
+            children=tuple(children),
+            locations=tuple(locations),
+            captures={capture: tuple(map(addr_from_pb2, locs.address)) for capture, locs in match.captures.items()},
+        )
+    else:
+        assert_never(node_type)
+
+
+def attack_from_pb2(pb: capa_pb2.AttackSpec) -> rd.AttackSpec:
+    return rd.AttackSpec(
+        parts=tuple(pb.parts),
+        tactic=pb.tactic,
+        technique=pb.technique,
+        subtechnique=pb.subtechnique,
+        id=pb.id,
+    )
+
+
+def mbc_from_pb2(pb: capa_pb2.MBCSpec) -> rd.MBCSpec:
+    return rd.MBCSpec(
+        parts=tuple(pb.parts),
+        objective=pb.objective,
+        behavior=pb.behavior,
+        method=pb.method,
+        id=pb.id,
+    )
+
+
+def maec_from_pb2(pb: capa_pb2.MaecMetadata) -> rd.MaecMetadata:
+    return rd.MaecMetadata(
+        analysis_conclusion=pb.analysis_conclusion or None,
+        analysis_conclusion_ov=pb.analysis_conclusion_ov or None,
+        malware_family=pb.malware_family or None,
+        malware_category=pb.malware_category or None,
+        malware_category_ov=pb.malware_category_ov or None,
+    )  # type: ignore
+    # Mypy is unable to recognise arguments due to alias
+
+
+def rule_metadata_from_pb2(pb: capa_pb2.RuleMetadata) -> rd.RuleMetadata:
+    return rd.RuleMetadata(
+        name=pb.name,
+        namespace=pb.namespace or None,
+        authors=tuple(pb.authors),
+        scope=scope_from_pb2(pb.scope),
+        attack=tuple([attack_from_pb2(attack) for attack in pb.attack]),
+        mbc=tuple([mbc_from_pb2(mbc) for mbc in pb.mbc]),
+        references=tuple(pb.references),
+        examples=tuple(pb.examples),
+        description=pb.description,
+        lib=pb.lib,
+        is_subscope_rule=pb.is_subscope_rule,
+        maec=maec_from_pb2(pb.maec),
+    )  # type: ignore
+    # Mypy is unable to recognise `attack` and `is_subscope_rule` as arguments due to alias
+
+
+def doc_from_pb2(doc: capa_pb2.ResultDocument) -> rd.ResultDocument:
+    rule_matches: Dict[str, rd.RuleMatches] = {}
+    for rule_name, matches in doc.rules.items():
+        m = rd.RuleMatches(
+            meta=rule_metadata_from_pb2(matches.meta),
+            source=matches.source,
+            matches=tuple([(addr_from_pb2(pair.address), match_from_pb2(pair.match)) for pair in matches.matches]),
+        )
+        rule_matches[rule_name] = m
+
+    return rd.ResultDocument(meta=metadata_from_pb2(doc.meta), rules=rule_matches)