Merge pull request #2370 from mandiant/release/v730

bump to v7.3.0

.devcontainer/Dockerfile

@@ -0,0 +1,21 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3/.devcontainer/base.Dockerfile
+
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+ARG VARIANT="3.10-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+# [Optional] Uncomment this line to install global node packages.
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.devcontainer/devcontainer.json

@@ -0,0 +1,51 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3
+{
+	"name": "Python 3",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": "..",
+		"args": { 
+			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+			// Append -bullseye or -buster to pin to an OS version.
+			// Use -bullseye variants on local on arm64/Apple Silicon.
+			"VARIANT": "3.10",
+			// Options
+			"NODE_VERSION": "none"
+		}
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": { 
+		"python.defaultInterpreterPath": "/usr/local/bin/python",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python",
+		"ms-python.vscode-pylance"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "git submodule update --init && pip3 install --user -e .[dev] && pre-commit install",
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"features": {
+		"git": "latest"
+	}
+}

.github/CONTRIBUTING.md

@@ -31,7 +31,7 @@ This project and everyone participating in it is governed by the [Capa Code of C
 
 ### Capa and its repositories
 
-We host the capa project as three Github repositories:
+We host the capa project as three GitHub repositories:
   - [capa](https://github.com/mandiant/capa)
   - [capa-rules](https://github.com/mandiant/capa-rules)
   - [capa-testfiles](https://github.com/mandiant/capa-testfiles)
@@ -57,7 +57,7 @@ When we make a significant decision in how we maintain the project and what we c
  we will document it in the [capa issues tracker](https://github.com/mandiant/capa/issues).
 This is the best place review our discussions about what/how/why we do things in the project.
 If you have a question, check to see if it is documented there.
-If it is *not* documented there, or you can't find an answer, please open a issue.
+If it is *not* documented there, or you can't find an answer, please open an issue.
 We'll link to existing issues when appropriate to keep discussions in one place.
 
 ## How Can I Contribute?
@@ -159,12 +159,25 @@ The process described here has several goals:
 
 Please follow these steps to have your contribution considered by the maintainers:
 
+0. Sign the [Contributor License Agreement](#contributor-license-agreement)
 1. Follow the [styleguides](#styleguides)
 2. Update the CHANGELOG and add tests and documentation. In case they are not needed, indicate it in [the PR template](pull_request_template.md).
 3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing? </summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
 
 While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
 
+### Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution,
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to <https://cla.developers.google.com/> to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
 ## Styleguides
 
 ### Git Commit Messages

.github/flake8.ini

@@ -0,0 +1,43 @@
+[flake8]
+max-line-length = 120
+
+extend-ignore =
+    # E203: whitespace before ':'  (black does this)
+    E203,
+    # F401: `foo` imported but unused  (prefer ruff)
+    F401,
+    # F811 Redefinition of unused `foo`  (prefer ruff)
+    F811,
+    # E501 line too long  (prefer black)
+    E501,
+    # E701 multiple statements on one line (colon)  (prefer black, see https://github.com/psf/black/issues/4173)
+    E701,
+    # B010 Do not call setattr with a constant attribute value
+    B010,
+    # G200 Logging statement uses exception in arguments
+    G200,
+    # SIM102 Use a single if-statement instead of nested if-statements
+    # doesn't provide a space for commenting or logical separation of conditions
+    SIM102,
+    # SIM114 Use logical or and a single body
+    # makes logic trees too complex
+    SIM114,
+    # SIM117 Use 'with Foo, Bar:' instead of multiple with statements
+    # makes lines too long
+    SIM117
+
+per-file-ignores =
+    # T201 print found.
+    #
+    # scripts are meant to print output
+    scripts/*: T201
+    # capa.exe is meant to print output
+    capa/main.py: T201
+    # IDA tests emit results to output window so need to print
+    tests/test_ida_features.py: T201
+    # utility used to find the Binary Ninja API via invoking python.exe
+    capa/features/extractors/binja/find_binja_api.py: T201
+
+copyright-check = True
+copyright-min-file-size = 1 
+copyright-regexp = Copyright \(C\) \d{4} Mandiant, Inc. All Rights Reserved.

.github/mypy/mypy.ini

@@ -1,8 +1,5 @@
 [mypy]
 
-[mypy-halo.*]
-ignore_missing_imports = True
-
 [mypy-tqdm.*]
 ignore_missing_imports = True
 
@@ -21,9 +18,6 @@ ignore_missing_imports = True
 [mypy-flirt.*]
 ignore_missing_imports = True
 
-[mypy-smda.*]
-ignore_missing_imports = True
-
 [mypy-lief.*]
 ignore_missing_imports = True
 
@@ -45,9 +39,15 @@ ignore_missing_imports = True
 [mypy-idautils.*]
 ignore_missing_imports = True
 
+[mypy-ida_auto.*]
+ignore_missing_imports = True
+
 [mypy-ida_bytes.*]
 ignore_missing_imports = True
 
+[mypy-ida_nalt.*]
+ignore_missing_imports = True
+
 [mypy-ida_kernwin.*]
 ignore_missing_imports = True
 
@@ -60,6 +60,9 @@ ignore_missing_imports = True
 [mypy-ida_loader.*]
 ignore_missing_imports = True
 
+[mypy-ida_segment.*]
+ignore_missing_imports = True
+
 [mypy-PyQt5.*]
 ignore_missing_imports = True
 
@@ -74,3 +77,12 @@ ignore_missing_imports = True
 
 [mypy-elftools.*]
 ignore_missing_imports = True
+
+[mypy-dncil.*]
+ignore_missing_imports = True
+
+[mypy-netnode.*]
+ignore_missing_imports = True
+
+[mypy-ghidra.*]
+ignore_missing_imports = True

.github/pyinstaller/hooks/hook-smda.py

@@ -1,5 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-import PyInstaller.utils.hooks
-
-# ref: https://groups.google.com/g/pyinstaller/c/amWi0-66uZI/m/miPoKfWjBAAJ
-binaries = PyInstaller.utils.hooks.collect_dynamic_libs("capstone")

.github/pyinstaller/hooks/hook-vivisect.py

@@ -24,7 +24,7 @@ excludedimports = [
     "pyqtwebengine",
     # the above are imported by these viv modules.
     # so really, we'd want to exclude these submodules of viv.
-    # but i dont think this works.
+    # but i don't think this works.
     "vqt",
     "vdb.qt",
     "envi.qt",
@@ -38,39 +38,36 @@ hiddenimports = [
     "vivisect",
     "vivisect.analysis",
     "vivisect.analysis.amd64",
-    "vivisect.analysis.amd64",
     "vivisect.analysis.amd64.emulation",
     "vivisect.analysis.amd64.golang",
     "vivisect.analysis.crypto",
-    "vivisect.analysis.crypto",
     "vivisect.analysis.crypto.constants",
     "vivisect.analysis.elf",
     "vivisect.analysis.elf.elfplt",
     "vivisect.analysis.elf.elfplt_late",
     "vivisect.analysis.elf.libc_start_main",
     "vivisect.analysis.generic",
-    "vivisect.analysis.generic",
     "vivisect.analysis.generic.codeblocks",
     "vivisect.analysis.generic.emucode",
     "vivisect.analysis.generic.entrypoints",
     "vivisect.analysis.generic.funcentries",
     "vivisect.analysis.generic.impapi",
+    "vivisect.analysis.generic.linker",
     "vivisect.analysis.generic.mkpointers",
+    "vivisect.analysis.generic.noret",
     "vivisect.analysis.generic.pointers",
     "vivisect.analysis.generic.pointertables",
     "vivisect.analysis.generic.relocations",
     "vivisect.analysis.generic.strconst",
     "vivisect.analysis.generic.switchcase",
+    "vivisect.analysis.generic.symswitchcase",
     "vivisect.analysis.generic.thunks",
-    "vivisect.analysis.generic.noret",
-    "vivisect.analysis.i386",
     "vivisect.analysis.i386",
     "vivisect.analysis.i386.calling",
     "vivisect.analysis.i386.golang",
     "vivisect.analysis.i386.importcalls",
     "vivisect.analysis.i386.instrhook",
-    "vivisect.analysis.i386.thunk_bx",
-    "vivisect.analysis.ms",
+    "vivisect.analysis.i386.thunk_reg",
     "vivisect.analysis.ms",
     "vivisect.analysis.ms.hotpatch",
     "vivisect.analysis.ms.localhints",
@@ -81,8 +78,40 @@ hiddenimports = [
     "vivisect.impapi.posix.amd64",
     "vivisect.impapi.posix.i386",
     "vivisect.impapi.windows",
+    "vivisect.impapi.windows.advapi_32",
+    "vivisect.impapi.windows.advapi_64",
     "vivisect.impapi.windows.amd64",
+    "vivisect.impapi.windows.gdi_32",
+    "vivisect.impapi.windows.gdi_64",
     "vivisect.impapi.windows.i386",
+    "vivisect.impapi.windows.kernel_32",
+    "vivisect.impapi.windows.kernel_64",
+    "vivisect.impapi.windows.msvcr100_32",
+    "vivisect.impapi.windows.msvcr100_64",
+    "vivisect.impapi.windows.msvcr110_32",
+    "vivisect.impapi.windows.msvcr110_64",
+    "vivisect.impapi.windows.msvcr120_32",
+    "vivisect.impapi.windows.msvcr120_64",
+    "vivisect.impapi.windows.msvcr71_32",
+    "vivisect.impapi.windows.msvcr80_32",
+    "vivisect.impapi.windows.msvcr80_64",
+    "vivisect.impapi.windows.msvcr90_32",
+    "vivisect.impapi.windows.msvcr90_64",
+    "vivisect.impapi.windows.msvcrt_32",
+    "vivisect.impapi.windows.msvcrt_64",
+    "vivisect.impapi.windows.ntdll_32",
+    "vivisect.impapi.windows.ntdll_64",
+    "vivisect.impapi.windows.ole_32",
+    "vivisect.impapi.windows.ole_64",
+    "vivisect.impapi.windows.rpcrt4_32",
+    "vivisect.impapi.windows.rpcrt4_64",
+    "vivisect.impapi.windows.shell_32",
+    "vivisect.impapi.windows.shell_64",
+    "vivisect.impapi.windows.user_32",
+    "vivisect.impapi.windows.user_64",
+    "vivisect.impapi.windows.ws2plus_32",
+    "vivisect.impapi.windows.ws2plus_64",
+    "vivisect.impapi.winkern",
     "vivisect.impapi.winkern.i386",
     "vivisect.impapi.winkern.amd64",
     "vivisect.parsers.blob",

.github/pyinstaller/pyinstaller.spec

@@ -1,56 +1,49 @@
 # -*- mode: python -*-
 # Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-import os.path
-import subprocess
+import sys
 
 import wcwidth
+import capa.rules.cache
 
+from pathlib import Path
 
-# when invoking pyinstaller from the project root,
-# this gets run from the project root.
-with open('./capa/version.py', 'wb') as f:
-    # git output will look like:
-    #
-    #     tags/v1.0.0-0-g3af38dc
-    #         ------- tag
-    #                 - commits since
-    #                   g------- git hash fragment
-    version = (subprocess.check_output(["git", "describe", "--always", "--tags", "--long"])
-               .decode("utf-8")
-               .strip()
-               .replace("tags/", ""))
-    f.write(("__version__ = '%s'" % version).encode("utf-8"))
+# SPECPATH is a global variable which points to .spec file path
+capa_dir = Path(SPECPATH).parent.parent
+rules_dir = capa_dir / 'rules'
+cache_dir = capa_dir / 'cache'
+
+if not capa.rules.cache.generate_rule_cache(rules_dir, cache_dir):
+    sys.exit(-1)
 
 a = Analysis(
     # when invoking pyinstaller from the project root,
     # this gets invoked from the directory of the spec file,
     # i.e. ./.github/pyinstaller
-    ['../../capa/main.py'],
-    pathex=['capa'],
+    ["../../capa/main.py"],
+    pathex=["capa"],
     binaries=None,
     datas=[
         # when invoking pyinstaller from the project root,
         # this gets invoked from the directory of the spec file,
         # i.e. ./.github/pyinstaller
-        ('../../rules', 'rules'),
-        ('../../sigs', 'sigs'),
-
+        ("../../rules", "rules"),
+        ("../../sigs", "sigs"),
+        ("../../cache", "cache"),
         # capa.render.default uses tabulate that depends on wcwidth.
         # it seems wcwidth uses a json file `version.json`
         # and this doesn't get picked up by pyinstaller automatically.
         # so we manually embed the wcwidth resources here.
         #
         # ref: https://stackoverflow.com/a/62278462/87207
-        (os.path.dirname(wcwidth.__file__), 'wcwidth')
+        (Path(wcwidth.__file__).parent, "wcwidth"),
     ],
     # when invoking pyinstaller from the project root,
     # this gets run from the project root.
-    hookspath=['.github/pyinstaller/hooks'],
+    hookspath=[".github/pyinstaller/hooks"],
     runtime_hooks=None,
     excludes=[
         # ignore packages that would otherwise be bundled with the .exe.
         # review: build/pyinstaller/xref-pyinstaller.html
-
         # we don't do any GUI stuff, so ignore these modules
         "tkinter",
         "_tkinter",
@@ -60,7 +53,6 @@ a = Analysis(
         # since we don't spawn a notebook, we can safely remove these.
         "IPython",
         "ipywidgets",
-
         # these are pulled in by networkx
         # but we don't need to compute the strongly connected components.
         "numpy",
@@ -68,7 +60,6 @@ a = Analysis(
         "matplotlib",
         "pandas",
         "pytest",
-
         # deps from viv that we don't use.
         # this duplicates the entries in `hook-vivisect`,
         # but works better this way.
@@ -78,35 +69,36 @@ a = Analysis(
         "PyQt5",
         "qt5",
         "pyqtwebengine",
-        "pyasn1"
-    ])
+        "pyasn1",
+        "binaryninja",
+    ],
+)
 
-a.binaries = a.binaries - TOC([
- ('tcl85.dll', None, None),
- ('tk85.dll', None, None),
- ('_tkinter', None, None)])
+a.binaries = a.binaries - TOC([("tcl85.dll", None, None), ("tk85.dll", None, None), ("_tkinter", None, None)])
 
 pyz = PYZ(a.pure, a.zipped_data)
 
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          exclude_binaries=False,
-          name='capa',
-          icon='logo.ico',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    exclude_binaries=False,
+    name="capa",
+    icon="logo.ico",
+    debug=False,
+    strip=False,
+    upx=True,
+    console=True,
+)
 
 # enable the following to debug the contents of the .exe
 #
-#coll = COLLECT(exe,
+# coll = COLLECT(exe,
 #               a.binaries,
 #               a.zipfiles,
 #               a.datas,
 #               strip=None,
 #               upx=True,
-#               name='capa-dat')
+#               name='capa-dat')

.github/ruff.toml

@@ -0,0 +1,43 @@
+# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+lint.select = ["E", "F"]
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+lint.fixable = ["ALL"]
+lint.unfixable = []
+
+# E402 module level import not at top of file
+# E722 do not use bare 'except'
+# E501 line too long
+lint.ignore = ["E402", "E722", "E501"]
+
+line-length = 120
+
+exclude = [
+    # Exclude a variety of commonly ignored directories.
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+    # protobuf generated files
+    "*_pb2.py", 
+    "*_pb2.pyi"
+]

.github/tox.ini

@@ -1,10 +0,0 @@
-[pycodestyle]
-; E402: module level import not at top of file
-; W503: line break before binary operator
-; E231 missing whitespace after ',' (emitted by black)
-; E203 whitespace before ':' (emitted by black)
-ignore = E402,W503,E203,E231
-max-line-length = 160
-statistics = True
-count = True
-exclude = .*

.github/workflows/build.yml

@@ -1,83 +1,104 @@
 name: build
 
 on:
-  push:
-    branches: [master]
+  pull_request:
+    branches: [ master ]
+    paths-ignore:
+      - 'web/**'
+      - 'doc/**'
+      - '**.md'
   release:
     types: [edited, published]
 
+permissions:
+  contents: write
+
 jobs:
   build:
-    name: PyInstaller for ${{ matrix.os }}
+    name: PyInstaller for ${{ matrix.os }} / Py ${{ matrix.python_version }}
     runs-on: ${{ matrix.os }}
     strategy:
+      # set to false for debugging
+      fail-fast: true
       matrix:
+        # using Python 3.8 to support running across multiple operating systems including Windows 7
         include:
-          - os: ubuntu-18.04
+          - os: ubuntu-20.04
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
+            python_version: 3.8
+          - os: ubuntu-20.04
+            artifact_name: capa
+            asset_name: linux-py311
+            python_version: 3.11
           - os: windows-2019
             artifact_name: capa.exe
             asset_name: windows
-          - os: macos-10.15
+            python_version: 3.8
+          - os: macos-12
+            # use older macOS for assumed better portability
             artifact_name: capa
             asset_name: macos
+            python_version: 3.8
     steps:
       - name: Checkout capa
-        uses: actions/checkout@v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           submodules: true
-      # using Python 3.8 to support running across multiple operating systems including Windows 7
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - name: Set up Python ${{ matrix.python_version }}
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
-          python-version: 3.8
-      - if: matrix.os == 'ubuntu-18.04'
+          python-version: ${{ matrix.python_version }}
+      - if: matrix.os == 'ubuntu-20.04'
         run: sudo apt-get install -y libyaml-dev
-      - name: Install PyInstaller
-        run: pip install 'pyinstaller==4.2'
-      - name: Install capa
-        run: pip install -e .
+      - name: Upgrade pip, setuptools
+        run: python -m pip install --upgrade pip setuptools
+      - name: Install capa with build requirements
+        run: |
+          pip install -r requirements.txt
+          pip install -e .[build]
       - name: Build standalone executable
-        run: pyinstaller .github/pyinstaller/pyinstaller.spec
+        run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
       - name: Does it run (PE)?
-        run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_"
+        run: dist/capa -d "tests/data/Practical Malware Analysis Lab 01-01.dll_"
       - name: Does it run (Shellcode)?
-        run: dist/capa "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
+        run: dist/capa -d "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
       - name: Does it run (ELF)?
-        run: dist/capa "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
-      - uses: actions/upload-artifact@v2
+        run: dist/capa -d "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
+      - name: Does it run (CAPE)?
+        run: |
+          7z e "tests/data/dynamic/cape/v2.2/d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json.gz"
+          dist/capa -d "d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json"
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: ${{ matrix.asset_name }}
           path: dist/${{ matrix.artifact_name }}
 
   test_run:
-    # test that binaries run on push to master
-    if: github.event_name == 'push'
-    name: Test run on ${{ matrix.os }}
+    name: Test run on ${{ matrix.os }} / ${{ matrix.asset_name }}
     runs-on: ${{ matrix.os }}
     needs: [build]
     strategy:
       matrix:
         include:
           # OSs not already tested above
-          - os: ubuntu-18.04
+          - os: ubuntu-22.04
             artifact_name: capa
             asset_name: linux
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             artifact_name: capa
-            asset_name: linux
-          - os: windows-2016
+            asset_name: linux-py311
+          - os: windows-2022
             artifact_name: capa.exe
             asset_name: windows
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.asset_name }}
       - name: Set executable flag
-        if: matrix.os != 'windows-2016'
+        if: matrix.os != 'windows-2022'
         run: chmod +x ${{ matrix.artifact_name }}
       - name: Run capa
         run: ./${{ matrix.artifact_name }} -h
@@ -93,13 +114,15 @@ jobs:
         include:
           - asset_name: linux
             artifact_name: capa
+          - asset_name: linux-py311
+            artifact_name: capa
           - asset_name: windows
             artifact_name: capa.exe
           - asset_name: macos
             artifact_name: capa
     steps:
       - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.asset_name }}
       - name: Set executable flag
@@ -109,7 +132,7 @@ jobs:
       - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
         run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
       - name: Upload ${{ env.zip_name }} to GH Release
-        uses: svenstaro/upload-release-action@v2
+        uses: svenstaro/upload-release-action@2728235f7dc9ff598bd86ce3c274b74f802d2208 # v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN}}
           file: ${{ env.zip_name }}

.github/workflows/changelog.yml

@@ -7,6 +7,9 @@ on:
   pull_request_target:
     types: [opened, edited, synchronize]
 
+permissions:
+  pull-requests: write
+
 jobs:
   check_changelog:
     # no need to check for dependency updates via dependabot
@@ -17,7 +20,7 @@ jobs:
     steps:
     - name: Get changed files
       id: files
-      uses: Ana06/get-changed-files@v1.2
+      uses: Ana06/get-changed-files@25f79e676e7ea1868813e21465014798211fad8c # v2.3.0
     - name: check changelog updated
       id: changelog_updated
       env:
@@ -27,14 +30,14 @@ jobs:
         echo $FILES | grep -qF 'CHANGELOG.md' || echo $PR_BODY | grep -qiF "$NO_CHANGELOG"
     - name: Reject pull request if no CHANGELOG update
       if: ${{ always() && steps.changelog_updated.outcome == 'failure' }}
-      uses: Ana06/automatic-pull-request-review@v0.1.0
+      uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: REQUEST_CHANGES
         body: "Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning to the `master (unreleased)` section of CHANGELOG.md. If no CHANGELOG update is needed add the following to the PR description: `${{ env.NO_CHANGELOG }}`"
         allow_duplicate: false
     - name: Dismiss previous review if CHANGELOG update
-      uses: Ana06/automatic-pull-request-review@v0.1.0
+      uses: Ana06/automatic-pull-request-review@76aaf9b15b116a54e1da7a28a46f91fe089600bf # v0.2.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         event: DISMISS

.github/workflows/pip-audit.yml

@@ -0,0 +1,21 @@
+name: PIP audit
+
+on:
+  schedule:
+  - cron: '0 8 * * 1'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - uses: pypa/gh-action-pip-audit@v1.0.8
+        with:
+          inputs: .

.github/workflows/publish.yml

@@ -1,30 +1,42 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
+# use PyPI trusted publishing, as described here: 
+# https://blog.trailofbits.com/2023/05/23/trusted-publishing-a-new-benchmark-for-packaging-security/
 name: publish to pypi
 
 on:
   release:
     types: [published]
 
+permissions:
+  contents: write
+
 jobs:
-  deploy:
-    runs-on: ubuntu-20.04
+  pypi-publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+    permissions:
+      id-token: write
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
         with:
-          python-version: '3.6'
+          python-version: '3.8'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+          pip install -r requirements.txt
+          pip install -e .[build]
+      - name: build package
         run: |
-          python setup.py sdist bdist_wheel
-          twine upload --skip-existing dist/*
-
+          python -m build
+      - name: upload package artifacts
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          path: dist/*
+      - name: publish package
+        uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0  # release/v1
+        with:
+          skip-existing: true
+          verbose: true
+          print-hash: true

.github/workflows/scorecard.yml

@@ -0,0 +1,72 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '43 4 * * 3'
+  push:
+    branches: [ "master" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11  # v4.1.1
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@8a470fddafa5cbb6266ee11b37ef4d8aae19c571  # v3.24.6
+        with:
+          sarif_file: results.sarif

.github/workflows/tag.yml

@@ -4,13 +4,15 @@ on:
   release:
     types: [published]
 
+permissions: read-all
+
 jobs:
   tag:
     name: Tag capa rules
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa-rules
-      uses: actions/checkout@v2
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         repository: mandiant/capa-rules
         token: ${{ secrets.CAPA_TOKEN }}
@@ -21,8 +23,9 @@ jobs:
         git config user.name 'Capa Bot'
         name=${{ github.event.release.tag_name }}
         git tag $name -m "https://github.com/mandiant/capa/releases/$name"
+        # TODO update branch name-major=${name%%.*}
     - name: Push tag to capa-rules
-      uses: ad-m/github-push-action@master
+      uses: ad-m/github-push-action@d91a481090679876dfc4178fef17f286781251df # v0.8.0
       with:
         repository: mandiant/capa-rules
         github_token: ${{ secrets.CAPA_TOKEN }}

.github/workflows/tests.yml

@@ -1,10 +1,24 @@
 name: CI
 
+# tests.yml workflow will run for all changes except:
+# any file or directory under web/ or doc/
+# any Markdown (.md) file anywhere in the repository
+
 on:
   push:
     branches: [ master ]
+    paths-ignore:
+      - 'web/**'
+      - 'doc/**'
+      - '**.md'
   pull_request:
     branches: [ master ]
+    paths-ignore:
+      - 'web/**'
+      - 'doc/**'
+      - '**.md'
+
+permissions: read-all
 
 # save workspaces to speed up testing
 env:
@@ -15,7 +29,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa
-      uses: actions/checkout@v2
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
     # The sync GH action in capa-rules relies on a single '- *$' in the CHANGELOG file
     - name: Ensure CHANGELOG has '- *$'
       run: |
@@ -26,33 +40,44 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa
-      uses: actions/checkout@v2
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+    # use latest available python to take advantage of best performance
+    - name: Set up Python 3.11
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
-        python-version: "3.8"
+        python-version: "3.11"
     - name: Install dependencies
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
+    - name: Lint with ruff
+      run: pre-commit run ruff
     - name: Lint with isort
-      run: isort --profile black --length-sort --line-width 120 -c .
+      run: pre-commit run isort --show-diff-on-failure
     - name: Lint with black
-      run: black -l 120 --check .
+      run: pre-commit run black --show-diff-on-failure
+    - name: Lint with flake8
+      run: pre-commit run flake8 --hook-stage manual
     - name: Check types with mypy
-      run: mypy --config-file .github/mypy/mypy.ini capa/ scripts/ tests/
+      run: pre-commit run mypy --hook-stage manual
+    - name: Check imports against dependencies
+      run: pre-commit run deptry --hook-stage manual
 
   rule_linter:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@v2
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
-        submodules: true
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+        submodules: recursive
+    - name: Set up Python 3.11
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
-        python-version: "3.8"
+        python-version: "3.11"
     - name: Install capa
-      run: pip install -e .
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
     - name: Run rule linter
       run: python scripts/lint.py rules/
 
@@ -63,30 +88,134 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-10.15]
+        os: [ubuntu-20.04, windows-2019, macos-12]
         # across all operating systems
-        python-version: ["3.6", "3.10"]
+        python-version: ["3.8", "3.11"]
         include:
           # on Ubuntu run these as well
-          - os: ubuntu-20.04
-            python-version: "3.7"
           - os: ubuntu-20.04
             python-version: "3.8"
           - os: ubuntu-20.04
             python-version: "3.9"
+          - os: ubuntu-20.04
+            python-version: "3.10"
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@v2
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
-        submodules: true
+        submodules: recursive
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install pyyaml
       if: matrix.os == 'ubuntu-20.04'
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
-      run: pip install -e .[dev]
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
+    - name: Run tests (fast)
+      # this set of tests runs about 80% of the cases in 20% of the time,
+      # and should catch most errors quickly.
+      run:  pre-commit run pytest-fast --all-files --hook-stage manual
     - name: Run tests
       run: pytest -v tests/
+
+  binja-tests:
+    name: Binary Ninja tests for ${{ matrix.python-version }}
+    env:
+      BN_SERIAL: ${{ secrets.BN_SERIAL }}
+    runs-on: ubuntu-22.04
+    needs: [tests]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.11"]
+    steps:
+    - name: Checkout capa with submodules
+      # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
+      if: ${{ env.BN_SERIAL != 0 }}
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
+    - name: Set up Python ${{ matrix.python-version }}
+      if: ${{ env.BN_SERIAL != 0 }}
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install pyyaml
+      if: ${{ env.BN_SERIAL != 0 }}
+      run: sudo apt-get install -y libyaml-dev
+    - name: Install capa
+      if: ${{ env.BN_SERIAL != 0 }}
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
+    - name: install Binary Ninja
+      if: ${{ env.BN_SERIAL != 0 }}
+      run: |
+        mkdir ./.github/binja
+        curl "https://raw.githubusercontent.com/Vector35/binaryninja-api/6812c97/scripts/download_headless.py" -o ./.github/binja/download_headless.py
+        python ./.github/binja/download_headless.py --serial ${{ env.BN_SERIAL }} --output .github/binja/BinaryNinja-headless.zip
+        unzip .github/binja/BinaryNinja-headless.zip -d .github/binja/
+        python .github/binja/binaryninja/scripts/install_api.py --install-on-root --silent
+    - name: Run tests
+      if: ${{ env.BN_SERIAL != 0 }}
+      env:
+        BN_LICENSE: ${{ secrets.BN_LICENSE }}
+      run: pytest -v tests/test_binja_features.py  # explicitly refer to the binja tests for performance. other tests run above.
+
+  ghidra-tests:
+    name: Ghidra tests for ${{ matrix.python-version }}
+    runs-on: ubuntu-20.04
+    needs: [tests]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.11"]
+        java-version: ["17"]
+        ghidra-version: ["11.0.1"]
+        public-version: ["PUBLIC_20240130"] # for ghidra releases
+        ghidrathon-version: ["4.0.0"] 
+    steps:
+    - name: Checkout capa with submodules
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: true
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Set up Java ${{ matrix.java-version }}
+      uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 # v4.0.0
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java-version }}
+    - name: Install Ghidra ${{ matrix.ghidra-version }} 
+      run: |
+        mkdir ./.github/ghidra
+        wget "https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_${{ matrix.ghidra-version }}_build/ghidra_${{ matrix.ghidra-version }}_${{ matrix.public-version }}.zip" -O ./.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip
+        unzip .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip -d .github/ghidra/
+    - name: Install Ghidrathon
+      run : |
+        mkdir ./.github/ghidrathon
+        wget "https://github.com/mandiant/Ghidrathon/releases/download/v${{ matrix.ghidrathon-version }}/Ghidrathon-v${{ matrix.ghidrathon-version}}.zip" -O ./.github/ghidrathon/ghidrathon-v${{ matrix.ghidrathon-version }}.zip
+        unzip .github/ghidrathon/ghidrathon-v${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/
+        python -m pip install -r .github/ghidrathon/requirements.txt
+        python .github/ghidrathon/ghidrathon_configure.py $(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC
+        unzip .github/ghidrathon/Ghidrathon-v${{ matrix.ghidrathon-version }}.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions
+    - name: Install pyyaml
+      run: sudo apt-get install -y libyaml-dev
+    - name: Install capa
+      run: |
+        pip install -r requirements.txt
+        pip install -e .[dev,scripts]
+    - name: Run tests
+      run: | 
+        mkdir ./.github/ghidra/project
+        .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/support/analyzeHeadless .github/ghidra/project ghidra_test -Import ./tests/data/mimikatz.exe_ -ScriptPath ./tests/ -PostScript test_ghidra_features.py > ../output.log
+        cat ../output.log
+        exit_code=$(cat ../output.log | grep exit | awk '{print $NF}')
+        exit $exit_code
+ 

.github/workflows/web-deploy.yml

@@ -0,0 +1,134 @@
+name: deploy web to GitHub Pages
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - 'web/**'
+
+  # Allows to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets the GITHUB_TOKEN permissions to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow one concurrent deployment
+concurrency:
+  group: 'pages'
+  cancel-in-progress: true
+
+jobs:
+  build-landing-page:
+    name: Build landing page
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/upload-artifact@v4
+        with:
+          name: landing-page
+          path: './web/public'
+
+  build-explorer:
+    name: Build capa Explorer Web
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          fetch-depth: 1
+          show-progress: true
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: './web/explorer/package-lock.json'
+      - name: Install dependencies
+        run: npm ci
+        working-directory: ./web/explorer
+      - name: Generate release bundle
+        run: npm run build:bundle
+        working-directory: ./web/explorer
+      - name: Zip release bundle
+        run: zip -r public/capa-explorer-web.zip capa-explorer-web
+        working-directory: ./web/explorer
+      - name: Build
+        run: npm run build
+        working-directory: ./web/explorer
+      - uses: actions/upload-artifact@v4
+        with:
+          name: explorer
+          path: './web/explorer/dist'
+
+  build-rules:
+    name: Build rules site
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          # full depth so that capa-rules has a full history
+          # and we can construct a timeline of rule updates.
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+        with:
+          python-version: '3.12'
+      - uses: extractions/setup-just@v2
+      - name: Install pagefind
+        uses: supplypike/setup-bin@v4
+        with:
+          uri: "https://github.com/CloudCannon/pagefind/releases/download/v1.1.0/pagefind-v1.1.0-x86_64-unknown-linux-musl.tar.gz"
+          name: "pagefind"
+          version: "1.1.0"
+      - name: Install dependencies
+        working-directory: ./web/rules
+        run: pip install -r requirements.txt
+      - name: Build the website
+        working-directory: ./web/rules
+        run: just build
+      - name: Index the website
+        working-directory: ./web/rules
+        run: pagefind --site "public"
+      # upload the build website to artifacts
+      # so that we can download and inspect, if desired.
+      - uses: actions/upload-artifact@v4
+        with:
+          name: rules
+          path: './web/rules/public'
+
+  deploy:
+    name: Deploy site to GitHub Pages
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: [build-landing-page, build-explorer, build-rules]
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: landing-page
+          path: './public/'
+      - uses: actions/download-artifact@v4
+        with:
+          name: explorer
+          path: './public/explorer'
+      - uses: actions/download-artifact@v4
+        with:
+          name: rules
+          path: './public/rules'
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: './public'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/web-tests.yml

@@ -0,0 +1,42 @@
+name: Capa Explorer Web tests 
+
+on:
+  pull_request:
+    branches: [ master ]
+    paths:
+      - 'web/explorer/**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: 'recursive'
+        fetch-depth: 1
+        show-progress: true
+
+    - name: Set up Node
+      uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: 'npm'
+        cache-dependency-path: './web/explorer/package-lock.json'
+
+    - name: Install dependencies
+      run: npm ci
+      working-directory: ./web/explorer
+
+    - name: Lint
+      run: npm run lint
+      working-directory: ./web/explorer
+
+    - name: Format
+      run: npm run format:check
+      working-directory: ./web/explorer
+
+    - name: Run unit tests
+      run: npm run test
+      working-directory: ./web/explorer

.gitignore

@@ -108,13 +108,22 @@ venv.bak/
 *.viv
 *.idb
 *.i64
+.vscode
+
 !rules/lib
 
-# hooks/ci.sh output
-isort-output.log
-black-output.log
-rule-linter-output.log
-.vscode
 scripts/perf/*.txt
 scripts/perf/*.svg
 scripts/perf/*.zip
+
+.direnv
+.envrc
+.DS_Store
+*/.DS_Store
+Pipfile
+Pipfile.lock
+/cache/
+.github/binja/binaryninja
+.github/binja/download_headless.py
+.github/binja/BinaryNinja-headless.zip
+justfile

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "rules"]
 	path = rules
-	url = ../capa-rules.git
+	url = ../../mandiant/capa-rules.git
 [submodule "tests/data"]
 	path = tests/data
-	url = ../capa-testfiles.git
+	url = ../../mandiant/capa-testfiles.git

.justfile

@@ -0,0 +1,25 @@
+@isort:
+    pre-commit run isort --show-diff-on-failure --all-files
+
+@black:
+    pre-commit run black --show-diff-on-failure --all-files
+
+@ruff:
+    pre-commit run ruff --all-files
+
+@flake8:
+    pre-commit run flake8 --hook-stage manual --all-files
+
+@mypy:
+    pre-commit run mypy --hook-stage manual --all-files
+
+@deptry:
+    pre-commit run deptry --hook-stage manual --all-files
+
+@lint:
+    -just isort
+    -just black
+    -just ruff
+    -just flake8
+    -just mypy
+    -just deptry

.pre-commit-config.yaml

@@ -0,0 +1,145 @@
+# install the pre-commit hooks:
+#
+#    ❯ pre-commit install --hook-type pre-commit
+#    pre-commit installed at .git/hooks/pre-commit
+#
+#    ❯ pre-commit install --hook-type pre-push
+#    pre-commit installed at .git/hooks/pre-push
+#
+# run all linters liks:
+#
+#    ❯ pre-commit run --all-files
+#    isort....................................................................Passed
+#    black....................................................................Passed
+#    ruff.....................................................................Passed
+#    flake8...................................................................Passed
+#    mypy.....................................................................Passed
+#
+# run a single linter like:
+#
+#    ❯ pre-commit run --all-files isort
+#    isort....................................................................Passed
+
+repos:
+-   repo: local
+    hooks:
+    -   id: isort
+        name: isort
+        stages: [commit, push, manual]
+        language: system
+        entry: isort
+        args: 
+        -   "--length-sort"
+        -   "--profile"
+        -   "black"
+        -   "--line-length=120"
+        -   "--skip-glob"
+        -   "*_pb2.py"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        -   "web/rules/scripts/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: black
+        name: black
+        stages: [commit, push, manual]
+        language: system
+        entry: black
+        args: 
+        -   "--line-length=120"
+        -   "--extend-exclude"
+        -   ".*_pb2.py"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        -   "web/rules/scripts/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: ruff
+        name: ruff
+        stages: [commit, push, manual]
+        language: system
+        entry: ruff
+        args: 
+        -   "check"
+        -   "--config"
+        -   ".github/ruff.toml"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        -   "web/rules/scripts/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: flake8
+        name: flake8
+        stages: [push, manual]
+        language: system
+        entry: flake8
+        args: 
+        -   "--config"
+        -   ".github/flake8.ini"
+        -   "--extend-exclude"
+        -   "capa/render/proto/capa_pb2.py,capa/features/extractors/binexport2/binexport2_pb2.py"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        -   "web/rules/scripts/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: mypy
+        name: mypy
+        stages: [push, manual]
+        language: system
+        entry: mypy
+        args: 
+        -   "--check-untyped-defs"
+        -   "--ignore-missing-imports"
+        -   "--config-file=.github/mypy/mypy.ini"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        -   "web/rules/scripts/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: deptry
+        name: deptry
+        stages: [push, manual]
+        language: system
+        entry: deptry .
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: pytest-fast
+        name: pytest (fast)
+        stages: [manual]
+        language: system
+        entry: pytest
+        args: 
+        -   "tests/"
+        -   "--ignore=tests/test_binja_features.py"
+        -   "--ignore=tests/test_ghidra_features.py"
+        -   "--ignore=tests/test_ida_features.py"
+        -   "--ignore=tests/test_viv_features.py"
+        -   "--ignore=tests/test_main.py"
+        -   "--ignore=tests/test_scripts.py"
+        always_run: true
+        pass_filenames: false
+

CHANGELOG.md

@@ -17,8 +17,777 @@
 ### Development
 
 ### Raw diffs
-- [capa v3.1.0...master](https://github.com/mandiant/capa/compare/v3.1.0...master)
-- [capa-rules v3.1.0...master](https://github.com/mandiant/capa-rules/compare/v3.1.0...master)
+- [capa v7.3.0...master](https://github.com/mandiant/capa/compare/v7.3.0...master)
+- [capa-rules v7.3.0...master](https://github.com/mandiant/capa-rules/compare/v7.3.0...master)
+
+## v7.3.0
+This release comes with the following three major enhancements:
+
+### 1. Support for VMRay sandbox analysis archives
+Unlock powerful malware analysis with capa's new [VMRay sandbox](https://www.vmray.com/) integration! Simply provide a VMRay analysis archive, and capa will automatically extract and match capabilities to streamline your workflow. This is the second support for the analysis of dynamic analysis results after [CAPE](https://www.mandiant.com/resources/blog/dynamic-capa-executable-behavior-cape-sandbox).
+
+### 2. Support for BinExport files generated by Ghidra
+[BinExport](https://github.com/google/binexport) files store disassembled data into a Protocol Buffer format. capa now supports the analysis of BinExport files generated by Ghidra. Using Ghidra and the BinExport file format users can now analyze ARM (AARCH64) ELF files targeting Android.
+
+### 3. Introducing the capa rules website
+You can now browse capa's default rule set at https://mandiant.github.io/capa/rules. In modern terminals the CLI capa tool hyperlinks to resources on the web, including entries on the capa rules website.
+Furthermore, https://mandiant.github.io/capa provides a landing page for the capa tool project.
+
+### Additional updates
+- [capa Explorer Web](https://mandiant.github.io/capa/explorer/) received several enhancements and bug fixes.
+- Support for the IDA Pro 9.0 IDAPython API while keeping compatibility to older IDA Pro versions
+- Six rules have been added and two rules have been updated
+
+Thanks to @r-sm2024 for their contribution in https://github.com/mandiant/capa/pull/2155 and their further work. And of course a big thanks to the community for reporting issues, participating in discussions, and supporting the capa tool and capa rules.
+
+### New Features
+- regenerate ruleset cache automatically on source change (only in dev mode) #2133 @s-ff
+- add landing page https://mandiant.github.io/capa/ @williballenthin #2310
+- add rules website https://mandiant.github.io/capa/rules @DeeyaSingh #2310
+- add .justfile @williballenthin #2325
+- dynamic: add support for VMRay dynamic sandbox traces #2208 @mike-hunhoff @r-sm2024 @mr-tz
+- cli: use modern terminal features to hyperlink to the rules website #2337 @williballenthin
+- support analyzing BinExport2 files generated by Ghidra #1950 @williballenthin @mehunhoff @mr-tz
+- add support for Android OS #1950 @williballenthin @mehunhoff @mr-tz
+- add support for aarch64 architecture via BinExport2 backend #1950 @williballenthin @mehunhoff @mr-tz
+
+### New Rules (6)
+
+- linking/static/minhook/linked-against-minhook jakub.jozwiak@mandiant.com
+- linking/static/sqlite3/linked-against-sqlcipher wballenthin@google.com
+- host-interaction/network/traffic/filter/delete-network-filter-via-wfp-api jakub.jozwiak@mandiant.com
+- host-interaction/network/traffic/filter/enumerate-network-filters-via-wfp-api jakub.jozwiak@mandiant.com
+- nursery/check-thread-suspend-count-exceeded ervinocampo@google.com
+- nursery/create-thread-bypassing-process-freeze ervinocampo@google.com
+
+### Bug Fixes
+
+- fix duplicate features shown in vverbose mode @williballenthin #2323
+- fix code path reference in linter @williballenthin #2350
+
+### capa explorer IDA Pro plugin
+- update IDAPython to IDA Pro 9.0 @mr-tz
+- fix byte search IDA Pro 7.5 compatibility @mr-tz #2371
+
+### Raw diffs
+- [capa v7.2.0...v7.3.0](https://github.com/mandiant/capa/compare/v7.2.0...v7.3.0)
+- [capa-rules v7.2.0...v7.3.0](https://github.com/mandiant/capa-rules/compare/v7.2.0...v7.3.0)
+
+## v7.2.0
+capa v7.2.0 introduces a first version of capa explorer web: a web-based user interface to inspect capa results using your browser. Users can inspect capa result JSON documents in an online web instance or a standalone HTML page for offline usage. capa explorer supports interactive exploring of capa results to make it easier to understand them. Users can filter, sort, and see the details of all identified capabilities. capa explorer web was worked on by @s-ff as part of a [GSoC project](https://summerofcode.withgoogle.com/programs/2024/projects/cR3hjbsq), and it is available at https://mandiant.github.io/capa/explorer/#/.
+
+This release also adds a feature extractor for output from the DRAKVUF sandbox. Now, analysts can pass the resulting `drakmon.log` file to capa and extract capabilities from the artifacts captured by the sandbox. This feature extractor will also be added to the DRAKVUF sandbox as a post-processing script, and it was worked on by @yelhamer as part of a [GSoC project](https://summerofcode.withgoogle.com/programs/2024/projects/fCnBGuEC).
+
+Additionally, we fixed several bugs handling ELF files, and added the ability to filter capa analysis by functions or processes. We also added support to the IDA Pro extractor to leverage analyst recovered API names.
+
+Special thanks to our repeat and new contributors:
+* @lakshayletsgo for their first contribution in https://github.com/mandiant/capa/pull/2248
+* @msm-cert for their first contribution in https://github.com/mandiant/capa/pull/2143
+* @VascoSch92 for their first contribution in https://github.com/mandiant/capa/pull/2143
+
+### New Features
+
+- webui: explore capa analysis results in a web-based UI online and offline #2224 @s-ff
+- support analyzing DRAKVUF traces #2143 @yelhamer
+- IDA extractor: extract names from dynamically resolved APIs stored in renamed global variables #2201 @Ana06
+- cli: add the ability to select which specific functions or processes to analyze @yelhamer
+
+### Breaking Changes
+
+### New Rules (5)
+
+- nursery/upload-file-to-onedrive jaredswilson@google.com ervinocampo@google.com
+- data-manipulation/encoding/base64/decode-data-using-base64-via-vbmi-lookup-table still@teamt5.org
+- communication/socket/attach-bpf-to-socket-on-linux jakub.jozwiak@mandiant.com
+- anti-analysis/anti-av/overwrite-dll-text-section-to-remove-hooks jakub.jozwiak@mandiant.com
+- nursery/delete-file-on-linux mehunhoff@google.com
+
+### Bug Fixes
+
+- elf: extract import / export symbols from stripped binaries #2096 @ygasparis
+- elf: fix handling of symbols in corrupt ELF files #2226 @williballenthin
+
+### capa explorer IDA Pro plugin
+
+### Development
+- CI: use macos-12 since macos-11 is deprecated and will be removed on June 28th, 2024 #2173 @mr-tz
+- CI: update Binary Ninja version to 4.1 and use Python 3.9 to test it #2211 @xusheng6
+- CI: update tests.yml workflow to exclude web and documentation files #2263 @s-ff
+- CI: update build.yml workflow to exclude web and documentation files #2270 @s-ff
+
+### Raw diffs
+- [capa v7.1.0...7.2.0](https://github.com/mandiant/capa/compare/v7.1.0...7.2.0)
+- [capa-rules v7.1.0...7.2.0](https://github.com/mandiant/capa-rules/compare/v7.1.0...7.2.0)
+
+## v7.1.0
+The v7.1.0 release brings large performance improvements to capa's rule matching engine.
+Additionally, we've fixed various bugs and added new features for people using and developing capa.
+
+Special thanks to our repeat and new contributors:
+* @sjha2048 made their first contribution in https://github.com/mandiant/capa/pull/2000
+* @Rohit1123 made their first contribution in https://github.com/mandiant/capa/pull/1990
+* @psahithireddy made their first contribution in https://github.com/mandiant/capa/pull/2020
+* @Atlas-64 made their first contribution in https://github.com/mandiant/capa/pull/2018
+* @s-ff made their first contribution in https://github.com/mandiant/capa/pull/2011
+* @samadpls made their first contribution in https://github.com/mandiant/capa/pull/2024
+* @acelynnzhang made their first contribution in https://github.com/mandiant/capa/pull/2044
+* @RainRat made their first contribution in https://github.com/mandiant/capa/pull/2058
+* @ReversingWithMe made their first contribution in https://github.com/mandiant/capa/pull/2093
+* @malwarefrank made their first contribution in https://github.com/mandiant/capa/pull/2037
+
+### New Features
+- Emit "dotnet" as format to ResultDocument when processing .NET files #2024 @samadpls
+- ELF: detect OS from statically-linked Go binaries #1978 @williballenthin
+- add function in capa/helpers to load plain and compressed JSON reports #1883 @Rohit1123
+- document Antivirus warnings and VirusTotal false positive detections #2028 @RionEV @mr-tz
+- Add json to sarif conversion script @reversingwithme
+- render maec/* fields #843 @s-ff
+- replace Halo spinner with Rich #2086 @s-ff
+- optimize rule matching #2080 @williballenthin
+- add aarch64 as a valid architecture #2144 mehunhoff@google.com @williballenthin
+- relax dependency version requirements for the capa library #2053 @williballenthin
+- add scripts dependency group and update documentation #2145 @mr-tz
+
+### New Rules (25)
+
+- impact/wipe-disk/delete-drive-layout-via-ioctl william.ballenthin@mandiant.com
+- host-interaction/driver/interact-with-driver-via-ioctl moritz.raabe@mandiant.com
+- host-interaction/driver/unload-driver moritz.raabe@mandiant.com
+- nursery/get-disk-information-via-ioctl william.ballenthin@mandiant.com
+- nursery/get-volume-information-via-ioctl william.ballenthin@mandiant.com
+- nursery/unmount-volume-via-ioctl william.ballenthin@mandiant.com
+- data-manipulation/encryption/rc4/encrypt-data-using-rc4-via-systemfunction033 daniel.stepanic@elastic.co
+- anti-analysis/anti-forensic/self-deletion/self-delete-using-alternate-data-streams daniel.stepanic@elastic.co
+- nursery/change-memory-permission-on-linux mehunhoff@google.com
+- nursery/check-file-permission-on-linux mehunhoff@google.com
+- nursery/check-if-process-is-running-under-android-emulator-on-android mehunhoff@google.com
+- nursery/map-or-unmap-memory-on-linux mehunhoff@google.com
+- persistence/act-as-share-provider-dll jakub.jozwiak@mandiant.com
+- persistence/act-as-windbg-extension jakub.jozwiak@mandiant.com
+- persistence/act-as-time-provider-dll jakub.jozwiak@mandiant.com
+- host-interaction/gui/window/hide/hide-graphical-window-from-taskbar jakub.jozwiak@mandiant.com
+- compiler/dart/compiled-with-dart jakub.jozwiak@mandiant.com
+- nursery/bypass-hidden-api-restrictions-via-jni-on-android mehunhoff@google.com
+- nursery/get-current-process-filesystem-mounts-on-linux mehunhoff@google.com
+- nursery/get-current-process-memory-mapping-on-linux mehunhoff@google.com
+- nursery/get-system-property-on-android mehunhoff@google.com
+- nursery/hook-routines-via-lsplant mehunhoff@google.com
+- nursery/load-packed-dex-via-jiagu-on-android mehunhoff@google.com
+- nursery/modify-api-blacklist-or-denylist-via-jni-on-android mehunhoff@google.com
+- nursery/truncate-file-on-linux mehunhoff@google.com
+
+### Bug Fixes
+
+- do some imports closer to where they are used #1810 @williballenthin
+- binja: fix and simplify stack string detection code after binja 4.0 @xusheng6
+- binja: add support for forwarded export #1646 @xusheng6
+- cape: support more report formats #2035 @mr-tz
+
+### capa explorer IDA Pro plugin
+- replace deprecated IDA API find_binary with bin_search #1606 @s-ff
+
+### Development
+
+- ci: Fix PR review in the changelog check GH action #2004 @Ana06
+- ci: use rules number badge stored in our bot gist and generated using `schneegans/dynamic-badges-action` #2001 capa-rules#882 @Ana06
+- ci: update github workflows to use latest version of actions that were using a deprecated version of node #1967 #2003 capa-rules#883 @sjha2048 @Ana06
+- ci: update binja version to stable 4.0 #2016 @xusheng6
+- ci: update github workflows to reflect the latest ghidrathon installation and bumped up jep, ghidra versions  #2020 @psahithireddy
+- ci: include rule caching in PyInstaller build process #2097 @s-ff
+- add deptry support #1497 @s-ff
+
+### Raw diffs
+- [capa v7.0.1...v7.1.0](https://github.com/mandiant/capa/compare/v7.0.1...v7.1.0)
+- [capa-rules v7.0.1...v7.1.0](https://github.com/mandiant/capa-rules/compare/v7.0.1...v7.1.0)
+
+## v7.0.1
+
+This release fixes a circular import error when using capa as a library.
+
+### Bug Fixes
+
+- fix potentially circular import errors #1969 @williballenthin
+
+### Raw diffs
+- [capa v7.0.0...v7.0.1](https://github.com/mandiant/capa/compare/v7.0.0...v7.0.1)
+- [capa-rules v7.0.0...v7.0.1](https://github.com/mandiant/capa-rules/compare/v7.0.0...v7.0.1)
+
+## v7.0.0
+This is the v7.0.0 release of capa which was mainly worked on during the Google Summer of Code (GSoC) 2023. A huge
+shoutout to our GSoC contributors @colton-gabertan and @yelhamer for their amazing work.
+
+Also, a big thanks to the other contributors: @aaronatp, @Aayush-Goel-04, @bkojusner, @doomedraven, @ruppde, @larchchen, @JCoonradt, and @xusheng6.
+
+### New Features
+
+- add Ghidra backend #1770 #1767 @colton-gabertan @mike-hunhoff
+- add Ghidra UI integration #1734 @colton-gabertan @mike-hunhoff
+- add dynamic analysis via CAPE sandbox reports #48 #1535 @yelhamer
+  - add call scope #771 @yelhamer
+  - add thread scope #1517 @yelhamer
+  - add process scope #1517 @yelhamer
+  - rules: change `meta.scope` to `meta.scopes` @yelhamer
+  - protobuf: add `Metadata.flavor` @williballenthin
+- binja: add support for forwarded exports #1646 @xusheng6
+- binja: add support for symtab names #1504 @xusheng6
+- add com class/interface features #322 @Aayush-goel-04
+- dotnet: emit enclosing class information for nested classes #1780 #1913 @bkojusner @mike-hunhoff
+
+### Breaking Changes
+
+- remove the `SCOPE_*` constants in favor of the `Scope` enum #1764 @williballenthin
+- protobuf: deprecate `RuleMetadata.scope` in favor of `RuleMetadata.scopes` @williballenthin
+- protobuf: deprecate `Metadata.analysis` in favor of `Metadata.analysis2` that is dynamic analysis aware @williballenthin
+- update freeze format to v3, adding support for dynamic analysis @williballenthin
+- extractor: ignore DLL name for api features #1815 @mr-tz
+- main: introduce wrapping routines within main for working with CLI args #1813 @williballenthin
+- move functions from `capa.main` to new `capa.loader` namespace #1821 @williballenthin
+- proto: add `package` declaration #1960 @larchchen
+
+### New Rules (41)
+
+- nursery/get-ntoskrnl-base-address @mr-tz
+- host-interaction/network/connectivity/set-tcp-connection-state @johnk3r
+- nursery/capture-process-snapshot-data @mr-tz
+- collection/network/capture-packets-using-sharppcap jakub.jozwiak@mandiant.com
+- nursery/communicate-with-kernel-module-via-netlink-socket-on-linux michael.hunhoff@mandiant.com
+- nursery/get-current-pid-on-linux michael.hunhoff@mandiant.com
+- nursery/get-file-system-information-on-linux michael.hunhoff@mandiant.com
+- nursery/get-password-database-entry-on-linux michael.hunhoff@mandiant.com
+- nursery/mark-thread-detached-on-linux michael.hunhoff@mandiant.com
+- nursery/persist-via-gnome-autostart-on-linux michael.hunhoff@mandiant.com
+- nursery/set-thread-name-on-linux michael.hunhoff@mandiant.com
+- load-code/dotnet/load-windows-common-language-runtime michael.hunhoff@mandiant.com blas.kojusner@mandiant.com jakub.jozwiak@mandiant.com
+- nursery/log-keystrokes-via-input-method-manager @mr-tz
+- nursery/encrypt-data-using-rc4-via-systemfunction032 richard.weiss@mandiant.com
+- nursery/add-value-to-global-atom-table @mr-tz
+- nursery/enumerate-processes-that-use-resource @Ana06
+- host-interaction/process/inject/allocate-or-change-rwx-memory @mr-tz
+- lib/allocate-or-change-rw-memory 0x534a@mailbox.org @mr-tz
+- lib/change-memory-protection @mr-tz
+- anti-analysis/anti-av/patch-antimalware-scan-interface-function jakub.jozwiak@mandiant.com
+- executable/dotnet-singlefile/bundled-with-dotnet-single-file-deployment sara.rincon@mandiant.com
+- internal/limitation/file/internal-dotnet-single-file-deployment-limitation sara.rincon@mandiant.com
+- data-manipulation/encoding/encode-data-using-add-xor-sub-operations jakub.jozwiak@mandiant.com
+- nursery/access-camera-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/capture-microphone-audio-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/capture-screenshot-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/check-for-incoming-call-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/check-for-outgoing-call-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/compiled-with-xamarin michael.hunhoff@mandiant.com
+- nursery/get-os-version-in-dotnet-on-android michael.hunhoff@mandiant.com
+- data-manipulation/compression/create-cabinet-on-windows michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com
+- data-manipulation/compression/extract-cabinet-on-windows jakub.jozwiak@mandiant.com
+- lib/create-file-decompression-interface-context-on-windows jakub.jozwiak@mandiant.com
+- nursery/enumerate-files-in-dotnet moritz.raabe@mandiant.com anushka.virgaonkar@mandiant.com
+- nursery/get-mac-address-in-dotnet moritz.raabe@mandiant.com michael.hunhoff@mandiant.com echernofsky@google.com
+- nursery/get-current-process-command-line william.ballenthin@mandiant.com
+- nursery/get-current-process-file-path william.ballenthin@mandiant.com
+- nursery/hook-routines-via-dlsym-rtld_next william.ballenthin@mandiant.com
+- nursery/linked-against-hp-socket still@teamt5.org
+- host-interaction/process/inject/process-ghostly-hollowing sara.rincon@mandiant.com
+
+### Bug Fixes
+- ghidra: fix `ints_to_bytes` performance #1761 @mike-hunhoff
+- binja: improve function call site detection @xusheng6
+- binja: use `binaryninja.load` to open files @xusheng6
+- binja: bump binja version to 3.5 #1789 @xusheng6
+- elf: better detect ELF OS via GCC .ident directives #1928 @williballenthin
+- elf: better detect ELF OS via Android dependencies #1947 @williballenthin
+- fix setuptools package discovery #1886 @gmacon @mr-tz
+- remove unnecessary scripts/vivisect-py2-vs-py3.sh file #1949 @JCoonradt
+
+### capa explorer IDA Pro plugin
+- various integration updates and minor bug fixes
+
+### Development
+- update ATT&CK/MBC data for linting #1932 @mr-tz
+
+#### Developer Notes
+With this new release, many classes and concepts have been split up into static (mostly identical to the
+prior implementations) and dynamic ones. For example, the legacy FeatureExtractor class has been renamed to
+StaticFeatureExtractor and the DynamicFeatureExtractor has been added.
+
+Starting from version 7.0, we have moved the component responsible for feature extractor from main to a new
+capabilities' module. Now, users wishing to utilize capa’s feature extraction abilities should use that module instead
+of importing the relevant logic from the main file.
+
+For sandbox-based feature extractors, we are using Pydantic models. Contributions of more models for other sandboxes
+are very welcome!
+
+With this release we've reorganized the logic found in `main()` to localize logic and ease readability and ease changes
+and integrations. The new "main routines" are expected to be used only within main functions, either capa main or
+related scripts. These functions should not be invoked from library code.
+
+Beyond copying code around, we've refined the handling of the input file/format/backend. The logic for picking the
+format and backend is more consistent. We've documented that the input file is not necessarily the sample itself
+(cape/freeze/etc.) inputs are not actually the sample.
+
+### Raw diffs
+- [capa v6.1.0...v7.0.0](https://github.com/mandiant/capa/compare/v6.1.0...v7.0.0)
+- [capa-rules v6.1.0...v7.0.0](https://github.com/mandiant/capa-rules/compare/v6.1.0...v7.0.0)
+
+## v6.1.0
+
+capa v6.1.0 is a bug fix release, most notably fixing unhandled exceptions in the capa explorer IDA Pro plugin.
+@Aayush-Goel-04 put a lot of effort into improving code quality and adding a script for rule authors.
+The script shows which features are present in a sample but not referenced by any existing rule.
+You could use this script to find opportunities for new rules.
+
+Speaking of new rules, we have eight additions, coming from Ronnie, Jakub, Moritz, Ervin, and still@teamt5.org!
+
+### New Features
+- ELF: implement import and export name extractor #1607 #1608 @Aayush-Goel-04
+- bump pydantic from 1.10.9 to 2.1.1 #1582 @Aayush-Goel-04
+- develop script to highlight features not used during matching #331 @Aayush-Goel-04
+
+### New Rules (8)
+
+- executable/pe/export/forwarded-export ronnie.salomonsen@mandiant.com
+- host-interaction/bootloader/get-uefi-variable jakub.jozwiak@mandiant.com
+- host-interaction/bootloader/set-uefi-variable jakub.jozwiak@mandiant.com
+- nursery/enumerate-device-drivers-on-linux @mr-tz
+- anti-analysis/anti-vm/vm-detection/check-for-foreground-window-switch ervin.ocampo@mandiant.com
+- linking/static/sqlite3/linked-against-cppsqlite3 still@teamt5.org
+- linking/static/sqlite3/linked-against-sqlite3 still@teamt5.org
+
+### Bug Fixes
+
+- rules: fix forwarded export characteristic #1656 @RonnieSalomonsen
+- Binary Ninja: Fix stack string detection #1473 @xusheng6
+- linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin 
+- OS: detect Android ELF files #1705 @williballenthin
+- ELF: fix parsing of symtab #1704 @williballenthin
+- result document: don't use deprecated pydantic functions #1718 @williballenthin
+- pytest: don't mark IDA tests as pytest tests #1719 @williballenthin
+
+### capa explorer IDA Pro plugin
+- fix unhandled exception when resolving rule path #1693 @mike-hunhoff
+
+### Raw diffs
+- [capa v6.0.0...v6.1.0](https://github.com/mandiant/capa/compare/v6.0.0...v6.1.0)
+- [capa-rules v6.0.0...v6.1.0](https://github.com/mandiant/capa-rules/compare/v6.0.0...v6.1.0)
+
+## v6.0.0
+
+capa v6.0 brings many bug fixes and quality improvements, including 64 rule updates and 26 new rules. We're now publishing to PyPI via [Trusted Publishing](https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/) and have migrated to using a `pyproject.toml` file. @Aayush-Goel-04 contributed a lot of new code across many files, so please welcome them to the project, along with @anders-v @crowface28 @dkelly2e @RonnieSalomonsen and @ejfocampo as first-time rule contributors!
+
+For those that use capa as a library, we've introduced some limited breaking changes that better represent data types (versus less-structured data like dictionaries and strings). With the recent deprecation, we've also dropped support for Python 3.7.
+
+### New Features
+- add script to detect feature overlap between new and existing capa rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
+- extract forwarded exports from PE files #1624 @williballenthin
+- extract function and API names from ELF symtab entries @yelhamer https://github.com/mandiant/capa-rules/issues/736
+- use fancy box drawing characters for default output #1586 @williballenthin
+
+### Breaking Changes
+- use a class to represent Metadata (not dict) #1411 @Aayush-Goel-04 @manasghandat
+- use pathlib.Path to represent file paths #1534 @Aayush-Goel-04
+- Python 3.8 is now the minimum supported Python version #1578 @williballenthin
+- Require a Contributor License Agreement (CLA) for PRs going forward #1642 @williballenthin
+
+### New Rules (26)
+
+- load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
+- nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
+- data-manipulation/encryption/aes/encrypt-data-using-aes-mixcolumns-step @mr-tz
+- linking/static/aplib/linked-against-aplib still@teamt5.org
+- communication/mailslot/read-from-mailslot nick.simonian@mandiant.com
+- nursery/hash-data-using-sha512managed-in-dotnet jonathanlepore@google.com
+- nursery/compiled-with-exescript jonathanlepore@google.com
+- nursery/check-for-sandbox-via-mac-address-ouis-in-dotnet jonathanlepore@google.com
+- host-interaction/hardware/enumerate-devices-by-category @mr-tz
+- host-interaction/service/continue-service @mr-tz
+- host-interaction/service/pause-service @mr-tz
+- persistence/exchange/act-as-exchange-transport-agent jakub.jozwiak@mandiant.com
+- host-interaction/file-system/create-virtual-file-system-in-dotnet jakub.jozwiak@mandiant.com
+- compiler/cx_freeze/compiled-with-cx_freeze @mr-tz jakub.jozwiak@mandiant.com
+- communication/socket/create-vmci-socket jakub.jozwiak@mandiant.com
+- persistence/office/act-as-excel-xll-add-in jakub.jozwiak@mandiant.com
+- persistence/office/act-as-office-com-add-in jakub.jozwiak@mandiant.com
+- persistence/office/act-as-word-wll-add-in jakub.jozwiak@mandiant.com
+- anti-analysis/anti-debugging/debugger-evasion/hide-thread-from-debugger michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com
+- host-interaction/memory/create-new-application-domain-in-dotnet jakub.jozwiak@mandiant.com
+- host-interaction/gui/switch-active-desktop jakub.jozwiak@mandiant.com
+- host-interaction/service/query-service-configuration @mr-tz
+- anti-analysis/anti-av/patch-event-tracing-for-windows-function jakub.jozwiak@mandiant.com
+- data-manipulation/encoding/xor/covertly-decode-and-write-data-to-windows-directory-using-indirect-calls dan.kelly@mandiant.com
+- linking/runtime-linking/resolve-function-by-brute-ratel-badger-hash jakub.jozwiak@mandiant.com
+
+### Bug Fixes
+- extractor: add a Binary Ninja test that asserts its version #1487 @xusheng6
+- extractor: update Binary Ninja stack string detection after the new constant outlining feature #1473 @xusheng6
+- extractor: update vivisect Arch extraction #1334 @mr-tz
+- extractor: avoid Binary Ninja exception when analyzing certain files #1441 @xusheng6 
+- symtab: fix struct.unpack() format for 64-bit ELF files @yelhamer
+- symtab: safeguard against ZeroDivisionError for files containing a symtab with a null entry size @yelhamer
+- improve ELF strtab and needed parsing @mr-tz
+- better handle exceptional cases when parsing ELF files #1458 @Aayush-Goel-04
+- improved testing coverage for Binary Ninja backend #1446 @Aayush-Goel-04
+- add logging and print redirect to tqdm for capa main #749 @Aayush-Goel-04
+- extractor: fix binja installation path detection does not work with Python 3.11
+- tests: refine the IDA test runner script #1513 @williballenthin
+- output: don't leave behind traces of progress bar @williballenthin
+- import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin
+- main: don't show spinner when emitting debug messages #1636 @williballenthin
+- rules: add forwarded export characteristics to rule syntax file scope #1653 @RonnieSalomonsen
+
+### capa explorer IDA Pro plugin
+
+### Development
+- update ATT&CK/MBC data for linting #1568 @mr-tz
+- log time taken to analyze each function #1290 @williballenthin
+- tests: make fixture available via conftest.py #1592 @williballenthin
+- publish via PyPI trusted publishing #1491 @williballenthin
+- migrate to pyproject.toml #1301 @williballenthin
+- use [pre-commit](https://pre-commit.com/) to invoke linters #1579 @williballenthin
+
+
+### Raw diffs
+- [capa v5.1.0...v6.0.0](https://github.com/mandiant/capa/compare/v5.1.0...v6.0.0)
+- [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0)
+
+## v5.1.0
+capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and an output color format that's easier to read.
+
+Over 25 capa rules have been added and improved.
+
+Thanks for all the support, especially to @xusheng6, @captainGeech42, @ggold7046, @manasghandat, @ooprathamm, @linpeiyu164, @yelhamer, @HongThatCong, @naikordian, @stevemk14ebr, @emtuls, @raymondlleong, @bkojusner, @joren485, and everyone else who submitted bugs and provided feedback!
+
+### New Features
+- add protobuf format for result documents #1219 @williballenthin @mr-tz 
+- extractor: add Binary Ninja feature extractor @xusheng6
+- new cli flag `--os` to override auto-detected operating system for a sample @captainGeech42
+- change colour/highlight to "cyan" instead of "blue" for better readability #1384 @ggold7046
+- add new format to parse output json back to capa #1396 @ooprathamm
+- parse ELF symbols' names to guess OS #1403 @yelhamer
+
+### New Rules (26)
+
+- persistence/scheduled-tasks/schedule-task-via-at joren485
+- data-manipulation/prng/generate-random-numbers-via-rtlgenrandom william.ballenthin@mandiant.com
+- communication/ip/convert-ip-address-from-string @mr-tz
+- data-manipulation/compression/compress-data-via-zlib-inflate-or-deflate blas.kojusner@mandiant.com
+- executable/installer/dotnet/packaged-as-single-file-dotnet-application michael.hunhoff@mandiant.com
+- communication/socket/create-raw-socket blas.kojusner@mandiant.com
+- communication/http/reference-http-user-agent-string @mr-tz
+- communication/http/get-http-content-length william.ballenthin@mandiant.com
+- nursery/move-directory michael.hunhoff@mandiant.com
+- nursery/get-http-request-uri william.ballenthin@mandiant.com
+- nursery/create-zip-archive-in-dotnet michael.hunhoff@mandiant.com
+- nursery/extract-zip-archive-in-dotnet anushka.virgaonkar@mandiant.com michael.hunhoff@mandiant.com
+- data-manipulation/encryption/tea/decrypt-data-using-tea william.ballenthin@mandiant.com raymond.leong@mandiant.com
+- data-manipulation/encryption/tea/encrypt-data-using-tea william.ballenthin@mandiant.com raymond.leong@mandiant.com
+- data-manipulation/encryption/xtea/encrypt-data-using-xtea raymond.leong@mandiant.com
+- data-manipulation/encryption/xxtea/encrypt-data-using-xxtea raymond.leong@mandiant.com
+- nursery/hash-data-using-ripemd128 raymond.leong@mandiant.com
+- nursery/hash-data-using-ripemd256 raymond.leong@mandiant.com
+- nursery/hash-data-using-ripemd320 raymond.leong@mandiant.com
+- nursery/set-web-proxy-in-dotnet michael.hunhoff@mandiant.com
+- nursery/check-for-windows-sandbox-via-subdirectory echernofsky@google.com
+- nursery/enumerate-pe-sections-in-dotnet @mr-tz
+- nursery/destroy-software-breakpoint-capability echernofsky@google.com
+- nursery/send-data-to-internet michael.hunhoff@mandiant.com
+- nursery/compiled-with-cx_freeze @mr-tz
+- nursery/contain-a-thread-local-storage-tls-section-in-dotnet michael.hunhoff@mandiant.com
+
+### Bug Fixes
+- extractor: interface of cache modified to prevent extracting file and global features multiple times @stevemk14ebr
+- extractor: removed '.dynsym' as the library name for ELF imports #1318 @stevemk14ebr 
+- extractor: fix vivisect loop detection corner case #1310 @mr-tz
+- match: extend OS characteristic to match OS_ANY to all supported OSes #1324 @mike-hunhoff
+- extractor: fix IDA and vivisect string and bytes features overlap and tests #1327 #1336 @xusheng6
+
+### capa explorer IDA Pro plugin
+- rule generator plugin now loads faster when jumping between functions @stevemk14ebr
+- fix exception when plugin loaded in IDA hosted under idat #1341 @mike-hunhoff
+- improve embedded PE detection performance and reduce FP potential #1344 @mike-hunhoff
+
+### Raw diffs
+- [capa v5.0.0...v5.1.0](https://github.com/mandiant/capa/compare/v5.0.0...v5.1.0)
+- [capa-rules v5.0.0...v5.1.0](https://github.com/mandiant/capa-rules/compare/v5.0.0...v5.1.0)
+
+
+## v5.0.0 (2023-02-08)
+This capa version comes with major improvements and additions to better handle .NET binaries. To showcase this we've updated and added over 30 .NET rules.  
+
+Additionally, capa now caches its rule set for better performance. The capa explorer also caches its analysis results, so that multiple IDA Pro or plugin invocations don't need to repeat the same analysis.
+
+We have removed the SMDA backend and changed the program return codes to be positive numbers.
+
+Other improvements to highlight include better ELF OS detection, various rendering bug fixes, and enhancements to the feature extraction. We've also added support for Python 3.11.
+
+Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardweiss80, @joren485, @ryantxu1, @mwilliams31, @anushkavirgaonkar, @MalwareMechanic, @Still34, @dzbeck, @johnk3r, and everyone else who submitted bugs and provided feedback!
+
+### New Features
+- verify rule metadata format on load #1160 @mr-tz
+- dotnet: emit property features #1168 @anushkavirgaonkar
+- dotnet: emit API features for objects created via the newobj instruction #1186 @mike-hunhoff
+- dotnet: emit API features for generic methods #1231 @mike-hunhoff
+- Python 3.11 support #1192 @williballenthin
+- dotnet: emit calls to/from MethodDef methods #1236 @mike-hunhoff
+- dotnet: emit namespace/class features for ldvirtftn/ldftn instructions #1241 @mike-hunhoff
+- dotnet: emit namespace/class features for type references #1242 @mike-hunhoff
+- dotnet: extract dotnet and pe format #1187 @mr-tz
+- don't render all library rule matches in vverbose output #1174 @mr-tz
+- cache the rule set across invocations for better performance #1212 @williballenthin
+- update ATT&CK/MBC data for linting #1297 @mr-tz
+
+### Breaking Changes
+- remove SMDA backend #1062 @williballenthin
+- error return codes are now positive numbers #1269 @mr-tz
+
+### New Rules (77)
+
+- collection/use-dotnet-library-sharpclipboard @johnk3r
+- data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r
+- data-manipulation/json/use-dotnet-library-newtonsoftjson @johnk3r
+- data-manipulation/svg/use-dotnet-library-sharpvectors @johnk3r
+- executable/resource/embed-dependencies-as-resources-using-fodycostura @johnk3r @mr-tz
+- communication/ftp/send/send-file-using-ftp michael.hunhof@mandiant.com anushka.virgaonkar@mandiant.com
+- nursery/extract-zip-archive anushka.virgaonkar@mandiant.com
+- nursery/allocate-unmanaged-memory-in-dotnet michael.hunhoff@mandiant.com
+- nursery/check-file-extension-in-dotnet michael.hunhoff@mandiant.com
+- nursery/decode-data-using-base64-in-dotnet michael.hunhoff@mandiant.com
+- nursery/deserialize-json-in-dotnet michael.hunhoff@mandiant.com
+- nursery/find-data-using-regex-in-dotnet michael.hunhoff@mandiant.com
+- nursery/generate-random-filename-in-dotnet michael.hunhoff@mandiant.com
+- nursery/get-os-version-in-dotnet michael.hunhoff@mandiant.com
+- nursery/load-xml-in-dotnet michael.hunhoff@mandiant.com
+- nursery/manipulate-unmanaged-memory-in-dotnet michael.hunhoff@mandiant.com
+- nursery/save-image-in-dotnet michael.hunhoff@mandiant.com
+- nursery/send-email-in-dotnet michael.hunhoff@mandiant.com
+- nursery/serialize-json-in-dotnet michael.hunhoff@mandiant.com
+- nursery/set-http-user-agent-in-dotnet michael.hunhoff@mandiant.com
+- nursery/compile-csharp-in-dotnet michael.hunhoff@mandiant.com
+- nursery/compile-visual-basic-in-dotnet michael.hunhoff@mandiant.com
+- nursery/compress-data-using-gzip-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-sqlite-statement-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-via-asynchronous-task-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-via-timer-in-dotnet michael.hunhoff@mandiant.com
+- nursery/execute-wmi-query-in-dotnet michael.hunhoff@mandiant.com
+- nursery/manipulate-network-credentials-in-dotnet michael.hunhoff@mandiant.com
+- nursery/encrypt-data-using-aes william.ballenthin@mandiant.com Ivan Kwiatkowski (@JusticeRage)
+- host-interaction/uac/bypass/bypass-uac-via-rpc david.cannings@pwc.com david@edeca.net
+- nursery/check-for-vm-using-instruction-vpcext richard.weiss@mandiant.com
+- nursery/get-windows-directory-from-kuser_shared_data david.cannings@pwc.com
+- nursery/encrypt-data-using-openssl-dsa Ana06
+- nursery/encrypt-data-using-openssl-ecdsa Ana06
+- nursery/encrypt-data-using-openssl-rsa Ana06
+- runtime/dotnet/execute-via-dotnet-startup-hook william.ballenthin@mandiant.com
+- host-interaction/console/manipulate-console-buffer william.ballenthin@mandiant.com michael.hunhoff@mandiant.com
+- nursery/access-wmi-data-in-dotnet michael.hunhoff@mandiant.com
+- nursery/allocate-unmanaged-memory-via-dotnet michael.hunhoff@mandiant.com
+- nursery/generate-random-bytes-in-dotnet michael.hunhoff@mandiant.com
+- nursery/manipulate-console-window michael.hunhoff@mandiant.com
+- nursery/obfuscated-with-koivm michael.hunhoff@mandiant.com
+- nursery/implement-com-dll moritz.raabe@mandiant.com
+- nursery/linked-against-libsodium @mr-tz
+- compiler/nuitka/compiled-with-nuitka @williballenthin
+- nursery/authenticate-data-with-md5-mac william.ballenthin@mandiant.com
+- nursery/resolve-function-by-djb2-hash still@teamt5.org
+- host-interaction/mutex/create-semaphore-on-linux @ramen0x3f
+- host-interaction/mutex/lock-semaphore-on-linux @ramen0x3f
+- host-interaction/mutex/unlock-semaphore-on-linux @ramen0x3f
+- data-manipulation/hashing/sha384/hash-data-using-sha384 william.ballenthin@mandiant.com
+- data-manipulation/hashing/sha512/hash-data-using-sha512 william.ballenthin@mandiant.com
+- nursery/decode-data-using-url-encoding michael.hunhoff@mandiant.com
+- nursery/manipulate-user-privileges michael.hunhoff@mandiant.com
+- lib/get-os-version @mr-tz
+- nursery/decrypt-data-using-tea william.ballenthin@mandiant.com
+- nursery/encrypt-data-using-tea william.ballenthin@mandiant.com
+- nursery/hash-data-using-whirlpool william.ballenthin@mandiant.com
+- nursery/reference-base58-string william.ballenthin@mandiant.com
+- communication/mailslot/create-mailslot william.ballenthin@mandiant.com
+- executable/resource/access-dotnet-resource @mr-tz
+- linking/static/linked-against-cpp-standard-library @mr-tz
+- data-manipulation/compression/compress-data-using-lzo david@edeca.net david.cannings@pwc.com
+- data-manipulation/compression/decompress-data-using-lzo david@edeca.net david.cannings@pwc.com
+- communication/socket/tcp/create-tcp-socket-via-raw-afd-driver william.ballenthin@mandiant.com
+- host-interaction/process/map-section-object william.ballenthin@mandiant.com
+- lib/create-or-open-section-object william.ballenthin@mandiant.com
+- load-code/dotnet/execute-dotnet-assembly-via-clr-host blas.kojusner@mandiant.com
+- load-code/execute-vbscript-javascript-or-jscript-in-memory blas.kojusner@mandiant.com
+- host-interaction/file-system/reference-absolute-stream-path-on-windows blas.kojusner@mandiant.com
+- nursery/generate-method-via-reflection-in-dotnet michael.hunhoff@mandiant.com
+- nursery/unmanaged-call-via-dynamic-pinvoke-in-dotnet michael.hunhoff@mandiant.com
+
+### Bug Fixes
+- render: convert feature attributes to aliased dictionary for vverbose #1152 @mike-hunhoff
+- decouple Token dependency / extractor and features #1139 @mr-tz
+- update pydantic model to guarantee type coercion #1176 @mike-hunhoff
+- do not overwrite version in version.py during PyInstaller build #1169 @mr-tz
+- render: fix vverbose rendering of offsets #1215 @williballenthin
+- elf: better detect OS via GLIBC ABI version needed and dependencies #1221 @williballenthin
+- dotnet: address unhandled exceptions with improved type checking #1230 @mike-hunhoff
+- fix import-to-ida script formatting #1208 @williballenthin
+- render: fix verbose rendering of scopes #1263 @williballenthin
+- rules: better detect invalid rules #1282 @williballenthin
+- show-features: better render strings with embedded whitespace #1267 @williballenthin
+- handle vivisect bug around strings at instruction level, use min length 4 #1271 @williballenthin @mr-tz
+- extractor: guard against invalid "calls from" features #1177 @mr-tz
+- extractor: add format to global features #1258 @mr-tz
+- extractor: discover all strings with length >= 4 #1280 @mr-tz
+- extractor: don't extract byte features for strings #1293 @mr-tz
+
+### capa explorer IDA Pro plugin
+- fix: display instruction items #1154 @mr-tz
+- fix: accept only plaintext pasted content #1194 @williballenthin
+- fix: UnboundLocalError #1217 @williballenthin
+- extractor: add support for COFF files and extern functions #1223 @mike-hunhoff
+- doc: improve error messaging and documentation related to capa rule set #1249 @mike-hunhoff
+- fix: assume 32-bit displacement for offsets #1250 @mike-hunhoff
+- generator: refactor caching and matching #1251 @mike-hunhoff
+- fix: improve exception handling to prevent IDA from locking up when errors occur #1262 @mike-hunhoff
+- verify rule metadata using Pydantic #1167 @mr-tz
+- extractor: make read consistent with file object behavior #1254 @mr-tz
+- fix: UnboundLocalError x2 #1302 @mike-hunhoff
+- cache capa results across IDA sessions #1279 @mr-tz
+
+### Raw diffs
+- [capa v4.0.1...v5.0.0](https://github.com/mandiant/capa/compare/v4.0.1...v5.0.0)
+- [capa-rules v4.0.1...v5.0.0](https://github.com/mandiant/capa-rules/compare/v4.0.1...v5.0.0)
+
+
+## v4.0.1 (2022-08-15)
+Some rules contained invalid metadata fields that caused an error when rendering rule hits. We've updated all rules and enhanced the rule linter to catch such issues.
+
+### New Rules (1)
+
+- anti-analysis/obfuscation/obfuscated-with-vs-obfuscation jakub.jozwiak@mandiant.com
+
+
+### Bug Fixes
+- linter: use pydantic to validate rule metadata #1141 @mike-hunhoff
+- build binaries using PyInstaller no longer overwrites functions in version.py #1136 @mr-tz
+
+### Raw diffs
+- [capa v4.0.0...v4.0.1](https://github.com/mandiant/capa/compare/v4.0.0...v4.0.1)
+- [capa-rules v4.0.0...v4.0.1](https://github.com/mandiant/capa-rules/compare/v4.0.0...v4.0.1)
+
+## v4.0.0 (2022-08-10)
+Version 4 adds support for analyzing .NET executables. capa will autodetect .NET modules, or you can explicitly invoke the new feature extractor via `--format dotnet`. We've also extended the rule syntax for .NET features including `namespace` and `class`.
+
+Additionally, new `instruction` scope and `operand` features enable users to create more explicit rules. These features are not backwards compatible. We removed the previously used `/x32` and `/x64` flavors of number and operand features.
+
+We updated 49 existing rules and added 22 new rules leveraging these new features and characteristics to detect capabilities seen in .NET malware.
+
+More breaking changes include updates to the JSON results document, freeze file format schema (now format version v2), and the internal handling of addresses.
+
+Thanks for all the support, especially to @htnhan, @jtothej, @sara-rn, @anushkavirgaonkar, and @_re_fox!
+
+*Deprecation warning: v4.0 will be the last capa version to support the SMDA backend.*
+
+### New Features
+
+ - add new scope "instruction" for matching mnemonics and operands #767 @williballenthin
+ - add new feature "operand[{0, 1, 2}].number" for matching instruction operand immediate values #767 @williballenthin
+ - add new feature "operand[{0, 1, 2}].offset" for matching instruction operand offsets #767 @williballenthin
+ - extract additional offset/number features in certain circumstances #320 @williballenthin
+ - add detection and basic feature extraction for dotnet #987 @mr-tz, @mike-hunhoff, @williballenthin
+ - add file string extraction for dotnet files #1012 @mike-hunhoff
+ - add file function-name extraction for dotnet files #1015 @mike-hunhoff
+ - add unmanaged call characteristic for dotnet files #1023 @mike-hunhoff
+ - add mixed mode characteristic feature extraction for dotnet files #1024 @mike-hunhoff
+ - emit class and namespace features for dotnet files #1030 @mike-hunhoff
+ - render: support Addresses that aren't simple integers, like .NET token+offset #981 @williballenthin
+ - document rule tags and branches #1006 @williballenthin, @mr-tz
+
+### Breaking Changes
+
+  - instruction scope and operand feature are new and are not backwards compatible with older versions of capa
+  - Python 3.7 is now the minimum supported Python version #866 @williballenthin
+  - remove /x32 and /x64 flavors of number and operand features #932 @williballenthin
+  - the tool now accepts multiple paths to rules, and JSON doc updated accordingly @williballenthin
+  - extractors must use handles to identify functions/basic blocks/instructions #981 @williballenthin
+  - the freeze file format schema was updated, including format version bump to v2 #986 @williballenthin
+
+Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issues/937), we plan to remove the SMDA backend for v5. If you rely on this backend, please reach out so we can discuss extending the support for SMDA or transitioning your workflow to use vivisect.
+
+### New Rules (30)
+
+- data-manipulation/encryption/aes/manually-build-aes-constants huynh.t.nhan@gmail.com
+- nursery/get-process-image-filename michael.hunhoff@mandiant.com
+- compiler/v/compiled-with-v jakub.jozwiak@mandiant.com
+- compiler/zig/compiled-with-zig jakub.jozwiak@mandiant.com
+- anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com
+- internal/limitation/file/internal-dotnet-file-limitation william.ballenthin@mandiant.com
+- nursery/get-os-information-via-kuser_shared_data @mr-tz
+- load-code/pe/resolve-function-by-parsing-PE-exports @sara-rn
+- anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com
+- nursery/execute-dotnet-assembly anushka.virgaonkar@mandiant.com
+- nursery/invoke-dotnet-assembly-method anushka.virgaonkar@mandiant.com
+- collection/screenshot/capture-screenshot-via-keybd-event @_re_fox
+- collection/browser/gather-chrome-based-browser-login-information @_re_fox
+- nursery/power-down-monitor michael.hunhoff@mandiant.com
+- nursery/hash-data-using-aphash @_re_fox
+- nursery/hash-data-using-jshash @_re_fox
+- host-interaction/file-system/files/list/enumerate-files-on-windows moritz.raabe@mandiant.com anushka.virgaonkar@mandiant.com
+- nursery/check-clipboard-data anushka.virgaonkar@mandiant.com
+- nursery/clear-clipboard-data anushka.virgaonkar@mandiant.com
+- nursery/compile-dotnet-assembly anushka.virgaonkar@mandiant.com
+- nursery/create-process-via-wmi anushka.virgaonkar@mandiant.com
+- nursery/display-service-notification-message-box anushka.virgaonkar@mandiant.com
+- nursery/find-process-by-name anushka.virgaonkar@mandiant.com
+- nursery/generate-random-numbers-in-dotnet anushka.virgaonkar@mandiant.com
+- nursery/send-keystrokes anushka.virgaonkar@mandiant.com
+- nursery/send-request-in-dotnet anushka.virgaonakr@mandiant.com
+- nursery/terminate-process-by-name-in-dotnet anushka.virgaonkar@mandiant.com
+- nursery/hash-data-using-rshash @_re_fox
+- persistence/authentication-process/act-as-credential-manager-dll jakub.jozwiak@mandiant.com
+- persistence/authentication-process/act-as-password-filter-dll jakub.jozwiak@mandiant.com
+
+### Bug Fixes
+- improve handling _ prefix compile/link artifact #924 @mike-hunhoff
+- better detect OS in ELF samples #988 @williballenthin
+- display number feature zero in vverbose #1097 @mike-hunhoff
+
+### capa explorer IDA Pro plugin
+- improve file format extraction #918 @mike-hunhoff
+- remove decorators added by IDA to ELF imports #919 @mike-hunhoff
+- bug fixes for Address abstraction #1091 @mike-hunhoff
+
+### Development
+
+### Raw diffs
+- [capa v3.2.0...v4.0.0](https://github.com/mandiant/capa/compare/v3.2.0...master)
+- [capa-rules v3.2.0...v4.0.0](https://github.com/mandiant/capa-rules/compare/v3.2.0...master)
+
+## v3.2.1 (2022-06-06)
+This out-of-band release bumps the SMDA dependency version to enable installation on Python 3.10.
+
+### Bug Fixes
+
+- update SMDA dependency @mike-hunhoff #922
+
+### Raw diffs
+- [capa v3.2.0...v3.2.1](https://github.com/mandiant/capa/compare/v3.2.0...v3.2.1)
+- [capa-rules v3.2.0...v3.2.1](https://github.com/mandiant/capa-rules/compare/v3.2.0...v3.2.1)
+
+## v3.2.0 (2022-03-03)
+This release adds a new characteristic `characteristic: call $+5` enabling users to create more explicit rules. The linter now also validates ATT&CK and MBC categories. Additionally, many dependencies, including the vivisect backend, have been updated.
+
+One rule has been added and many more have been improved.
+
+Thanks for all the support, especially to @kn0wl3dge and first time contributor @uckelman-sf!
+
+### New Features
+
+- linter: validate ATT&CK/MBC categories and IDs #103 @kn0wl3dge
+- extractor: add characteristic "call $+5" feature #366 @kn0wl3dge
+
+### New Rules (1)
+
+- anti-analysis/obfuscation/obfuscated-with-advobfuscator jakub.jozwiak@mandiant.com
+
+### Bug Fixes
+
+- remove typing package as a requirement for Python 3.7+ compatibility #901 @uckelman-sf
+- elf: fix OS detection for Linux kernel modules #867 @williballenthin
+
+### Raw diffs
+- [capa v3.1.0...v3.2.0](https://github.com/mandiant/capa/compare/v3.1.0...v3.2.0)
+- [capa-rules v3.1.0...v3.2.0](https://github.com/mandiant/capa-rules/compare/v3.1.0...v3.2.0)
 
 ## v3.1.0 (2022-01-10)
 This release improves the performance of capa while also adding 23 new rules and many code quality enhancements. We profiled capa's CPU usage and optimized the way that it matches rules, such as by short circuiting when appropriate. According to our testing, the matching phase is approximately 66% faster than v3.0.3! We also added support for Python 3.10, aarch64 builds, and additional MAEC metadata in the rule headers.
@@ -878,7 +1647,7 @@ The IDA Pro integration is now distributed as a real plugin, instead of a script
   - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%`
   - generally doing thing the "right way"
 
-How to get this new version? Its easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. 
+How to get this new version? It's easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. 
 
 Please refer to the plugin [readme](https://github.com/mandiant/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin.
 
@@ -1103,7 +1872,7 @@ Download a standalone binary below and checkout the readme [here on GitHub](http
   - setup: pin vivisect version @williballenthin
   - setup: bump vivisect dependency version @williballenthin
   - setup: set Python project name to `flare-capa` @williballenthin
-  - ci: run tests and linter via Github Actions @Ana06
+  - ci: run tests and linter via GitHub Actions @Ana06
   - hooks: run style checkers and hide stashed output @Ana06
   - linter: ignore period in rule filename @williballenthin
   - linter: warn on nursery rule with no changes needed @williballenthin

CITATION.cff

@@ -0,0 +1,8 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - name: "The FLARE Team"
+title: "capa, a tool to identify capabilities in programs and sandbox traces."
+date-released: 2020-07-16
+url: "https://github.com/mandiant/capa"
+

README.md

@@ -1,21 +1,40 @@
-![capa](https://github.com/mandiant/capa/blob/master/.github/logo.png)
+<br />
+<div align="center">
+<a href="https://mandiant.github.io/capa/" target="_blank">
+  <img src="https://github.com/mandiant/capa/blob/master/.github/logo.png">
+</a>
+<p align="center">
+  <a href="https://mandiant.github.io/capa/" target="_blank">Website</a>
+  |
+  <a href="https://github.com/mandiant/capa/releases/latest" target="_blank">Download</a>
+  |
+  <a href="https://mandiant.github.io/capa/explorer/" target="_blank">Web Interface</a>
+</p>
+<div align="center">
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-658-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://gist.githubusercontent.com/capa-bot/6d7960e911f48b3b74916df8988cf0f3/raw/rules_badge.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
+</div>
+</div>
+
+---
+
 capa detects capabilities in executable files.
-You run it against a PE, ELF, or shellcode file and it tells you what it thinks the program can do.
+You run it against a PE, ELF, .NET module, shellcode file, or a sandbox report and it tells you what it thinks the program can do.
 For example, it might suggest that the file is a backdoor, is capable of installing services, or relies on HTTP to communicate.
 
-Check out:
-- the overview in our first [capa blog post](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html)
-- the major version 2.0 updates described in our [second blog post](https://www.fireeye.com/blog/threat-research/2021/07/capa-2-better-stronger-faster.html)
-- the major version 3.0 (ELF support) described in the [third blog post](https://www.fireeye.com/blog/threat-research/2021/09/elfant-in-the-room-capa-v3.html)
+To interactively inspect capa results in your browser use the [capa Explorer Web](https://mandiant.github.io/capa/explorer/).
 
+If you want to inspect or write capa rules, head on over to the [capa-rules repository](https://github.com/mandiant/capa-rules). Otherwise, keep reading.
+
+Below you find a list of [our capa blog posts with more details.](#blog-posts)
+
+# example capa output
 ```
 $ capa.exe suspicious.exe
 
@@ -70,16 +89,23 @@ Download stable releases of the standalone capa binaries [here](https://github.c
 
 To use capa as a library or integrate with another tool, see [doc/installation.md](https://github.com/mandiant/capa/blob/master/doc/installation.md) for further setup instructions.
 
-For more information about how to use capa, see [doc/usage.md](https://github.com/mandiant/capa/blob/master/doc/usage.md).
+# capa Explorer Web
+The [capa Explorer Web](https://mandiant.github.io/capa/explorer/) enables you to interactively explore capa results in your web browser. Besides the online version you can download a standalone HTML file for local offline usage.
+
+![capa Explorer Web screenshot](https://github.com/mandiant/capa/blob/master/doc/img/capa_web_explorer.png)
+
+More details on the web UI is available in the [capa Explorer Web README](https://github.com/mandiant/capa/blob/master/web/explorer/README.md).
 
 # example
 
-In the above sample output, we ran capa against an unknown binary (`suspicious.exe`),
-and the tool reported that the program can send HTTP requests, decode data via XOR and Base64,
+In the above sample output, we run capa against an unknown binary (`suspicious.exe`),
+and the tool reports that the program can send HTTP requests, decode data via XOR and Base64,
 install services, and spawn new processes.
 Taken together, this makes us think that `suspicious.exe` could be a persistent backdoor.
 Therefore, our next analysis step might be to run `suspicious.exe` in a sandbox and try to recover the command and control server.
 
+## detailed results
+
 By passing the `-vv` flag (for very verbose), capa reports exactly where it found evidence of these capabilities.
 This is useful for at least two reasons:
 
@@ -95,26 +121,131 @@ author      matthew.williams@mandiant.com
 scope       function
 att&ck      Execution::Command and Scripting Interpreter::Windows Command Shell [T1059.003]
 references  https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/ns-processthreadsapi-startupinfoa
-examples    Practical Malware Analysis Lab 14-02.exe_:0x4011C0
-function @ 0x10003A13
+function @ 0x4011C0
   and:
-    match: create a process with modified I/O handles and window @ 0x10003A13
+    match: create a process with modified I/O handles and window @ 0x4011C0
       and:
+        number: 257 = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW @ 0x4012B8
         or:
-          api: kernel32.CreateProcess @ 0x10003D6D
-        number: 0x101 @ 0x10003B03
-        or:
-          number: 0x44 @ 0x10003ADC
-        optional:
-          api: kernel32.GetStartupInfo @ 0x10003AE4
-    match: create pipe @ 0x10003A13
+          number: 68 = StartupInfo.cb (size) @ 0x401282
+        or: = API functions that accept a pointer to a STARTUPINFO structure
+          api: kernel32.CreateProcess @ 0x401343
+    match: create pipe @ 0x4011C0
       or:
-        api: kernel32.CreatePipe @ 0x10003ACB
+        api: kernel32.CreatePipe @ 0x40126F, 0x401280
+    optional:
+      match: create thread @ 0x40136A, 0x4013BA
+        or:
+          and:
+            os: windows
+            or:
+              api: kernel32.CreateThread @ 0x4013D7
+        or:
+          and:
+            os: windows
+            or:
+              api: kernel32.CreateThread @ 0x401395
     or:
-      string: cmd.exe /c  @ 0x10003AED
+      string: "cmd.exe" @ 0x4012FD
 ...
 ```
 
+capa also supports dynamic capabilities detection for multiple sandboxes including:
+* [CAPE](https://github.com/kevoreilly/CAPEv2) (supported report formats: `.json`, `.json_`, `.json.gz`)
+* [DRAKVUF](https://github.com/CERT-Polska/drakvuf-sandbox/) (supported report formats: `.log`, `.log.gz`)
+* [VMRay](https://www.vmray.com/) (supported report formats: analysis archive `.zip`)
+
+
+To use this feature, submit your file to a supported sandbox and then download and run capa against the generated report file. This feature enables capa to match capabilities against dynamic and static features that the sandbox captured during execution.
+
+Here's an example of running capa against a packed file, and then running capa against the CAPE report generated for the same packed file:
+
+```yaml
+$ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.exe
+WARNING:capa.capabilities.common:--------------------------------------------------------------------------------
+WARNING:capa.capabilities.common: This sample appears to be packed.
+WARNING:capa.capabilities.common: 
+WARNING:capa.capabilities.common: Packed samples have often been obfuscated to hide their logic.
+WARNING:capa.capabilities.common: capa cannot handle obfuscation well using static analysis. This means the results may be misleading or incomplete.
+WARNING:capa.capabilities.common: If possible, you should try to unpack this input file before analyzing it with capa.
+WARNING:capa.capabilities.common: Alternatively, run the sample in a supported sandbox and invoke capa against the report to obtain dynamic analysis results.
+WARNING:capa.capabilities.common: 
+WARNING:capa.capabilities.common: Identified via rule: (internal) packer file limitation
+WARNING:capa.capabilities.common: 
+WARNING:capa.capabilities.common: Use -v or -vv if you really want to see the capabilities identified by capa.
+WARNING:capa.capabilities.common:--------------------------------------------------------------------------------
+
+$ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.json
+
+┍━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
+│ ATT&CK Tactic          │ ATT&CK Technique                                                                   │
+┝━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
+│ CREDENTIAL ACCESS      │ Credentials from Password Stores T1555                                             │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ DEFENSE EVASION        │ File and Directory Permissions Modification T1222                                  │
+│                        │ Modify Registry T1112                                                              │
+│                        │ Obfuscated Files or Information T1027                                              │
+│                        │ Virtualization/Sandbox Evasion::User Activity Based Checks T1497.002               │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ DISCOVERY              │ Account Discovery T1087                                                            │
+│                        │ Application Window Discovery T1010                                                 │
+│                        │ File and Directory Discovery T1083                                                 │
+│                        │ Query Registry T1012                                                               │
+│                        │ System Information Discovery T1082                                                 │
+│                        │ System Location Discovery::System Language Discovery T1614.001                     │
+│                        │ System Owner/User Discovery T1033                                                  │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ EXECUTION              │ System Services::Service Execution T1569.002                                       │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ PERSISTENCE            │ Boot or Logon Autostart Execution::Registry Run Keys / Startup Folder T1547.001    │
+│                        │ Boot or Logon Autostart Execution::Winlogon Helper DLL T1547.004                   │
+│                        │ Create or Modify System Process::Windows Service T1543.003                         │
+┕━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
+
+┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
+│ Capability                                           │ Namespace                                            │
+┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
+│ check for unmoving mouse cursor (3 matches)          │ anti-analysis/anti-vm/vm-detection                   │
+│ gather bitkinex information                          │ collection/file-managers                             │
+│ gather classicftp information                        │ collection/file-managers                             │
+│ gather filezilla information                         │ collection/file-managers                             │
+│ gather total-commander information                   │ collection/file-managers                             │
+│ gather ultrafxp information                          │ collection/file-managers                             │
+│ resolve DNS (23 matches)                             │ communication/dns                                    │
+│ initialize Winsock library (7 matches)               │ communication/socket                                 │
+│ act as TCP client (3 matches)                        │ communication/tcp/client                             │
+│ create new key via CryptAcquireContext               │ data-manipulation/encryption                         │
+│ encrypt or decrypt via WinCrypt                      │ data-manipulation/encryption                         │
+│ hash data via WinCrypt                               │ data-manipulation/hashing                            │
+│ initialize hashing via WinCrypt                      │ data-manipulation/hashing                            │
+│ hash data with MD5                                   │ data-manipulation/hashing/md5                        │
+│ generate random numbers via WinAPI                   │ data-manipulation/prng                               │
+│ extract resource via kernel32 functions (2 matches)  │ executable/resource                                  │
+│ interact with driver via control codes (2 matches)   │ host-interaction/driver                              │
+│ get Program Files directory (18 matches)             │ host-interaction/file-system                         │
+│ get common file path (575 matches)                   │ host-interaction/file-system                         │
+│ create directory (2 matches)                         │ host-interaction/file-system/create                  │
+│ delete file                                          │ host-interaction/file-system/delete                  │
+│ get file attributes (122 matches)                    │ host-interaction/file-system/meta                    │
+│ set file attributes (8 matches)                      │ host-interaction/file-system/meta                    │
+│ move file                                            │ host-interaction/file-system/move                    │
+│ find taskbar (3 matches)                             │ host-interaction/gui/taskbar/find                    │
+│ get keyboard layout (12 matches)                     │ host-interaction/hardware/keyboard                   │
+│ get disk size                                        │ host-interaction/hardware/storage                    │
+│ get hostname (4 matches)                             │ host-interaction/os/hostname                         │
+│ allocate or change RWX memory (3 matches)            │ host-interaction/process/inject                      │
+│ query or enumerate registry key (3 matches)          │ host-interaction/registry                            │
+│ query or enumerate registry value (8 matches)        │ host-interaction/registry                            │
+│ delete registry key                                  │ host-interaction/registry/delete                     │
+│ start service                                        │ host-interaction/service/start                       │
+│ get session user name                                │ host-interaction/session                             │
+│ persist via Run registry key                         │ persistence/registry/run                             │
+│ persist via Winlogon Helper DLL registry key         │ persistence/registry/winlogon-helper                 │
+│ persist via Windows service (2 matches)              │ persistence/service                                  │
+┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
+```
+
+# capa rules
 capa uses a collection of rules to identify capabilities within a program.
 These rules are easy to write, even for those new to reverse engineering.
 By authoring rules, you can extend the capabilities that capa recognizes.
@@ -125,31 +256,54 @@ Here's an example rule used by capa:
 ```yaml
 rule:
   meta:
-    name: hash data with CRC32
-    namespace: data-manipulation/checksum/crc32
-    author: moritz.raabe@mandiant.com
-    scope: function
+    name: create TCP socket
+    namespace: communication/socket/tcp
+    authors:
+      - william.ballenthin@mandiant.com
+      - joakim@intezer.com
+      - anushka.virgaonkar@mandiant.com
+    scopes:
+      static: basic block
+      dynamic: call
+    mbc:
+      - Communication::Socket Communication::Create TCP Socket [C0001.011]
     examples:
-      - 2D3EDC218A90F03089CC01715A9F047F:0x403CBD
-      - 7D28CB106CB54876B2A5C111724A07CD:0x402350  # RtlComputeCrc32
+      - Practical Malware Analysis Lab 01-01.dll_:0x10001010
   features:
     - or:
       - and:
-        - mnemonic: shr
-        - number: 0xEDB88320
-        - number: 8
-        - characteristic: nzxor
-      - api: RtlComputeCrc32
+        - number: 6 = IPPROTO_TCP
+        - number: 1 = SOCK_STREAM
+        - number: 2 = AF_INET
+        - or:
+          - api: ws2_32.socket
+          - api: ws2_32.WSASocket
+          - api: socket
+      - property/read: System.Net.Sockets.TcpClient::Client
 ```
 
-The [github.com/mandiant/capa-rules](https://github.com/mandiant/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
+The [github.com/mandiant/capa-rules](https://github.com/mandiant/capa-rules) repository contains hundreds of standard rules that are distributed with capa.
 Please learn to write rules and contribute new entries as you find interesting techniques in malware.
 
+# IDA Pro plugin: capa explorer
 If you use IDA Pro, then you can use the [capa explorer](https://github.com/mandiant/capa/tree/master/capa/ida/plugin) plugin.
 capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted directly from your IDA Pro database.
+It also uses your local changes to the .idb to extract better features, such as when you rename a global variable that contains a dynamically resolved API address.
 
 ![capa + IDA Pro integration](https://github.com/mandiant/capa/blob/master/doc/img/explorer_expanded.png)
 
+# Ghidra integration
+If you use Ghidra, then you can use the [capa + Ghidra integration](/capa/ghidra/) to run capa's analysis directly on your Ghidra database and render the results in Ghidra's user interface.
+
+<img src="https://github.com/mandiant/capa/assets/66766340/eeae33f4-99d4-42dc-a5e8-4c1b8c661492" width=300>
+
+# blog posts
+- [Dynamic capa: Exploring Executable Run-Time Behavior with the CAPE Sandbox](https://www.mandiant.com/resources/blog/dynamic-capa-executable-behavior-cape-sandbox)
+- [capa v4: casting a wider .NET](https://www.mandiant.com/resources/blog/capa-v4-casting-wider-net) (.NET support)
+- [ELFant in the Room – capa v3](https://www.mandiant.com/resources/elfant-in-the-room-capa-v3) (ELF support)
+- [capa 2.0: Better, Stronger, Faster](https://www.mandiant.com/resources/capa-2-better-stronger-faster)
+- [capa: Automatically Identify Malware Capabilities](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities)
+
 # further information
 ## capa
 - [Installation](https://github.com/mandiant/capa/blob/master/doc/installation.md)

capa/capabilities/common.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+import itertools
+import collections
+from typing import Any, Tuple
+
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.features.address import NO_ADDRESS
+from capa.features.extractors.base_extractor import FeatureExtractor, StaticFeatureExtractor, DynamicFeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def find_file_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, function_features: FeatureSet):
+    file_features: FeatureSet = collections.defaultdict(set)
+
+    for feature, va in itertools.chain(extractor.extract_file_features(), extractor.extract_global_features()):
+        # not all file features may have virtual addresses.
+        # if not, then at least ensure the feature shows up in the index.
+        # the set of addresses will still be empty.
+        if va:
+            file_features[feature].add(va)
+        else:
+            if feature not in file_features:
+                file_features[feature] = set()
+
+    logger.debug("analyzed file and extracted %d features", len(file_features))
+
+    file_features.update(function_features)
+
+    _, matches = ruleset.match(Scope.FILE, file_features, NO_ADDRESS)
+    return matches, len(file_features)
+
+
+def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalone=True) -> bool:
+    file_limitation_rules = list(filter(lambda r: r.is_file_limitation_rule(), rules.rules.values()))
+
+    for file_limitation_rule in file_limitation_rules:
+        if file_limitation_rule.name not in capabilities:
+            continue
+
+        logger.warning("-" * 80)
+        for line in file_limitation_rule.meta.get("description", "").split("\n"):
+            logger.warning(" %s", line)
+        logger.warning(" Identified via rule: %s", file_limitation_rule.name)
+        if is_standalone:
+            logger.warning(" ")
+            logger.warning(" Use -v or -vv if you really want to see the capabilities identified by capa.")
+        logger.warning("-" * 80)
+
+        # bail on first file limitation
+        return True
+
+    return False
+
+
+def find_capabilities(
+    ruleset: RuleSet, extractor: FeatureExtractor, disable_progress=None, **kwargs
+) -> Tuple[MatchResults, Any]:
+    from capa.capabilities.static import find_static_capabilities
+    from capa.capabilities.dynamic import find_dynamic_capabilities
+
+    if isinstance(extractor, StaticFeatureExtractor):
+        # for the time being, extractors are either static or dynamic.
+        # Remove this assertion once that has changed
+        assert not isinstance(extractor, DynamicFeatureExtractor)
+        return find_static_capabilities(ruleset, extractor, disable_progress=disable_progress, **kwargs)
+    if isinstance(extractor, DynamicFeatureExtractor):
+        return find_dynamic_capabilities(ruleset, extractor, disable_progress=disable_progress, **kwargs)
+
+    raise ValueError(f"unexpected extractor type: {extractor.__class__.__name__}")

capa/capabilities/dynamic.py

@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import sys
+import logging
+import itertools
+import collections
+from typing import Any, Tuple
+
+import tqdm
+
+import capa.perf
+import capa.features.freeze as frz
+import capa.render.result_document as rdoc
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.helpers import redirecting_print_to_tqdm
+from capa.capabilities.common import find_file_capabilities
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle, DynamicFeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def find_call_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+) -> Tuple[FeatureSet, MatchResults]:
+    """
+    find matches for the given rules for the given call.
+
+    returns: tuple containing (features for call, match results for call)
+    """
+    # all features found for the call.
+    features: FeatureSet = collections.defaultdict(set)
+
+    for feature, addr in itertools.chain(
+        extractor.extract_call_features(ph, th, ch), extractor.extract_global_features()
+    ):
+        features[feature].add(addr)
+
+    # matches found at this thread.
+    _, matches = ruleset.match(Scope.CALL, features, ch.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for addr, _ in res:
+            capa.engine.index_rule_matches(features, rule, [addr])
+
+    return features, matches
+
+
+def find_thread_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle
+) -> Tuple[FeatureSet, MatchResults, MatchResults]:
+    """
+    find matches for the given rules within the given thread.
+
+    returns: tuple containing (features for thread, match results for thread, match results for calls)
+    """
+    # all features found within this thread,
+    # includes features found within calls.
+    features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the call scope.
+    # might be found at different calls, that's ok.
+    call_matches: MatchResults = collections.defaultdict(list)
+
+    for ch in extractor.get_calls(ph, th):
+        ifeatures, imatches = find_call_capabilities(ruleset, extractor, ph, th, ch)
+        for feature, vas in ifeatures.items():
+            features[feature].update(vas)
+
+        for rule_name, res in imatches.items():
+            call_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(extractor.extract_thread_features(ph, th), extractor.extract_global_features()):
+        features[feature].add(va)
+
+    # matches found within this thread.
+    _, matches = ruleset.match(Scope.THREAD, features, th.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for va, _ in res:
+            capa.engine.index_rule_matches(features, rule, [va])
+
+    return features, matches, call_matches
+
+
+def find_process_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle
+) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
+    """
+    find matches for the given rules within the given process.
+
+    returns: tuple containing (match results for process, match results for threads, match results for calls, number of features)
+    """
+    # all features found within this process,
+    # includes features found within threads (and calls).
+    process_features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the basic threads.
+    # might be found at different threads, that's ok.
+    thread_matches: MatchResults = collections.defaultdict(list)
+
+    # matches found at the call scope.
+    # might be found at different calls, that's ok.
+    call_matches: MatchResults = collections.defaultdict(list)
+
+    for th in extractor.get_threads(ph):
+        features, tmatches, cmatches = find_thread_capabilities(ruleset, extractor, ph, th)
+        for feature, vas in features.items():
+            process_features[feature].update(vas)
+
+        for rule_name, res in tmatches.items():
+            thread_matches[rule_name].extend(res)
+
+        for rule_name, res in cmatches.items():
+            call_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(extractor.extract_process_features(ph), extractor.extract_global_features()):
+        process_features[feature].add(va)
+
+    _, process_matches = ruleset.match(Scope.PROCESS, process_features, ph.address)
+    return process_matches, thread_matches, call_matches, len(process_features)
+
+
+def find_dynamic_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, disable_progress=None
+) -> Tuple[MatchResults, Any]:
+    all_process_matches: MatchResults = collections.defaultdict(list)
+    all_thread_matches: MatchResults = collections.defaultdict(list)
+    all_call_matches: MatchResults = collections.defaultdict(list)
+
+    feature_counts = rdoc.DynamicFeatureCounts(file=0, processes=())
+
+    assert isinstance(extractor, DynamicFeatureExtractor)
+    with redirecting_print_to_tqdm(disable_progress):
+        with tqdm.contrib.logging.logging_redirect_tqdm():
+            pbar = tqdm.tqdm
+            if disable_progress:
+                # do not use tqdm to avoid unnecessary side effects when caller intends
+                # to disable progress completely
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            elif not sys.stderr.isatty():
+                # don't display progress bar when stderr is redirected to a file
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            processes = list(extractor.get_processes())
+
+            pb = pbar(processes, desc="matching", unit=" processes", leave=False)
+            for p in pb:
+                process_matches, thread_matches, call_matches, feature_count = find_process_capabilities(
+                    ruleset, extractor, p
+                )
+                feature_counts.processes += (
+                    rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count),
+                )
+                logger.debug("analyzed %s and extracted %d features", p.address, feature_count)
+
+                for rule_name, res in process_matches.items():
+                    all_process_matches[rule_name].extend(res)
+                for rule_name, res in thread_matches.items():
+                    all_thread_matches[rule_name].extend(res)
+                for rule_name, res in call_matches.items():
+                    all_call_matches[rule_name].extend(res)
+
+    # collection of features that captures the rule matches within process and thread scopes.
+    # mapping from feature (matched rule) to set of addresses at which it matched.
+    process_and_lower_features: FeatureSet = collections.defaultdict(set)
+    for rule_name, results in itertools.chain(
+        all_process_matches.items(), all_thread_matches.items(), all_call_matches.items()
+    ):
+        locations = {p[0] for p in results}
+        rule = ruleset[rule_name]
+        capa.engine.index_rule_matches(process_and_lower_features, rule, locations)
+
+    all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, process_and_lower_features)
+    feature_counts.file = feature_count
+
+    matches = dict(
+        itertools.chain(
+            # each rule exists in exactly one scope,
+            # so there won't be any overlap among these following MatchResults,
+            # and we can merge the dictionaries naively.
+            all_thread_matches.items(),
+            all_process_matches.items(),
+            all_call_matches.items(),
+            all_file_matches.items(),
+        )
+    )
+
+    meta = {
+        "feature_counts": feature_counts,
+    }
+
+    return matches, meta

capa/capabilities/static.py

@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import sys
+import time
+import logging
+import itertools
+import collections
+from typing import Any, Tuple
+
+import tqdm.contrib.logging
+
+import capa.perf
+import capa.features.freeze as frz
+import capa.render.result_document as rdoc
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.helpers import redirecting_print_to_tqdm
+from capa.capabilities.common import find_file_capabilities
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, StaticFeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def find_instruction_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
+) -> Tuple[FeatureSet, MatchResults]:
+    """
+    find matches for the given rules for the given instruction.
+
+    returns: tuple containing (features for instruction, match results for instruction)
+    """
+    # all features found for the instruction.
+    features: FeatureSet = collections.defaultdict(set)
+
+    for feature, addr in itertools.chain(
+        extractor.extract_insn_features(f, bb, insn), extractor.extract_global_features()
+    ):
+        features[feature].add(addr)
+
+    # matches found at this instruction.
+    _, matches = ruleset.match(Scope.INSTRUCTION, features, insn.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for addr, _ in res:
+            capa.engine.index_rule_matches(features, rule, [addr])
+
+    return features, matches
+
+
+def find_basic_block_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle
+) -> Tuple[FeatureSet, MatchResults, MatchResults]:
+    """
+    find matches for the given rules within the given basic block.
+
+    returns: tuple containing (features for basic block, match results for basic block, match results for instructions)
+    """
+    # all features found within this basic block,
+    # includes features found within instructions.
+    features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the instruction scope.
+    # might be found at different instructions, that's ok.
+    insn_matches: MatchResults = collections.defaultdict(list)
+
+    for insn in extractor.get_instructions(f, bb):
+        ifeatures, imatches = find_instruction_capabilities(ruleset, extractor, f, bb, insn)
+        for feature, vas in ifeatures.items():
+            features[feature].update(vas)
+
+        for rule_name, res in imatches.items():
+            insn_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(
+        extractor.extract_basic_block_features(f, bb), extractor.extract_global_features()
+    ):
+        features[feature].add(va)
+
+    # matches found within this basic block.
+    _, matches = ruleset.match(Scope.BASIC_BLOCK, features, bb.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for va, _ in res:
+            capa.engine.index_rule_matches(features, rule, [va])
+
+    return features, matches, insn_matches
+
+
+def find_code_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, fh: FunctionHandle
+) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
+    """
+    find matches for the given rules within the given function.
+
+    returns: tuple containing (match results for function, match results for basic blocks, match results for instructions, number of features)
+    """
+    # all features found within this function,
+    # includes features found within basic blocks (and instructions).
+    function_features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the basic block scope.
+    # might be found at different basic blocks, that's ok.
+    bb_matches: MatchResults = collections.defaultdict(list)
+
+    # matches found at the instruction scope.
+    # might be found at different instructions, that's ok.
+    insn_matches: MatchResults = collections.defaultdict(list)
+
+    for bb in extractor.get_basic_blocks(fh):
+        features, bmatches, imatches = find_basic_block_capabilities(ruleset, extractor, fh, bb)
+        for feature, vas in features.items():
+            function_features[feature].update(vas)
+
+        for rule_name, res in bmatches.items():
+            bb_matches[rule_name].extend(res)
+
+        for rule_name, res in imatches.items():
+            insn_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(extractor.extract_function_features(fh), extractor.extract_global_features()):
+        function_features[feature].add(va)
+
+    _, function_matches = ruleset.match(Scope.FUNCTION, function_features, fh.address)
+    return function_matches, bb_matches, insn_matches, len(function_features)
+
+
+def find_static_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, disable_progress=None
+) -> Tuple[MatchResults, Any]:
+    all_function_matches: MatchResults = collections.defaultdict(list)
+    all_bb_matches: MatchResults = collections.defaultdict(list)
+    all_insn_matches: MatchResults = collections.defaultdict(list)
+
+    feature_counts = rdoc.StaticFeatureCounts(file=0, functions=())
+    library_functions: Tuple[rdoc.LibraryFunction, ...] = ()
+
+    assert isinstance(extractor, StaticFeatureExtractor)
+    with redirecting_print_to_tqdm(disable_progress):
+        with tqdm.contrib.logging.logging_redirect_tqdm():
+            pbar = tqdm.tqdm
+            if capa.helpers.is_runtime_ghidra():
+                # Ghidrathon interpreter cannot properly handle
+                # the TMonitor thread that is created via a monitor_interval
+                # > 0
+                pbar.monitor_interval = 0
+            if disable_progress:
+                # do not use tqdm to avoid unnecessary side effects when caller intends
+                # to disable progress completely
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            elif not sys.stderr.isatty():
+                # don't display progress bar when stderr is redirected to a file
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            functions = list(extractor.get_functions())
+            n_funcs = len(functions)
+
+            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False)
+            for f in pb:
+                t0 = time.time()
+                if extractor.is_library_function(f.address):
+                    function_name = extractor.get_function_name(f.address)
+                    logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
+                    library_functions += (
+                        rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name),
+                    )
+                    n_libs = len(library_functions)
+                    percentage = round(100 * (n_libs / n_funcs))
+                    if isinstance(pb, tqdm.tqdm):
+                        pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
+                    continue
+
+                function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(
+                    ruleset, extractor, f
+                )
+                feature_counts.functions += (
+                    rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count),
+                )
+                t1 = time.time()
+
+                match_count = 0
+                for name, matches_ in itertools.chain(
+                    function_matches.items(), bb_matches.items(), insn_matches.items()
+                ):
+                    # in practice, most matches are derived rules,
+                    # like "check OS version/5bf4c7f39fd4492cbed0f6dc7d596d49"
+                    # but when we log to the human, they really care about "real" rules.
+                    if not ruleset.rules[name].is_subscope_rule():
+                        match_count += len(matches_)
+
+                logger.debug(
+                    "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
+                    f.address,
+                    feature_count,
+                    match_count,
+                    t1 - t0,
+                )
+
+                for rule_name, res in function_matches.items():
+                    all_function_matches[rule_name].extend(res)
+                for rule_name, res in bb_matches.items():
+                    all_bb_matches[rule_name].extend(res)
+                for rule_name, res in insn_matches.items():
+                    all_insn_matches[rule_name].extend(res)
+
+    # collection of features that captures the rule matches within function, BB, and instruction scopes.
+    # mapping from feature (matched rule) to set of addresses at which it matched.
+    function_and_lower_features: FeatureSet = collections.defaultdict(set)
+    for rule_name, results in itertools.chain(
+        all_function_matches.items(), all_bb_matches.items(), all_insn_matches.items()
+    ):
+        locations = {p[0] for p in results}
+        rule = ruleset[rule_name]
+        capa.engine.index_rule_matches(function_and_lower_features, rule, locations)
+
+    all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, function_and_lower_features)
+    feature_counts.file = feature_count
+
+    matches: MatchResults = dict(
+        itertools.chain(
+            # each rule exists in exactly one scope,
+            # so there won't be any overlap among these following MatchResults,
+            # and we can merge the dictionaries naively.
+            all_insn_matches.items(),
+            all_bb_matches.items(),
+            all_function_matches.items(),
+            all_file_matches.items(),
+        )
+    )
+
+    meta = {
+        "feature_counts": feature_counts,
+        "library_functions": library_functions,
+    }
+
+    return matches, meta

capa/engine.py

@@ -8,11 +8,12 @@
 
 import copy
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Mapping, Iterable
+from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator
 
 import capa.perf
 import capa.features.common
 from capa.features.common import Result, Feature
+from capa.features.address import Address
 
 if TYPE_CHECKING:
     # circular import, otherwise
@@ -26,7 +27,7 @@ if TYPE_CHECKING:
 # to collect the locations of a feature, do: `features[Number(0x10)]`
 #
 # aliased here so that the type can be documented and xref'd.
-FeatureSet = Dict[Feature, Set[int]]
+FeatureSet = Dict[Feature, Set[Address]]
 
 
 class Statement:
@@ -37,15 +38,17 @@ class Statement:
     """
 
     def __init__(self, description=None):
-        super(Statement, self).__init__()
+        super().__init__()
         self.name = self.__class__.__name__
         self.description = description
 
     def __str__(self):
+        name = self.name.lower()
+        children = ",".join(map(str, self.get_children()))
         if self.description:
-            return "%s(%s = %s)" % (self.name.lower(), ",".join(map(str, self.get_children())), self.description)
+            return f"{name}({children} = {self.description})"
         else:
-            return "%s(%s)" % (self.name.lower(), ",".join(map(str, self.get_children())))
+            return f"{name}({children})"
 
     def __repr__(self):
         return str(self)
@@ -59,21 +62,28 @@ class Statement:
         """
         raise NotImplementedError()
 
-    def get_children(self):
+    def get_children(self) -> Iterator[Union["Statement", Feature]]:
         if hasattr(self, "child"):
-            yield self.child
+            # this really confuses mypy because the property may not exist
+            # since its defined in the subclasses.
+            child = self.child  # type: ignore
+            assert isinstance(child, (Statement, Feature))
+            yield child
 
         if hasattr(self, "children"):
-            for child in getattr(self, "children"):
+            for child in self.children:
+                assert isinstance(child, (Statement, Feature))
                 yield child
 
     def replace_child(self, existing, new):
         if hasattr(self, "child"):
-            if self.child is existing:
+            # this really confuses mypy because the property may not exist
+            # since its defined in the subclasses.
+            if self.child is existing:  # type: ignore
                 self.child = new
 
         if hasattr(self, "children"):
-            children = getattr(self, "children")
+            children = self.children
             for i, child in enumerate(children):
                 if child is existing:
                     children[i] = new
@@ -89,17 +99,17 @@ class And(Statement):
     """
 
     def __init__(self, children, description=None):
-        super(And, self).__init__(description=description)
+        super().__init__(description=description)
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.and"] += 1
 
         if short_circuit:
             results = []
             for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
+                result = child.evaluate(features, short_circuit=short_circuit)
                 results.append(result)
                 if not result:
                     # short circuit
@@ -107,7 +117,7 @@ class And(Statement):
 
             return Result(True, self, results)
         else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
+            results = [child.evaluate(features, short_circuit=short_circuit) for child in self.children]
             success = all(results)
             return Result(success, self, results)
 
@@ -122,17 +132,17 @@ class Or(Statement):
     """
 
     def __init__(self, children, description=None):
-        super(Or, self).__init__(description=description)
+        super().__init__(description=description)
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.or"] += 1
 
         if short_circuit:
             results = []
             for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
+                result = child.evaluate(features, short_circuit=short_circuit)
                 results.append(result)
                 if result:
                     # short circuit as soon as we hit one match
@@ -140,7 +150,7 @@ class Or(Statement):
 
             return Result(False, self, results)
         else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
+            results = [child.evaluate(features, short_circuit=short_circuit) for child in self.children]
             success = any(results)
             return Result(success, self, results)
 
@@ -149,14 +159,14 @@ class Not(Statement):
     """match only if the child evaluates to False."""
 
     def __init__(self, child, description=None):
-        super(Not, self).__init__(description=description)
+        super().__init__(description=description)
         self.child = child
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.not"] += 1
 
-        results = [self.child.evaluate(ctx, short_circuit=short_circuit)]
+        results = [self.child.evaluate(features, short_circuit=short_circuit)]
         success = not results[0]
         return Result(success, self, results)
 
@@ -171,11 +181,11 @@ class Some(Statement):
     """
 
     def __init__(self, count, children, description=None):
-        super(Some, self).__init__(description=description)
+        super().__init__(description=description)
         self.count = count
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.some"] += 1
 
@@ -183,7 +193,7 @@ class Some(Statement):
             results = []
             satisfied_children_count = 0
             for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
+                result = child.evaluate(features, short_circuit=short_circuit)
                 results.append(result)
                 if result:
                     satisfied_children_count += 1
@@ -194,7 +204,7 @@ class Some(Statement):
 
             return Result(False, self, results)
         else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
+            results = [child.evaluate(features, short_circuit=short_circuit) for child in self.children]
             # note that here we cast the child result as a bool
             # because we've overridden `__bool__` above.
             #
@@ -204,29 +214,29 @@ class Some(Statement):
 
 
 class Range(Statement):
-    """match if the child is contained in the ctx set with a count in the given range."""
+    """match if the child is contained in the feature set with a count in the given range."""
 
     def __init__(self, child, min=None, max=None, description=None):
-        super(Range, self).__init__(description=description)
+        super().__init__(description=description)
         self.child = child
         self.min = min if min is not None else 0
         self.max = max if max is not None else (1 << 64 - 1)
 
-    def evaluate(self, ctx, **kwargs):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.range"] += 1
 
-        count = len(ctx.get(self.child, []))
+        count = len(features.get(self.child, []))
         if self.min == 0 and count == 0:
             return Result(True, self, [])
 
-        return Result(self.min <= count <= self.max, self, [], locations=ctx.get(self.child))
+        return Result(self.min <= count <= self.max, self, [], locations=features.get(self.child))
 
     def __str__(self):
         if self.max == (1 << 64 - 1):
-            return "range(%s, min=%d, max=infinity)" % (str(self.child), self.min)
+            return f"range({str(self.child)}, min={self.min}, max=infinity)"
         else:
-            return "range(%s, min=%d, max=%d)" % (str(self.child), self.min, self.max)
+            return f"range({str(self.child)}, min={self.min}, max={self.max})"
 
 
 class Subscope(Statement):
@@ -235,12 +245,12 @@ class Subscope(Statement):
     the engine should preprocess rules to extract subscope statements into their own rules.
     """
 
-    def __init__(self, scope, child):
-        super(Subscope, self).__init__()
+    def __init__(self, scope, child, description=None):
+        super().__init__(description=description)
         self.scope = scope
         self.child = child
 
-    def evaluate(self, ctx, **kwargs):
+    def evaluate(self, features: FeatureSet, short_circuit=True):
         raise ValueError("cannot evaluate a subscope directly!")
 
 
@@ -257,10 +267,18 @@ class Subscope(Statement):
 #         inspect(match_details)
 #
 # aliased here so that the type can be documented and xref'd.
-MatchResults = Mapping[str, List[Tuple[int, Result]]]
+MatchResults = Mapping[str, List[Tuple[Address, Result]]]
 
 
-def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations: Iterable[int]):
+def get_rule_namespaces(rule: "capa.rules.Rule") -> Iterator[str]:
+    namespace = rule.meta.get("namespace")
+    if namespace:
+        while namespace:
+            yield namespace
+            namespace, _, _ = namespace.rpartition("/")
+
+
+def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations: Iterable[Address]):
     """
     record into the given featureset that the given rule matched at the given locations.
 
@@ -270,14 +288,11 @@ def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations:
     updates `features` in-place. doesn't modify the remaining arguments.
     """
     features[capa.features.common.MatchedRule(rule.name)].update(locations)
-    namespace = rule.meta.get("namespace")
-    if namespace:
-        while namespace:
-            features[capa.features.common.MatchedRule(namespace)].update(locations)
-            namespace, _, _ = namespace.rpartition("/")
+    for namespace in get_rule_namespaces(rule):
+        features[capa.features.common.MatchedRule(namespace)].update(locations)
 
 
-def match(rules: List["capa.rules.Rule"], features: FeatureSet, va: int) -> Tuple[FeatureSet, MatchResults]:
+def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -> Tuple[FeatureSet, MatchResults]:
     """
     match the given rules against the given features,
     returning an updated set of features and the matches.
@@ -294,7 +309,7 @@ def match(rules: List["capa.rules.Rule"], features: FeatureSet, va: int) -> Tupl
     other strategies can be imagined that match differently; implement these elsewhere.
     specifically, this routine does "top down" matching of the given rules against the feature set.
     """
-    results = collections.defaultdict(list)  # type: MatchResults
+    results: MatchResults = collections.defaultdict(list)
 
     # copy features so that we can modify it
     # without affecting the caller (keep this function pure)
@@ -315,10 +330,10 @@ def match(rules: List["capa.rules.Rule"], features: FeatureSet, va: int) -> Tupl
             # sanity check
             assert bool(res) is True
 
-            results[rule.name].append((va, res))
+            results[rule.name].append((addr, res))
             # we need to update the current `features`
             # because subsequent iterations of this loop may use newly added features,
             # such as rule or namespace matches.
-            index_rule_matches(features, rule, [va])
+            index_rule_matches(features, rule, [addr])
 
     return (features, results)

capa/exceptions.py

@@ -0,0 +1,37 @@
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+class UnsupportedRuntimeError(RuntimeError):
+    pass
+
+
+class UnsupportedFormatError(ValueError):
+    pass
+
+
+class UnsupportedArchError(ValueError):
+    pass
+
+
+class UnsupportedOSError(ValueError):
+    pass
+
+
+class EmptyReportError(ValueError):
+    pass
+
+
+class InvalidArgument(ValueError):
+    pass
+
+
+class NonExistantFunctionError(ValueError):
+    pass
+
+
+class NonExistantProcessError(ValueError):
+    pass

capa/features/address.py

@@ -0,0 +1,193 @@
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import abc
+
+
+class Address(abc.ABC):
+    @abc.abstractmethod
+    def __eq__(self, other): ...
+
+    @abc.abstractmethod
+    def __lt__(self, other):
+        # implement < so that addresses can be sorted from low to high
+        ...
+
+    @abc.abstractmethod
+    def __hash__(self):
+        # implement hash so that addresses can be used in sets and dicts
+        ...
+
+    @abc.abstractmethod
+    def __repr__(self):
+        # implement repr to help during debugging
+        ...
+
+
+class AbsoluteVirtualAddress(int, Address):
+    """an absolute memory address"""
+
+    def __new__(cls, v):
+        assert v >= 0
+        return int.__new__(cls, v)
+
+    def __repr__(self):
+        return f"absolute(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
+
+
+class ProcessAddress(Address):
+    """an address of a process in a dynamic execution trace"""
+
+    def __init__(self, pid: int, ppid: int = 0):
+        assert ppid >= 0
+        assert pid > 0
+        self.ppid = ppid
+        self.pid = pid
+
+    def __repr__(self):
+        return "process(%s%s)" % (
+            f"ppid: {self.ppid}, " if self.ppid > 0 else "",
+            f"pid: {self.pid}",
+        )
+
+    def __hash__(self):
+        return hash((self.ppid, self.pid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) == (other.ppid, other.pid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) < (other.ppid, other.pid)
+
+
+class ThreadAddress(Address):
+    """addresses a thread in a dynamic execution trace"""
+
+    def __init__(self, process: ProcessAddress, tid: int):
+        assert tid >= 0
+        self.process = process
+        self.tid = tid
+
+    def __repr__(self):
+        return f"{self.process}, thread(tid: {self.tid})"
+
+    def __hash__(self):
+        return hash((self.process, self.tid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) == (other.process, other.tid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) < (other.process, other.tid)
+
+
+class DynamicCallAddress(Address):
+    """addresses a call in a dynamic execution trace"""
+
+    def __init__(self, thread: ThreadAddress, id: int):
+        assert id >= 0
+        self.thread = thread
+        self.id = id
+
+    def __repr__(self):
+        return f"{self.thread}, call(id: {self.id})"
+
+    def __hash__(self):
+        return hash((self.thread, self.id))
+
+    def __eq__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) == (other.thread, other.id)
+
+    def __lt__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) < (other.thread, other.id)
+
+
+class RelativeVirtualAddress(int, Address):
+    """a memory address relative to a base address"""
+
+    def __repr__(self):
+        return f"relative(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
+
+
+class FileOffsetAddress(int, Address):
+    """an address relative to the start of a file"""
+
+    def __new__(cls, v):
+        assert v >= 0
+        return int.__new__(cls, v)
+
+    def __repr__(self):
+        return f"file(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
+
+
+class DNTokenAddress(int, Address):
+    """a .NET token"""
+
+    def __new__(cls, token: int):
+        return int.__new__(cls, token)
+
+    def __repr__(self):
+        return f"token(0x{self:x})"
+
+    def __hash__(self):
+        return int.__hash__(self)
+
+
+class DNTokenOffsetAddress(Address):
+    """an offset into an object specified by a .NET token"""
+
+    def __init__(self, token: int, offset: int):
+        assert offset >= 0
+        self.token = token
+        self.offset = offset
+
+    def __eq__(self, other):
+        return (self.token, self.offset) == (other.token, other.offset)
+
+    def __lt__(self, other):
+        return (self.token, self.offset) < (other.token, other.offset)
+
+    def __hash__(self):
+        return hash((self.token, self.offset))
+
+    def __repr__(self):
+        return f"token(0x{self.token:x})+(0x{self.offset:x})"
+
+    def __index__(self):
+        return self.token + self.offset
+
+
+class _NoAddress(Address):
+    def __eq__(self, other):
+        return True
+
+    def __lt__(self, other):
+        return False
+
+    def __hash__(self):
+        return hash(0)
+
+    def __repr__(self):
+        return "no address"
+
+
+NO_ADDRESS = _NoAddress()

capa/features/basicblock.py

@@ -10,18 +10,11 @@ from capa.features.common import Feature
 
 
 class BasicBlock(Feature):
-    def __init__(self):
-        super(BasicBlock, self).__init__(None)
+    def __init__(self, description=None):
+        super().__init__(0, description=description)
 
     def __str__(self):
         return "basic block"
 
     def get_value_str(self):
         return ""
-
-    def freeze_serialize(self):
-        return (self.__class__.__name__, [])
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        return cls()

capa/features/com/__init__.py

@@ -0,0 +1,36 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from enum import Enum
+from typing import Dict, List
+
+from capa.helpers import assert_never
+
+
+class ComType(Enum):
+    CLASS = "class"
+    INTERFACE = "interface"
+
+
+COM_PREFIXES = {
+    ComType.CLASS: "CLSID_",
+    ComType.INTERFACE: "IID_",
+}
+
+
+def load_com_database(com_type: ComType) -> Dict[str, List[str]]:
+    # lazy load these python files since they are so large.
+    # that is, don't load them unless a COM feature is being handled.
+    import capa.features.com.classes
+    import capa.features.com.interfaces
+
+    if com_type == ComType.CLASS:
+        return capa.features.com.classes.COM_CLASSES
+    elif com_type == ComType.INTERFACE:
+        return capa.features.com.interfaces.COM_INTERFACES
+    else:
+        assert_never(com_type)

capa/features/common.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,10 +7,12 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import re
+import abc
 import codecs
+import typing
 import logging
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Union
+from typing import TYPE_CHECKING, Set, Dict, List, Union, Optional
 
 if TYPE_CHECKING:
     # circular import, otherwise
@@ -19,6 +21,7 @@ if TYPE_CHECKING:
 import capa.perf
 import capa.features
 import capa.features.extractors.elf
+from capa.features.address import Address
 
 logger = logging.getLogger(__name__)
 MAX_BYTES_FEATURE_SIZE = 0x100
@@ -27,6 +30,14 @@ MAX_BYTES_FEATURE_SIZE = 0x100
 THUNK_CHAIN_DEPTH_DELTA = 5
 
 
+class FeatureAccess:
+    READ = "read"
+    WRITE = "write"
+
+
+VALID_FEATURE_ACCESS = (FeatureAccess.READ, FeatureAccess.WRITE)
+
+
 def bytes_to_str(b: bytes) -> str:
     return str(codecs.encode(b, "hex").decode("utf-8"))
 
@@ -69,20 +80,13 @@ class Result:
         success: bool,
         statement: Union["capa.engine.Statement", "Feature"],
         children: List["Result"],
-        locations=None,
+        locations: Optional[Set[Address]] = None,
     ):
-        """
-        args:
-          success (bool)
-          statement (capa.engine.Statement or capa.features.Feature)
-          children (list[Result])
-          locations (iterable[VA])
-        """
-        super(Result, self).__init__()
+        super().__init__()
         self.success = success
         self.statement = statement
         self.children = children
-        self.locations = locations if locations is not None else ()
+        self.locations = locations if locations is not None else set()
 
     def __eq__(self, other):
         if isinstance(other, bool):
@@ -96,111 +100,123 @@ class Result:
         return self.success
 
 
-class Feature:
-    def __init__(self, value: Union[str, int, bytes], bitness=None, description=None):
+class Feature(abc.ABC):  # noqa: B024
+    # this is an abstract class, since we don't want anyone to instantiate it directly,
+    # but it doesn't have any abstract methods.
+
+    def __init__(
+        self,
+        value: Union[str, int, float, bytes],
+        description: Optional[str] = None,
+    ):
         """
         Args:
           value (any): the value of the feature, such as the number or string.
-          bitness (str): one of the VALID_BITNESS values, or None.
-            When None, then the feature applies to any bitness.
-            Modifies the feature name from `feature` to `feature/bitness`, like `offset/x32`.
           description (str): a human-readable description that explains the feature value.
         """
-        super(Feature, self).__init__()
-
-        if bitness is not None:
-            if bitness not in VALID_BITNESS:
-                raise ValueError("bitness '%s' must be one of %s" % (bitness, VALID_BITNESS))
-            self.name = self.__class__.__name__.lower() + "/" + bitness
-        else:
-            self.name = self.__class__.__name__.lower()
+        super().__init__()
 
+        self.name = self.__class__.__name__.lower()
         self.value = value
-        self.bitness = bitness
         self.description = description
 
     def __hash__(self):
-        return hash((self.name, self.value, self.bitness))
+        return hash((self.name, self.value))
 
     def __eq__(self, other):
-        return self.name == other.name and self.value == other.value and self.bitness == other.bitness
+        return self.name == other.name and self.value == other.value
+
+    def __lt__(self, other):
+        # implementing sorting by serializing to JSON is a huge hack.
+        # it's slow, inelegant, and probably doesn't work intuitively;
+        # however, we only use it for deterministic output, so it's good enough for now.
+
+        # circular import
+        # we should fix if this wasn't already a huge hack.
+        import capa.features.freeze.features
+
+        return (
+            capa.features.freeze.features.feature_from_capa(self).model_dump_json()
+            < capa.features.freeze.features.feature_from_capa(other).model_dump_json()
+        )
+
+    def get_name_str(self) -> str:
+        """
+        render the name of this feature, for use by `__str__` and friends.
+        subclasses should override to customize the rendering.
+        """
+        return self.name
 
     def get_value_str(self) -> str:
         """
         render the value of this feature, for use by `__str__` and friends.
         subclasses should override to customize the rendering.
-
-        Returns: any
         """
         return str(self.value)
 
     def __str__(self):
         if self.value is not None:
             if self.description:
-                return "%s(%s = %s)" % (self.name, self.get_value_str(), self.description)
+                return f"{self.get_name_str()}({self.get_value_str()} = {self.description})"
             else:
-                return "%s(%s)" % (self.name, self.get_value_str())
+                return f"{self.get_name_str()}({self.get_value_str()})"
         else:
-            return "%s" % self.name
+            return f"{self.get_name_str()}"
 
     def __repr__(self):
         return str(self)
 
-    def evaluate(self, ctx: Dict["Feature", Set[int]], **kwargs) -> Result:
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True) -> Result:
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature." + self.name] += 1
-        return Result(self in ctx, self, [], locations=ctx.get(self, []))
-
-    def freeze_serialize(self):
-        if self.bitness is not None:
-            return (self.__class__.__name__, [self.value, {"bitness": self.bitness}])
-        else:
-            return (self.__class__.__name__, [self.value])
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        # as you can see below in code,
-        # if the last argument is a dictionary,
-        # consider it to be kwargs passed to the feature constructor.
-        if len(args) == 1:
-            return cls(*args)
-        elif isinstance(args[-1], dict):
-            kwargs = args[-1]
-            args = args[:-1]
-            return cls(*args, **kwargs)
+        return Result(self in features, self, [], locations=features.get(self, set()))
 
 
 class MatchedRule(Feature):
     def __init__(self, value: str, description=None):
-        super(MatchedRule, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "match"
 
 
 class Characteristic(Feature):
     def __init__(self, value: str, description=None):
-
-        super(Characteristic, self).__init__(value, description=description)
+        super().__init__(value, description=description)
 
 
 class String(Feature):
     def __init__(self, value: str, description=None):
-        super(String, self).__init__(value, description=description)
+        super().__init__(value, description=description)
+
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, str)
+        return escape_string(self.value)
+
+
+class Class(Feature):
+    def __init__(self, value: str, description=None):
+        super().__init__(value, description=description)
+
+
+class Namespace(Feature):
+    def __init__(self, value: str, description=None):
+        super().__init__(value, description=description)
 
 
 class Substring(String):
     def __init__(self, value: str, description=None):
-        super(Substring, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.value = value
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.substring"] += 1
 
         # mapping from string value to list of locations.
         # will unique the locations later on.
-        matches = collections.defaultdict(list)
+        matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
 
-        for feature, locations in ctx.items():
+        assert isinstance(self.value, str)
+        for feature, locations in features.items():
             if not isinstance(feature, (String,)):
                 continue
 
@@ -209,32 +225,32 @@ class Substring(String):
                 raise ValueError("unexpected feature value type")
 
             if self.value in feature.value:
-                matches[feature.value].extend(locations)
+                matches[feature.value].update(locations)
                 if short_circuit:
-                    # we found one matching string, thats sufficient to match.
+                    # we found one matching string, that's sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 
         if matches:
-            # finalize: defaultdict -> dict
-            # which makes json serialization easier
-            matches = dict(matches)
-
             # collect all locations
             locations = set()
-            for s in matches.keys():
-                matches[s] = list(set(matches[s]))
-                locations.update(matches[s])
+            for locs in matches.values():
+                locations.update(locs)
 
             # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
             # this is because `self` may match on many strings, so we can't stuff the matched value into it.
             # instead, return a new instance that has a reference to both the substring and the matched values.
-            return Result(True, _MatchedSubstring(self, matches), [], locations=locations)
+            return Result(True, _MatchedSubstring(self, dict(matches)), [], locations=locations)
         else:
-            return Result(False, _MatchedSubstring(self, None), [])
+            return Result(False, _MatchedSubstring(self, {}), [])
+
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, str)
+        return escape_string(self.value)
 
     def __str__(self):
-        return "substring(%s)" % self.value
+        assert isinstance(self.value, str)
+        return f"substring({escape_string(self.value)})"
 
 
 class _MatchedSubstring(Substring):
@@ -245,13 +261,13 @@ class _MatchedSubstring(Substring):
     note: this type should only ever be constructed by `Substring.evaluate()`. it is not part of the public API.
     """
 
-    def __init__(self, substring: Substring, matches):
+    def __init__(self, substring: Substring, matches: Dict[str, Set[Address]]):
         """
         args:
-          substring (Substring): the substring feature that matches.
-          match (Dict[string, List[int]]|None): mapping from matching string to its locations.
+          substring: the substring feature that matches.
+          match: mapping from matching string to its locations.
         """
-        super(_MatchedSubstring, self).__init__(str(substring.value), description=substring.description)
+        super().__init__(str(substring.value), description=substring.description)
         # we want this to collide with the name of `Substring` above,
         # so that it works nicely with the renderers.
         self.name = "substring"
@@ -259,15 +275,14 @@ class _MatchedSubstring(Substring):
         self.matches = matches
 
     def __str__(self):
-        return 'substring("%s", matches = %s)' % (
-            self.value,
-            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
-        )
+        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
+        assert isinstance(self.value, str)
+        return f'substring("{self.value}", matches = {matches})'
 
 
 class Regex(String):
     def __init__(self, value: str, description=None):
-        super(Regex, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.value = value
 
         pat = self.value[len("/") : -len("/")]
@@ -277,22 +292,22 @@ class Regex(String):
             flags |= re.IGNORECASE
         try:
             self.re = re.compile(pat, flags)
-        except re.error:
+        except re.error as exc:
             if value.endswith("/i"):
                 value = value[: -len("i")]
             raise ValueError(
-                "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value
-            )
+                f"invalid regular expression: {value} it should use Python syntax, try it at https://pythex.org"
+            ) from exc
 
-    def evaluate(self, ctx, short_circuit=True):
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.regex"] += 1
 
         # mapping from string value to list of locations.
         # will unique the locations later on.
-        matches = collections.defaultdict(list)
+        matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
 
-        for feature, locations in ctx.items():
+        for feature, locations in features.items():
             if not isinstance(feature, (String,)):
                 continue
 
@@ -305,33 +320,29 @@ class Regex(String):
             # using this mode cleans is more convenient for rule authors,
             # so that they don't have to prefix/suffix their terms like: /.*foo.*/.
             if self.re.search(feature.value):
-                matches[feature.value].extend(locations)
+                matches[feature.value].update(locations)
                 if short_circuit:
-                    # we found one matching string, thats sufficient to match.
+                    # we found one matching string, that's sufficient to match.
                     # don't collect other matching strings in this mode.
                     break
 
         if matches:
-            # finalize: defaultdict -> dict
-            # which makes json serialization easier
-            matches = dict(matches)
-
             # collect all locations
             locations = set()
-            for s in matches.keys():
-                matches[s] = list(set(matches[s]))
-                locations.update(matches[s])
+            for locs in matches.values():
+                locations.update(locs)
 
             # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
             # this is because `self` may match on many strings, so we can't stuff the matched value into it.
             # instead, return a new instance that has a reference to both the regex and the matched values.
             # see #262.
-            return Result(True, _MatchedRegex(self, matches), [], locations=locations)
+            return Result(True, _MatchedRegex(self, dict(matches)), [], locations=locations)
         else:
-            return Result(False, _MatchedRegex(self, None), [])
+            return Result(False, _MatchedRegex(self, {}), [])
 
     def __str__(self):
-        return "regex(string =~ %s)" % self.value
+        assert isinstance(self.value, str)
+        return f"regex(string =~ {self.value})"
 
 
 class _MatchedRegex(Regex):
@@ -342,13 +353,13 @@ class _MatchedRegex(Regex):
     note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
     """
 
-    def __init__(self, regex: Regex, matches):
+    def __init__(self, regex: Regex, matches: Dict[str, Set[Address]]):
         """
         args:
-          regex (Regex): the regex feature that matches.
-          match (Dict[string, List[int]]|None): mapping from matching string to its locations.
+          regex: the regex feature that matches.
+          matches: mapping from matching string to its locations.
         """
-        super(_MatchedRegex, self).__init__(str(regex.value), description=regex.description)
+        super().__init__(str(regex.value), description=regex.description)
         # we want this to collide with the name of `Regex` above,
         # so that it works nicely with the renderers.
         self.name = "regex"
@@ -356,10 +367,9 @@ class _MatchedRegex(Regex):
         self.matches = matches
 
     def __str__(self):
-        return "regex(string =~ %s, matches = %s)" % (
-            self.value,
-            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
-        )
+        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
+        assert isinstance(self.value, str)
+        return f"regex(string =~ {self.value}, matches = {matches})"
 
 
 class StringFactory:
@@ -371,79 +381,121 @@ class StringFactory:
 
 class Bytes(Feature):
     def __init__(self, value: bytes, description=None):
-        super(Bytes, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.value = value
 
-    def evaluate(self, ctx, **kwargs):
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
+        assert isinstance(self.value, bytes)
+
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.bytes"] += 1
+        capa.perf.counters["evaluate.feature.bytes." + str(len(self.value))] += 1
 
-        for feature, locations in ctx.items():
+        for feature, locations in features.items():
             if not isinstance(feature, (Bytes,)):
                 continue
 
+            assert isinstance(feature.value, bytes)
             if feature.value.startswith(self.value):
                 return Result(True, self, [], locations=locations)
 
         return Result(False, self, [])
 
     def get_value_str(self):
+        assert isinstance(self.value, bytes)
         return hex_string(bytes_to_str(self.value))
 
-    def freeze_serialize(self):
-        return (self.__class__.__name__, [bytes_to_str(self.value).upper()])
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        return cls(*[codecs.decode(x, "hex") for x in args])
-
-
-# identifiers for supported bitness names that tweak a feature
-# for example, offset/x32
-BITNESS_X32 = "x32"
-BITNESS_X64 = "x64"
-VALID_BITNESS = (BITNESS_X32, BITNESS_X64)
-
 
 # other candidates here: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types
 ARCH_I386 = "i386"
 ARCH_AMD64 = "amd64"
-VALID_ARCH = (ARCH_I386, ARCH_AMD64)
+ARCH_AARCH64 = "aarch64"
+# dotnet
+ARCH_ANY = "any"
+VALID_ARCH = (ARCH_I386, ARCH_AMD64, ARCH_AARCH64, ARCH_ANY)
 
 
 class Arch(Feature):
     def __init__(self, value: str, description=None):
-        super(Arch, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "arch"
 
 
 OS_WINDOWS = "windows"
 OS_LINUX = "linux"
 OS_MACOS = "macos"
+OS_ANDROID = "android"
+# dotnet
+OS_ANY = "any"
 VALID_OS = {os.value for os in capa.features.extractors.elf.OS}
-VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS})
+VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS, OS_ANY, OS_ANDROID})
+# internal only, not to be used in rules
+OS_AUTO = "auto"
 
 
 class OS(Feature):
     def __init__(self, value: str, description=None):
-        super(OS, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "os"
 
+    def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True):
+        capa.perf.counters["evaluate.feature"] += 1
+        capa.perf.counters["evaluate.feature." + self.name] += 1
+
+        for feature, locations in features.items():
+            if not isinstance(feature, (OS,)):
+                continue
+
+            assert isinstance(feature.value, str)
+            if OS_ANY in (self.value, feature.value) or self.value == feature.value:
+                return Result(True, self, [], locations=locations)
+
+        return Result(False, self, [])
+
 
 FORMAT_PE = "pe"
 FORMAT_ELF = "elf"
-VALID_FORMAT = (FORMAT_PE, FORMAT_ELF)
+FORMAT_DOTNET = "dotnet"
+VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET)
+# internal only, not to be used in rules
+FORMAT_AUTO = "auto"
+FORMAT_SC32 = "sc32"
+FORMAT_SC64 = "sc64"
+FORMAT_CAPE = "cape"
+FORMAT_DRAKVUF = "drakvuf"
+FORMAT_VMRAY = "vmray"
+FORMAT_BINEXPORT2 = "binexport2"
+FORMAT_FREEZE = "freeze"
+FORMAT_RESULT = "result"
+STATIC_FORMATS = {
+    FORMAT_SC32,
+    FORMAT_SC64,
+    FORMAT_PE,
+    FORMAT_ELF,
+    FORMAT_DOTNET,
+    FORMAT_FREEZE,
+    FORMAT_RESULT,
+    FORMAT_BINEXPORT2,
+}
+DYNAMIC_FORMATS = {
+    FORMAT_CAPE,
+    FORMAT_DRAKVUF,
+    FORMAT_VMRAY,
+    FORMAT_FREEZE,
+    FORMAT_RESULT,
+}
+FORMAT_UNKNOWN = "unknown"
 
 
 class Format(Feature):
     def __init__(self, value: str, description=None):
-        super(Format, self).__init__(value, description=description)
+        super().__init__(value, description=description)
         self.name = "format"
 
 
 def is_global_feature(feature):
     """
     is this a feature that is extracted at every scope?
-    today, these are OS and arch features.
+    today, these are OS, arch, and format features.
     """
-    return isinstance(feature, (OS, Arch))
+    return isinstance(feature, (OS, Arch, Format))

capa/features/extractors/base_extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,35 +7,99 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import abc
-from typing import Tuple, Iterator, SupportsInt
+import hashlib
+import dataclasses
+from copy import copy
+from types import MethodType
+from typing import Any, Set, Dict, Tuple, Union, Iterator
+from dataclasses import dataclass
 
+# TODO(williballenthin): use typing.TypeAlias directly when Python 3.9 is deprecated
+# https://github.com/mandiant/capa/issues/1699
+from typing_extensions import TypeAlias
+
+import capa.features.address
 from capa.features.common import Feature
+from capa.features.address import Address, ThreadAddress, ProcessAddress, DynamicCallAddress, AbsoluteVirtualAddress
 
 # feature extractors may reference functions, BBs, insns by opaque handle values.
-# the only requirement of these handles are that they support `__int__`,
-# so that they can be rendered as addresses.
+# you can use the `.address` property to get and render the address of the feature.
 #
 # these handles are only consumed by routines on
 # the feature extractor from which they were created.
-#
-# int(FunctionHandle) -> function start address
-# int(BBHandle) -> BasicBlock start address
-# int(InsnHandle) -> instruction address
-FunctionHandle = SupportsInt
-BBHandle = SupportsInt
-InsnHandle = SupportsInt
 
 
-class FeatureExtractor:
+@dataclass
+class SampleHashes:
+    md5: str
+    sha1: str
+    sha256: str
+
+    @classmethod
+    def from_bytes(cls, buf: bytes) -> "SampleHashes":
+        md5 = hashlib.md5()
+        sha1 = hashlib.sha1()
+        sha256 = hashlib.sha256()
+        md5.update(buf)
+        sha1.update(buf)
+        sha256.update(buf)
+
+        return cls(md5=md5.hexdigest(), sha1=sha1.hexdigest(), sha256=sha256.hexdigest())
+
+
+@dataclass
+class FunctionHandle:
+    """reference to a function recognized by a feature extractor.
+
+    Attributes:
+        address: the address of the function.
+        inner: extractor-specific data.
+        ctx: a context object for the extractor.
     """
-    FeatureExtractor defines the interface for fetching features from a sample.
+
+    address: Address
+    inner: Any
+    ctx: Dict[str, Any] = dataclasses.field(default_factory=dict)
+
+
+@dataclass
+class BBHandle:
+    """reference to a basic block recognized by a feature extractor.
+
+    Attributes:
+        address: the address of the basic block start address.
+        inner: extractor-specific data.
+    """
+
+    address: Address
+    inner: Any
+
+
+@dataclass
+class InsnHandle:
+    """reference to an instruction recognized by a feature extractor.
+
+    Attributes:
+        address: the address of the instruction address.
+        inner: extractor-specific data.
+    """
+
+    address: Address
+    inner: Any
+
+
+class StaticFeatureExtractor:
+    """
+    StaticFeatureExtractor defines the interface for fetching features from a
+    sample without running it; extractors that rely on the execution trace of
+    a sample must implement the other sibling class, DynamicFeatureExtracor.
 
     There may be multiple backends that support fetching features for capa.
     For example, we use vivisect by default, but also want to support saving
      and restoring features from a JSON file.
     When we restore the features, we'd like to use exactly the same matching logic
      to find matching rules.
-    Therefore, we can define a FeatureExtractor that provides features from the
+    Therefore, we can define a StaticFeatureExtractor that provides features from the
      serialized JSON file and do matching without a binary analysis pass.
     Also, this provides a way to hook in an IDA backend.
 
@@ -44,23 +108,34 @@ class FeatureExtractor:
 
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self):
+    def __init__(self, hashes: SampleHashes):
         #
         # note: a subclass should define ctor parameters for its own use.
         #  for example, the Vivisect feature extract might require the vw and/or path.
         # this base class doesn't know what to do with that info, though.
         #
-        super(FeatureExtractor, self).__init__()
+        super().__init__()
+        self._sample_hashes = hashes
 
     @abc.abstractmethod
-    def get_base_address(self) -> int:
+    def get_base_address(self) -> Union[AbsoluteVirtualAddress, capa.features.address._NoAddress]:
         """
         fetch the preferred load address at which the sample was analyzed.
+
+        when the base address is `NO_ADDRESS`, then the loader has no concept of a preferred load address.
+        such as: shellcode, .NET modules, etc.
+        in these scenarios, RelativeVirtualAddresses aren't used.
         """
         raise NotImplementedError()
 
+    def get_sample_hashes(self) -> SampleHashes:
+        """
+        fetch the hashes for the sample contained within the extractor.
+        """
+        return self._sample_hashes
+
     @abc.abstractmethod
-    def extract_global_features(self) -> Iterator[Tuple[Feature, int]]:
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
         """
         extract features found at every scope ("global").
 
@@ -71,12 +146,12 @@ class FeatureExtractor:
                 print('0x%x: %s', va, feature)
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def extract_file_features(self) -> Iterator[Tuple[Feature, int]]:
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
         """
         extract file-scope features.
 
@@ -87,7 +162,7 @@ class FeatureExtractor:
                 print('0x%x: %s', va, feature)
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
 
@@ -99,32 +174,33 @@ class FeatureExtractor:
         """
         raise NotImplementedError()
 
-    def is_library_function(self, va: int) -> bool:
+    def is_library_function(self, addr: Address) -> bool:
         """
         is the given address a library function?
         the backend may implement its own function matching algorithm, or none at all.
-        we accept a VA here, rather than function object, to handle addresses identified in instructions.
+        we accept an address here, rather than function object,
+         to handle addresses identified in instructions.
 
         this information is used to:
           - filter out matches in library functions (by default), and
           - recognize when to fetch symbol names for called (non-API) functions
 
         args:
-          va (int): the virtual address of a function.
+          addr (Address): the address of a function.
 
         returns:
           bool: True if the given address is the start of a library function.
         """
         return False
 
-    def get_function_name(self, va: int) -> str:
+    def get_function_name(self, addr: Address) -> str:
         """
         fetch any recognized name for the given address.
         this is only guaranteed to return a value when the given function is a recognized library function.
         we accept a VA here, rather than function object, to handle addresses identified in instructions.
 
         args:
-          va (int): the virtual address of a function.
+          addr (Address): the address of a function.
 
         returns:
           str: the function name
@@ -132,10 +208,10 @@ class FeatureExtractor:
         raises:
           KeyError: when the given function does not have a name.
         """
-        raise KeyError(va)
+        raise KeyError(addr)
 
     @abc.abstractmethod
-    def extract_function_features(self, f: FunctionHandle) -> Iterator[Tuple[Feature, int]]:
+    def extract_function_features(self, f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
         """
         extract function-scope features.
         the arguments are opaque values previously provided by `.get_functions()`, etc.
@@ -144,14 +220,14 @@ class FeatureExtractor:
 
             extractor = VivisectFeatureExtractor(vw, path)
             for function in extractor.get_functions():
-                for feature, va in extractor.extract_function_features(function):
-                    print('0x%x: %s', va, feature)
+                for feature, address in extractor.extract_function_features(function):
+                    print('0x%x: %s', address, feature)
 
         args:
           f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
 
@@ -164,7 +240,7 @@ class FeatureExtractor:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, int]]:
+    def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
         """
         extract basic block-scope features.
         the arguments are opaque values previously provided by `.get_functions()`, etc.
@@ -174,15 +250,15 @@ class FeatureExtractor:
             extractor = VivisectFeatureExtractor(vw, path)
             for function in extractor.get_functions():
                 for bb in extractor.get_basic_blocks(function):
-                    for feature, va in extractor.extract_basic_block_features(function, bb):
-                        print('0x%x: %s', va, feature)
+                    for feature, address in extractor.extract_basic_block_features(function, bb):
+                        print('0x%x: %s', address, feature)
 
         args:
           f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
           bb [BBHandle]: an opaque value previously fetched from `.get_basic_blocks()`.
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
 
@@ -195,7 +271,9 @@ class FeatureExtractor:
         raise NotImplementedError()
 
     @abc.abstractmethod
-    def extract_insn_features(self, f: FunctionHandle, bb: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, int]]:
+    def extract_insn_features(
+        self, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
         """
         extract instruction-scope features.
         the arguments are opaque values previously provided by `.get_functions()`, etc.
@@ -206,8 +284,8 @@ class FeatureExtractor:
             for function in extractor.get_functions():
                 for bb in extractor.get_basic_blocks(function):
                     for insn in extractor.get_instructions(function, bb):
-                        for feature, va in extractor.extract_insn_features(function, bb, insn):
-                            print('0x%x: %s', va, feature)
+                        for feature, address in extractor.extract_insn_features(function, bb, insn):
+                            print('0x%x: %s', address, feature)
 
         args:
           f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
@@ -215,123 +293,212 @@ class FeatureExtractor:
           insn [InsnHandle]: an opaque value previously fetched from `.get_instructions()`.
 
         yields:
-          Tuple[Feature, int]: feature and its location
+          Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
 
 
-class NullFeatureExtractor(FeatureExtractor):
+def FunctionFilter(extractor: StaticFeatureExtractor, functions: Set) -> StaticFeatureExtractor:
+    original_get_functions = extractor.get_functions
+
+    def filtered_get_functions(self):
+        yield from (f for f in original_get_functions() if f.address in functions)
+
+    # we make a copy of the original extractor object and then update its get_functions() method with the decorated filter one.
+    # this is in order to preserve the original extractor object's get_functions() method, in case it is used elsewhere in the code.
+    # an example where this is important is in our testfiles where we may use the same extractor object with different tests,
+    # with some of these tests needing to install a functions filter on the extractor object.
+    new_extractor = copy(extractor)
+    new_extractor.get_functions = MethodType(filtered_get_functions, extractor)  # type: ignore
+
+    return new_extractor
+
+
+@dataclass
+class ProcessHandle:
     """
-    An extractor that extracts some user-provided features.
-    The structure of the single parameter is demonstrated in the example below.
+    reference to a process extracted by the sandbox.
 
-    This is useful for testing, as we can provide expected values and see if matching works.
-    Also, this is how we represent features deserialized from a freeze file.
-
-    example::
-
-        extractor = NullFeatureExtractor({
-            'base address: 0x401000,
-            'global features': [
-                (0x0, capa.features.Arch('i386')),
-                (0x0, capa.features.OS('linux')),
-            ],
-            'file features': [
-                (0x402345, capa.features.Characteristic('embedded pe')),
-            ],
-            'functions': {
-                0x401000: {
-                    'features': [
-                        (0x401000, capa.features.Characteristic('nzxor')),
-                    ],
-                    'basic blocks': {
-                        0x401000: {
-                            'features': [
-                                (0x401000, capa.features.Characteristic('tight-loop')),
-                            ],
-                            'instructions': {
-                                0x401000: {
-                                    'features': [
-                                        (0x401000, capa.features.Characteristic('nzxor')),
-                                    ],
-                                },
-                                0x401002: ...
-                            }
-                        },
-                        0x401005: ...
-                    }
-                },
-                0x40200: ...
-            }
-        )
+    Attributes:
+        address: process's address (pid)
+        inner: sandbox-specific data
     """
 
-    def __init__(self, features):
-        super(NullFeatureExtractor, self).__init__()
-        self.features = features
+    address: ProcessAddress
+    inner: Any
 
-    def get_base_address(self):
-        return self.features["base address"]
 
-    def extract_global_features(self):
-        for p in self.features.get("global features", []):
-            va, feature = p
-            yield feature, va
+@dataclass
+class ThreadHandle:
+    """
+    reference to a thread extracted by the sandbox.
 
-    def extract_file_features(self):
-        for p in self.features.get("file features", []):
-            va, feature = p
-            yield feature, va
+    Attributes:
+        address: thread's address (tid)
+        inner: sandbox-specific data
+    """
 
-    def get_functions(self):
-        for va in sorted(self.features["functions"].keys()):
-            yield va
+    address: ThreadAddress
+    inner: Any
 
-    def extract_function_features(self, f):
-        for p in self.features.get("functions", {}).get(f, {}).get("features", []):  # noqa: E127 line over-indented
-            va, feature = p
-            yield feature, va
 
-    def get_basic_blocks(self, f):
-        for va in sorted(
-            self.features.get("functions", {})  # noqa: E127 line over-indented
-            .get(f, {})
-            .get("basic blocks", {})
-            .keys()
-        ):
-            yield va
+@dataclass
+class CallHandle:
+    """
+    reference to an api call extracted by the sandbox.
 
-    def extract_basic_block_features(self, f, bb):
-        for p in (
-            self.features.get("functions", {})  # noqa: E127 line over-indented
-            .get(f, {})
-            .get("basic blocks", {})
-            .get(bb, {})
-            .get("features", [])
-        ):
-            va, feature = p
-            yield feature, va
+    Attributes:
+        address: call's address, such as event index or id
+        inner: sandbox-specific data
+    """
 
-    def get_instructions(self, f, bb):
-        for va in sorted(
-            self.features.get("functions", {})  # noqa: E127 line over-indented
-            .get(f, {})
-            .get("basic blocks", {})
-            .get(bb, {})
-            .get("instructions", {})
-            .keys()
-        ):
-            yield va
+    address: DynamicCallAddress
+    inner: Any
 
-    def extract_insn_features(self, f, bb, insn):
-        for p in (
-            self.features.get("functions", {})  # noqa: E127 line over-indented
-            .get(f, {})
-            .get("basic blocks", {})
-            .get(bb, {})
-            .get("instructions", {})
-            .get(insn, {})
-            .get("features", [])
-        ):
-            va, feature = p
-            yield feature, va
+
+class DynamicFeatureExtractor:
+    """
+    DynamicFeatureExtractor defines the interface for fetching features from a
+    sandbox' analysis of a sample; extractors that rely on statically analyzing
+    a sample must implement the sibling extractor, StaticFeatureExtractor.
+
+    Features are grouped mainly into threads that alongside their meta-features are also grouped into
+    processes (that also have their own features). Other scopes (such as function and file) may also apply
+    for a specific sandbox.
+
+    This class is not instantiated directly; it is the base class for other implementations.
+    """
+
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, hashes: SampleHashes):
+        #
+        # note: a subclass should define ctor parameters for its own use.
+        #  for example, the Vivisect feature extract might require the vw and/or path.
+        # this base class doesn't know what to do with that info, though.
+        #
+        super().__init__()
+        self._sample_hashes = hashes
+
+    def get_sample_hashes(self) -> SampleHashes:
+        """
+        fetch the hashes for the sample contained within the extractor.
+        """
+        return self._sample_hashes
+
+    @abc.abstractmethod
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
+        """
+        extract features found at every scope ("global").
+
+        example::
+
+            extractor = CapeFeatureExtractor.from_report(json.loads(buf))
+            for feature, addr in extractor.get_global_features():
+                print(addr, feature)
+
+        yields:
+          Tuple[Feature, Address]: feature and its location
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
+        """
+        extract file-scope features.
+
+        example::
+
+            extractor = CapeFeatureExtractor.from_report(json.loads(buf))
+            for feature, addr in extractor.get_file_features():
+                print(addr, feature)
+
+        yields:
+          Tuple[Feature, Address]: feature and its location
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_processes(self) -> Iterator[ProcessHandle]:
+        """
+        Enumerate processes in the trace.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+        """
+        Yields all the features of a process. These include:
+        - file features of the process' image
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_process_name(self, ph: ProcessHandle) -> str:
+        """
+        Returns the human-readable name for the given process,
+        such as the filename.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
+        """
+        Enumerate threads in the given process.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
+        """
+        Yields all the features of a thread. These include:
+        - sequenced api traces
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+        """
+        Enumerate calls in the given thread
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_call_features(
+        self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
+        """
+        Yields all features of a call. These include:
+        - api name
+        - bytes/strings/numbers extracted from arguments
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_call_name(self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> str:
+        """
+        Returns the human-readable name for the given call,
+        such as as rendered API log entry, like:
+
+            Foo(1, "two", b"\x00\x11") -> -1
+        """
+        raise NotImplementedError()
+
+
+def ProcessFilter(extractor: DynamicFeatureExtractor, processes: Set) -> DynamicFeatureExtractor:
+    original_get_processes = extractor.get_processes
+
+    def filtered_get_processes(self):
+        yield from (f for f in original_get_processes() if f.address.pid in processes)
+
+    # we make a copy of the original extractor object and then update its get_processes() method with the decorated filter one.
+    # this is in order to preserve the original extractor object's get_processes() method, in case it is used elsewhere in the code.
+    # an example where this is important is in our testfiles where we may use the same extractor object with different tests,
+    # with some of these tests needing to install a processes filter on the extractor object.
+    new_extractor = copy(extractor)
+    new_extractor.get_processes = MethodType(filtered_get_processes, extractor)  # type: ignore
+
+    return new_extractor
+
+
+FeatureExtractor: TypeAlias = Union[StaticFeatureExtractor, DynamicFeatureExtractor]

capa/features/extractors/binexport2/__init__.py

@@ -0,0 +1,416 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+"""
+Proto files generated via protobuf v24.4:
+
+    protoc --python_out=. --mypy_out=. binexport2.proto
+
+from BinExport2 at 6916731d5f6693c4a4f0a052501fd3bd92cfd08b
+https://github.com/google/binexport/blob/6916731/binexport2.proto
+"""
+import io
+import hashlib
+import logging
+import contextlib
+from typing import Set, Dict, List, Tuple, Iterator
+from pathlib import Path
+from collections import defaultdict
+from dataclasses import dataclass
+
+from pefile import PE
+from elftools.elf.elffile import ELFFile
+
+import capa.features.common
+import capa.features.extractors.common
+import capa.features.extractors.binexport2.helpers
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+
+logger = logging.getLogger(__name__)
+
+
+def get_binexport2(sample: Path) -> BinExport2:
+    be2: BinExport2 = BinExport2()
+    be2.ParseFromString(sample.read_bytes())
+    return be2
+
+
+def compute_common_prefix_length(m: str, n: str) -> int:
+    # ensure #m < #n
+    if len(n) < len(m):
+        m, n = n, m
+
+    for i, c in enumerate(m):
+        if n[i] != c:
+            return i
+
+    return len(m)
+
+
+def get_sample_from_binexport2(input_file: Path, be2: BinExport2, search_paths: List[Path]) -> Path:
+    """attempt to find the sample file, given a BinExport2 file.
+
+    searches in the same directory as the BinExport2 file, and then in search_paths.
+    """
+
+    def filename_similarity_key(p: Path) -> Tuple[int, str]:
+        # note closure over input_file.
+        # sort first by length of common prefix, then by name (for stability)
+        return (compute_common_prefix_length(p.name, input_file.name), p.name)
+
+    wanted_sha256: str = be2.meta_information.executable_id.lower()
+
+    input_directory: Path = input_file.parent
+    siblings: List[Path] = [p for p in input_directory.iterdir() if p.is_file()]
+    siblings.sort(key=filename_similarity_key, reverse=True)
+    for sibling in siblings:
+        # e.g. with open IDA files in the same directory on Windows
+        with contextlib.suppress(PermissionError):
+            if hashlib.sha256(sibling.read_bytes()).hexdigest().lower() == wanted_sha256:
+                return sibling
+
+    for search_path in search_paths:
+        candidates: List[Path] = [p for p in search_path.iterdir() if p.is_file()]
+        candidates.sort(key=filename_similarity_key, reverse=True)
+        for candidate in candidates:
+            with contextlib.suppress(PermissionError):
+                if hashlib.sha256(candidate.read_bytes()).hexdigest().lower() == wanted_sha256:
+                    return candidate
+
+    raise ValueError("cannot find sample, you may specify the path using the CAPA_SAMPLES_DIR environment variable")
+
+
+class BinExport2Index:
+    def __init__(self, be2: BinExport2):
+        self.be2: BinExport2 = be2
+
+        self.callers_by_vertex_index: Dict[int, List[int]] = defaultdict(list)
+        self.callees_by_vertex_index: Dict[int, List[int]] = defaultdict(list)
+
+        # note: flow graph != call graph (vertex)
+        self.flow_graph_index_by_address: Dict[int, int] = {}
+        self.flow_graph_address_by_index: Dict[int, int] = {}
+
+        # edges that come from the given basic block
+        self.source_edges_by_basic_block_index: Dict[int, List[BinExport2.FlowGraph.Edge]] = defaultdict(list)
+        # edges that end up at the given basic block
+        self.target_edges_by_basic_block_index: Dict[int, List[BinExport2.FlowGraph.Edge]] = defaultdict(list)
+
+        self.vertex_index_by_address: Dict[int, int] = {}
+
+        self.data_reference_index_by_source_instruction_index: Dict[int, List[int]] = defaultdict(list)
+        self.data_reference_index_by_target_address: Dict[int, List[int]] = defaultdict(list)
+        self.string_reference_index_by_source_instruction_index: Dict[int, List[int]] = defaultdict(list)
+
+        self.insn_address_by_index: Dict[int, int] = {}
+        self.insn_index_by_address: Dict[int, int] = {}
+        self.insn_by_address: Dict[int, BinExport2.Instruction] = {}
+
+        # must index instructions first
+        self._index_insn_addresses()
+        self._index_vertex_edges()
+        self._index_flow_graph_nodes()
+        self._index_flow_graph_edges()
+        self._index_call_graph_vertices()
+        self._index_data_references()
+        self._index_string_references()
+
+    def get_insn_address(self, insn_index: int) -> int:
+        assert insn_index in self.insn_address_by_index, f"insn must be indexed, missing {insn_index}"
+        return self.insn_address_by_index[insn_index]
+
+    def get_basic_block_address(self, basic_block_index: int) -> int:
+        basic_block: BinExport2.BasicBlock = self.be2.basic_block[basic_block_index]
+        first_instruction_index: int = next(self.instruction_indices(basic_block))
+        return self.get_insn_address(first_instruction_index)
+
+    def _index_vertex_edges(self):
+        for edge in self.be2.call_graph.edge:
+            if not edge.source_vertex_index:
+                continue
+            if not edge.target_vertex_index:
+                continue
+
+            self.callers_by_vertex_index[edge.target_vertex_index].append(edge.source_vertex_index)
+            self.callees_by_vertex_index[edge.source_vertex_index].append(edge.target_vertex_index)
+
+    def _index_flow_graph_nodes(self):
+        for flow_graph_index, flow_graph in enumerate(self.be2.flow_graph):
+            function_address: int = self.get_basic_block_address(flow_graph.entry_basic_block_index)
+            self.flow_graph_index_by_address[function_address] = flow_graph_index
+            self.flow_graph_address_by_index[flow_graph_index] = function_address
+
+    def _index_flow_graph_edges(self):
+        for flow_graph in self.be2.flow_graph:
+            for edge in flow_graph.edge:
+                if not edge.HasField("source_basic_block_index") or not edge.HasField("target_basic_block_index"):
+                    continue
+
+                self.source_edges_by_basic_block_index[edge.source_basic_block_index].append(edge)
+                self.target_edges_by_basic_block_index[edge.target_basic_block_index].append(edge)
+
+    def _index_call_graph_vertices(self):
+        for vertex_index, vertex in enumerate(self.be2.call_graph.vertex):
+            if not vertex.HasField("address"):
+                continue
+
+            vertex_address: int = vertex.address
+            self.vertex_index_by_address[vertex_address] = vertex_index
+
+    def _index_data_references(self):
+        for data_reference_index, data_reference in enumerate(self.be2.data_reference):
+            self.data_reference_index_by_source_instruction_index[data_reference.instruction_index].append(
+                data_reference_index
+            )
+            self.data_reference_index_by_target_address[data_reference.address].append(data_reference_index)
+
+    def _index_string_references(self):
+        for string_reference_index, string_reference in enumerate(self.be2.string_reference):
+            self.string_reference_index_by_source_instruction_index[string_reference.instruction_index].append(
+                string_reference_index
+            )
+
+    def _index_insn_addresses(self):
+        # see https://github.com/google/binexport/blob/39f6445c232bb5caf5c4a2a996de91dfa20c48e8/binexport.cc#L45
+        if len(self.be2.instruction) == 0:
+            return
+
+        assert self.be2.instruction[0].HasField("address"), "first insn must have explicit address"
+
+        addr: int = 0
+        next_addr: int = 0
+        for idx, insn in enumerate(self.be2.instruction):
+            if insn.HasField("address"):
+                addr = insn.address
+                next_addr = addr + len(insn.raw_bytes)
+            else:
+                addr = next_addr
+                next_addr += len(insn.raw_bytes)
+            self.insn_address_by_index[idx] = addr
+            self.insn_index_by_address[addr] = idx
+            self.insn_by_address[addr] = insn
+
+    @staticmethod
+    def instruction_indices(basic_block: BinExport2.BasicBlock) -> Iterator[int]:
+        """
+        For a given basic block, enumerate the instruction indices.
+        """
+        for index_range in basic_block.instruction_index:
+            if not index_range.HasField("end_index"):
+                yield index_range.begin_index
+                continue
+            else:
+                yield from range(index_range.begin_index, index_range.end_index)
+
+    def basic_block_instructions(
+        self, basic_block: BinExport2.BasicBlock
+    ) -> Iterator[Tuple[int, BinExport2.Instruction, int]]:
+        """
+        For a given basic block, enumerate the instruction indices,
+        the instruction instances, and their addresses.
+        """
+        for instruction_index in self.instruction_indices(basic_block):
+            instruction: BinExport2.Instruction = self.be2.instruction[instruction_index]
+            instruction_address: int = self.get_insn_address(instruction_index)
+
+            yield instruction_index, instruction, instruction_address
+
+    def get_function_name_by_vertex(self, vertex_index: int) -> str:
+        vertex: BinExport2.CallGraph.Vertex = self.be2.call_graph.vertex[vertex_index]
+        name: str = f"sub_{vertex.address:x}"
+        if vertex.HasField("mangled_name"):
+            name = vertex.mangled_name
+
+        if vertex.HasField("demangled_name"):
+            name = vertex.demangled_name
+
+        if vertex.HasField("library_index"):
+            library: BinExport2.Library = self.be2.library[vertex.library_index]
+            if library.HasField("name"):
+                name = f"{library.name}!{name}"
+
+        return name
+
+    def get_function_name_by_address(self, address: int) -> str:
+        if address not in self.vertex_index_by_address:
+            return ""
+
+        vertex_index: int = self.vertex_index_by_address[address]
+        return self.get_function_name_by_vertex(vertex_index)
+
+    def get_instruction_by_address(self, address: int) -> BinExport2.Instruction:
+        assert address in self.insn_by_address, f"address must be indexed, missing {address:x}"
+        return self.insn_by_address[address]
+
+
+class BinExport2Analysis:
+    def __init__(self, be2: BinExport2, idx: BinExport2Index, buf: bytes):
+        self.be2: BinExport2 = be2
+        self.idx: BinExport2Index = idx
+        self.buf: bytes = buf
+        self.base_address: int = 0
+        self.thunks: Dict[int, int] = {}
+
+        self._find_base_address()
+        self._compute_thunks()
+
+    def _find_base_address(self):
+        sections_with_perms: Iterator[BinExport2.Section] = filter(
+            lambda s: s.flag_r or s.flag_w or s.flag_x, self.be2.section
+        )
+        # assume the lowest address is the base address.
+        # this works as long as BinExport doesn't record other
+        # libraries mapped into memory.
+        self.base_address = min(s.address for s in sections_with_perms)
+
+        logger.debug("found base address: %x", self.base_address)
+
+    def _compute_thunks(self):
+        for addr, idx in self.idx.vertex_index_by_address.items():
+            vertex: BinExport2.CallGraph.Vertex = self.be2.call_graph.vertex[idx]
+            if not capa.features.extractors.binexport2.helpers.is_vertex_type(
+                vertex, BinExport2.CallGraph.Vertex.Type.THUNK
+            ):
+                continue
+
+            curr_idx: int = idx
+            for _ in range(capa.features.common.THUNK_CHAIN_DEPTH_DELTA):
+                thunk_callees: List[int] = self.idx.callees_by_vertex_index[curr_idx]
+                # if this doesn't hold, then it doesn't seem like this is a thunk,
+                # because either, len is:
+                #    0 and the thunk doesn't point to anything, or
+                #   >1 and the thunk may end up at many functions.
+                assert len(thunk_callees) == 1, f"thunk @ {hex(addr)} failed"
+
+                thunked_idx: int = thunk_callees[0]
+                thunked_vertex: BinExport2.CallGraph.Vertex = self.be2.call_graph.vertex[thunked_idx]
+
+                if not capa.features.extractors.binexport2.helpers.is_vertex_type(
+                    thunked_vertex, BinExport2.CallGraph.Vertex.Type.THUNK
+                ):
+                    assert thunked_vertex.HasField("address")
+
+                    self.thunks[addr] = thunked_vertex.address
+                    break
+
+                curr_idx = thunked_idx
+
+
+@dataclass
+class MemoryRegion:
+    # location of the bytes, potentially relative to a base address
+    address: int
+    buf: bytes
+
+    @property
+    def end(self) -> int:
+        return self.address + len(self.buf)
+
+    def contains(self, address: int) -> bool:
+        # note: address must be relative to any base address
+        return self.address <= address < self.end
+
+
+class ReadMemoryError(ValueError): ...
+
+
+class AddressNotMappedError(ReadMemoryError): ...
+
+
+@dataclass
+class AddressSpace:
+    base_address: int
+    memory_regions: Tuple[MemoryRegion, ...]
+
+    def read_memory(self, address: int, length: int) -> bytes:
+        rva: int = address - self.base_address
+        for region in self.memory_regions:
+            if region.contains(rva):
+                offset: int = rva - region.address
+                return region.buf[offset : offset + length]
+
+        raise AddressNotMappedError(address)
+
+    @classmethod
+    def from_pe(cls, pe: PE, base_address: int):
+        regions: List[MemoryRegion] = []
+        for section in pe.sections:
+            address: int = section.VirtualAddress
+            size: int = section.Misc_VirtualSize
+            buf: bytes = section.get_data()
+
+            if len(buf) != size:
+                # pad the section with NULLs
+                # assume page alignment is already handled.
+                # might need more hardening here.
+                buf += b"\x00" * (size - len(buf))
+
+            regions.append(MemoryRegion(address, buf))
+
+        return cls(base_address, tuple(regions))
+
+    @classmethod
+    def from_elf(cls, elf: ELFFile, base_address: int):
+        regions: List[MemoryRegion] = []
+
+        # ELF segments are for runtime data,
+        # ELF sections are for link-time data.
+        for segment in elf.iter_segments():
+            # assume p_align is consistent with addresses here.
+            # otherwise, should harden this loader.
+            segment_rva: int = segment.header.p_vaddr
+            segment_size: int = segment.header.p_memsz
+            segment_data: bytes = segment.data()
+
+            if len(segment_data) < segment_size:
+                # pad the section with NULLs
+                # assume page alignment is already handled.
+                # might need more hardening here.
+                segment_data += b"\x00" * (segment_size - len(segment_data))
+
+            regions.append(MemoryRegion(segment_rva, segment_data))
+
+        return cls(base_address, tuple(regions))
+
+    @classmethod
+    def from_buf(cls, buf: bytes, base_address: int):
+        if buf.startswith(capa.features.extractors.common.MATCH_PE):
+            pe: PE = PE(data=buf)
+            return cls.from_pe(pe, base_address)
+        elif buf.startswith(capa.features.extractors.common.MATCH_ELF):
+            elf: ELFFile = ELFFile(io.BytesIO(buf))
+            return cls.from_elf(elf, base_address)
+        else:
+            raise NotImplementedError("file format address space")
+
+
+@dataclass
+class AnalysisContext:
+    sample_bytes: bytes
+    be2: BinExport2
+    idx: BinExport2Index
+    analysis: BinExport2Analysis
+    address_space: AddressSpace
+
+
+@dataclass
+class FunctionContext:
+    ctx: AnalysisContext
+    flow_graph_index: int
+    format: Set[str]
+    os: Set[str]
+    arch: Set[str]
+
+
+@dataclass
+class BasicBlockContext:
+    basic_block_index: int
+
+
+@dataclass
+class InstructionContext:
+    instruction_index: int

capa/features/extractors/binexport2/arch/arm/helpers.py

@@ -0,0 +1,15 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+
+
+def is_stack_register_expression(be2: BinExport2, expression: BinExport2.Expression) -> bool:
+    return bool(
+        expression and expression.type == BinExport2.Expression.REGISTER and expression.symbol.lower().endswith("sp")
+    )

capa/features/extractors/binexport2/arch/arm/insn.py

@@ -0,0 +1,155 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import List, Tuple, Iterator, Optional
+
+import capa.features.extractors.binexport2.helpers
+from capa.features.insn import MAX_STRUCTURE_SIZE, Number, Offset, OperandNumber, OperandOffset
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.extractors.binexport2 import FunctionContext, InstructionContext
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+from capa.features.extractors.binexport2.helpers import (
+    BinExport2InstructionPatternMatcher,
+    mask_immediate,
+    is_address_mapped,
+    get_instruction_mnemonic,
+    get_operand_register_expression,
+    get_operand_immediate_expression,
+)
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+from capa.features.extractors.binexport2.arch.arm.helpers import is_stack_register_expression
+
+logger = logging.getLogger(__name__)
+
+
+def extract_insn_number_features(
+    fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+
+    instruction_index: int = ii.instruction_index
+    instruction: BinExport2.Instruction = be2.instruction[instruction_index]
+
+    if len(instruction.operand_index) == 0:
+        # skip things like:
+        #   .text:0040116e leave
+        return
+
+    mnemonic: str = get_instruction_mnemonic(be2, instruction)
+
+    if mnemonic in ("add", "sub"):
+        assert len(instruction.operand_index) == 3
+
+        operand1_expression: Optional[BinExport2.Expression] = get_operand_register_expression(
+            be2, be2.operand[instruction.operand_index[1]]
+        )
+        if operand1_expression and is_stack_register_expression(be2, operand1_expression):
+            # skip things like:
+            # add x0,sp,#0x8
+            return
+
+    for i, operand_index in enumerate(instruction.operand_index):
+        operand: BinExport2.Operand = be2.operand[operand_index]
+
+        immediate_expression: Optional[BinExport2.Expression] = get_operand_immediate_expression(be2, operand)
+        if not immediate_expression:
+            continue
+
+        value: int = mask_immediate(fhi.arch, immediate_expression.immediate)
+        if is_address_mapped(be2, value):
+            continue
+
+        yield Number(value), ih.address
+        yield OperandNumber(i, value), ih.address
+
+        if mnemonic == "add" and i == 2:
+            if 0 < value < MAX_STRUCTURE_SIZE:
+                yield Offset(value), ih.address
+                yield OperandOffset(i, value), ih.address
+
+
+OFFSET_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
+    """
+    ldr|ldrb|ldrh|ldrsb|ldrsh|ldrex|ldrd|str|strb|strh|strex|strd reg, [reg(not-stack),  #int]                                 ; capture #int
+    ldr|ldrb|ldrh|ldrsb|ldrsh|ldrex|ldrd|str|strb|strh|strex|strd reg, [reg(not-stack),  #int]!                                ; capture #int
+    ldr|ldrb|ldrh|ldrsb|ldrsh|ldrex|ldrd|str|strb|strh|strex|strd reg, [reg(not-stack)],        #int                           ; capture #int
+    ldp|ldpd|stp|stpd                                             reg, reg,                     [reg(not-stack), #int]         ; capture #int
+    ldp|ldpd|stp|stpd                                             reg, reg,                     [reg(not-stack), #int]!        ; capture #int
+    ldp|ldpd|stp|stpd                                             reg, reg,                     [reg(not-stack)],       #int   ; capture #int
+    """
+)
+
+
+def extract_insn_offset_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+
+    match = OFFSET_PATTERNS.match_with_be2(be2, ii.instruction_index)
+    if not match:
+        return
+
+    value = match.expression.immediate
+
+    value = mask_immediate(fhi.arch, value)
+    if not is_address_mapped(be2, value):
+        value = capa.features.extractors.binexport2.helpers.twos_complement(fhi.arch, value)
+        yield Offset(value), ih.address
+        yield OperandOffset(match.operand_index, value), ih.address
+
+
+NZXOR_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
+    """
+    eor reg, reg, reg
+    eor reg, reg, #int
+    """
+)
+
+
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+    be2: BinExport2 = fhi.ctx.be2
+
+    if NZXOR_PATTERNS.match_with_be2(be2, ii.instruction_index) is None:
+        return
+
+    instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
+    # guaranteed to be simple int/reg operands
+    # so we don't have to realize the tree/list.
+    operands: List[BinExport2.Operand] = [be2.operand[operand_index] for operand_index in instruction.operand_index]
+
+    if operands[1] != operands[2]:
+        yield Characteristic("nzxor"), ih.address
+
+
+INDIRECT_CALL_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
+    """
+    blx|bx|blr reg
+    """
+)
+
+
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+    be2: BinExport2 = fhi.ctx.be2
+
+    if INDIRECT_CALL_PATTERNS.match_with_be2(be2, ii.instruction_index) is not None:
+        yield Characteristic("indirect call"), ih.address

capa/features/extractors/binexport2/arch/intel/helpers.py

@@ -0,0 +1,135 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Optional
+from dataclasses import dataclass
+
+from capa.features.extractors.binexport2.helpers import get_operand_expressions
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+
+# security cookie checks may perform non-zeroing XORs, these are expected within a certain
+# byte range within the first and returning basic blocks, this helps to reduce FP features
+SECURITY_COOKIE_BYTES_DELTA: int = 0x40
+
+
+@dataclass
+class OperandPhraseInfo:
+    scale: Optional[BinExport2.Expression] = None
+    index: Optional[BinExport2.Expression] = None
+    base: Optional[BinExport2.Expression] = None
+    displacement: Optional[BinExport2.Expression] = None
+
+
+def get_operand_phrase_info(be2: BinExport2, operand: BinExport2.Operand) -> Optional[OperandPhraseInfo]:
+    # assume the following (see https://blog.yossarian.net/2020/06/13/How-x86_64-addresses-memory):
+    #
+    # Scale: A 2-bit constant factor
+    # Index: Any general purpose register
+    # Base: Any general purpose register
+    # Displacement: An integral offset
+
+    expressions: List[BinExport2.Expression] = get_operand_expressions(be2, operand)
+
+    # skip expression up to and including BinExport2.Expression.DEREFERENCE, assume caller
+    # has checked for BinExport2.Expression.DEREFERENCE
+    for i, expression in enumerate(expressions):
+        if expression.type == BinExport2.Expression.DEREFERENCE:
+            expressions = expressions[i + 1 :]
+            break
+
+    expression0: BinExport2.Expression
+    expression1: BinExport2.Expression
+    expression2: BinExport2.Expression
+    expression3: BinExport2.Expression
+    expression4: BinExport2.Expression
+
+    if len(expressions) == 1:
+        expression0 = expressions[0]
+
+        assert (
+            expression0.type == BinExport2.Expression.IMMEDIATE_INT
+            or expression0.type == BinExport2.Expression.REGISTER
+        )
+
+        if expression0.type == BinExport2.Expression.IMMEDIATE_INT:
+            # Displacement
+            return OperandPhraseInfo(displacement=expression0)
+        elif expression0.type == BinExport2.Expression.REGISTER:
+            # Base
+            return OperandPhraseInfo(base=expression0)
+
+    elif len(expressions) == 3:
+        expression0 = expressions[0]
+        expression1 = expressions[1]
+        expression2 = expressions[2]
+
+        assert expression0.type == BinExport2.Expression.REGISTER
+        assert expression1.type == BinExport2.Expression.OPERATOR
+        assert (
+            expression2.type == BinExport2.Expression.IMMEDIATE_INT
+            or expression2.type == BinExport2.Expression.REGISTER
+        )
+
+        if expression2.type == BinExport2.Expression.REGISTER:
+            # Base + Index
+            return OperandPhraseInfo(base=expression0, index=expression2)
+        elif expression2.type == BinExport2.Expression.IMMEDIATE_INT:
+            # Base + Displacement
+            return OperandPhraseInfo(base=expression0, displacement=expression2)
+
+    elif len(expressions) == 5:
+        expression0 = expressions[0]
+        expression1 = expressions[1]
+        expression2 = expressions[2]
+        expression3 = expressions[3]
+        expression4 = expressions[4]
+
+        assert expression0.type == BinExport2.Expression.REGISTER
+        assert expression1.type == BinExport2.Expression.OPERATOR
+        assert (
+            expression2.type == BinExport2.Expression.REGISTER
+            or expression2.type == BinExport2.Expression.IMMEDIATE_INT
+        )
+        assert expression3.type == BinExport2.Expression.OPERATOR
+        assert expression4.type == BinExport2.Expression.IMMEDIATE_INT
+
+        if expression1.symbol == "+" and expression3.symbol == "+":
+            # Base + Index + Displacement
+            return OperandPhraseInfo(base=expression0, index=expression2, displacement=expression4)
+        elif expression1.symbol == "+" and expression3.symbol == "*":
+            # Base + (Index * Scale)
+            return OperandPhraseInfo(base=expression0, index=expression2, scale=expression3)
+        elif expression1.symbol == "*" and expression3.symbol == "+":
+            # (Index * Scale) + Displacement
+            return OperandPhraseInfo(index=expression0, scale=expression2, displacement=expression3)
+        else:
+            raise NotImplementedError(expression1.symbol, expression3.symbol)
+
+    elif len(expressions) == 7:
+        expression0 = expressions[0]
+        expression1 = expressions[1]
+        expression2 = expressions[2]
+        expression3 = expressions[3]
+        expression4 = expressions[4]
+        expression5 = expressions[5]
+        expression6 = expressions[6]
+
+        assert expression0.type == BinExport2.Expression.REGISTER
+        assert expression1.type == BinExport2.Expression.OPERATOR
+        assert expression2.type == BinExport2.Expression.REGISTER
+        assert expression3.type == BinExport2.Expression.OPERATOR
+        assert expression4.type == BinExport2.Expression.IMMEDIATE_INT
+        assert expression5.type == BinExport2.Expression.OPERATOR
+        assert expression6.type == BinExport2.Expression.IMMEDIATE_INT
+
+        # Base + (Index * Scale) + Displacement
+        return OperandPhraseInfo(base=expression0, index=expression2, scale=expression4, displacement=expression6)
+
+    else:
+        raise NotImplementedError(len(expressions))
+
+    return None

capa/features/extractors/binexport2/arch/intel/insn.py

@@ -0,0 +1,248 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import List, Tuple, Iterator
+
+import capa.features.extractors.strings
+import capa.features.extractors.binexport2.helpers
+from capa.features.insn import MAX_STRUCTURE_SIZE, Number, Offset, OperandNumber, OperandOffset
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.extractors.binexport2 import BinExport2Index, FunctionContext, BasicBlockContext, InstructionContext
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+from capa.features.extractors.binexport2.helpers import (
+    BinExport2InstructionPatternMatcher,
+    mask_immediate,
+    is_address_mapped,
+    get_instruction_mnemonic,
+)
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+from capa.features.extractors.binexport2.arch.intel.helpers import SECURITY_COOKIE_BYTES_DELTA
+
+logger = logging.getLogger(__name__)
+
+
+IGNORE_NUMBER_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
+    """
+    ret  #int
+    retn #int
+    add  reg(stack), #int
+    sub  reg(stack), #int
+    """
+)
+
+NUMBER_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
+    """
+    push #int0            ; capture #int0
+
+    # its a little tedious to enumerate all the address forms
+    # but at least we are explicit
+    cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar  reg,                      #int0  ; capture #int0
+    cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg],                     #int0  ; capture #int0
+    cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [#int],                    #int0  ; capture #int0
+    cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg + #int],              #int0  ; capture #int0
+    cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg + reg + #int],        #int0  ; capture #int0
+    cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg + reg * #int],        #int0  ; capture #int0
+    cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg + reg * #int + #int], #int0  ; capture #int0
+
+    imul reg, reg, #int  ; capture #int
+    # note that int is first
+    cmp|test #int0, reg   ; capture #int0
+
+    # imagine reg is zero'd out, then this is like `mov reg, #int`
+    # which is not uncommon.
+    lea reg, [reg + #int]  ; capture #int
+    """
+)
+
+
+def extract_insn_number_features(
+    fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+
+    if IGNORE_NUMBER_PATTERNS.match_with_be2(be2, ii.instruction_index):
+        return
+
+    match = NUMBER_PATTERNS.match_with_be2(be2, ii.instruction_index)
+    if not match:
+        return
+
+    value: int = mask_immediate(fhi.arch, match.expression.immediate)
+    if is_address_mapped(be2, value):
+        return
+
+    yield Number(value), ih.address
+    yield OperandNumber(match.operand_index, value), ih.address
+
+    instruction_index: int = ii.instruction_index
+    instruction: BinExport2.Instruction = be2.instruction[instruction_index]
+
+    mnemonic: str = get_instruction_mnemonic(be2, instruction)
+    if mnemonic.startswith("add"):
+        if 0 < value < MAX_STRUCTURE_SIZE:
+            yield Offset(value), ih.address
+            yield OperandOffset(match.operand_index, value), ih.address
+
+
+OFFSET_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
+    """
+    mov|movzx|movsb|cmp [reg            +  reg * #int + #int0], #int  ; capture #int0
+    mov|movzx|movsb|cmp [reg            * #int + #int0],        #int  ; capture #int0
+    mov|movzx|movsb|cmp [reg            +  reg + #int0],        #int  ; capture #int0
+    mov|movzx|movsb|cmp [reg(not-stack) + #int0],               #int  ; capture #int0
+    mov|movzx|movsb|cmp [reg            +  reg * #int + #int0], reg   ; capture #int0
+    mov|movzx|movsb|cmp [reg            * #int + #int0],        reg   ; capture #int0
+    mov|movzx|movsb|cmp [reg            +  reg + #int0],        reg   ; capture #int0
+    mov|movzx|movsb|cmp [reg(not-stack) + #int0],               reg   ; capture #int0
+    mov|movzx|movsb|cmp|lea reg, [reg            +  reg * #int + #int0]  ; capture #int0
+    mov|movzx|movsb|cmp|lea reg, [reg            * #int + #int0]         ; capture #int0
+    mov|movzx|movsb|cmp|lea reg, [reg            +  reg + #int0]         ; capture #int0
+    mov|movzx|movsb|cmp|lea reg, [reg(not-stack) + #int0]                ; capture #int0
+    """
+)
+
+# these are patterns that access offset 0 from some pointer
+# (pointer is not the stack pointer).
+OFFSET_ZERO_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
+    """
+    mov|movzx|movsb [reg(not-stack)], reg
+    mov|movzx|movsb [reg(not-stack)], #int
+    lea             reg,              [reg(not-stack)]
+    """
+)
+
+
+def extract_insn_offset_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+
+    match = OFFSET_PATTERNS.match_with_be2(be2, ii.instruction_index)
+    if not match:
+        match = OFFSET_ZERO_PATTERNS.match_with_be2(be2, ii.instruction_index)
+        if not match:
+            return
+
+        yield Offset(0), ih.address
+        yield OperandOffset(match.operand_index, 0), ih.address
+
+    value = mask_immediate(fhi.arch, match.expression.immediate)
+    if is_address_mapped(be2, value):
+        return
+
+    value = capa.features.extractors.binexport2.helpers.twos_complement(fhi.arch, value, 32)
+    yield Offset(value), ih.address
+    yield OperandOffset(match.operand_index, value), ih.address
+
+
+def is_security_cookie(
+    fhi: FunctionContext,
+    bbi: BasicBlockContext,
+    instruction_address: int,
+    instruction: BinExport2.Instruction,
+) -> bool:
+    """
+    check if an instruction is related to security cookie checks.
+    """
+    be2: BinExport2 = fhi.ctx.be2
+    idx: BinExport2Index = fhi.ctx.idx
+
+    # security cookie check should use SP or BP
+    op1: BinExport2.Operand = be2.operand[instruction.operand_index[1]]
+    op1_exprs: List[BinExport2.Expression] = [be2.expression[expr_i] for expr_i in op1.expression_index]
+    if all(expr.symbol.lower() not in ("bp", "esp", "ebp", "rbp", "rsp") for expr in op1_exprs):
+        return False
+
+    # check_nzxor_security_cookie_delta
+    # if insn falls at the start of first entry block of the parent function.
+    flow_graph: BinExport2.FlowGraph = be2.flow_graph[fhi.flow_graph_index]
+    basic_block_index: int = bbi.basic_block_index
+    bb: BinExport2.BasicBlock = be2.basic_block[basic_block_index]
+    if flow_graph.entry_basic_block_index == basic_block_index:
+        first_addr: int = min((idx.insn_address_by_index[ir.begin_index] for ir in bb.instruction_index))
+        if instruction_address < first_addr + SECURITY_COOKIE_BYTES_DELTA:
+            return True
+    # or insn falls at the end before return in a terminal basic block.
+    if basic_block_index not in (e.source_basic_block_index for e in flow_graph.edge):
+        last_addr: int = max((idx.insn_address_by_index[ir.end_index - 1] for ir in bb.instruction_index))
+        if instruction_address > last_addr - SECURITY_COOKIE_BYTES_DELTA:
+            return True
+    return False
+
+
+NZXOR_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
+    """
+    xor|xorpd|xorps|pxor reg, reg
+    xor|xorpd|xorps|pxor reg, #int
+    """
+)
+
+
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse non-zeroing XOR instruction from the given instruction.
+    ignore expected non-zeroing XORs, e.g. security cookies.
+    """
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+    idx: BinExport2Index = fhi.ctx.idx
+
+    if NZXOR_PATTERNS.match_with_be2(be2, ii.instruction_index) is None:
+        return
+
+    instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
+    # guaranteed to be simple int/reg operands
+    # so we don't have to realize the tree/list.
+    operands: List[BinExport2.Operand] = [be2.operand[operand_index] for operand_index in instruction.operand_index]
+
+    if operands[0] == operands[1]:
+        return
+
+    instruction_address: int = idx.insn_address_by_index[ii.instruction_index]
+    if is_security_cookie(fhi, bbh.inner, instruction_address, instruction):
+        return
+
+    yield Characteristic("nzxor"), ih.address
+
+
+INDIRECT_CALL_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
+    """
+    call|jmp reg0
+    call|jmp [reg + reg * #int + #int]
+    call|jmp [reg + reg * #int]
+    call|jmp [reg * #int + #int]
+    call|jmp [reg + reg + #int]
+    call|jmp [reg + #int]
+    call|jmp [reg]
+    """
+)
+
+
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+    be2: BinExport2 = fhi.ctx.be2
+
+    match = INDIRECT_CALL_PATTERNS.match_with_be2(be2, ii.instruction_index)
+    if match is None:
+        return
+
+    yield Characteristic("indirect call"), ih.address

capa/features/extractors/binexport2/basicblock.py

@@ -0,0 +1,40 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from typing import List, Tuple, Iterator
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.binexport2 import FunctionContext, BasicBlockContext
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+
+
+def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    bbi: BasicBlockContext = bbh.inner
+
+    idx = fhi.ctx.idx
+
+    basic_block_index: int = bbi.basic_block_index
+    target_edges: List[BinExport2.FlowGraph.Edge] = idx.target_edges_by_basic_block_index[basic_block_index]
+    if basic_block_index in (e.source_basic_block_index for e in target_edges):
+        basic_block_address: int = idx.get_basic_block_address(basic_block_index)
+        yield Characteristic("tight loop"), AbsoluteVirtualAddress(basic_block_address)
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract basic block features"""
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, addr in bb_handler(fh, bbh):
+            yield feature, addr
+    yield BasicBlock(), bbh.address
+
+
+BASIC_BLOCK_HANDLERS = (extract_bb_tight_loop,)

capa/features/extractors/binexport2/binexport2_pb2.pyi

@@ -0,0 +1,784 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+The representation is generic to accommodate various source architectures.
+In particular 32 and 64 bit versions of x86, ARM, PowerPC and MIPS have been
+tested.
+
+Multiple levels of deduping have been applied to make the format more compact
+and avoid redundant data duplication. Some of this due to hard-earned
+experience trying to cope with intentionally obfuscated malicious binaries.
+Note in particular that the same instruction may occur in multiple basic
+blocks and the same basic block in multiple functions (instruction and basic
+block sharing). Implemented naively, malware can use this to cause
+combinatorial explosion in memory usage, DOSing the analyst. This format
+should store every unique expression, mnemonic, operand, instruction and
+basic block only once instead of duplicating the information for every
+instance of it.
+
+This format does _not_ try to be 100% backwards compatible with the old
+version. In particular, we do not store IDA's comment types, making lossless
+porting of IDA comments impossible. We do however, store comments and
+expression substitutions, so porting the actual data is possible, just not
+the exact IDA type.
+
+While it would be more natural to use addresses when defining call graph and
+flow graph edges and other such references, it is more efficient to employ
+one more level of indirection and use indices into the basic block or
+function arrays instead. This is because addresses will usually use most of
+the available 64 bit space while indices will be much smaller and compress
+much better (less randomly distributed).
+
+We omit all fields that are set to their default value anyways. Note that
+this has two side effects:
+  - changing the defaults in this proto file will, in effect, change what's
+    read from disk
+  - the generated code has_* methods are somewhat less useful
+WARNING: We omit the defaults manually in the code writing the data. Do not
+         change the defaults here without changing the code!
+
+TODO(cblichmann): Link flow graphs to call graph nodes. The connection is
+                  there via the address, but tricky to extract.
+"""
+import builtins
+import collections.abc
+import google.protobuf.descriptor
+import google.protobuf.internal.containers
+import google.protobuf.internal.enum_type_wrapper
+import google.protobuf.message
+import sys
+import typing
+
+if sys.version_info >= (3, 10):
+    import typing as typing_extensions
+else:
+    import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+@typing_extensions.final
+class BinExport2(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    @typing_extensions.final
+    class Meta(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        EXECUTABLE_NAME_FIELD_NUMBER: builtins.int
+        EXECUTABLE_ID_FIELD_NUMBER: builtins.int
+        ARCHITECTURE_NAME_FIELD_NUMBER: builtins.int
+        TIMESTAMP_FIELD_NUMBER: builtins.int
+        executable_name: builtins.str
+        """Input binary filename including file extension but excluding file path.
+        example: "insider_gcc.exe"
+        """
+        executable_id: builtins.str
+        """Application defined executable id. Often the SHA256 hash of the input
+        binary.
+        """
+        architecture_name: builtins.str
+        """Input architecture name, e.g. x86-32."""
+        timestamp: builtins.int
+        """When did this file get created? Unix time. This may be used for some
+        primitive versioning in case the file format ever changes.
+        """
+        def __init__(
+            self,
+            *,
+            executable_name: builtins.str | None = ...,
+            executable_id: builtins.str | None = ...,
+            architecture_name: builtins.str | None = ...,
+            timestamp: builtins.int | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["architecture_name", b"architecture_name", "executable_id", b"executable_id", "executable_name", b"executable_name", "timestamp", b"timestamp"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["architecture_name", b"architecture_name", "executable_id", b"executable_id", "executable_name", b"executable_name", "timestamp", b"timestamp"]) -> None: ...
+
+    @typing_extensions.final
+    class CallGraph(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        @typing_extensions.final
+        class Vertex(google.protobuf.message.Message):
+            DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+            class _Type:
+                ValueType = typing.NewType("ValueType", builtins.int)
+                V: typing_extensions.TypeAlias = ValueType
+
+            class _TypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[BinExport2.CallGraph.Vertex._Type.ValueType], builtins.type):
+                DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+                NORMAL: BinExport2.CallGraph.Vertex._Type.ValueType  # 0
+                """Regular function with full disassembly."""
+                LIBRARY: BinExport2.CallGraph.Vertex._Type.ValueType  # 1
+                """This function is a well known library function."""
+                IMPORTED: BinExport2.CallGraph.Vertex._Type.ValueType  # 2
+                """Imported from a dynamic link library (e.g. dll)."""
+                THUNK: BinExport2.CallGraph.Vertex._Type.ValueType  # 3
+                """A thunk function, forwarding its work via an unconditional jump."""
+                INVALID: BinExport2.CallGraph.Vertex._Type.ValueType  # 4
+                """An invalid function (a function that contained invalid code or was
+                considered invalid by some heuristics).
+                """
+
+            class Type(_Type, metaclass=_TypeEnumTypeWrapper): ...
+            NORMAL: BinExport2.CallGraph.Vertex.Type.ValueType  # 0
+            """Regular function with full disassembly."""
+            LIBRARY: BinExport2.CallGraph.Vertex.Type.ValueType  # 1
+            """This function is a well known library function."""
+            IMPORTED: BinExport2.CallGraph.Vertex.Type.ValueType  # 2
+            """Imported from a dynamic link library (e.g. dll)."""
+            THUNK: BinExport2.CallGraph.Vertex.Type.ValueType  # 3
+            """A thunk function, forwarding its work via an unconditional jump."""
+            INVALID: BinExport2.CallGraph.Vertex.Type.ValueType  # 4
+            """An invalid function (a function that contained invalid code or was
+            considered invalid by some heuristics).
+            """
+
+            ADDRESS_FIELD_NUMBER: builtins.int
+            TYPE_FIELD_NUMBER: builtins.int
+            MANGLED_NAME_FIELD_NUMBER: builtins.int
+            DEMANGLED_NAME_FIELD_NUMBER: builtins.int
+            LIBRARY_INDEX_FIELD_NUMBER: builtins.int
+            MODULE_INDEX_FIELD_NUMBER: builtins.int
+            address: builtins.int
+            """The function's entry point address. Messages need to be sorted, see
+            comment below on `vertex`.
+            """
+            type: global___BinExport2.CallGraph.Vertex.Type.ValueType
+            mangled_name: builtins.str
+            """If the function has a user defined, real name it will be given here.
+            main() is a proper name, sub_BAADF00D is not (auto generated dummy
+            name).
+            """
+            demangled_name: builtins.str
+            """Demangled name if the function is a mangled C++ function and we could
+            demangle it.
+            """
+            library_index: builtins.int
+            """If this is a library function, what is its index in library arrays."""
+            module_index: builtins.int
+            """If module name, such as class name for DEX files, is present - index in
+            module table.
+            """
+            def __init__(
+                self,
+                *,
+                address: builtins.int | None = ...,
+                type: global___BinExport2.CallGraph.Vertex.Type.ValueType | None = ...,
+                mangled_name: builtins.str | None = ...,
+                demangled_name: builtins.str | None = ...,
+                library_index: builtins.int | None = ...,
+                module_index: builtins.int | None = ...,
+            ) -> None: ...
+            def HasField(self, field_name: typing_extensions.Literal["address", b"address", "demangled_name", b"demangled_name", "library_index", b"library_index", "mangled_name", b"mangled_name", "module_index", b"module_index", "type", b"type"]) -> builtins.bool: ...
+            def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "demangled_name", b"demangled_name", "library_index", b"library_index", "mangled_name", b"mangled_name", "module_index", b"module_index", "type", b"type"]) -> None: ...
+
+        @typing_extensions.final
+        class Edge(google.protobuf.message.Message):
+            DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+            SOURCE_VERTEX_INDEX_FIELD_NUMBER: builtins.int
+            TARGET_VERTEX_INDEX_FIELD_NUMBER: builtins.int
+            source_vertex_index: builtins.int
+            """source and target index into the vertex repeated field."""
+            target_vertex_index: builtins.int
+            def __init__(
+                self,
+                *,
+                source_vertex_index: builtins.int | None = ...,
+                target_vertex_index: builtins.int | None = ...,
+            ) -> None: ...
+            def HasField(self, field_name: typing_extensions.Literal["source_vertex_index", b"source_vertex_index", "target_vertex_index", b"target_vertex_index"]) -> builtins.bool: ...
+            def ClearField(self, field_name: typing_extensions.Literal["source_vertex_index", b"source_vertex_index", "target_vertex_index", b"target_vertex_index"]) -> None: ...
+
+        VERTEX_FIELD_NUMBER: builtins.int
+        EDGE_FIELD_NUMBER: builtins.int
+        @property
+        def vertex(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.CallGraph.Vertex]:
+            """vertices == functions in the call graph.
+            Important: Most downstream tooling (notably BinDiff), need these to be
+                       sorted by `Vertex::address` (ascending). For C++, the
+                       `BinExport2Writer` class enforces this invariant.
+            """
+        @property
+        def edge(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.CallGraph.Edge]:
+            """edges == calls in the call graph."""
+        def __init__(
+            self,
+            *,
+            vertex: collections.abc.Iterable[global___BinExport2.CallGraph.Vertex] | None = ...,
+            edge: collections.abc.Iterable[global___BinExport2.CallGraph.Edge] | None = ...,
+        ) -> None: ...
+        def ClearField(self, field_name: typing_extensions.Literal["edge", b"edge", "vertex", b"vertex"]) -> None: ...
+
+    @typing_extensions.final
+    class Expression(google.protobuf.message.Message):
+        """An operand consists of 1 or more expressions, linked together as a tree."""
+
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        class _Type:
+            ValueType = typing.NewType("ValueType", builtins.int)
+            V: typing_extensions.TypeAlias = ValueType
+
+        class _TypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[BinExport2.Expression._Type.ValueType], builtins.type):
+            DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+            SYMBOL: BinExport2.Expression._Type.ValueType  # 1
+            IMMEDIATE_INT: BinExport2.Expression._Type.ValueType  # 2
+            IMMEDIATE_FLOAT: BinExport2.Expression._Type.ValueType  # 3
+            OPERATOR: BinExport2.Expression._Type.ValueType  # 4
+            REGISTER: BinExport2.Expression._Type.ValueType  # 5
+            SIZE_PREFIX: BinExport2.Expression._Type.ValueType  # 6
+            DEREFERENCE: BinExport2.Expression._Type.ValueType  # 7
+
+        class Type(_Type, metaclass=_TypeEnumTypeWrapper): ...
+        SYMBOL: BinExport2.Expression.Type.ValueType  # 1
+        IMMEDIATE_INT: BinExport2.Expression.Type.ValueType  # 2
+        IMMEDIATE_FLOAT: BinExport2.Expression.Type.ValueType  # 3
+        OPERATOR: BinExport2.Expression.Type.ValueType  # 4
+        REGISTER: BinExport2.Expression.Type.ValueType  # 5
+        SIZE_PREFIX: BinExport2.Expression.Type.ValueType  # 6
+        DEREFERENCE: BinExport2.Expression.Type.ValueType  # 7
+
+        TYPE_FIELD_NUMBER: builtins.int
+        SYMBOL_FIELD_NUMBER: builtins.int
+        IMMEDIATE_FIELD_NUMBER: builtins.int
+        PARENT_INDEX_FIELD_NUMBER: builtins.int
+        IS_RELOCATION_FIELD_NUMBER: builtins.int
+        type: global___BinExport2.Expression.Type.ValueType
+        """IMMEDIATE_INT is by far the most common type and thus we can save some
+        space by omitting it as the default.
+        """
+        symbol: builtins.str
+        """Symbol for this expression. Interpretation depends on type. Examples
+        include: "eax", "[", "+"
+        """
+        immediate: builtins.int
+        """If the expression can be interpreted as an integer value (IMMEDIATE_INT)
+        the value is given here.
+        """
+        parent_index: builtins.int
+        """The parent expression. Example expression tree for the second operand of:
+        mov eax, b4 [ebx + 12]
+        "b4" --- "[" --- "+" --- "ebx"
+                              \\  "12"
+        """
+        is_relocation: builtins.bool
+        """true if the expression has entry in relocation table"""
+        def __init__(
+            self,
+            *,
+            type: global___BinExport2.Expression.Type.ValueType | None = ...,
+            symbol: builtins.str | None = ...,
+            immediate: builtins.int | None = ...,
+            parent_index: builtins.int | None = ...,
+            is_relocation: builtins.bool | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["immediate", b"immediate", "is_relocation", b"is_relocation", "parent_index", b"parent_index", "symbol", b"symbol", "type", b"type"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["immediate", b"immediate", "is_relocation", b"is_relocation", "parent_index", b"parent_index", "symbol", b"symbol", "type", b"type"]) -> None: ...
+
+    @typing_extensions.final
+    class Operand(google.protobuf.message.Message):
+        """An instruction may have 0 or more operands."""
+
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        EXPRESSION_INDEX_FIELD_NUMBER: builtins.int
+        @property
+        def expression_index(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
+            """Contains all expressions constituting this operand. All expressions
+            should be linked into a single tree, i.e. there should only be one
+            expression in this list with parent_index == NULL and all others should
+            descend from that. Rendering order for expressions on the same tree level
+            (siblings) is implicitly given by the order they are referenced in this
+            repeated field.
+            Implicit: expression sequence
+            """
+        def __init__(
+            self,
+            *,
+            expression_index: collections.abc.Iterable[builtins.int] | None = ...,
+        ) -> None: ...
+        def ClearField(self, field_name: typing_extensions.Literal["expression_index", b"expression_index"]) -> None: ...
+
+    @typing_extensions.final
+    class Mnemonic(google.protobuf.message.Message):
+        """An instruction has exactly 1 mnemonic."""
+
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        NAME_FIELD_NUMBER: builtins.int
+        name: builtins.str
+        """Literal representation of the mnemonic, e.g.: "mov"."""
+        def __init__(
+            self,
+            *,
+            name: builtins.str | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["name", b"name"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["name", b"name"]) -> None: ...
+
+    @typing_extensions.final
+    class Instruction(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        ADDRESS_FIELD_NUMBER: builtins.int
+        CALL_TARGET_FIELD_NUMBER: builtins.int
+        MNEMONIC_INDEX_FIELD_NUMBER: builtins.int
+        OPERAND_INDEX_FIELD_NUMBER: builtins.int
+        RAW_BYTES_FIELD_NUMBER: builtins.int
+        COMMENT_INDEX_FIELD_NUMBER: builtins.int
+        address: builtins.int
+        """This will only be filled for instructions that do not just flow from the
+        immediately preceding instruction. Regular instructions will have to
+        calculate their own address by adding raw_bytes.size() to the previous
+        instruction's address.
+        """
+        @property
+        def call_target(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
+            """If this is a call instruction and call targets could be determined
+            they'll be given here. Note that we may or may not have a flow graph for
+            the target and thus cannot use an index into the flow graph table here.
+            We could potentially use call graph nodes, but linking instructions to
+            the call graph directly does not seem a good choice.
+            """
+        mnemonic_index: builtins.int
+        """Index into the mnemonic array of strings. Used for de-duping the data.
+        The default value is used for the most common mnemonic in the executable.
+        """
+        @property
+        def operand_index(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
+            """Indices into the operand tree. On X86 this can be 0, 1 or 2 elements
+            long, 3 elements with VEX/EVEX.
+            Implicit: operand sequence
+            """
+        raw_bytes: builtins.bytes
+        """The unmodified input bytes corresponding to this instruction."""
+        @property
+        def comment_index(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
+            """Implicit: comment sequence"""
+        def __init__(
+            self,
+            *,
+            address: builtins.int | None = ...,
+            call_target: collections.abc.Iterable[builtins.int] | None = ...,
+            mnemonic_index: builtins.int | None = ...,
+            operand_index: collections.abc.Iterable[builtins.int] | None = ...,
+            raw_bytes: builtins.bytes | None = ...,
+            comment_index: collections.abc.Iterable[builtins.int] | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["address", b"address", "mnemonic_index", b"mnemonic_index", "raw_bytes", b"raw_bytes"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "call_target", b"call_target", "comment_index", b"comment_index", "mnemonic_index", b"mnemonic_index", "operand_index", b"operand_index", "raw_bytes", b"raw_bytes"]) -> None: ...
+
+    @typing_extensions.final
+    class BasicBlock(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        @typing_extensions.final
+        class IndexRange(google.protobuf.message.Message):
+            """This is a space optimization. The instructions for an individual basic
+            block will usually be in a continuous index range. Thus it is more
+            efficient to store the range instead of individual indices. However, this
+            does not hold true for all basic blocks, so we need to be able to store
+            multiple index ranges per block.
+            """
+
+            DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+            BEGIN_INDEX_FIELD_NUMBER: builtins.int
+            END_INDEX_FIELD_NUMBER: builtins.int
+            begin_index: builtins.int
+            """These work like begin and end iterators, i.e. the sequence is
+            [begin_index, end_index). If the sequence only contains a single
+            element end_index will be omitted.
+            """
+            end_index: builtins.int
+            def __init__(
+                self,
+                *,
+                begin_index: builtins.int | None = ...,
+                end_index: builtins.int | None = ...,
+            ) -> None: ...
+            def HasField(self, field_name: typing_extensions.Literal["begin_index", b"begin_index", "end_index", b"end_index"]) -> builtins.bool: ...
+            def ClearField(self, field_name: typing_extensions.Literal["begin_index", b"begin_index", "end_index", b"end_index"]) -> None: ...
+
+        INSTRUCTION_INDEX_FIELD_NUMBER: builtins.int
+        @property
+        def instruction_index(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.BasicBlock.IndexRange]:
+            """Implicit: instruction sequence"""
+        def __init__(
+            self,
+            *,
+            instruction_index: collections.abc.Iterable[global___BinExport2.BasicBlock.IndexRange] | None = ...,
+        ) -> None: ...
+        def ClearField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index"]) -> None: ...
+
+    @typing_extensions.final
+    class FlowGraph(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        @typing_extensions.final
+        class Edge(google.protobuf.message.Message):
+            DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+            class _Type:
+                ValueType = typing.NewType("ValueType", builtins.int)
+                V: typing_extensions.TypeAlias = ValueType
+
+            class _TypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[BinExport2.FlowGraph.Edge._Type.ValueType], builtins.type):
+                DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+                CONDITION_TRUE: BinExport2.FlowGraph.Edge._Type.ValueType  # 1
+                CONDITION_FALSE: BinExport2.FlowGraph.Edge._Type.ValueType  # 2
+                UNCONDITIONAL: BinExport2.FlowGraph.Edge._Type.ValueType  # 3
+                SWITCH: BinExport2.FlowGraph.Edge._Type.ValueType  # 4
+
+            class Type(_Type, metaclass=_TypeEnumTypeWrapper): ...
+            CONDITION_TRUE: BinExport2.FlowGraph.Edge.Type.ValueType  # 1
+            CONDITION_FALSE: BinExport2.FlowGraph.Edge.Type.ValueType  # 2
+            UNCONDITIONAL: BinExport2.FlowGraph.Edge.Type.ValueType  # 3
+            SWITCH: BinExport2.FlowGraph.Edge.Type.ValueType  # 4
+
+            SOURCE_BASIC_BLOCK_INDEX_FIELD_NUMBER: builtins.int
+            TARGET_BASIC_BLOCK_INDEX_FIELD_NUMBER: builtins.int
+            TYPE_FIELD_NUMBER: builtins.int
+            IS_BACK_EDGE_FIELD_NUMBER: builtins.int
+            source_basic_block_index: builtins.int
+            """Source instruction will always be the last instruction of the source
+            basic block, target instruction the first instruction of the target
+            basic block.
+            """
+            target_basic_block_index: builtins.int
+            type: global___BinExport2.FlowGraph.Edge.Type.ValueType
+            is_back_edge: builtins.bool
+            """Indicates whether this is a loop edge as determined by Lengauer-Tarjan."""
+            def __init__(
+                self,
+                *,
+                source_basic_block_index: builtins.int | None = ...,
+                target_basic_block_index: builtins.int | None = ...,
+                type: global___BinExport2.FlowGraph.Edge.Type.ValueType | None = ...,
+                is_back_edge: builtins.bool | None = ...,
+            ) -> None: ...
+            def HasField(self, field_name: typing_extensions.Literal["is_back_edge", b"is_back_edge", "source_basic_block_index", b"source_basic_block_index", "target_basic_block_index", b"target_basic_block_index", "type", b"type"]) -> builtins.bool: ...
+            def ClearField(self, field_name: typing_extensions.Literal["is_back_edge", b"is_back_edge", "source_basic_block_index", b"source_basic_block_index", "target_basic_block_index", b"target_basic_block_index", "type", b"type"]) -> None: ...
+
+        BASIC_BLOCK_INDEX_FIELD_NUMBER: builtins.int
+        ENTRY_BASIC_BLOCK_INDEX_FIELD_NUMBER: builtins.int
+        EDGE_FIELD_NUMBER: builtins.int
+        @property
+        def basic_block_index(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
+            """Basic blocks are sorted by address."""
+        entry_basic_block_index: builtins.int
+        """The flow graph's entry point address is the first instruction of the
+        entry_basic_block.
+        """
+        @property
+        def edge(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.FlowGraph.Edge]: ...
+        def __init__(
+            self,
+            *,
+            basic_block_index: collections.abc.Iterable[builtins.int] | None = ...,
+            entry_basic_block_index: builtins.int | None = ...,
+            edge: collections.abc.Iterable[global___BinExport2.FlowGraph.Edge] | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["entry_basic_block_index", b"entry_basic_block_index"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["basic_block_index", b"basic_block_index", "edge", b"edge", "entry_basic_block_index", b"entry_basic_block_index"]) -> None: ...
+
+    @typing_extensions.final
+    class Reference(google.protobuf.message.Message):
+        """Generic reference class used for address comments (deprecated), string
+        references and expression substitutions. It allows referencing from an
+        instruction, operand, expression subtree tuple to a de-duped string in the
+        string table.
+        """
+
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        INSTRUCTION_INDEX_FIELD_NUMBER: builtins.int
+        INSTRUCTION_OPERAND_INDEX_FIELD_NUMBER: builtins.int
+        OPERAND_EXPRESSION_INDEX_FIELD_NUMBER: builtins.int
+        STRING_TABLE_INDEX_FIELD_NUMBER: builtins.int
+        instruction_index: builtins.int
+        """Index into the global instruction table."""
+        instruction_operand_index: builtins.int
+        """Index into the operand array local to an instruction."""
+        operand_expression_index: builtins.int
+        """Index into the expression array local to an operand."""
+        string_table_index: builtins.int
+        """Index into the global string table."""
+        def __init__(
+            self,
+            *,
+            instruction_index: builtins.int | None = ...,
+            instruction_operand_index: builtins.int | None = ...,
+            operand_expression_index: builtins.int | None = ...,
+            string_table_index: builtins.int | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index", "instruction_operand_index", b"instruction_operand_index", "operand_expression_index", b"operand_expression_index", "string_table_index", b"string_table_index"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index", "instruction_operand_index", b"instruction_operand_index", "operand_expression_index", b"operand_expression_index", "string_table_index", b"string_table_index"]) -> None: ...
+
+    @typing_extensions.final
+    class DataReference(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        INSTRUCTION_INDEX_FIELD_NUMBER: builtins.int
+        ADDRESS_FIELD_NUMBER: builtins.int
+        instruction_index: builtins.int
+        """Index into the global instruction table."""
+        address: builtins.int
+        """Address being referred."""
+        def __init__(
+            self,
+            *,
+            instruction_index: builtins.int | None = ...,
+            address: builtins.int | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["address", b"address", "instruction_index", b"instruction_index"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "instruction_index", b"instruction_index"]) -> None: ...
+
+    @typing_extensions.final
+    class Comment(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        class _Type:
+            ValueType = typing.NewType("ValueType", builtins.int)
+            V: typing_extensions.TypeAlias = ValueType
+
+        class _TypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[BinExport2.Comment._Type.ValueType], builtins.type):
+            DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+            DEFAULT: BinExport2.Comment._Type.ValueType  # 0
+            """A regular instruction comment. Typically displayed next to the
+            instruction disassembly.
+            """
+            ANTERIOR: BinExport2.Comment._Type.ValueType  # 1
+            """A comment line that is typically displayed before (above) the
+            instruction it refers to.
+            """
+            POSTERIOR: BinExport2.Comment._Type.ValueType  # 2
+            """Like ANTERIOR, but a typically displayed after (below)."""
+            FUNCTION: BinExport2.Comment._Type.ValueType  # 3
+            """Similar to an ANTERIOR comment, but applies to the beginning of an
+            identified function. Programs displaying the proto may choose to render
+            these differently (e.g. above an inferred function signature).
+            """
+            ENUM: BinExport2.Comment._Type.ValueType  # 4
+            """Named constants, bitfields and similar."""
+            LOCATION: BinExport2.Comment._Type.ValueType  # 5
+            """Named locations, usually the target of a jump."""
+            GLOBAL_REFERENCE: BinExport2.Comment._Type.ValueType  # 6
+            """Data cross references."""
+            LOCAL_REFERENCE: BinExport2.Comment._Type.ValueType  # 7
+            """Local/stack variables."""
+
+        class Type(_Type, metaclass=_TypeEnumTypeWrapper): ...
+        DEFAULT: BinExport2.Comment.Type.ValueType  # 0
+        """A regular instruction comment. Typically displayed next to the
+        instruction disassembly.
+        """
+        ANTERIOR: BinExport2.Comment.Type.ValueType  # 1
+        """A comment line that is typically displayed before (above) the
+        instruction it refers to.
+        """
+        POSTERIOR: BinExport2.Comment.Type.ValueType  # 2
+        """Like ANTERIOR, but a typically displayed after (below)."""
+        FUNCTION: BinExport2.Comment.Type.ValueType  # 3
+        """Similar to an ANTERIOR comment, but applies to the beginning of an
+        identified function. Programs displaying the proto may choose to render
+        these differently (e.g. above an inferred function signature).
+        """
+        ENUM: BinExport2.Comment.Type.ValueType  # 4
+        """Named constants, bitfields and similar."""
+        LOCATION: BinExport2.Comment.Type.ValueType  # 5
+        """Named locations, usually the target of a jump."""
+        GLOBAL_REFERENCE: BinExport2.Comment.Type.ValueType  # 6
+        """Data cross references."""
+        LOCAL_REFERENCE: BinExport2.Comment.Type.ValueType  # 7
+        """Local/stack variables."""
+
+        INSTRUCTION_INDEX_FIELD_NUMBER: builtins.int
+        INSTRUCTION_OPERAND_INDEX_FIELD_NUMBER: builtins.int
+        OPERAND_EXPRESSION_INDEX_FIELD_NUMBER: builtins.int
+        STRING_TABLE_INDEX_FIELD_NUMBER: builtins.int
+        REPEATABLE_FIELD_NUMBER: builtins.int
+        TYPE_FIELD_NUMBER: builtins.int
+        instruction_index: builtins.int
+        """Index into the global instruction table. This is here to enable
+        comment processing without having to iterate over all instructions.
+        There is an N:M mapping of instructions to comments.
+        """
+        instruction_operand_index: builtins.int
+        """Index into the operand array local to an instruction."""
+        operand_expression_index: builtins.int
+        """Index into the expression array local to an operand, like in Reference.
+        This is not currently used, but allows to implement expression
+        substitutions.
+        """
+        string_table_index: builtins.int
+        """Index into the global string table."""
+        repeatable: builtins.bool
+        """Comment is propagated to all locations that reference the original
+        location.
+        """
+        type: global___BinExport2.Comment.Type.ValueType
+        def __init__(
+            self,
+            *,
+            instruction_index: builtins.int | None = ...,
+            instruction_operand_index: builtins.int | None = ...,
+            operand_expression_index: builtins.int | None = ...,
+            string_table_index: builtins.int | None = ...,
+            repeatable: builtins.bool | None = ...,
+            type: global___BinExport2.Comment.Type.ValueType | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index", "instruction_operand_index", b"instruction_operand_index", "operand_expression_index", b"operand_expression_index", "repeatable", b"repeatable", "string_table_index", b"string_table_index", "type", b"type"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index", "instruction_operand_index", b"instruction_operand_index", "operand_expression_index", b"operand_expression_index", "repeatable", b"repeatable", "string_table_index", b"string_table_index", "type", b"type"]) -> None: ...
+
+    @typing_extensions.final
+    class Section(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        ADDRESS_FIELD_NUMBER: builtins.int
+        SIZE_FIELD_NUMBER: builtins.int
+        FLAG_R_FIELD_NUMBER: builtins.int
+        FLAG_W_FIELD_NUMBER: builtins.int
+        FLAG_X_FIELD_NUMBER: builtins.int
+        address: builtins.int
+        """Section start address."""
+        size: builtins.int
+        """Section size."""
+        flag_r: builtins.bool
+        """Read flag of the section, True when section is readable."""
+        flag_w: builtins.bool
+        """Write flag of the section, True when section is writable."""
+        flag_x: builtins.bool
+        """Execute flag of the section, True when section is executable."""
+        def __init__(
+            self,
+            *,
+            address: builtins.int | None = ...,
+            size: builtins.int | None = ...,
+            flag_r: builtins.bool | None = ...,
+            flag_w: builtins.bool | None = ...,
+            flag_x: builtins.bool | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["address", b"address", "flag_r", b"flag_r", "flag_w", b"flag_w", "flag_x", b"flag_x", "size", b"size"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "flag_r", b"flag_r", "flag_w", b"flag_w", "flag_x", b"flag_x", "size", b"size"]) -> None: ...
+
+    @typing_extensions.final
+    class Library(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        IS_STATIC_FIELD_NUMBER: builtins.int
+        LOAD_ADDRESS_FIELD_NUMBER: builtins.int
+        NAME_FIELD_NUMBER: builtins.int
+        is_static: builtins.bool
+        """If this library is statically linked."""
+        load_address: builtins.int
+        """Address where this library was loaded, 0 if unknown."""
+        name: builtins.str
+        """Name of the library (format is platform-dependent)."""
+        def __init__(
+            self,
+            *,
+            is_static: builtins.bool | None = ...,
+            load_address: builtins.int | None = ...,
+            name: builtins.str | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["is_static", b"is_static", "load_address", b"load_address", "name", b"name"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["is_static", b"is_static", "load_address", b"load_address", "name", b"name"]) -> None: ...
+
+    @typing_extensions.final
+    class Module(google.protobuf.message.Message):
+        DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+        NAME_FIELD_NUMBER: builtins.int
+        name: builtins.str
+        """Name, such as Java class name. Platform-dependent."""
+        def __init__(
+            self,
+            *,
+            name: builtins.str | None = ...,
+        ) -> None: ...
+        def HasField(self, field_name: typing_extensions.Literal["name", b"name"]) -> builtins.bool: ...
+        def ClearField(self, field_name: typing_extensions.Literal["name", b"name"]) -> None: ...
+
+    META_INFORMATION_FIELD_NUMBER: builtins.int
+    EXPRESSION_FIELD_NUMBER: builtins.int
+    OPERAND_FIELD_NUMBER: builtins.int
+    MNEMONIC_FIELD_NUMBER: builtins.int
+    INSTRUCTION_FIELD_NUMBER: builtins.int
+    BASIC_BLOCK_FIELD_NUMBER: builtins.int
+    FLOW_GRAPH_FIELD_NUMBER: builtins.int
+    CALL_GRAPH_FIELD_NUMBER: builtins.int
+    STRING_TABLE_FIELD_NUMBER: builtins.int
+    ADDRESS_COMMENT_FIELD_NUMBER: builtins.int
+    COMMENT_FIELD_NUMBER: builtins.int
+    STRING_REFERENCE_FIELD_NUMBER: builtins.int
+    EXPRESSION_SUBSTITUTION_FIELD_NUMBER: builtins.int
+    SECTION_FIELD_NUMBER: builtins.int
+    LIBRARY_FIELD_NUMBER: builtins.int
+    DATA_REFERENCE_FIELD_NUMBER: builtins.int
+    MODULE_FIELD_NUMBER: builtins.int
+    @property
+    def meta_information(self) -> global___BinExport2.Meta: ...
+    @property
+    def expression(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Expression]: ...
+    @property
+    def operand(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Operand]: ...
+    @property
+    def mnemonic(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Mnemonic]: ...
+    @property
+    def instruction(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Instruction]: ...
+    @property
+    def basic_block(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.BasicBlock]: ...
+    @property
+    def flow_graph(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.FlowGraph]: ...
+    @property
+    def call_graph(self) -> global___BinExport2.CallGraph: ...
+    @property
+    def string_table(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+    @property
+    def address_comment(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Reference]:
+        """No longer written. This is here so that BinDiff can work with older
+        BinExport files.
+        """
+    @property
+    def comment(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Comment]:
+        """Rich comment index used for BinDiff's comment porting."""
+    @property
+    def string_reference(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Reference]: ...
+    @property
+    def expression_substitution(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Reference]: ...
+    @property
+    def section(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Section]: ...
+    @property
+    def library(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Library]: ...
+    @property
+    def data_reference(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.DataReference]: ...
+    @property
+    def module(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Module]: ...
+    def __init__(
+        self,
+        *,
+        meta_information: global___BinExport2.Meta | None = ...,
+        expression: collections.abc.Iterable[global___BinExport2.Expression] | None = ...,
+        operand: collections.abc.Iterable[global___BinExport2.Operand] | None = ...,
+        mnemonic: collections.abc.Iterable[global___BinExport2.Mnemonic] | None = ...,
+        instruction: collections.abc.Iterable[global___BinExport2.Instruction] | None = ...,
+        basic_block: collections.abc.Iterable[global___BinExport2.BasicBlock] | None = ...,
+        flow_graph: collections.abc.Iterable[global___BinExport2.FlowGraph] | None = ...,
+        call_graph: global___BinExport2.CallGraph | None = ...,
+        string_table: collections.abc.Iterable[builtins.str] | None = ...,
+        address_comment: collections.abc.Iterable[global___BinExport2.Reference] | None = ...,
+        comment: collections.abc.Iterable[global___BinExport2.Comment] | None = ...,
+        string_reference: collections.abc.Iterable[global___BinExport2.Reference] | None = ...,
+        expression_substitution: collections.abc.Iterable[global___BinExport2.Reference] | None = ...,
+        section: collections.abc.Iterable[global___BinExport2.Section] | None = ...,
+        library: collections.abc.Iterable[global___BinExport2.Library] | None = ...,
+        data_reference: collections.abc.Iterable[global___BinExport2.DataReference] | None = ...,
+        module: collections.abc.Iterable[global___BinExport2.Module] | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["call_graph", b"call_graph", "meta_information", b"meta_information"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["address_comment", b"address_comment", "basic_block", b"basic_block", "call_graph", b"call_graph", "comment", b"comment", "data_reference", b"data_reference", "expression", b"expression", "expression_substitution", b"expression_substitution", "flow_graph", b"flow_graph", "instruction", b"instruction", "library", b"library", "meta_information", b"meta_information", "mnemonic", b"mnemonic", "module", b"module", "operand", b"operand", "section", b"section", "string_reference", b"string_reference", "string_table", b"string_table"]) -> None: ...
+
+global___BinExport2 = BinExport2

capa/features/extractors/binexport2/extractor.py

@@ -0,0 +1,130 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import Set, List, Tuple, Iterator
+
+import capa.features.extractors.elf
+import capa.features.extractors.common
+import capa.features.extractors.binexport2.file
+import capa.features.extractors.binexport2.insn
+import capa.features.extractors.binexport2.helpers
+import capa.features.extractors.binexport2.function
+import capa.features.extractors.binexport2.basicblock
+from capa.features.common import OS, Arch, Format, Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.binexport2 import (
+    AddressSpace,
+    AnalysisContext,
+    BinExport2Index,
+    FunctionContext,
+    BasicBlockContext,
+    BinExport2Analysis,
+    InstructionContext,
+)
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+
+logger = logging.getLogger(__name__)
+
+
+class BinExport2FeatureExtractor(StaticFeatureExtractor):
+    def __init__(self, be2: BinExport2, buf: bytes):
+        super().__init__(hashes=SampleHashes.from_bytes(buf))
+        self.be2: BinExport2 = be2
+        self.buf: bytes = buf
+        self.idx: BinExport2Index = BinExport2Index(self.be2)
+        self.analysis: BinExport2Analysis = BinExport2Analysis(self.be2, self.idx, self.buf)
+        address_space: AddressSpace = AddressSpace.from_buf(buf, self.analysis.base_address)
+        self.ctx: AnalysisContext = AnalysisContext(self.buf, self.be2, self.idx, self.analysis, address_space)
+
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(list(capa.features.extractors.common.extract_format(self.buf)))
+        self.global_features.extend(list(capa.features.extractors.common.extract_os(self.buf)))
+        self.global_features.extend(list(capa.features.extractors.common.extract_arch(self.buf)))
+
+        self.format: Set[str] = set()
+        self.os: Set[str] = set()
+        self.arch: Set[str] = set()
+
+        for feature, _ in self.global_features:
+            assert isinstance(feature.value, str)
+
+            if isinstance(feature, Format):
+                self.format.add(feature.value)
+            elif isinstance(feature, OS):
+                self.os.add(feature.value)
+            elif isinstance(feature, Arch):
+                self.arch.add(feature.value)
+            else:
+                raise ValueError("unexpected global feature: %s", feature)
+
+    def get_base_address(self) -> AbsoluteVirtualAddress:
+        return AbsoluteVirtualAddress(self.analysis.base_address)
+
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from self.global_features
+
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binexport2.file.extract_features(self.be2, self.buf)
+
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        for flow_graph_index, flow_graph in enumerate(self.be2.flow_graph):
+            entry_basic_block_index: int = flow_graph.entry_basic_block_index
+            flow_graph_address: int = self.idx.get_basic_block_address(entry_basic_block_index)
+
+            vertex_idx: int = self.idx.vertex_index_by_address[flow_graph_address]
+            be2_vertex: BinExport2.CallGraph.Vertex = self.be2.call_graph.vertex[vertex_idx]
+
+            # skip thunks
+            if capa.features.extractors.binexport2.helpers.is_vertex_type(
+                be2_vertex, BinExport2.CallGraph.Vertex.Type.THUNK
+            ):
+                continue
+
+            yield FunctionHandle(
+                AbsoluteVirtualAddress(flow_graph_address),
+                inner=FunctionContext(self.ctx, flow_graph_index, self.format, self.os, self.arch),
+            )
+
+    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binexport2.function.extract_features(fh)
+
+    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
+        fhi: FunctionContext = fh.inner
+        flow_graph_index: int = fhi.flow_graph_index
+        flow_graph: BinExport2.FlowGraph = self.be2.flow_graph[flow_graph_index]
+
+        for basic_block_index in flow_graph.basic_block_index:
+            basic_block_address: int = self.idx.get_basic_block_address(basic_block_index)
+            yield BBHandle(
+                address=AbsoluteVirtualAddress(basic_block_address),
+                inner=BasicBlockContext(basic_block_index),
+            )
+
+    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binexport2.basicblock.extract_features(fh, bbh)
+
+    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
+        bbi: BasicBlockContext = bbh.inner
+        basic_block: BinExport2.BasicBlock = self.be2.basic_block[bbi.basic_block_index]
+        for instruction_index, _, instruction_address in self.idx.basic_block_instructions(basic_block):
+            yield InsnHandle(
+                address=AbsoluteVirtualAddress(instruction_address),
+                inner=InstructionContext(instruction_index),
+            )
+
+    def extract_insn_features(
+        self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binexport2.insn.extract_features(fh, bbh, ih)

capa/features/extractors/binexport2/file.py

@@ -0,0 +1,80 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import io
+import logging
+from typing import Tuple, Iterator
+
+import pefile
+from elftools.elf.elffile import ELFFile
+
+import capa.features.common
+import capa.features.extractors.common
+import capa.features.extractors.pefile
+import capa.features.extractors.elffile
+from capa.features.common import Feature
+from capa.features.address import Address
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+
+logger = logging.getLogger(__name__)
+
+
+def extract_file_export_names(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
+    if buf.startswith(capa.features.extractors.common.MATCH_PE):
+        pe: pefile.PE = pefile.PE(data=buf)
+        yield from capa.features.extractors.pefile.extract_file_export_names(pe)
+    elif buf.startswith(capa.features.extractors.common.MATCH_ELF):
+        elf: ELFFile = ELFFile(io.BytesIO(buf))
+        yield from capa.features.extractors.elffile.extract_file_export_names(elf)
+    else:
+        logger.warning("unsupported format")
+
+
+def extract_file_import_names(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
+    if buf.startswith(capa.features.extractors.common.MATCH_PE):
+        pe: pefile.PE = pefile.PE(data=buf)
+        yield from capa.features.extractors.pefile.extract_file_import_names(pe)
+    elif buf.startswith(capa.features.extractors.common.MATCH_ELF):
+        elf: ELFFile = ELFFile(io.BytesIO(buf))
+        yield from capa.features.extractors.elffile.extract_file_import_names(elf)
+    else:
+        logger.warning("unsupported format")
+
+
+def extract_file_section_names(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
+    if buf.startswith(capa.features.extractors.common.MATCH_PE):
+        pe: pefile.PE = pefile.PE(data=buf)
+        yield from capa.features.extractors.pefile.extract_file_section_names(pe)
+    elif buf.startswith(capa.features.extractors.common.MATCH_ELF):
+        elf: ELFFile = ELFFile(io.BytesIO(buf))
+        yield from capa.features.extractors.elffile.extract_file_section_names(elf)
+    else:
+        logger.warning("unsupported format")
+
+
+def extract_file_strings(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
+    yield from capa.features.extractors.common.extract_file_strings(buf)
+
+
+def extract_file_format(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
+    yield from capa.features.extractors.common.extract_format(buf)
+
+
+def extract_features(be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
+    """extract file features"""
+    for file_handler in FILE_HANDLERS:
+        for feature, addr in file_handler(be2, buf):
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_strings,
+    extract_file_section_names,
+    extract_file_format,
+)

capa/features/extractors/binexport2/function.py

@@ -0,0 +1,72 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Tuple, Iterator
+
+from capa.features.file import FunctionName
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors import loops
+from capa.features.extractors.binexport2 import BinExport2Index, FunctionContext
+from capa.features.extractors.base_extractor import FunctionHandle
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+
+
+def extract_function_calls_to(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+    idx: BinExport2Index = fhi.ctx.idx
+
+    flow_graph_index: int = fhi.flow_graph_index
+    flow_graph_address: int = idx.flow_graph_address_by_index[flow_graph_index]
+    vertex_index: int = idx.vertex_index_by_address[flow_graph_address]
+
+    for caller_index in idx.callers_by_vertex_index[vertex_index]:
+        caller: BinExport2.CallGraph.Vertex = be2.call_graph.vertex[caller_index]
+        caller_address: int = caller.address
+        yield Characteristic("calls to"), AbsoluteVirtualAddress(caller_address)
+
+
+def extract_function_loop(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+
+    flow_graph_index: int = fhi.flow_graph_index
+    flow_graph: BinExport2.FlowGraph = be2.flow_graph[flow_graph_index]
+
+    edges: List[Tuple[int, int]] = []
+    for edge in flow_graph.edge:
+        edges.append((edge.source_basic_block_index, edge.target_basic_block_index))
+
+    if loops.has_loop(edges):
+        yield Characteristic("loop"), fh.address
+
+
+def extract_function_name(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+    idx: BinExport2Index = fhi.ctx.idx
+    flow_graph_index: int = fhi.flow_graph_index
+
+    flow_graph_address: int = idx.flow_graph_address_by_index[flow_graph_index]
+    vertex_index: int = idx.vertex_index_by_address[flow_graph_address]
+    vertex: BinExport2.CallGraph.Vertex = be2.call_graph.vertex[vertex_index]
+
+    if vertex.HasField("mangled_name"):
+        yield FunctionName(vertex.mangled_name), fh.address
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_function_name)

capa/features/extractors/binexport2/helpers.py

@@ -0,0 +1,650 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import re
+from typing import Set, Dict, List, Tuple, Union, Iterator, Optional
+from collections import defaultdict
+from dataclasses import dataclass
+
+import capa.features.extractors.helpers
+import capa.features.extractors.binexport2.helpers
+from capa.features.common import ARCH_I386, ARCH_AMD64, ARCH_AARCH64
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+
+HAS_ARCH32 = {ARCH_I386}
+HAS_ARCH64 = {ARCH_AARCH64, ARCH_AMD64}
+
+HAS_ARCH_INTEL = {ARCH_I386, ARCH_AMD64}
+HAS_ARCH_ARM = {ARCH_AARCH64}
+
+
+def mask_immediate(arch: Set[str], immediate: int) -> int:
+    if arch & HAS_ARCH64:
+        immediate &= 0xFFFFFFFFFFFFFFFF
+    elif arch & HAS_ARCH32:
+        immediate &= 0xFFFFFFFF
+    return immediate
+
+
+def twos_complement(arch: Set[str], immediate: int, default: Optional[int] = None) -> int:
+    if default is not None:
+        return capa.features.extractors.helpers.twos_complement(immediate, default)
+    elif arch & HAS_ARCH64:
+        return capa.features.extractors.helpers.twos_complement(immediate, 64)
+    elif arch & HAS_ARCH32:
+        return capa.features.extractors.helpers.twos_complement(immediate, 32)
+    return immediate
+
+
+def is_address_mapped(be2: BinExport2, address: int) -> bool:
+    """return True if the given address is mapped"""
+    sections_with_perms: Iterator[BinExport2.Section] = filter(lambda s: s.flag_r or s.flag_w or s.flag_x, be2.section)
+    return any(section.address <= address < section.address + section.size for section in sections_with_perms)
+
+
+def is_vertex_type(vertex: BinExport2.CallGraph.Vertex, type_: BinExport2.CallGraph.Vertex.Type.ValueType) -> bool:
+    return vertex.HasField("type") and vertex.type == type_
+
+
+# internal to `build_expression_tree`
+# this is unstable: it is subject to change, so don't rely on it!
+def _prune_expression_tree_empty_shifts(
+    be2: BinExport2,
+    operand: BinExport2.Operand,
+    expression_tree: List[List[int]],
+    tree_index: int,
+):
+    expression_index = operand.expression_index[tree_index]
+    expression = be2.expression[expression_index]
+    children_tree_indexes: List[int] = expression_tree[tree_index]
+
+    if expression.type == BinExport2.Expression.OPERATOR:
+        if len(children_tree_indexes) == 0 and expression.symbol in ("lsl", "lsr"):
+            # Ghidra may emit superfluous lsl nodes with no children.
+            # https://github.com/mandiant/capa/pull/2340/files#r1750003919
+            # Which is maybe: https://github.com/NationalSecurityAgency/ghidra/issues/6821#issuecomment-2295394697
+            #
+            # Which seems to be as if the shift wasn't there (shift of #0)
+            # so we want to remove references to this node from any parent nodes.
+            for tree_node in expression_tree:
+                if tree_index in tree_node:
+                    tree_node.remove(tree_index)
+
+            return
+
+    for child_tree_index in children_tree_indexes:
+        _prune_expression_tree_empty_shifts(be2, operand, expression_tree, child_tree_index)
+
+
+# internal to `build_expression_tree`
+# this is unstable: it is subject to change, so don't rely on it!
+def _prune_expression_tree_empty_commas(
+    be2: BinExport2,
+    operand: BinExport2.Operand,
+    expression_tree: List[List[int]],
+    tree_index: int,
+):
+    expression_index = operand.expression_index[tree_index]
+    expression = be2.expression[expression_index]
+    children_tree_indexes: List[int] = expression_tree[tree_index]
+
+    if expression.type == BinExport2.Expression.OPERATOR:
+        if len(children_tree_indexes) == 1 and expression.symbol == ",":
+            # Due to the above pruning of empty LSL or LSR expressions,
+            # the parents might need to be fixed up.
+            #
+            # Specifically, if the pruned node was part of a comma list with two children,
+            # now there's only a single child, which renders as an extra comma,
+            # so we replace references to the comma node with the immediate child.
+            #
+            # A more correct way of doing this might be to walk up the parents and do fixups,
+            # but I'm not quite sure how to do this yet. Just do two passes right now.
+            child = children_tree_indexes[0]
+
+            for tree_node in expression_tree:
+                tree_node.index
+                if tree_index in tree_node:
+                    tree_node[tree_node.index(tree_index)] = child
+
+            return
+
+    for child_tree_index in children_tree_indexes:
+        _prune_expression_tree_empty_commas(be2, operand, expression_tree, child_tree_index)
+
+
+# internal to `build_expression_tree`
+# this is unstable: it is subject to change, so don't rely on it!
+def _prune_expression_tree(
+    be2: BinExport2,
+    operand: BinExport2.Operand,
+    expression_tree: List[List[int]],
+):
+    _prune_expression_tree_empty_shifts(be2, operand, expression_tree, 0)
+    _prune_expression_tree_empty_commas(be2, operand, expression_tree, 0)
+
+
+# this is unstable: it is subject to change, so don't rely on it!
+def _build_expression_tree(
+    be2: BinExport2,
+    operand: BinExport2.Operand,
+) -> List[List[int]]:
+    # The reconstructed expression tree layout, linking parent nodes to their children.
+    #
+    # There is one list of integers for each expression in the operand.
+    # These integers are indexes of other expressions in the same operand,
+    # which are the children of that expression.
+    #
+    # So:
+    #
+    #   [ [1, 3], [2], [], [4], [5], []]
+    #
+    # means the first expression has two children, at index 1 and 3,
+    # and the tree looks like:
+    #
+    #        0
+    #       / \
+    #      1   3
+    #      |   |
+    #      2   4
+    #          |
+    #          5
+    #
+    # Remember, these are the indices into the entries in operand.expression_index.
+    if len(operand.expression_index) == 0:
+        # Ghidra bug where empty operands (no expressions) may
+        # exist (see https://github.com/NationalSecurityAgency/ghidra/issues/6817)
+        return []
+
+    tree: List[List[int]] = []
+    for i, expression_index in enumerate(operand.expression_index):
+        children = []
+
+        # scan all subsequent expressions, looking for those that have parent_index == current.expression_index
+        for j, candidate_index in enumerate(operand.expression_index[i + 1 :]):
+            candidate = be2.expression[candidate_index]
+
+            if candidate.parent_index == expression_index:
+                children.append(i + j + 1)
+
+        tree.append(children)
+
+    _prune_expression_tree(be2, operand, tree)
+    _prune_expression_tree(be2, operand, tree)
+
+    return tree
+
+
+def _fill_operand_expression_list(
+    be2: BinExport2,
+    operand: BinExport2.Operand,
+    expression_tree: List[List[int]],
+    tree_index: int,
+    expression_list: List[BinExport2.Expression],
+):
+    """
+    Walk the given expression tree and collect the expression nodes in-order.
+    """
+    expression_index = operand.expression_index[tree_index]
+    expression = be2.expression[expression_index]
+    children_tree_indexes: List[int] = expression_tree[tree_index]
+
+    if expression.type == BinExport2.Expression.REGISTER:
+        assert len(children_tree_indexes) == 0
+        expression_list.append(expression)
+        return
+
+    elif expression.type == BinExport2.Expression.SYMBOL:
+        assert len(children_tree_indexes) <= 1
+        expression_list.append(expression)
+
+        if len(children_tree_indexes) == 0:
+            return
+        elif len(children_tree_indexes) == 1:
+            # like: v
+            # from: mov v0.D[0x1], x9
+            #           |
+            #           0
+            #           .
+            #           |
+            #           D
+            child_index = children_tree_indexes[0]
+            _fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
+            return
+        else:
+            raise NotImplementedError(len(children_tree_indexes))
+
+    elif expression.type == BinExport2.Expression.IMMEDIATE_INT:
+        assert len(children_tree_indexes) == 0
+        expression_list.append(expression)
+        return
+
+    elif expression.type == BinExport2.Expression.SIZE_PREFIX:
+        # like: b4
+        #
+        # We might want to use this occasionally, such as to disambiguate the
+        # size of MOVs into/out of memory. But I'm not sure when/where we need that yet.
+        #
+        # IDA spams this size prefix hint *everywhere*, so we can't rely on the exporter
+        # to provide it only when necessary.
+        assert len(children_tree_indexes) == 1
+        child_index = children_tree_indexes[0]
+        _fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
+        return
+
+    elif expression.type == BinExport2.Expression.OPERATOR:
+        if len(children_tree_indexes) == 1:
+            # prefix operator, like "ds:"
+            expression_list.append(expression)
+            child_index = children_tree_indexes[0]
+            _fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
+            return
+
+        elif len(children_tree_indexes) == 2:
+            # infix operator: like "+" in "ebp+10"
+            child_a = children_tree_indexes[0]
+            child_b = children_tree_indexes[1]
+            _fill_operand_expression_list(be2, operand, expression_tree, child_a, expression_list)
+            expression_list.append(expression)
+            _fill_operand_expression_list(be2, operand, expression_tree, child_b, expression_list)
+            return
+
+        elif len(children_tree_indexes) == 3:
+            # infix operator: like "+" in "ebp+ecx+10"
+            child_a = children_tree_indexes[0]
+            child_b = children_tree_indexes[1]
+            child_c = children_tree_indexes[2]
+            _fill_operand_expression_list(be2, operand, expression_tree, child_a, expression_list)
+            expression_list.append(expression)
+            _fill_operand_expression_list(be2, operand, expression_tree, child_b, expression_list)
+            expression_list.append(expression)
+            _fill_operand_expression_list(be2, operand, expression_tree, child_c, expression_list)
+            return
+
+        else:
+            raise NotImplementedError(len(children_tree_indexes))
+
+    elif expression.type == BinExport2.Expression.DEREFERENCE:
+        assert len(children_tree_indexes) == 1
+        expression_list.append(expression)
+
+        child_index = children_tree_indexes[0]
+        _fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
+        return
+
+    elif expression.type == BinExport2.Expression.IMMEDIATE_FLOAT:
+        raise NotImplementedError(expression.type)
+
+    else:
+        raise NotImplementedError(expression.type)
+
+
+def get_operand_expressions(be2: BinExport2, op: BinExport2.Operand) -> List[BinExport2.Expression]:
+    tree = _build_expression_tree(be2, op)
+
+    expressions: List[BinExport2.Expression] = []
+    _fill_operand_expression_list(be2, op, tree, 0, expressions)
+
+    return expressions
+
+
+def get_operand_register_expression(be2: BinExport2, operand: BinExport2.Operand) -> Optional[BinExport2.Expression]:
+    if len(operand.expression_index) == 1:
+        expression: BinExport2.Expression = be2.expression[operand.expression_index[0]]
+        if expression.type == BinExport2.Expression.REGISTER:
+            return expression
+    return None
+
+
+def get_operand_immediate_expression(be2: BinExport2, operand: BinExport2.Operand) -> Optional[BinExport2.Expression]:
+    if len(operand.expression_index) == 1:
+        # - type: IMMEDIATE_INT
+        #   immediate: 20588728364
+        #   parent_index: 0
+        expression: BinExport2.Expression = be2.expression[operand.expression_index[0]]
+        if expression.type == BinExport2.Expression.IMMEDIATE_INT:
+            return expression
+
+    elif len(operand.expression_index) == 2:
+        # from IDA, which provides a size hint for every operand,
+        # we get the following pattern for immediate constants:
+        #
+        # - type: SIZE_PREFIX
+        #   symbol: "b8"
+        # - type: IMMEDIATE_INT
+        #   immediate: 20588728364
+        #   parent_index: 0
+        expression0: BinExport2.Expression = be2.expression[operand.expression_index[0]]
+        expression1: BinExport2.Expression = be2.expression[operand.expression_index[1]]
+
+        if expression0.type == BinExport2.Expression.SIZE_PREFIX:
+            if expression1.type == BinExport2.Expression.IMMEDIATE_INT:
+                return expression1
+
+    return None
+
+
+def get_instruction_mnemonic(be2: BinExport2, instruction: BinExport2.Instruction) -> str:
+    return be2.mnemonic[instruction.mnemonic_index].name.lower()
+
+
+def get_instruction_operands(be2: BinExport2, instruction: BinExport2.Instruction) -> List[BinExport2.Operand]:
+    return [be2.operand[operand_index] for operand_index in instruction.operand_index]
+
+
+def split_with_delimiters(s: str, delimiters: Tuple[str, ...]) -> Iterator[str]:
+    """
+    Splits a string by any of the provided delimiter characters,
+    including the delimiters in the results.
+
+    Args:
+        string: The string to split.
+        delimiters: A string containing the characters to use as delimiters.
+    """
+    start = 0
+    for i, char in enumerate(s):
+        if char in delimiters:
+            yield s[start:i]
+            yield char
+            start = i + 1
+
+    if start < len(s):
+        yield s[start:]
+
+
+BinExport2OperandPattern = Union[str, Tuple[str, ...]]
+
+
+@dataclass
+class BinExport2InstructionPattern:
+    """
+    This describes a way to match disassembled instructions, with mnemonics and operands.
+
+    You can specify constraints on the instruction, via:
+      - the mnemonics, like "mov",
+      - number of operands, and
+      - format of each operand, "[reg, reg, #int]".
+
+    During matching, you can also capture a single element, to see its concrete value.
+    For example, given the pattern:
+
+        mov reg0, #int0  ; capture int0
+
+    and the instruction:
+
+        mov eax, 1
+
+    Then the capture will contain the immediate integer 1.
+
+    This matcher uses the BinExport2 data layout under the hood.
+    """
+
+    mnemonics: Tuple[str, ...]
+    operands: Tuple[Union[str, BinExport2OperandPattern], ...]
+    capture: Optional[str]
+
+    @classmethod
+    def from_str(cls, query: str):
+        """
+        Parse a pattern string into a Pattern instance.
+        The supported syntax is like this:
+
+            br      reg
+            br      reg                          ; capture reg
+            br      reg(stack)                   ; capture reg
+            br      reg(not-stack)               ; capture reg
+            mov     reg0, reg1                   ; capture reg0
+            adrp    reg, #int                    ; capture #int
+            add     reg, reg, #int               ; capture #int
+            ldr     reg0, [reg1]                 ; capture reg1
+            ldr|str reg, [reg, #int]             ; capture #int
+            ldr|str reg, [reg(stack), #int]      ; capture #int
+            ldr|str reg, [reg(not-stack), #int]  ; capture #int
+            ldr|str reg, [reg, #int]!            ; capture #int
+            ldr|str reg, [reg], #int             ; capture #int
+            ldp|stp reg, reg, [reg, #int]        ; capture #int
+            ldp|stp reg, reg, [reg, #int]!       ; capture #int
+            ldp|stp reg, reg, [reg], #int        ; capture #int
+        """
+        #
+        # The implementation of the parser here is obviously ugly.
+        # Its handwritten and probably fragile. But since we don't
+        # expect this to be widely used, its probably ok.
+        # Don't hesitate to rewrite this if it becomes more important.
+        #
+        # Note that this doesn't have to be very performant.
+        # We expect these patterns to be parsed once upfront and then reused
+        # (globally at the module level?) rather than within any loop.
+        #
+
+        pattern, _, comment = query.strip().partition(";")
+
+        # we don't support fs: yet
+        assert ":" not in pattern
+
+        # from "capture #int" to "#int"
+        if comment:
+            comment = comment.strip()
+            assert comment.startswith("capture ")
+            capture = comment[len("capture ") :]
+        else:
+            capture = None
+
+        # from "ldr|str ..." to ["ldr", "str"]
+        pattern = pattern.strip()
+        mnemonic, _, rest = pattern.partition(" ")
+        mnemonics = mnemonic.split("|")
+
+        operands: List[Union[str, Tuple[str, ...]]] = []
+        while rest:
+            rest = rest.strip()
+            if not rest.startswith("["):
+                # If its not a dereference, which looks like `[op, op, op, ...]`,
+                # then its a simple operand, which we can split by the next comma.
+                operand, _, rest = rest.partition(", ")
+                rest = rest.strip()
+                operands.append(operand)
+
+            else:
+                # This looks like a dereference, something like `[op, op, op, ...]`.
+                # Since these can't be nested, look for the next ] and then parse backwards.
+                deref_end = rest.index("]")
+                try:
+                    deref_end = rest.index(", ", deref_end)
+                    deref_end += len(", ")
+                except ValueError:
+                    deref = rest
+                    rest = ""
+                else:
+                    deref = rest[:deref_end]
+                    rest = rest[deref_end:]
+                    rest = rest.strip()
+                    deref = deref.rstrip(" ")
+                    deref = deref.rstrip(",")
+
+                # like: [reg, #int]!
+                has_postindex_writeback = deref.endswith("!")
+
+                deref = deref.rstrip("!")
+                deref = deref.rstrip("]")
+                deref = deref.lstrip("[")
+
+                parts = tuple(split_with_delimiters(deref, (",", "+", "*")))
+                parts = tuple(s.strip() for s in parts)
+
+                # emit operands in this order to match
+                # how BinExport2 expressions are flatted
+                # by get_operand_expressions
+                if has_postindex_writeback:
+                    operands.append(("!", "[") + parts)
+                else:
+                    operands.append(("[",) + parts)
+
+        for operand in operands:  # type: ignore
+            # Try to ensure we've parsed the operands correctly.
+            # This is just sanity checking.
+            for o in (operand,) if isinstance(operand, str) else operand:
+                # operands can look like:
+                #  - reg
+                #  - reg0
+                #  - reg(stack)
+                #  - reg0(stack)
+                #  - reg(not-stack)
+                #  - reg0(not-stack)
+                #  - #int
+                #  - #int0
+                # and a limited set of supported operators.
+                # use an inline regex so that its easy to read. not perf critical.
+                assert re.match(r"^(reg|#int)[0-9]?(\(stack\)|\(not-stack\))?$", o) or o in ("[", ",", "!", "+", "*")
+
+        return cls(tuple(mnemonics), tuple(operands), capture)
+
+    @dataclass
+    class MatchResult:
+        operand_index: int
+        expression_index: int
+        expression: BinExport2.Expression
+
+    def match(
+        self, mnemonic: str, operand_expressions: List[List[BinExport2.Expression]]
+    ) -> Optional["BinExport2InstructionPattern.MatchResult"]:
+        """
+        Match the given BinExport2 data against this pattern.
+
+        The BinExport2 expression tree must have been flattened, such as with
+        capa.features.extractors.binexport2.helpers.get_operand_expressions.
+
+        If there's a match, the captured Expression instance is returned.
+        Otherwise, you get None back.
+        """
+        if mnemonic not in self.mnemonics:
+            return None
+
+        if len(self.operands) != len(operand_expressions):
+            return None
+
+        captured = None
+
+        for operand_index, found_expressions in enumerate(operand_expressions):
+            wanted_expressions = self.operands[operand_index]
+
+            # from `"reg"` to `("reg", )`
+            if isinstance(wanted_expressions, str):
+                wanted_expressions = (wanted_expressions,)
+            assert isinstance(wanted_expressions, tuple)
+
+            if len(wanted_expressions) != len(found_expressions):
+                return None
+
+            for expression_index, (wanted_expression, found_expression) in enumerate(
+                zip(wanted_expressions, found_expressions)
+            ):
+                if wanted_expression.startswith("reg"):
+                    if found_expression.type != BinExport2.Expression.REGISTER:
+                        return None
+
+                    if wanted_expression.endswith(")"):
+                        if wanted_expression.endswith("(not-stack)"):
+                            # intel 64: rsp, esp, sp,
+                            # intel 32: ebp, ebp, bp
+                            # arm: sp
+                            register_name = found_expression.symbol.lower()
+                            if register_name in ("rsp", "esp", "sp", "rbp", "ebp", "bp"):
+                                return None
+
+                        elif wanted_expression.endswith("(stack)"):
+                            register_name = found_expression.symbol.lower()
+                            if register_name not in ("rsp", "esp", "sp", "rbp", "ebp", "bp"):
+                                return None
+
+                        else:
+                            raise ValueError("unexpected expression suffix", wanted_expression)
+
+                    if self.capture == wanted_expression:
+                        captured = BinExport2InstructionPattern.MatchResult(
+                            operand_index, expression_index, found_expression
+                        )
+
+                elif wanted_expression.startswith("#int"):
+                    if found_expression.type != BinExport2.Expression.IMMEDIATE_INT:
+                        return None
+
+                    if self.capture == wanted_expression:
+                        captured = BinExport2InstructionPattern.MatchResult(
+                            operand_index, expression_index, found_expression
+                        )
+
+                elif wanted_expression == "[":
+                    if found_expression.type != BinExport2.Expression.DEREFERENCE:
+                        return None
+
+                elif wanted_expression in (",", "!", "+", "*"):
+                    if found_expression.type != BinExport2.Expression.OPERATOR:
+                        return None
+
+                    if found_expression.symbol != wanted_expression:
+                        return None
+
+                else:
+                    raise ValueError(found_expression)
+
+        if captured:
+            return captured
+        else:
+            # There were no captures, so
+            # return arbitrary non-None expression
+            return BinExport2InstructionPattern.MatchResult(operand_index, expression_index, found_expression)
+
+
+class BinExport2InstructionPatternMatcher:
+    """Index and match a collection of instruction patterns."""
+
+    def __init__(self, queries: List[BinExport2InstructionPattern]):
+        self.queries = queries
+        # shard the patterns by (mnemonic, #operands)
+        self._index: Dict[Tuple[str, int], List[BinExport2InstructionPattern]] = defaultdict(list)
+
+        for query in queries:
+            for mnemonic in query.mnemonics:
+                self._index[(mnemonic.lower(), len(query.operands))].append(query)
+
+    @classmethod
+    def from_str(cls, patterns: str):
+        return cls(
+            [
+                BinExport2InstructionPattern.from_str(line)
+                for line in filter(
+                    lambda line: not line.startswith("#"), (line.strip() for line in patterns.split("\n"))
+                )
+            ]
+        )
+
+    def match(
+        self, mnemonic: str, operand_expressions: List[List[BinExport2.Expression]]
+    ) -> Optional[BinExport2InstructionPattern.MatchResult]:
+        queries = self._index.get((mnemonic.lower(), len(operand_expressions)), [])
+        for query in queries:
+            captured = query.match(mnemonic.lower(), operand_expressions)
+            if captured:
+                return captured
+
+        return None
+
+    def match_with_be2(
+        self, be2: BinExport2, instruction_index: int
+    ) -> Optional[BinExport2InstructionPattern.MatchResult]:
+        instruction: BinExport2.Instruction = be2.instruction[instruction_index]
+        mnemonic: str = get_instruction_mnemonic(be2, instruction)
+
+        if (mnemonic.lower(), len(instruction.operand_index)) not in self._index:
+            # verify that we might have a hit before we realize the operand expression list
+            return None
+
+        operands = []
+        for operand_index in instruction.operand_index:
+            operands.append(get_operand_expressions(be2, be2.operand[operand_index]))
+
+        return self.match(mnemonic, operands)

capa/features/extractors/binexport2/insn.py

@@ -0,0 +1,254 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import List, Tuple, Iterator
+
+import capa.features.extractors.helpers
+import capa.features.extractors.strings
+import capa.features.extractors.binexport2.helpers
+import capa.features.extractors.binexport2.arch.arm.insn
+import capa.features.extractors.binexport2.arch.intel.insn
+from capa.features.insn import API, Mnemonic
+from capa.features.common import Bytes, String, Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.binexport2 import (
+    AddressSpace,
+    AnalysisContext,
+    BinExport2Index,
+    FunctionContext,
+    ReadMemoryError,
+    BinExport2Analysis,
+    InstructionContext,
+)
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+from capa.features.extractors.binexport2.helpers import HAS_ARCH_ARM, HAS_ARCH_INTEL
+from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
+
+logger = logging.getLogger(__name__)
+
+
+def extract_insn_api_features(fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+    be2_index: BinExport2Index = fhi.ctx.idx
+    be2_analysis: BinExport2Analysis = fhi.ctx.analysis
+    insn: BinExport2.Instruction = be2.instruction[ii.instruction_index]
+
+    for addr in insn.call_target:
+        addr = be2_analysis.thunks.get(addr, addr)
+
+        if addr not in be2_index.vertex_index_by_address:
+            # disassembler did not define function at address
+            logger.debug("0x%x is not a vertex", addr)
+            continue
+
+        vertex_idx: int = be2_index.vertex_index_by_address[addr]
+        vertex: BinExport2.CallGraph.Vertex = be2.call_graph.vertex[vertex_idx]
+
+        if not capa.features.extractors.binexport2.helpers.is_vertex_type(
+            vertex, BinExport2.CallGraph.Vertex.Type.IMPORTED
+        ):
+            continue
+
+        if not vertex.HasField("mangled_name"):
+            logger.debug("vertex %d does not have mangled_name", vertex_idx)
+            continue
+
+        api_name: str = vertex.mangled_name
+        for name in capa.features.extractors.helpers.generate_symbols("", api_name):
+            yield API(name), ih.address
+
+
+def extract_insn_number_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+
+    if fhi.arch & HAS_ARCH_INTEL:
+        yield from capa.features.extractors.binexport2.arch.intel.insn.extract_insn_number_features(fh, bbh, ih)
+    elif fhi.arch & HAS_ARCH_ARM:
+        yield from capa.features.extractors.binexport2.arch.arm.insn.extract_insn_number_features(fh, bbh, ih)
+
+
+def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    ctx: AnalysisContext = fhi.ctx
+    be2: BinExport2 = ctx.be2
+    idx: BinExport2Index = ctx.idx
+    address_space: AddressSpace = ctx.address_space
+
+    instruction_index: int = ii.instruction_index
+
+    if instruction_index in idx.string_reference_index_by_source_instruction_index:
+        # disassembler already identified string reference from instruction
+        return
+
+    reference_addresses: List[int] = []
+
+    if instruction_index in idx.data_reference_index_by_source_instruction_index:
+        for data_reference_index in idx.data_reference_index_by_source_instruction_index[instruction_index]:
+            data_reference: BinExport2.DataReference = be2.data_reference[data_reference_index]
+            data_reference_address: int = data_reference.address
+
+            if data_reference_address in idx.insn_address_by_index:
+                # appears to be code
+                continue
+
+            reference_addresses.append(data_reference_address)
+
+    for reference_address in reference_addresses:
+        try:
+            # if at end of segment then there might be an overrun here.
+            buf: bytes = address_space.read_memory(reference_address, 0x100)
+        except ReadMemoryError:
+            logger.debug("failed to read memory: 0x%x", reference_address)
+            continue
+
+        if capa.features.extractors.helpers.all_zeros(buf):
+            continue
+
+        is_string: bool = False
+
+        # note: we *always* break after the first iteration
+        for s in capa.features.extractors.strings.extract_ascii_strings(buf):
+            if s.offset != 0:
+                break
+
+            yield String(s.s), ih.address
+            is_string = True
+            break
+
+        # note: we *always* break after the first iteration
+        for s in capa.features.extractors.strings.extract_unicode_strings(buf):
+            if s.offset != 0:
+                break
+
+            yield String(s.s), ih.address
+            is_string = True
+            break
+
+        if not is_string:
+            yield Bytes(buf), ih.address
+
+
+def extract_insn_string_features(
+    fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+    idx: BinExport2Index = fhi.ctx.idx
+
+    instruction_index: int = ii.instruction_index
+
+    if instruction_index in idx.string_reference_index_by_source_instruction_index:
+        for string_reference_index in idx.string_reference_index_by_source_instruction_index[instruction_index]:
+            string_reference: BinExport2.Reference = be2.string_reference[string_reference_index]
+            string_index: int = string_reference.string_table_index
+            string: str = be2.string_table[string_index]
+            yield String(string), ih.address
+
+
+def extract_insn_offset_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+
+    if fhi.arch & HAS_ARCH_INTEL:
+        yield from capa.features.extractors.binexport2.arch.intel.insn.extract_insn_offset_features(fh, bbh, ih)
+    elif fhi.arch & HAS_ARCH_ARM:
+        yield from capa.features.extractors.binexport2.arch.arm.insn.extract_insn_offset_features(fh, bbh, ih)
+
+
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+
+    if fhi.arch & HAS_ARCH_INTEL:
+        yield from capa.features.extractors.binexport2.arch.intel.insn.extract_insn_nzxor_characteristic_features(
+            fh, bbh, ih
+        )
+    elif fhi.arch & HAS_ARCH_ARM:
+        yield from capa.features.extractors.binexport2.arch.arm.insn.extract_insn_nzxor_characteristic_features(
+            fh, bbh, ih
+        )
+
+
+def extract_insn_mnemonic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+
+    instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
+    mnemonic: BinExport2.Mnemonic = be2.mnemonic[instruction.mnemonic_index]
+    mnemonic_name: str = mnemonic.name.lower()
+    yield Mnemonic(mnemonic_name), ih.address
+
+
+def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract functions calls from features
+
+    most relevant at the function scope;
+    however, its most efficient to extract at the instruction scope.
+    """
+    fhi: FunctionContext = fh.inner
+    ii: InstructionContext = ih.inner
+
+    be2: BinExport2 = fhi.ctx.be2
+
+    instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
+    for call_target_address in instruction.call_target:
+        addr: AbsoluteVirtualAddress = AbsoluteVirtualAddress(call_target_address)
+        yield Characteristic("calls from"), addr
+
+        if fh.address == addr:
+            yield Characteristic("recursive call"), addr
+
+
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    fhi: FunctionContext = fh.inner
+
+    if fhi.arch & HAS_ARCH_INTEL:
+        yield from capa.features.extractors.binexport2.arch.intel.insn.extract_function_indirect_call_characteristic_features(
+            fh, bbh, ih
+        )
+    elif fhi.arch & HAS_ARCH_ARM:
+        yield from capa.features.extractors.binexport2.arch.arm.insn.extract_function_indirect_call_characteristic_features(
+            fh, bbh, ih
+        )
+
+
+def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract instruction features"""
+    for inst_handler in INSTRUCTION_HANDLERS:
+        for feature, ea in inst_handler(f, bbh, insn):
+            yield feature, ea
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_bytes_features,
+    extract_insn_string_features,
+    extract_insn_offset_features,
+    extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    extract_function_calls_from,
+    extract_function_indirect_call_characteristic_features,
+)

capa/features/extractors/binja/basicblock.py

@@ -0,0 +1,127 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import string
+from typing import Tuple, Iterator
+
+from binaryninja import Function
+from binaryninja import BasicBlock as BinjaBasicBlock
+from binaryninja import (
+    BinaryView,
+    SymbolType,
+    RegisterValueType,
+    VariableSourceType,
+    MediumLevelILOperation,
+    MediumLevelILBasicBlock,
+    MediumLevelILInstruction,
+)
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
+
+
+def get_printable_len_ascii(s: bytes) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    count = 0
+    for c in s:
+        if c == 0:
+            return count
+        if c < 127 and chr(c) in string.printable:
+            count += 1
+    return count
+
+
+def get_printable_len_wide(s: bytes) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    if all(c == 0x00 for c in s[1::2]):
+        return get_printable_len_ascii(s[::2])
+    return 0
+
+
+def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
+    bv: BinaryView = f.view
+
+    if il.operation != MediumLevelILOperation.MLIL_CALL:
+        return 0
+
+    target = il.dest
+    if target.operation not in [MediumLevelILOperation.MLIL_CONST, MediumLevelILOperation.MLIL_CONST_PTR]:
+        return 0
+
+    addr = target.value.value
+    sym = bv.get_symbol_at(addr)
+    if not sym or sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.SymbolicFunctionSymbol]:
+        return 0
+
+    if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
+        return 0
+
+    if len(il.params) < 2:
+        return 0
+
+    dest = il.params[0]
+    if dest.operation in [MediumLevelILOperation.MLIL_ADDRESS_OF, MediumLevelILOperation.MLIL_VAR]:
+        var = dest.src
+    else:
+        return 0
+
+    if var.source_type != VariableSourceType.StackVariableSourceType:
+        return 0
+
+    src = il.params[1]
+    if src.value.type != RegisterValueType.ConstantDataAggregateValue:
+        return 0
+
+    s = f.get_constant_data(RegisterValueType.ConstantDataAggregateValue, src.value.value)
+    return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
+
+
+def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
+    """check basic block for stackstring indicators
+
+    true if basic block contains enough moves of constant bytes to the stack
+    """
+    count = 0
+    for il in bb:
+        count += get_stack_string_len(f, il)
+        if count > MIN_STACKSTRING_LEN:
+            return True
+
+    return False
+
+
+def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract stackstring indicators from basic block"""
+    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
+    if bb[1] is not None and bb_contains_stackstring(fh.inner, bb[1]):
+        yield Characteristic("stack string"), bbh.address
+
+
+def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract tight loop indicators from a basic block"""
+    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
+    for edge in bb[0].outgoing_edges:
+        if edge.target.start == bb[0].start:
+            yield Characteristic("tight loop"), bbh.address
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract basic block features"""
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, addr in bb_handler(fh, bbh):
+            yield feature, addr
+    yield BasicBlock(), bbh.address
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    extract_bb_stackstring,
+)

capa/features/extractors/binja/extractor.py

@@ -0,0 +1,81 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Tuple, Iterator
+
+import binaryninja as binja
+
+import capa.features.extractors.elf
+import capa.features.extractors.binja.file
+import capa.features.extractors.binja.insn
+import capa.features.extractors.binja.global_
+import capa.features.extractors.binja.function
+import capa.features.extractors.binja.basicblock
+from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
+
+
+class BinjaFeatureExtractor(StaticFeatureExtractor):
+    def __init__(self, bv: binja.BinaryView):
+        super().__init__(hashes=SampleHashes.from_bytes(bv.file.raw.read(0, bv.file.raw.length)))
+        self.bv = bv
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv))
+        self.global_features.extend(capa.features.extractors.binja.global_.extract_os(self.bv))
+        self.global_features.extend(capa.features.extractors.binja.global_.extract_arch(self.bv))
+
+    def get_base_address(self):
+        return AbsoluteVirtualAddress(self.bv.start)
+
+    def extract_global_features(self):
+        yield from self.global_features
+
+    def extract_file_features(self):
+        yield from capa.features.extractors.binja.file.extract_features(self.bv)
+
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        for f in self.bv.functions:
+            yield FunctionHandle(address=AbsoluteVirtualAddress(f.start), inner=f)
+
+    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binja.function.extract_features(fh)
+
+    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
+        f: binja.Function = fh.inner
+        # Set up a MLIL basic block dict look up to associate the disassembly basic block with its MLIL basic block
+        mlil_lookup = {}
+        for mlil_bb in f.mlil.basic_blocks:
+            mlil_lookup[mlil_bb.source_block.start] = mlil_bb
+
+        for bb in f.basic_blocks:
+            mlil_bb = mlil_lookup.get(bb.start)
+
+            yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=(bb, mlil_bb))
+
+    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binja.basicblock.extract_features(fh, bbh)
+
+    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
+        import capa.features.extractors.binja.helpers as binja_helpers
+
+        bb: Tuple[binja.BasicBlock, binja.MediumLevelILBasicBlock] = bbh.inner
+        addr = bb[0].start
+
+        for text, length in bb[0]:
+            insn = binja_helpers.DisassemblyInstruction(addr, length, text)
+            yield InsnHandle(address=AbsoluteVirtualAddress(addr), inner=insn)
+            addr += length
+
+    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
+        yield from capa.features.extractors.binja.insn.extract_features(fh, bbh, ih)

capa/features/extractors/binja/file.py

@@ -0,0 +1,192 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import struct
+from typing import Tuple, Iterator
+
+from binaryninja import Segment, BinaryView, SymbolType, SymbolBinding
+
+import capa.features.extractors.common
+import capa.features.extractors.helpers
+import capa.features.extractors.strings
+from capa.features.file import Export, Import, Section, FunctionName
+from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
+from capa.features.extractors.binja.helpers import read_c_string, unmangle_c_name
+
+
+def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[int, int]]:
+    """check segment for embedded PE
+
+    adapted for binja from:
+    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
+    """
+    mz_xor = [
+        (
+            capa.features.extractors.helpers.xor_static(b"MZ", i),
+            capa.features.extractors.helpers.xor_static(b"PE", i),
+            i,
+        )
+        for i in range(256)
+    ]
+
+    todo = []
+    # If this is the first segment of the binary, skip the first bytes. Otherwise, there will always be a matched
+    # PE at the start of the binaryview.
+    start = seg.start
+    if bv.view_type == "PE" and start == bv.start:
+        start += 1
+
+    for mzx, pex, i in mz_xor:
+        for off, _ in bv.find_all_data(start, seg.end, mzx):
+            todo.append((off, mzx, pex, i))
+
+    while len(todo):
+        off, mzx, pex, i = todo.pop()
+
+        # The MZ header has one field we will check e_lfanew is at 0x3c
+        e_lfanew = off + 0x3C
+
+        if seg.end < (e_lfanew + 4):
+            continue
+
+        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(bv.read(e_lfanew, 4), i))[0]
+
+        peoff = off + newoff
+        if seg.end < (peoff + 2):
+            continue
+
+        if bv.read(peoff, 2) == pex:
+            yield off, i
+
+
+def extract_file_embedded_pe(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract embedded PE features"""
+    for seg in bv.segments:
+        for ea, _ in check_segment_for_pe(bv, seg):
+            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
+
+
+def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract function exports"""
+    for sym in bv.get_symbols_of_type(SymbolType.FunctionSymbol) + bv.get_symbols_of_type(SymbolType.DataSymbol):
+        if sym.binding in [SymbolBinding.GlobalBinding, SymbolBinding.WeakBinding]:
+            name = sym.short_name
+            if name.startswith("__forwarder_name(") and name.endswith(")"):
+                yield Export(name[17:-1]), AbsoluteVirtualAddress(sym.address)
+                yield Characteristic("forwarded export"), AbsoluteVirtualAddress(sym.address)
+            else:
+                yield Export(name), AbsoluteVirtualAddress(sym.address)
+
+                unmangled_name = unmangle_c_name(name)
+                if name != unmangled_name:
+                    yield Export(unmangled_name), AbsoluteVirtualAddress(sym.address)
+
+    for sym in bv.get_symbols_of_type(SymbolType.DataSymbol):
+        if sym.binding not in [SymbolBinding.GlobalBinding]:
+            continue
+
+        name = sym.short_name
+        if not name.startswith("__forwarder_name"):
+            continue
+
+        # Due to https://github.com/Vector35/binaryninja-api/issues/4641, in binja version 3.5, the symbol's name
+        # does not contain the DLL name. As a workaround, we read the C string at the symbol's address, which contains
+        # both the DLL name and the function name.
+        # Once the above issue is closed in the next binjs stable release, we can update the code here to use the
+        # symbol name directly.
+        name = read_c_string(bv, sym.address, 1024)
+        forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(name)
+        yield Export(forwarded_name), AbsoluteVirtualAddress(sym.address)
+        yield Characteristic("forwarded export"), AbsoluteVirtualAddress(sym.address)
+
+
+def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract function imports
+
+    1. imports by ordinal:
+     - modulename.#ordinal
+
+    2. imports by name, results in two features to support importname-only
+       matching:
+     - modulename.importname
+     - importname
+    """
+    for sym in bv.get_symbols_of_type(SymbolType.ImportAddressSymbol):
+        lib_name = str(sym.namespace)
+        addr = AbsoluteVirtualAddress(sym.address)
+        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym.short_name, include_dll=True):
+            yield Import(name), addr
+
+        ordinal = sym.ordinal
+        if ordinal != 0 and (lib_name != ""):
+            ordinal_name = f"#{ordinal}"
+            for name in capa.features.extractors.helpers.generate_symbols(lib_name, ordinal_name, include_dll=True):
+                yield Import(name), addr
+
+
+def extract_file_section_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract section names"""
+    for name, section in bv.sections.items():
+        yield Section(name), AbsoluteVirtualAddress(section.start)
+
+
+def extract_file_strings(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract ASCII and UTF-16 LE strings"""
+    for s in bv.strings:
+        yield String(s.value), FileOffsetAddress(s.start)
+
+
+def extract_file_function_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract the names of statically-linked library functions.
+    """
+    for sym_name in bv.symbols:
+        for sym in bv.symbols[sym_name]:
+            if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
+                continue
+
+            name = sym.short_name
+            yield FunctionName(name), sym.address
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                yield FunctionName(name[1:]), sym.address
+
+
+def extract_file_format(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    view_type = bv.view_type
+    if view_type in ["PE", "COFF"]:
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif view_type == "ELF":
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    elif view_type == "Raw":
+        # no file type to return when processing a binary file, but we want to continue processing
+        return
+    else:
+        raise NotImplementedError(f"unexpected file format: {view_type}")
+
+
+def extract_features(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract file features"""
+    for file_handler in FILE_HANDLERS:
+        for feature, addr in file_handler(bv):
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_strings,
+    extract_file_section_names,
+    extract_file_embedded_pe,
+    extract_file_function_names,
+    extract_file_format,
+)

capa/features/extractors/binja/find_binja_api.py

@@ -0,0 +1,35 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import subprocess
+from pathlib import Path
+
+# When the script gets executed as a standalone executable (via PyInstaller), `import binaryninja` does not work because
+# we have excluded the binaryninja module in `pyinstaller.spec`. The trick here is to call the system Python and try
+# to find out the path of the binaryninja module that has been installed.
+# Note, including the binaryninja module in the `pyinstaller.spec` would not work, since the binaryninja module tries to
+# find the binaryninja core e.g., `libbinaryninjacore.dylib`, using a relative path. And this does not work when the
+# binaryninja module is extracted by the PyInstaller.
+code = r"""
+from pathlib import Path
+from importlib import util
+spec = util.find_spec('binaryninja')
+if spec is not None:
+    if len(spec.submodule_search_locations) > 0:
+            path = Path(spec.submodule_search_locations[0])
+            # encode the path with utf8 then convert to hex, make sure it can be read and restored properly
+            print(str(path.parent).encode('utf8').hex())
+"""
+
+
+def find_binja_path() -> Path:
+    raw_output = subprocess.check_output(["python", "-c", code]).decode("ascii").strip()
+    return Path(bytes.fromhex(raw_output).decode("utf8"))
+
+
+if __name__ == "__main__":
+    print(find_binja_path())

capa/features/extractors/binja/function.py

@@ -0,0 +1,104 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Tuple, Iterator
+
+from binaryninja import Function, BinaryView, SymbolType, RegisterValueType, LowLevelILOperation
+
+from capa.features.file import FunctionName
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors import loops
+from capa.features.extractors.base_extractor import FunctionHandle
+
+
+def extract_function_calls_to(fh: FunctionHandle):
+    """extract callers to a function"""
+    func: Function = fh.inner
+
+    for caller in func.caller_sites:
+        # Everything that is a code reference to the current function is considered a caller, which actually includes
+        # many other references that are NOT a caller. For example, an instruction `push function_start` will also be
+        # considered a caller to the function
+        llil = caller.llil
+        if (llil is None) or llil.operation not in [
+            LowLevelILOperation.LLIL_CALL,
+            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+            LowLevelILOperation.LLIL_JUMP,
+            LowLevelILOperation.LLIL_TAILCALL,
+        ]:
+            continue
+
+        if llil.dest.value.type not in [
+            RegisterValueType.ImportedAddressValue,
+            RegisterValueType.ConstantValue,
+            RegisterValueType.ConstantPointerValue,
+        ]:
+            continue
+
+        address = llil.dest.value.value
+        if address != func.start:
+            continue
+
+        yield Characteristic("calls to"), AbsoluteVirtualAddress(caller.address)
+
+
+def extract_function_loop(fh: FunctionHandle):
+    """extract loop indicators from a function"""
+    func: Function = fh.inner
+
+    edges = []
+
+    # construct control flow graph
+    for bb in func.basic_blocks:
+        for edge in bb.outgoing_edges:
+            edges.append((bb.start, edge.target.start))
+
+    if loops.has_loop(edges):
+        yield Characteristic("loop"), fh.address
+
+
+def extract_recursive_call(fh: FunctionHandle):
+    """extract recursive function call"""
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+    if bv is None:
+        return
+
+    for ref in bv.get_code_refs(func.start):
+        if ref.function == func:
+            yield Characteristic("recursive call"), fh.address
+
+
+def extract_function_name(fh: FunctionHandle):
+    """extract function names (e.g., symtab names)"""
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+    if bv is None:
+        return
+
+    for sym in bv.get_symbols(func.start):
+        if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
+            continue
+
+        name = sym.short_name
+        yield FunctionName(name), sym.address
+        if name.startswith("_"):
+            # some linkers may prefix linked routines with a `_` to avoid name collisions.
+            # extract features for both the mangled and un-mangled representations.
+            # e.g. `_fwrite` -> `fwrite`
+            # see: https://stackoverflow.com/a/2628384/87207
+            yield FunctionName(name[1:]), sym.address
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call, extract_function_name)

capa/features/extractors/binja/global_.py

@@ -0,0 +1,60 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import Tuple, Iterator
+
+from binaryninja import BinaryView
+
+from capa.features.common import OS, OS_MACOS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
+from capa.features.address import NO_ADDRESS, Address
+
+logger = logging.getLogger(__name__)
+
+
+def extract_os(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    name = bv.platform.name
+    if "-" in name:
+        name = name.split("-")[0]
+
+    if name == "windows":
+        yield OS(OS_WINDOWS), NO_ADDRESS
+
+    elif name == "macos":
+        yield OS(OS_MACOS), NO_ADDRESS
+
+    elif name in ["linux", "freebsd", "decree"]:
+        yield OS(name), NO_ADDRESS
+
+    else:
+        # we likely end up here:
+        #  1. handling shellcode, or
+        #  2. handling a new file format (e.g. macho)
+        #
+        # for (1) we can't do much - its shellcode and all bets are off.
+        # we could maybe accept a further CLI argument to specify the OS,
+        # but i think this would be rarely used.
+        # rules that rely on OS conditions will fail to match on shellcode.
+        #
+        # for (2), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported file format: %s, will not guess OS", name)
+        return
+
+
+def extract_arch(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    arch = bv.arch.name
+    if arch == "x86_64":
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+    elif arch == "x86":
+        yield Arch(ARCH_I386), NO_ADDRESS
+    else:
+        # we likely end up here:
+        #  1. handling a new architecture (e.g. aarch64)
+        #
+        # for (1), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported architecture: %s", arch)
+        return

capa/features/extractors/binja/helpers.py

@@ -0,0 +1,69 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import re
+from typing import List, Callable
+from dataclasses import dataclass
+
+from binaryninja import BinaryView, LowLevelILInstruction
+from binaryninja.architecture import InstructionTextToken
+
+
+@dataclass
+class DisassemblyInstruction:
+    address: int
+    length: int
+    text: List[InstructionTextToken]
+
+
+LLIL_VISITOR = Callable[[LowLevelILInstruction, LowLevelILInstruction, int], bool]
+
+
+def visit_llil_exprs(il: LowLevelILInstruction, func: LLIL_VISITOR):
+    # BN does not really support operand index at the disassembly level, so use the LLIL operand index as a substitute.
+    # Note, this is NOT always guaranteed to be the same as disassembly operand.
+    for i, op in enumerate(il.operands):
+        if isinstance(op, LowLevelILInstruction) and func(op, il, i):
+            visit_llil_exprs(op, func)
+
+
+def unmangle_c_name(name: str) -> str:
+    # https://learn.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170#FormatC
+    # Possible variations for BaseThreadInitThunk:
+    # @BaseThreadInitThunk@12
+    # _BaseThreadInitThunk
+    # _BaseThreadInitThunk@12
+    # It is also possible for a function to have a `Stub` appended to its name:
+    # _lstrlenWStub@4
+
+    # A small optimization to avoid running the regex too many times
+    # this still increases the unit test execution time from 170s to 200s, should be able to accelerate it
+    #
+    # TODO(xusheng): performance optimizations to improve test execution time
+    # https://github.com/mandiant/capa/issues/1610
+    if name[0] in ["@", "_"]:
+        match = re.match(r"^[@|_](.*?)(Stub)?(@\d+)?$", name)
+        if match:
+            return match.group(1)
+
+    return name
+
+
+def read_c_string(bv: BinaryView, offset: int, max_len: int) -> str:
+    s: List[str] = []
+    while len(s) < max_len:
+        try:
+            c = bv.read(offset + len(s), 1)[0]
+        except Exception:
+            break
+
+        if c == 0:
+            break
+
+        s.append(chr(c))
+
+    return "".join(s)

capa/features/extractors/binja/insn.py

@@ -0,0 +1,586 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Any, List, Tuple, Iterator, Optional
+
+from binaryninja import Function
+from binaryninja import BasicBlock as BinjaBasicBlock
+from binaryninja import (
+    BinaryView,
+    ILRegister,
+    SymbolType,
+    BinaryReader,
+    RegisterValueType,
+    LowLevelILOperation,
+    LowLevelILInstruction,
+)
+
+import capa.features.extractors.helpers
+from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
+from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.binja.helpers import DisassemblyInstruction, visit_llil_exprs
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+# security cookie checks may perform non-zeroing XORs, these are expected within a certain
+# byte range within the first and returning basic blocks, this helps to reduce FP features
+SECURITY_COOKIE_BYTES_DELTA = 0x40
+
+
+# check if a function is a stub function to another function/symbol. The criteria is:
+# 1. The function must only have one basic block
+# 2. The function must only make one call/jump to another address
+# If the function being checked is a stub function, returns the target address. Otherwise, return None.
+def is_stub_function(bv: BinaryView, addr: int) -> Optional[int]:
+    funcs = bv.get_functions_at(addr)
+    for func in funcs:
+        if len(func.basic_blocks) != 1:
+            continue
+
+        call_count = 0
+        call_target = None
+        for il in func.llil.instructions:
+            if il.operation in [
+                LowLevelILOperation.LLIL_CALL,
+                LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+                LowLevelILOperation.LLIL_JUMP,
+                LowLevelILOperation.LLIL_TAILCALL,
+            ]:
+                call_count += 1
+                if il.dest.value.type in [
+                    RegisterValueType.ImportedAddressValue,
+                    RegisterValueType.ConstantValue,
+                    RegisterValueType.ConstantPointerValue,
+                ]:
+                    call_target = il.dest.value.value
+
+        if call_count == 1 and call_target is not None:
+            return call_target
+
+    return None
+
+
+def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction API features
+
+    example:
+       call dword [0x00473038]
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    for llil in func.get_llils_at(ih.address):
+        if llil.operation in [
+            LowLevelILOperation.LLIL_CALL,
+            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+            LowLevelILOperation.LLIL_JUMP,
+            LowLevelILOperation.LLIL_TAILCALL,
+        ]:
+            if llil.dest.value.type not in [
+                RegisterValueType.ImportedAddressValue,
+                RegisterValueType.ConstantValue,
+                RegisterValueType.ConstantPointerValue,
+            ]:
+                continue
+            address = llil.dest.value.value
+            candidate_addrs = [address]
+            stub_addr = is_stub_function(bv, address)
+            if stub_addr is not None:
+                candidate_addrs.append(stub_addr)
+
+            for address in candidate_addrs:
+                for sym in func.view.get_symbols(address):
+                    if sym is None or sym.type not in [
+                        SymbolType.ImportAddressSymbol,
+                        SymbolType.ImportedFunctionSymbol,
+                        SymbolType.FunctionSymbol,
+                    ]:
+                        continue
+
+                    sym_name = sym.short_name
+
+                    lib_name = ""
+                    import_lib = bv.lookup_imported_object_library(sym.address)
+                    if import_lib is not None:
+                        lib_name = import_lib[0].name
+                        if lib_name.endswith(".dll"):
+                            lib_name = lib_name[:-4]
+                        elif lib_name.endswith(".so"):
+                            lib_name = lib_name[:-3]
+
+                    for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name):
+                        yield API(name), ih.address
+
+                    if sym_name.startswith("_"):
+                        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name[1:]):
+                            yield API(name), ih.address
+
+
+def extract_insn_number_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction number features
+    example:
+        push    3136B0h         ; dwControlCode
+    """
+    func: Function = fh.inner
+
+    results: List[Tuple[Any[Number, OperandNumber], Address]] = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation == LowLevelILOperation.LLIL_LOAD:
+            return False
+
+        if il.operation not in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            return True
+
+        for op in parent.operands:
+            if isinstance(op, ILRegister) and op.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
+                return False
+            elif isinstance(op, LowLevelILInstruction) and op.operation == LowLevelILOperation.LLIL_REG:
+                if op.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
+                    return False
+
+        raw_value = il.value.value
+        if parent.operation == LowLevelILOperation.LLIL_SUB:
+            raw_value = -raw_value
+
+        results.append((Number(raw_value), ih.address))
+        results.append((OperandNumber(index, raw_value), ih.address))
+
+        return False
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse referenced byte sequences
+    example:
+        push    offset iid_004118d4_IShellLinkA ; riid
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    candidate_addrs = set()
+
+    llil = func.get_llil_at(ih.address)
+    if llil is None or llil.operation in [LowLevelILOperation.LLIL_CALL, LowLevelILOperation.LLIL_CALL_STACK_ADJUST]:
+        return
+
+    for ref in bv.get_code_refs_from(ih.address):
+        if ref == ih.address:
+            continue
+
+        if len(bv.get_functions_containing(ref)) > 0:
+            continue
+
+        candidate_addrs.add(ref)
+
+    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            value = il.value.value
+            if value > 0:
+                candidate_addrs.add(value)
+            return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    for addr in candidate_addrs:
+        extracted_bytes = bv.read(addr, MAX_BYTES_FEATURE_SIZE)
+        if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
+            if bv.get_string_at(addr) is None:
+                # don't extract byte features for obvious strings
+                yield Bytes(extracted_bytes), ih.address
+
+
+def extract_insn_string_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction string features
+
+    example:
+        push offset aAcr     ; "ACR  > "
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    candidate_addrs = set()
+
+    # collect candidate address from code refs directly
+    for ref in bv.get_code_refs_from(ih.address):
+        if ref == ih.address:
+            continue
+
+        if len(bv.get_functions_containing(ref)) > 0:
+            continue
+
+        candidate_addrs.add(ref)
+
+    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            value = il.value.value
+            if value > 0:
+                candidate_addrs.add(value)
+            return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    # Now we have all the candidate address, check them for string or pointer to string
+    br = BinaryReader(bv)
+    for addr in candidate_addrs:
+        found = bv.get_string_at(addr)
+        if found:
+            yield String(found.value), ih.address
+
+        br.seek(addr)
+        pointer = None
+        if bv.arch.address_size == 4:
+            pointer = br.read32()
+        elif bv.arch.address_size == 8:
+            pointer = br.read64()
+
+        if pointer is not None:
+            found = bv.get_string_at(pointer)
+            if found:
+                yield String(found.value), ih.address
+
+
+def extract_insn_offset_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction structure offset features
+
+    example:
+        .text:0040112F cmp [esi+4], ebx
+    """
+    func: Function = fh.inner
+
+    results: List[Tuple[Any[Offset, OperandOffset], Address]] = []
+    address_size = func.view.arch.address_size * 8
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        #  The most common case, read/write dereference to something like `dword [eax+0x28]`
+        if il.operation in [LowLevelILOperation.LLIL_ADD, LowLevelILOperation.LLIL_SUB]:
+            left = il.left
+            right = il.right
+            # Exclude offsets based on stack/franme pointers
+            if left.operation == LowLevelILOperation.LLIL_REG and left.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
+                return True
+
+            if right.operation != LowLevelILOperation.LLIL_CONST:
+                return True
+
+            raw_value = right.value.value
+            # If this is not a dereference, then this must be an add and the offset must be in the range \
+            # [0, MAX_STRUCTURE_SIZE]. For example,
+            # add eax, 0x10,
+            # lea ebx, [eax + 1]
+            if parent.operation not in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
+                if il.operation != LowLevelILOperation.LLIL_ADD or (not 0 < raw_value < MAX_STRUCTURE_SIZE):
+                    return False
+
+            if address_size > 0:
+                # BN also encodes the constant value as two's complement, we need to restore its original value
+                value = capa.features.extractors.helpers.twos_complement(raw_value, address_size)
+            else:
+                value = raw_value
+
+            results.append((Offset(value), ih.address))
+            results.append((OperandOffset(index, value), ih.address))
+            return False
+
+        # An edge case: for code like `push dword [esi]`, we need to generate a feature for offset 0x0
+        elif il.operation in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
+            if il.operands[0].operation == LowLevelILOperation.LLIL_REG:
+                results.append((Offset(0), ih.address))
+                results.append((OperandOffset(index, 0), ih.address))
+                return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def is_nzxor_stack_cookie(f: Function, bb: BinjaBasicBlock, llil: LowLevelILInstruction) -> bool:
+    """check if nzxor exists within stack cookie delta"""
+    # TODO(xusheng): use LLIL SSA to do more accurate analysis
+    # https://github.com/mandiant/capa/issues/1609
+
+    reg_names = []
+    if llil.left.operation == LowLevelILOperation.LLIL_REG:
+        reg_names.append(llil.left.src.name)
+
+    if llil.right.operation == LowLevelILOperation.LLIL_REG:
+        reg_names.append(llil.right.src.name)
+
+    # stack cookie reg should be stack/frame pointer
+    if not any(reg in ["ebp", "esp", "rbp", "rsp", "sp"] for reg in reg_names):
+        return False
+
+    # expect security cookie init in first basic block within first bytes (instructions)
+    if len(bb.incoming_edges) == 0 and llil.address < (bb.start + SECURITY_COOKIE_BYTES_DELTA):
+        return True
+
+    # ... or within last bytes (instructions) before a return
+    if len(bb.outgoing_edges) == 0 and llil.address > (bb.end - SECURITY_COOKIE_BYTES_DELTA):
+        return True
+
+    return False
+
+
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction non-zeroing XOR instruction
+    ignore expected non-zeroing XORs, e.g. security cookies
+    """
+    func: Function = fh.inner
+
+    results = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        # If the two operands of the xor instruction are the same, the LLIL will be translated to other instructions,
+        # e.g., <llil: eax = 0>, (LLIL_SET_REG). So we do not need to check whether the two operands are the same.
+        if il.operation == LowLevelILOperation.LLIL_XOR:
+            # Exclude cases related to the stack cookie
+            if is_nzxor_stack_cookie(fh.inner, bbh.inner[0], il):
+                return False
+            results.append((Characteristic("nzxor"), ih.address))
+            return False
+        else:
+            return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_mnemonic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction mnemonic features"""
+    insn: DisassemblyInstruction = ih.inner
+    yield Mnemonic(insn.text[0].text), ih.address
+
+
+def extract_insn_obfs_call_plus_5_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse call $+5 instruction from the given instruction.
+    """
+    insn: DisassemblyInstruction = ih.inner
+    if insn.text[0].text == "call" and insn.text[2].text == "$+5" and insn.length == 5:
+        yield Characteristic("call $+5"), ih.address
+
+
+def extract_insn_peb_access_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction peb access
+
+    fs:[0x30] on x86, gs:[0x60] on x64
+    """
+    func: Function = fh.inner
+
+    results = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILOperation, index: int) -> bool:
+        if il.operation != LowLevelILOperation.LLIL_LOAD:
+            return True
+
+        src = il.src
+        if src.operation != LowLevelILOperation.LLIL_ADD:
+            return True
+
+        left = src.left
+        right = src.right
+
+        if left.operation != LowLevelILOperation.LLIL_REG:
+            return True
+
+        reg = left.src.name
+
+        if right.operation != LowLevelILOperation.LLIL_CONST:
+            return True
+
+        value = right.value.value
+        if (reg, value) not in (("fsbase", 0x30), ("gsbase", 0x60)):
+            return True
+
+        results.append((Characteristic("peb access"), ih.address))
+        return False
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_segment_access_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction fs or gs access"""
+    func: Function = fh.inner
+
+    results = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation == LowLevelILOperation.LLIL_REG:
+            reg = il.src.name
+            if reg == "fsbase":
+                results.append((Characteristic("fs access"), ih.address))
+                return False
+            elif reg == "gsbase":
+                results.append((Characteristic("gs access"), ih.address))
+                return False
+            return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_cross_section_cflow(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    if bv is None:
+        return
+
+    seg1 = bv.get_segment_at(ih.address)
+    sections1 = bv.get_sections_at(ih.address)
+    for ref in bv.get_code_refs_from(ih.address):
+        if len(bv.get_functions_at(ref)) == 0:
+            continue
+
+        seg2 = bv.get_segment_at(ref)
+        sections2 = bv.get_sections_at(ref)
+        if seg1 != seg2 or sections1 != sections2:
+            yield Characteristic("cross section flow"), ih.address
+
+
+def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract functions calls from features
+
+    most relevant at the function scope, however, its most efficient to extract at the instruction scope
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    if bv is None:
+        return
+
+    for il in func.get_llils_at(ih.address):
+        if il.operation not in [
+            LowLevelILOperation.LLIL_CALL,
+            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+            LowLevelILOperation.LLIL_TAILCALL,
+        ]:
+            continue
+
+        dest = il.dest
+        if dest.operation == LowLevelILOperation.LLIL_CONST_PTR:
+            value = dest.value.value
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
+        elif dest.operation == LowLevelILOperation.LLIL_CONST:
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value)
+        elif dest.operation == LowLevelILOperation.LLIL_LOAD:
+            indirect_src = dest.src
+            if indirect_src.operation == LowLevelILOperation.LLIL_CONST_PTR:
+                value = indirect_src.value.value
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
+            elif indirect_src.operation == LowLevelILOperation.LLIL_CONST:
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(indirect_src.value)
+        elif dest.operation == LowLevelILOperation.LLIL_REG:
+            if dest.value.type in [
+                RegisterValueType.ImportedAddressValue,
+                RegisterValueType.ConstantValue,
+                RegisterValueType.ConstantPointerValue,
+            ]:
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value.value)
+
+
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
+    does not include calls like => call ds:dword_ABD4974
+
+    most relevant at the function or basic block scope;
+    however, its most efficient to extract at the instruction scope
+    """
+    func: Function = fh.inner
+
+    llil = func.get_llil_at(ih.address)
+    if llil is None or llil.operation not in [
+        LowLevelILOperation.LLIL_CALL,
+        LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+        LowLevelILOperation.LLIL_TAILCALL,
+    ]:
+        return
+
+    if llil.dest.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+        return
+
+    if llil.dest.operation == LowLevelILOperation.LLIL_LOAD:
+        src = llil.dest.src
+        if src.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            return
+
+    yield Characteristic("indirect call"), ih.address
+
+
+def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract instruction features"""
+    for inst_handler in INSTRUCTION_HANDLERS:
+        for feature, ea in inst_handler(f, bbh, insn):
+            yield feature, ea
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_bytes_features,
+    extract_insn_string_features,
+    extract_insn_offset_features,
+    extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    extract_insn_obfs_call_plus_5_characteristic_features,
+    extract_insn_peb_access_characteristic_features,
+    extract_insn_cross_section_cflow,
+    extract_insn_segment_access_features,
+    extract_function_calls_from,
+    extract_function_indirect_call_characteristic_features,
+)

capa/features/extractors/cape/call.py

@@ -0,0 +1,62 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.helpers import assert_never
+from capa.features.insn import API, Number
+from capa.features.common import String, Feature
+from capa.features.address import Address
+from capa.features.extractors.cape.models import Call
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    this method extracts the given call's features (such as API name and arguments),
+    and returns them as API, Number, and String features.
+
+    args:
+      ph: process handle (for defining the extraction scope)
+      th: thread handle (for defining the extraction scope)
+      ch: call handle (for defining the extraction scope)
+
+    yields:
+      Feature, address; where Feature is either: API, Number, or String.
+    """
+    call: Call = ch.inner
+
+    # list similar to disassembly: arguments right-to-left, call
+    for arg in reversed(call.arguments):
+        value = arg.value
+        if isinstance(value, list) and len(value) == 0:
+            # unsure why CAPE captures arguments as empty lists?
+            continue
+
+        elif isinstance(value, str):
+            yield String(value), ch.address
+
+        elif isinstance(value, int):
+            yield Number(value), ch.address
+
+        else:
+            assert_never(value)
+
+    yield API(call.api), ch.address
+
+
+def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
+    for handler in CALL_HANDLERS:
+        for feature, addr in handler(ph, th, ch):
+            yield feature, addr
+
+
+CALL_HANDLERS = (extract_call_features,)

capa/features/extractors/cape/extractor.py

@@ -0,0 +1,153 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Dict, Tuple, Union, Iterator
+
+import capa.features.extractors.cape.call
+import capa.features.extractors.cape.file
+import capa.features.extractors.cape.thread
+import capa.features.extractors.cape.global_
+import capa.features.extractors.cape.process
+from capa.exceptions import EmptyReportError, UnsupportedFormatError
+from capa.features.common import Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress, _NoAddress
+from capa.features.extractors.cape.models import Call, Static, Process, CapeReport
+from capa.features.extractors.base_extractor import (
+    CallHandle,
+    SampleHashes,
+    ThreadHandle,
+    ProcessHandle,
+    DynamicFeatureExtractor,
+)
+
+logger = logging.getLogger(__name__)
+
+TESTED_VERSIONS = {"2.2-CAPE", "2.4-CAPE"}
+
+
+class CapeExtractor(DynamicFeatureExtractor):
+    def __init__(self, report: CapeReport):
+        super().__init__(
+            hashes=SampleHashes(
+                md5=report.target.file.md5.lower(),
+                sha1=report.target.file.sha1.lower(),
+                sha256=report.target.file.sha256.lower(),
+            )
+        )
+        self.report: CapeReport = report
+
+        # pre-compute these because we'll yield them at *every* scope.
+        self.global_features = list(capa.features.extractors.cape.global_.extract_features(self.report))
+
+    def get_base_address(self) -> Union[AbsoluteVirtualAddress, _NoAddress, None]:
+        # value according to the PE header, the actual trace may use a different imagebase
+        assert self.report.static is not None and self.report.static.pe is not None
+        return AbsoluteVirtualAddress(self.report.static.pe.imagebase)
+
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from self.global_features
+
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.cape.file.extract_features(self.report)
+
+    def get_processes(self) -> Iterator[ProcessHandle]:
+        yield from capa.features.extractors.cape.file.get_processes(self.report)
+
+    def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.cape.process.extract_features(ph)
+
+    def get_process_name(self, ph) -> str:
+        process: Process = ph.inner
+        return process.process_name
+
+    def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
+        yield from capa.features.extractors.cape.process.get_threads(ph)
+
+    def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
+        if False:
+            # force this routine to be a generator,
+            # but we don't actually have any elements to generate.
+            yield Characteristic("never"), NO_ADDRESS
+        return
+
+    def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+        yield from capa.features.extractors.cape.thread.get_calls(ph, th)
+
+    def extract_call_features(
+        self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.cape.call.extract_features(ph, th, ch)
+
+    def get_call_name(self, ph, th, ch) -> str:
+        call: Call = ch.inner
+
+        parts = []
+        parts.append(call.api)
+        parts.append("(")
+        for argument in call.arguments:
+            parts.append(argument.name)
+            parts.append("=")
+
+            if argument.pretty_value:
+                parts.append(argument.pretty_value)
+            else:
+                if isinstance(argument.value, int):
+                    parts.append(hex(argument.value))
+                elif isinstance(argument.value, str):
+                    parts.append('"')
+                    parts.append(argument.value)
+                    parts.append('"')
+                elif isinstance(argument.value, list):
+                    pass
+                else:
+                    capa.helpers.assert_never(argument.value)
+
+            parts.append(", ")
+        if call.arguments:
+            # remove the trailing comma
+            parts.pop()
+        parts.append(")")
+        parts.append(" -> ")
+        if call.pretty_return:
+            parts.append(call.pretty_return)
+        else:
+            parts.append(hex(call.return_))
+
+        return "".join(parts)
+
+    @classmethod
+    def from_report(cls, report: Dict) -> "CapeExtractor":
+        cr = CapeReport.model_validate(report)
+
+        if cr.info.version not in TESTED_VERSIONS:
+            logger.warning("CAPE version '%s' not tested/supported yet", cr.info.version)
+
+        # TODO(mr-tz): support more file types
+        # https://github.com/mandiant/capa/issues/1933
+        if "PE" not in cr.target.file.type:
+            logger.error(
+                "capa currently only supports PE target files, this target file's type is: '%s'.\nPlease report this at: https://github.com/mandiant/capa/issues/1933",
+                cr.target.file.type,
+            )
+
+        # observed in 2.4-CAPE reports from capesandbox.com
+        if cr.static is None and cr.target.file.pe is not None:
+            cr.static = Static()
+            cr.static.pe = cr.target.file.pe
+
+        if cr.static is None:
+            raise UnsupportedFormatError("CAPE report missing static analysis")
+
+        if cr.static.pe is None:
+            raise UnsupportedFormatError("CAPE report missing PE analysis")
+
+        if len(cr.behavior.processes) == 0:
+            raise EmptyReportError("CAPE did not capture any processes")
+
+        return cls(cr)

capa/features/extractors/cape/file.py

@@ -0,0 +1,132 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.file import Export, Import, Section
+from capa.features.common import String, Feature
+from capa.features.address import NO_ADDRESS, Address, ProcessAddress, AbsoluteVirtualAddress
+from capa.features.extractors.helpers import generate_symbols
+from capa.features.extractors.cape.models import CapeReport
+from capa.features.extractors.base_extractor import ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def get_processes(report: CapeReport) -> Iterator[ProcessHandle]:
+    """
+    get all the created processes for a sample
+    """
+    seen_processes = {}
+    for process in report.behavior.processes:
+        addr = ProcessAddress(pid=process.process_id, ppid=process.parent_id)
+        yield ProcessHandle(address=addr, inner=process)
+
+        # check for pid and ppid reuse
+        if addr not in seen_processes:
+            seen_processes[addr] = [process]
+        else:
+            logger.warning(
+                "pid and ppid reuse detected between process %s and process%s: %s",
+                process,
+                "es" if len(seen_processes[addr]) > 1 else "",
+                seen_processes[addr],
+            )
+            seen_processes[addr].append(process)
+
+
+def extract_import_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract imported function names
+    """
+    assert report.static is not None and report.static.pe is not None
+    imports = report.static.pe.imports
+
+    if isinstance(imports, dict):
+        imports = list(imports.values())
+
+    assert isinstance(imports, list)
+
+    for library in imports:
+        for function in library.imports:
+            if not function.name:
+                continue
+
+            for name in generate_symbols(library.dll, function.name, include_dll=True):
+                yield Import(name), AbsoluteVirtualAddress(function.address)
+
+
+def extract_export_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    assert report.static is not None and report.static.pe is not None
+    for function in report.static.pe.exports:
+        yield Export(function.name), AbsoluteVirtualAddress(function.address)
+
+
+def extract_section_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    assert report.static is not None and report.static.pe is not None
+    for section in report.static.pe.sections:
+        yield Section(section.name), AbsoluteVirtualAddress(section.virtual_address)
+
+
+def extract_file_strings(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    if report.strings is not None:
+        for string in report.strings:
+            yield String(string), NO_ADDRESS
+
+
+def extract_used_regkeys(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for regkey in report.behavior.summary.keys:
+        yield String(regkey), NO_ADDRESS
+
+
+def extract_used_files(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for file in report.behavior.summary.files:
+        yield String(file), NO_ADDRESS
+
+
+def extract_used_mutexes(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for mutex in report.behavior.summary.mutexes:
+        yield String(mutex), NO_ADDRESS
+
+
+def extract_used_commands(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for cmd in report.behavior.summary.executed_commands:
+        yield String(cmd), NO_ADDRESS
+
+
+def extract_used_apis(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for symbol in report.behavior.summary.resolved_apis:
+        yield String(symbol), NO_ADDRESS
+
+
+def extract_used_services(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for svc in report.behavior.summary.created_services:
+        yield String(svc), NO_ADDRESS
+    for svc in report.behavior.summary.started_services:
+        yield String(svc), NO_ADDRESS
+
+
+def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for handler in FILE_HANDLERS:
+        for feature, addr in handler(report):
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_import_names,
+    extract_export_names,
+    extract_section_names,
+    extract_file_strings,
+    extract_used_regkeys,
+    extract_used_files,
+    extract_used_mutexes,
+    extract_used_commands,
+    extract_used_apis,
+    extract_used_services,
+)

capa/features/extractors/cape/global_.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.common import (
+    OS,
+    OS_ANY,
+    OS_LINUX,
+    ARCH_I386,
+    FORMAT_PE,
+    ARCH_AMD64,
+    FORMAT_ELF,
+    OS_WINDOWS,
+    Arch,
+    Format,
+    Feature,
+)
+from capa.features.address import NO_ADDRESS, Address
+from capa.features.extractors.cape.models import CapeReport
+
+logger = logging.getLogger(__name__)
+
+
+def extract_arch(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    if "Intel 80386" in report.target.file.type:
+        yield Arch(ARCH_I386), NO_ADDRESS
+    elif "x86-64" in report.target.file.type:
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+    else:
+        logger.warning("unrecognized Architecture: %s", report.target.file.type)
+        raise ValueError(
+            f"unrecognized Architecture from the CAPE report; output of file command: {report.target.file.type}"
+        )
+
+
+def extract_format(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    if "PE" in report.target.file.type:
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif "ELF" in report.target.file.type:
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    else:
+        logger.warning("unknown file format, file command output: %s", report.target.file.type)
+        raise ValueError(
+            f"unrecognized file format from the CAPE report; output of file command: {report.target.file.type}"
+        )
+
+
+def extract_os(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    # this variable contains the output of the file command
+    file_output = report.target.file.type
+
+    if "windows" in file_output.lower():
+        yield OS(OS_WINDOWS), NO_ADDRESS
+    elif "elf" in file_output.lower():
+        # operating systems recognized by the file command: https://github.com/file/file/blob/master/src/readelf.c#L609
+        if "Linux" in file_output:
+            yield OS(OS_LINUX), NO_ADDRESS
+        elif "Hurd" in file_output:
+            yield OS("hurd"), NO_ADDRESS
+        elif "Solaris" in file_output:
+            yield OS("solaris"), NO_ADDRESS
+        elif "kFreeBSD" in file_output:
+            yield OS("freebsd"), NO_ADDRESS
+        elif "kNetBSD" in file_output:
+            yield OS("netbsd"), NO_ADDRESS
+        else:
+            # if the operating system information is missing from the cape report, it's likely a bug
+            logger.warning("unrecognized OS: %s", file_output)
+            raise ValueError(f"unrecognized OS from the CAPE report; output of file command: {file_output}")
+    else:
+        # the sample is shellcode
+        logger.debug("unsupported file format, file command output: %s", file_output)
+        yield OS(OS_ANY), NO_ADDRESS
+
+
+def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for global_handler in GLOBAL_HANDLER:
+        for feature, addr in global_handler(report):
+            yield feature, addr
+
+
+GLOBAL_HANDLER = (
+    extract_format,
+    extract_os,
+    extract_arch,
+)

capa/features/extractors/cape/helpers.py

@@ -0,0 +1,29 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from typing import Any, Dict, List
+
+from capa.features.extractors.base_extractor import ProcessHandle
+
+
+def find_process(processes: List[Dict[str, Any]], ph: ProcessHandle) -> Dict[str, Any]:
+    """
+    find a specific process identified by a process handler.
+
+    args:
+      processes: a list of processes extracted by CAPE
+      ph: handle of the sought process
+
+    return:
+      a CAPE-defined dictionary for the sought process' information
+    """
+
+    for process in processes:
+        if ph.address.ppid == process["parent_id"] and ph.address.pid == process["process_id"]:
+            return process
+    return {}

capa/features/extractors/cape/models.py

@@ -0,0 +1,446 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import binascii
+from typing import Any, Dict, List, Union, Literal, Optional
+
+from pydantic import Field, BaseModel, ConfigDict
+from typing_extensions import Annotated, TypeAlias
+from pydantic.functional_validators import BeforeValidator
+
+
+def validate_hex_int(value):
+    if isinstance(value, str):
+        return int(value, 16) if value.startswith("0x") else int(value, 10)
+    else:
+        return value
+
+
+def validate_hex_bytes(value):
+    return binascii.unhexlify(value) if isinstance(value, str) else value
+
+
+HexInt = Annotated[int, BeforeValidator(validate_hex_int)]
+HexBytes = Annotated[bytes, BeforeValidator(validate_hex_bytes)]
+
+
+# a model that *cannot* have extra fields
+# if they do, pydantic raises an exception.
+# use this for models we rely upon and cannot change.
+#
+# for things that may be extended and we don't care,
+# use FlexibleModel.
+class ExactModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+# a model that can have extra fields that we ignore.
+# use this if we don't want to raise an exception for extra
+# data fields that we didn't expect.
+class FlexibleModel(BaseModel):
+    pass
+
+
+# use this type to indicate that we won't model this data.
+# because it's not relevant to our use in capa.
+#
+# while its nice to have full coverage of the data shape,
+# it can easily change and break our parsing.
+# so we really only want to describe what we'll use.
+Skip: TypeAlias = Optional[Any]
+
+
+# mark fields that we haven't seen yet and need to model.
+# pydantic should raise an error when encountering data
+# in a field with this type.
+# then we can update the model with the discovered shape.
+TODO: TypeAlias = None
+ListTODO: TypeAlias = List[None]
+DictTODO: TypeAlias = ExactModel
+
+EmptyDict: TypeAlias = BaseModel
+EmptyList: TypeAlias = List[Any]
+
+
+class Info(FlexibleModel):
+    version: str
+
+
+class ImportedSymbol(ExactModel):
+    address: HexInt
+    name: Optional[str] = None
+
+
+class ImportedDll(ExactModel):
+    dll: str
+    imports: List[ImportedSymbol]
+
+
+class DirectoryEntry(ExactModel):
+    name: str
+    virtual_address: HexInt
+    size: HexInt
+
+
+class Section(ExactModel):
+    name: str
+    raw_address: HexInt
+    virtual_address: HexInt
+    virtual_size: HexInt
+    size_of_data: HexInt
+    characteristics: str
+    characteristics_raw: HexInt
+    entropy: float
+
+
+class Resource(ExactModel):
+    name: str
+    language: Optional[str] = None
+    sublanguage: str
+    filetype: Optional[str]
+    offset: HexInt
+    size: HexInt
+    entropy: float
+
+
+class DigitalSigner(FlexibleModel):
+    md5_fingerprint: str
+    not_after: str
+    not_before: str
+    serial_number: str
+    sha1_fingerprint: str
+    sha256_fingerprint: str
+
+    issuer_commonName: Optional[str] = None
+    issuer_countryName: Optional[str] = None
+    issuer_localityName: Optional[str] = None
+    issuer_organizationName: Optional[str] = None
+    issuer_stateOrProvinceName: Optional[str] = None
+
+    subject_commonName: Optional[str] = None
+    subject_countryName: Optional[str] = None
+    subject_localityName: Optional[str] = None
+    subject_organizationName: Optional[str] = None
+    subject_stateOrProvinceName: Optional[str] = None
+
+    extensions_authorityInfoAccess_caIssuers: Optional[str] = None
+    extensions_authorityKeyIdentifier: Optional[str] = None
+    extensions_cRLDistributionPoints_0: Optional[str] = None
+    extensions_certificatePolicies_0: Optional[str] = None
+    extensions_subjectAltName_0: Optional[str] = None
+    extensions_subjectKeyIdentifier: Optional[str] = None
+
+
+class AuxSigner(ExactModel):
+    name: str
+    issued_to: str = Field(alias="Issued to")
+    issued_by: str = Field(alias="Issued by")
+    expires: str = Field(alias="Expires")
+    sha1_hash: str = Field(alias="SHA1 hash")
+
+
+class Signer(ExactModel):
+    aux_sha1: Optional[str] = None
+    aux_timestamp: Optional[str] = None
+    aux_valid: Optional[bool] = None
+    aux_error: Optional[bool] = None
+    aux_error_desc: Optional[str] = None
+    aux_signers: Optional[List[AuxSigner]] = None
+
+
+class Overlay(ExactModel):
+    offset: HexInt
+    size: HexInt
+
+
+class KV(ExactModel):
+    name: str
+    value: str
+
+
+class ExportedSymbol(ExactModel):
+    address: HexInt
+    name: str
+    ordinal: int
+
+
+class PE(ExactModel):
+    peid_signatures: TODO
+    imagebase: HexInt
+    entrypoint: HexInt
+    reported_checksum: HexInt
+    actual_checksum: HexInt
+    osversion: str
+    pdbpath: Optional[str] = None
+    timestamp: str
+
+    # List[ImportedDll], or Dict[basename(dll), ImportedDll]
+    imports: Union[List[ImportedDll], Dict[str, ImportedDll]]
+    imported_dll_count: Optional[int] = None
+    imphash: str
+
+    exported_dll_name: Optional[str] = None
+    exports: List[ExportedSymbol]
+
+    dirents: List[DirectoryEntry]
+    sections: List[Section]
+
+    ep_bytes: Optional[HexBytes] = None
+
+    overlay: Optional[Overlay] = None
+    resources: List[Resource]
+    versioninfo: List[KV]
+
+    # base64 encoded data
+    icon: Optional[str] = None
+    # MD5-like hash
+    icon_hash: Optional[str] = None
+    # MD5-like hash
+    icon_fuzzy: Optional[str] = None
+    # short hex string
+    icon_dhash: Optional[str] = None
+
+    digital_signers: List[DigitalSigner]
+    guest_signers: Signer
+
+
+# TODO(mr-tz): target.file.dotnet, target.file.extracted_files, target.file.extracted_files_tool,
+#  target.file.extracted_files_time
+# https://github.com/mandiant/capa/issues/1814
+class File(FlexibleModel):
+    type: str
+    cape_type_code: Optional[int] = None
+    cape_type: Optional[str] = None
+
+    pid: Optional[Union[int, Literal[""]]] = None
+    name: Union[List[str], str]
+    path: str
+    guest_paths: Union[List[str], str, None]
+    timestamp: Optional[str] = None
+
+    #
+    # hashes
+    #
+    crc32: str
+    md5: str
+    sha1: str
+    sha256: str
+    sha512: str
+    sha3_384: Optional[str] = None
+    ssdeep: str
+    # unsure why this would ever be "False"
+    tlsh: Optional[Union[str, bool]] = None
+    rh_hash: Optional[str] = None
+
+    #
+    # other metadata, static analysis
+    #
+    size: int
+    pe: Optional[PE] = None
+    ep_bytes: Optional[HexBytes] = None
+    entrypoint: Optional[int] = None
+    data: Optional[str] = None
+    strings: Optional[List[str]] = None
+
+    #
+    # detections (skip)
+    #
+    yara: Skip = None
+    cape_yara: Skip = None
+    clamav: Skip = None
+    virustotal: Skip = None
+
+
+class ProcessFile(File):
+    #
+    # like a File, but also has dynamic analysis results
+    #
+    pid: Optional[int] = None
+    process_path: Optional[str] = None
+    process_name: Optional[str] = None
+    module_path: Optional[str] = None
+    virtual_address: Optional[HexInt] = None
+    target_pid: Optional[Union[int, str]] = None
+    target_path: Optional[str] = None
+    target_process: Optional[str] = None
+
+
+class Argument(ExactModel):
+    name: str
+    # unsure why empty list is provided here
+    value: Union[HexInt, int, str, EmptyList]
+    pretty_value: Optional[str] = None
+
+
+class Call(ExactModel):
+    timestamp: str
+    thread_id: int
+    category: str
+
+    api: str
+
+    arguments: List[Argument]
+    status: bool
+    return_: HexInt = Field(alias="return")
+    pretty_return: Optional[str] = None
+
+    repeated: int
+
+    # virtual addresses
+    caller: HexInt
+    parentcaller: HexInt
+
+    # index into calls array
+    id: int
+
+
+class Process(ExactModel):
+    process_id: int
+    process_name: str
+    parent_id: int
+    module_path: str
+    first_seen: str
+    calls: List[Call]
+    threads: List[int]
+    environ: Dict[str, str]
+
+
+class ProcessTree(ExactModel):
+    name: str
+    pid: int
+    parent_id: int
+    module_path: str
+    threads: List[int]
+    environ: Dict[str, str]
+    children: List["ProcessTree"]
+
+
+class Summary(ExactModel):
+    files: List[str]
+    read_files: List[str]
+    write_files: List[str]
+    delete_files: List[str]
+    keys: List[str]
+    read_keys: List[str]
+    write_keys: List[str]
+    delete_keys: List[str]
+    executed_commands: List[str]
+    resolved_apis: List[str]
+    mutexes: List[str]
+    created_services: List[str]
+    started_services: List[str]
+
+
+class EncryptedBuffer(ExactModel):
+    process_name: str
+    pid: int
+
+    api_call: str
+    buffer: str
+    buffer_size: Optional[int] = None
+    crypt_key: Optional[Union[HexInt, str]] = None
+
+
+class Behavior(ExactModel):
+    summary: Summary
+
+    # list of processes, of threads, of calls
+    processes: List[Process]
+    # tree of processes
+    processtree: List[ProcessTree]
+
+    anomaly: List[str]
+    encryptedbuffers: List[EncryptedBuffer]
+    # these are small objects that describe atomic events,
+    # like file move, registry access.
+    # we'll detect the same with our API call analysis.
+    enhanced: Skip = None
+
+
+class Target(ExactModel):
+    category: str
+    file: File
+    pe: Optional[PE] = None
+
+
+class Static(ExactModel):
+    pe: Optional[PE] = None
+    flare_capa: Skip = None
+
+
+class Cape(ExactModel):
+    payloads: List[ProcessFile]
+    configs: Skip = None
+
+
+# flexible because there may be more sorts of analysis
+# but we only care about the ones described here.
+class CapeReport(FlexibleModel):
+    # the input file, I think
+    target: Target
+    # info about the processing job, like machine and distributed metadata.
+    info: Info
+
+    #
+    # static analysis results
+    #
+    static: Optional[Static] = None
+    strings: Optional[List[str]] = None
+
+    #
+    # dynamic analysis results
+    #
+    # post-processed results: process tree, anomalies, etc
+    behavior: Behavior
+
+    # post-processed results: payloads and extracted configs
+    CAPE: Optional[Union[Cape, List]] = None
+    dropped: Optional[List[File]] = None
+    procdump: Optional[List[ProcessFile]] = None
+    procmemory: ListTODO
+
+    # =========================================================================
+    # information we won't use in capa
+    #
+
+    #
+    # NBIs and HBIs
+    # these are super interesting, but they don't enable use to detect behaviors.
+    # they take a lot of code to model and details to maintain.
+    #
+    # if we come up with a future use for this, go ahead and re-enable!
+    #
+    network: Skip = None
+    suricata: Skip = None
+    curtain: Skip = None
+    sysmon: Skip = None
+    url_analysis: Skip = None
+
+    # screenshot hash values
+    deduplicated_shots: Skip = None
+    # k-v pairs describing the time it took to run each stage.
+    statistics: Skip = None
+    # k-v pairs of ATT&CK ID to signature name or similar.
+    ttps: Skip = None
+    # debug log messages
+    debug: Skip = None
+
+    # various signature matches
+    # we could potentially extend capa to use this info one day,
+    # though it would be quite sandbox-specific,
+    # and more detection-oriented than capability detection.
+    signatures: Skip = None
+    malfamily_tag: Optional[str] = None
+    malscore: float
+    detections: Skip = None
+    detections2pid: Optional[Dict[int, List[str]]] = None
+    # AV detections for the sample.
+    virustotal: Skip = None
+
+    @classmethod
+    def from_buf(cls, buf: bytes) -> "CapeReport":
+        return cls.model_validate_json(buf)

capa/features/extractors/cape/process.py

@@ -0,0 +1,48 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import List, Tuple, Iterator
+
+from capa.features.common import String, Feature
+from capa.features.address import Address, ThreadAddress
+from capa.features.extractors.cape.models import Process
+from capa.features.extractors.base_extractor import ThreadHandle, ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def get_threads(ph: ProcessHandle) -> Iterator[ThreadHandle]:
+    """
+    get the threads associated with a given process
+    """
+    process: Process = ph.inner
+    threads: List[int] = process.threads
+
+    for thread in threads:
+        address: ThreadAddress = ThreadAddress(process=ph.address, tid=thread)
+        yield ThreadHandle(address=address, inner={})
+
+
+def extract_environ_strings(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract strings from a process' provided environment variables.
+    """
+    process: Process = ph.inner
+
+    for value in (value for value in process.environ.values() if value):
+        yield String(value), ph.address
+
+
+def extract_features(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+    for handler in PROCESS_HANDLERS:
+        for feature, addr in handler(ph):
+            yield feature, addr
+
+
+PROCESS_HANDLERS = (extract_environ_strings,)

capa/features/extractors/cape/thread.py

@@ -0,0 +1,32 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Iterator
+
+from capa.features.address import DynamicCallAddress
+from capa.features.extractors.helpers import generate_symbols
+from capa.features.extractors.cape.models import Process
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def get_calls(ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+    process: Process = ph.inner
+
+    tid = th.address.tid
+    for call_index, call in enumerate(process.calls):
+        if call.thread_id != tid:
+            continue
+
+        for symbol in generate_symbols("", call.api):
+            call.api = symbol
+
+            addr = DynamicCallAddress(thread=th.address, id=call_index)
+            yield CallHandle(address=addr, inner=call)

capa/features/extractors/common.py

@@ -1,48 +1,92 @@
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import io
+import re
 import logging
 import binascii
 import contextlib
+from typing import Tuple, Iterator
 
 import pefile
 
 import capa.features
 import capa.features.extractors.elf
 import capa.features.extractors.pefile
-from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Arch, Format, String
+import capa.features.extractors.strings
+from capa.features.common import (
+    OS,
+    OS_ANY,
+    OS_AUTO,
+    ARCH_ANY,
+    FORMAT_PE,
+    FORMAT_ELF,
+    OS_WINDOWS,
+    FORMAT_FREEZE,
+    FORMAT_RESULT,
+    Arch,
+    Format,
+    String,
+    Feature,
+)
+from capa.features.freeze import is_freeze
+from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress
 
 logger = logging.getLogger(__name__)
 
+# match strings for formats
+MATCH_PE = b"MZ"
+MATCH_ELF = b"\x7fELF"
+MATCH_RESULT = b'{"meta":'
+MATCH_JSON_OBJECT = b'{"'
 
-def extract_file_strings(buf, **kwargs):
+
+def extract_file_strings(buf: bytes, **kwargs) -> Iterator[Tuple[String, Address]]:
     """
     extract ASCII and UTF-16 LE strings from file
     """
     for s in capa.features.extractors.strings.extract_ascii_strings(buf):
-        yield String(s.s), s.offset
+        yield String(s.s), FileOffsetAddress(s.offset)
 
     for s in capa.features.extractors.strings.extract_unicode_strings(buf):
-        yield String(s.s), s.offset
+        yield String(s.s), FileOffsetAddress(s.offset)
 
 
-def extract_format(buf):
-    if buf.startswith(b"MZ"):
-        yield Format(FORMAT_PE), 0x0
-    elif buf.startswith(b"\x7fELF"):
-        yield Format(FORMAT_ELF), 0x0
+def extract_format(buf: bytes) -> Iterator[Tuple[Feature, Address]]:
+    if buf.startswith(MATCH_PE):
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif buf.startswith(MATCH_ELF):
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    elif is_freeze(buf):
+        yield Format(FORMAT_FREEZE), NO_ADDRESS
+    elif buf.startswith(MATCH_RESULT):
+        yield Format(FORMAT_RESULT), NO_ADDRESS
+    elif re.sub(rb"\s", b"", buf[:20]).startswith(MATCH_JSON_OBJECT):
+        # potential start of JSON object data without whitespace
+        # we don't know what it is exactly, but may support it (e.g. a dynamic CAPE sandbox report)
+        # skip verdict here and let subsequent code analyze this further
+        return
     else:
         # we likely end up here:
         #  1. handling a file format (e.g. macho)
         #
         # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s", binascii.hexlify(buf[:4]).decode("ascii"))
+        logger.debug("unknown file format: %s", buf[:4].hex())
         return
 
 
-def extract_arch(buf):
-    if buf.startswith(b"MZ"):
+def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
+    if buf.startswith(MATCH_PE):
         yield from capa.features.extractors.pefile.extract_file_arch(pe=pefile.PE(data=buf))
 
-    elif buf.startswith(b"\x7fELF"):
+    elif buf.startswith(MATCH_RESULT):
+        yield Arch(ARCH_ANY), NO_ADDRESS
+
+    elif buf.startswith(MATCH_ELF):
         with contextlib.closing(io.BytesIO(buf)) as f:
             arch = capa.features.extractors.elf.detect_elf_arch(f)
 
@@ -50,7 +94,7 @@ def extract_arch(buf):
             logger.debug("unsupported arch: %s", arch)
             return
 
-        yield Arch(arch), 0x0
+        yield Arch(arch), NO_ADDRESS
 
     else:
         # we likely end up here:
@@ -58,7 +102,7 @@ def extract_arch(buf):
         #  2. handling a new file format (e.g. macho)
         #
         # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the arch,
+        # we could maybe accept a further CLI argument to specify the arch,
         # but i think this would be rarely used.
         # rules that rely on arch conditions will fail to match on shellcode.
         #
@@ -67,10 +111,15 @@ def extract_arch(buf):
         return
 
 
-def extract_os(buf):
-    if buf.startswith(b"MZ"):
-        yield OS(OS_WINDOWS), 0x0
-    elif buf.startswith(b"\x7fELF"):
+def extract_os(buf, os=OS_AUTO) -> Iterator[Tuple[Feature, Address]]:
+    if os != OS_AUTO:
+        yield OS(os), NO_ADDRESS
+
+    if buf.startswith(MATCH_PE):
+        yield OS(OS_WINDOWS), NO_ADDRESS
+    elif buf.startswith(MATCH_RESULT):
+        yield OS(OS_ANY), NO_ADDRESS
+    elif buf.startswith(MATCH_ELF):
         with contextlib.closing(io.BytesIO(buf)) as f:
             os = capa.features.extractors.elf.detect_elf_os(f)
 
@@ -78,7 +127,7 @@ def extract_os(buf):
             logger.debug("unsupported os: %s", os)
             return
 
-        yield OS(os), 0x0
+        yield OS(os), NO_ADDRESS
 
     else:
         # we likely end up here:
@@ -86,8 +135,6 @@ def extract_os(buf):
         #  2. handling a new file format (e.g. macho)
         #
         # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the OS,
-        # but i think this would be rarely used.
         # rules that rely on OS conditions will fail to match on shellcode.
         #
         # for (2), this logic will need to be updated as the format is implemented.

capa/features/extractors/dnfile/extractor.py

@@ -0,0 +1,161 @@
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+from typing import Dict, List, Tuple, Union, Iterator, Optional
+from pathlib import Path
+
+import dnfile
+from dncil.cil.opcode import OpCodes
+
+import capa.features.extractors
+import capa.features.extractors.dotnetfile
+import capa.features.extractors.dnfile.file
+import capa.features.extractors.dnfile.insn
+import capa.features.extractors.dnfile.function
+from capa.features.common import Feature
+from capa.features.address import NO_ADDRESS, Address, DNTokenAddress, DNTokenOffsetAddress
+from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
+from capa.features.extractors.dnfile.helpers import (
+    get_dotnet_types,
+    get_dotnet_fields,
+    get_dotnet_managed_imports,
+    get_dotnet_managed_methods,
+    get_dotnet_unmanaged_imports,
+    get_dotnet_managed_method_bodies,
+)
+
+
+class DnFileFeatureExtractorCache:
+    def __init__(self, pe: dnfile.dnPE):
+        self.imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.native_imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.methods: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.fields: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+        self.types: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
+
+        for import_ in get_dotnet_managed_imports(pe):
+            self.imports[import_.token] = import_
+        for native_import in get_dotnet_unmanaged_imports(pe):
+            self.native_imports[native_import.token] = native_import
+        for method in get_dotnet_managed_methods(pe):
+            self.methods[method.token] = method
+        for field in get_dotnet_fields(pe):
+            self.fields[field.token] = field
+        for type_ in get_dotnet_types(pe):
+            self.types[type_.token] = type_
+
+    def get_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.imports.get(token)
+
+    def get_native_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.native_imports.get(token)
+
+    def get_method(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.methods.get(token)
+
+    def get_field(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.fields.get(token)
+
+    def get_type(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+        return self.types.get(token)
+
+
+class DnfileFeatureExtractor(StaticFeatureExtractor):
+    def __init__(self, path: Path):
+        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
+
+        # pre-compute .NET token lookup tables; each .NET method has access to this cache for feature extraction
+        # most relevant at instruction scope
+        self.token_cache: DnFileFeatureExtractorCache = DnFileFeatureExtractorCache(self.pe)
+
+        # pre-compute these because we'll yield them at *every* scope.
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_format())
+        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_os(pe=self.pe))
+        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_arch(pe=self.pe))
+
+    def get_base_address(self):
+        return NO_ADDRESS
+
+    def extract_global_features(self):
+        yield from self.global_features
+
+    def extract_file_features(self):
+        yield from capa.features.extractors.dnfile.file.extract_features(self.pe)
+
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        # create a method lookup table
+        methods: Dict[Address, FunctionHandle] = {}
+        for token, method in get_dotnet_managed_method_bodies(self.pe):
+            fh: FunctionHandle = FunctionHandle(
+                address=DNTokenAddress(token),
+                inner=method,
+                ctx={"pe": self.pe, "calls_from": set(), "calls_to": set(), "cache": self.token_cache},
+            )
+
+            # method tokens should be unique
+            assert fh.address not in methods.keys()
+            methods[fh.address] = fh
+
+        # calculate unique calls to/from each method
+        for fh in methods.values():
+            for insn in fh.inner.instructions:
+                if insn.opcode not in (
+                    OpCodes.Call,
+                    OpCodes.Callvirt,
+                    OpCodes.Jmp,
+                    OpCodes.Newobj,
+                ):
+                    continue
+
+                address: DNTokenAddress = DNTokenAddress(insn.operand.value)
+
+                # record call to destination method; note: we only consider MethodDef methods for destinations
+                dest: Optional[FunctionHandle] = methods.get(address)
+                if dest is not None:
+                    dest.ctx["calls_to"].add(fh.address)
+
+                # record call from source method; note: we record all unique calls from a MethodDef method, not just
+                # those calls to other MethodDef methods e.g. calls to imported MemberRef methods
+                fh.ctx["calls_from"].add(address)
+
+        yield from methods.values()
+
+    def extract_function_features(self, fh) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.dnfile.function.extract_features(fh)
+
+    def get_basic_blocks(self, f) -> Iterator[BBHandle]:
+        # each dotnet method is considered 1 basic block
+        yield BBHandle(
+            address=f.address,
+            inner=f.inner,
+        )
+
+    def extract_basic_block_features(self, fh, bbh):
+        # we don't support basic block features
+        yield from []
+
+    def get_instructions(self, fh, bbh):
+        for insn in bbh.inner.instructions:
+            yield InsnHandle(
+                address=DNTokenOffsetAddress(bbh.address, insn.offset - (fh.inner.offset + fh.inner.header_size)),
+                inner=insn,
+            )
+
+    def extract_insn_features(self, fh, bbh, ih) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.dnfile.insn.extract_features(fh, bbh, ih)

capa/features/extractors/dnfile/file.py

@@ -0,0 +1,63 @@
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+from typing import Tuple, Iterator
+
+import dnfile
+
+import capa.features.extractors.dotnetfile
+from capa.features.file import Import, FunctionName
+from capa.features.common import Class, Format, String, Feature, Namespace, Characteristic
+from capa.features.address import Address
+
+
+def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[Tuple[Import, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_import_names(pe=pe)
+
+
+def extract_file_format(pe: dnfile.dnPE) -> Iterator[Tuple[Format, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_format(pe=pe)
+
+
+def extract_file_function_names(pe: dnfile.dnPE) -> Iterator[Tuple[FunctionName, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_function_names(pe=pe)
+
+
+def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe)
+
+
+def extract_file_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[Tuple[Characteristic, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_mixed_mode_characteristic_features(pe=pe)
+
+
+def extract_file_namespace_features(pe: dnfile.dnPE) -> Iterator[Tuple[Namespace, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_namespace_features(pe=pe)
+
+
+def extract_file_class_features(pe: dnfile.dnPE) -> Iterator[Tuple[Class, Address]]:
+    yield from capa.features.extractors.dotnetfile.extract_file_class_features(pe=pe)
+
+
+def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
+    for file_handler in FILE_HANDLERS:
+        for feature, address in file_handler(pe):
+            yield feature, address
+
+
+FILE_HANDLERS = (
+    extract_file_import_names,
+    extract_file_function_names,
+    extract_file_strings,
+    extract_file_format,
+    extract_file_mixed_mode_characteristic_features,
+    extract_file_namespace_features,
+    extract_file_class_features,
+)

capa/features/extractors/dnfile/function.py

@@ -0,0 +1,50 @@
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.extractors.base_extractor import FunctionHandle
+
+logger = logging.getLogger(__name__)
+
+
+def extract_function_calls_to(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract callers to a function"""
+    for dest in fh.ctx["calls_to"]:
+        yield Characteristic("calls to"), dest
+
+
+def extract_function_calls_from(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract callers from a function"""
+    for src in fh.ctx["calls_from"]:
+        yield Characteristic("calls from"), src
+
+
+def extract_recursive_call(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract recursive function call"""
+    if fh.address in fh.ctx["calls_to"]:
+        yield Characteristic("recursive call"), fh.address
+
+
+def extract_function_loop(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
+    """extract loop indicators from a function"""
+    raise NotImplementedError()
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_calls_from, extract_recursive_call)

capa/features/extractors/dnfile/helpers.py

@@ -0,0 +1,451 @@
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import Dict, Tuple, Union, Iterator, Optional
+
+import dnfile
+from dncil.cil.body import CilMethodBody
+from dncil.cil.error import MethodBodyFormatError
+from dncil.clr.token import Token, StringToken, InvalidToken
+from dncil.cil.body.reader import CilMethodBodyReaderBase
+
+from capa.features.common import FeatureAccess
+from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
+
+logger = logging.getLogger(__name__)
+
+
+class DnfileMethodBodyReader(CilMethodBodyReaderBase):
+    def __init__(self, pe: dnfile.dnPE, row: dnfile.mdtable.MethodDefRow):
+        self.pe: dnfile.dnPE = pe
+        self.offset: int = self.pe.get_offset_from_rva(row.Rva)
+
+    def read(self, n: int) -> bytes:
+        data: bytes = self.pe.get_data(self.pe.get_rva_from_offset(self.offset), n)
+        self.offset += n
+        return data
+
+    def tell(self) -> int:
+        return self.offset
+
+    def seek(self, offset: int) -> int:
+        self.offset = offset
+        return self.offset
+
+
+def resolve_dotnet_token(pe: dnfile.dnPE, token: Token) -> Union[dnfile.base.MDTableRow, InvalidToken, str]:
+    """map generic token to string or table row"""
+    assert pe.net is not None
+    assert pe.net.mdtables is not None
+
+    if isinstance(token, StringToken):
+        user_string: Optional[str] = read_dotnet_user_string(pe, token)
+        if user_string is None:
+            return InvalidToken(token.value)
+        return user_string
+
+    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(token.table)
+    if table is None:
+        # table index is not valid
+        return InvalidToken(token.value)
+
+    try:
+        return table.rows[token.rid - 1]
+    except IndexError:
+        # table index is valid but row index is not valid
+        return InvalidToken(token.value)
+
+
+def read_dotnet_method_body(pe: dnfile.dnPE, row: dnfile.mdtable.MethodDefRow) -> Optional[CilMethodBody]:
+    """read dotnet method body"""
+    try:
+        return CilMethodBody(DnfileMethodBodyReader(pe, row))
+    except MethodBodyFormatError as e:
+        logger.debug("failed to parse managed method body @ 0x%08x (%s)", row.Rva, e)
+        return None
+
+
+def read_dotnet_user_string(pe: dnfile.dnPE, token: StringToken) -> Optional[str]:
+    """read user string from #US stream"""
+    assert pe.net is not None
+
+    if pe.net.user_strings is None:
+        # stream may not exist (seen in obfuscated .NET)
+        logger.debug("#US stream does not exist for stream index 0x%08x", token.rid)
+        return None
+
+    try:
+        user_string: Optional[dnfile.stream.UserString] = pe.net.user_strings.get(token.rid)
+    except UnicodeDecodeError as e:
+        logger.debug("failed to decode #US stream index 0x%08x (%s)", token.rid, e)
+        return None
+
+    if user_string is None:
+        return None
+
+    return user_string.value
+
+
+def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get managed imports from MemberRef table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    10 - MemberRef Table
+        Each row represents an imported method
+            Class (index into the TypeRef, ModuleRef, MethodDef, TypeSpec or TypeDef tables)
+            Name (index into String heap)
+    01 - TypeRef Table
+        Each row represents an imported class, its namespace and the assembly which contains it
+            TypeName (index into String heap)
+            TypeNamespace (index into String heap)
+    """
+    for rid, member_ref in iter_dotnet_table(pe, dnfile.mdtable.MemberRef.number):
+        assert isinstance(member_ref, dnfile.mdtable.MemberRefRow)
+
+        if not isinstance(member_ref.Class.row, dnfile.mdtable.TypeRefRow):
+            # only process class imports from TypeRef table
+            continue
+
+        token: int = calculate_dotnet_token_value(dnfile.mdtable.MemberRef.number, rid)
+        access: Optional[str]
+
+        # assume .NET imports starting with get_/set_ are used to access a property
+        member_ref_name: str = str(member_ref.Name)
+        if member_ref_name.startswith("get_"):
+            access = FeatureAccess.READ
+        elif member_ref_name.startswith("set_"):
+            access = FeatureAccess.WRITE
+        else:
+            access = None
+
+        if member_ref_name.startswith(("get_", "set_")):
+            # remove get_/set_ from MemberRef name
+            member_ref_name = member_ref_name[4:]
+
+        typerefnamespace, typerefname = resolve_nested_typeref_name(
+            member_ref.Class.row_index, member_ref.Class.row, pe
+        )
+
+        yield DnType(
+            token,
+            typerefname,
+            namespace=typerefnamespace,
+            member=member_ref_name,
+            access=access,
+        )
+
+
+def get_dotnet_methoddef_property_accessors(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
+    """get MethodDef methods used to access properties
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    24 - MethodSemantics Table
+        Links Events and Properties to specific methods. For example one Event can be associated to more methods. A property uses this table to associate get/set methods.
+            Semantics (a 2-byte bitmask of type MethodSemanticsAttributes)
+            Method (index into the MethodDef table)
+            Association (index into the Event or Property table; more precisely, a HasSemantics coded index)
+    """
+    for rid, method_semantics in iter_dotnet_table(pe, dnfile.mdtable.MethodSemantics.number):
+        assert isinstance(method_semantics, dnfile.mdtable.MethodSemanticsRow)
+
+        if method_semantics.Association.row is None:
+            logger.debug("MethodSemantics[0x%X] Association row is None", rid)
+            continue
+
+        if isinstance(method_semantics.Association.row, dnfile.mdtable.EventRow):
+            # ignore events
+            logger.debug("MethodSemantics[0x%X] ignoring Event", rid)
+            continue
+
+        if method_semantics.Method.table is None:
+            logger.debug("MethodSemantics[0x%X] Method table is None", rid)
+            continue
+
+        token: int = calculate_dotnet_token_value(
+            method_semantics.Method.table.number, method_semantics.Method.row_index
+        )
+
+        if method_semantics.Semantics.msSetter:
+            yield token, FeatureAccess.WRITE
+        elif method_semantics.Semantics.msGetter:
+            yield token, FeatureAccess.READ
+
+
+def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get managed method names from TypeDef table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    02 - TypeDef Table
+        Each row represents a class in the current assembly.
+            TypeName (index into String heap)
+            TypeNamespace (index into String heap)
+            MethodList (index into MethodDef table; it marks the first of a contiguous run of Methods owned by this Type)
+    """
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
+    accessor_map: Dict[int, str] = {}
+    for methoddef, methoddef_access in get_dotnet_methoddef_property_accessors(pe):
+        accessor_map[methoddef] = methoddef_access
+
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        for idx, method in enumerate(typedef.MethodList):
+            if method.table is None:
+                logger.debug("TypeDef[0x%X] MethodList[0x%X] table is None", rid, idx)
+                continue
+            if method.row is None:
+                logger.debug("TypeDef[0x%X] MethodList[0x%X] row is None", rid, idx)
+                continue
+
+            token: int = calculate_dotnet_token_value(method.table.number, method.row_index)
+            access: Optional[str] = accessor_map.get(token)
+
+            method_name: str = str(method.row.Name)
+            if method_name.startswith(("get_", "set_")):
+                # remove get_/set_
+                method_name = method_name[4:]
+
+            typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
+            yield DnType(token, typedefname, namespace=typedefnamespace, member=method_name, access=access)
+
+
+def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get fields from TypeDef table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    02 - TypeDef Table
+        Each row represents a class in the current assembly.
+            TypeName (index into String heap)
+            TypeNamespace (index into String heap)
+            FieldList (index into Field table; it marks the first of a contiguous run of Fields owned by this Type)
+    """
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        for idx, field in enumerate(typedef.FieldList):
+            if field.table is None:
+                logger.debug("TypeDef[0x%X] FieldList[0x%X] table is None", rid, idx)
+                continue
+            if field.row is None:
+                logger.debug("TypeDef[0x%X] FieldList[0x%X] row is None", rid, idx)
+                continue
+
+            typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
+            token: int = calculate_dotnet_token_value(field.table.number, field.row_index)
+            yield DnType(token, typedefname, namespace=typedefnamespace, member=field.row.Name)
+
+
+def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
+    """get managed methods from MethodDef table"""
+    for rid, method_def in iter_dotnet_table(pe, dnfile.mdtable.MethodDef.number):
+        assert isinstance(method_def, dnfile.mdtable.MethodDefRow)
+
+        if not method_def.ImplFlags.miIL or any((method_def.Flags.mdAbstract, method_def.Flags.mdPinvokeImpl)):
+            # skip methods that do not have a method body
+            continue
+
+        body: Optional[CilMethodBody] = read_dotnet_method_body(pe, method_def)
+        if body is None:
+            logger.debug("MethodDef[0x%X] method body is None", rid)
+            continue
+
+        token: int = calculate_dotnet_token_value(dnfile.mdtable.MethodDef.number, rid)
+        yield token, body
+
+
+def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[DnUnmanagedMethod]:
+    """get unmanaged imports from ImplMap table
+
+    see https://www.ntcore.com/files/dotnetformat.htm
+
+    28 - ImplMap Table
+        ImplMap table holds information about unmanaged methods that can be reached from managed code, using PInvoke dispatch
+            MemberForwarded (index into the Field or MethodDef table; more precisely, a MemberForwarded coded index)
+            ImportName (index into the String heap)
+            ImportScope (index into the ModuleRef table)
+    """
+    for rid, impl_map in iter_dotnet_table(pe, dnfile.mdtable.ImplMap.number):
+        assert isinstance(impl_map, dnfile.mdtable.ImplMapRow)
+
+        module: str
+        if impl_map.ImportScope.row is None:
+            logger.debug("ImplMap[0x%X] ImportScope row is None", rid)
+            module = ""
+        else:
+            module = str(impl_map.ImportScope.row.Name)
+        method: str = str(impl_map.ImportName)
+
+        member_forward_table: int
+        if impl_map.MemberForwarded.table is None:
+            logger.debug("ImplMap[0x%X] MemberForwarded table is None", rid)
+            continue
+        else:
+            member_forward_table = impl_map.MemberForwarded.table.number
+        member_forward_row: int = impl_map.MemberForwarded.row_index
+
+        # ECMA says "Each row of the ImplMap table associates a row in the MethodDef table (MemberForwarded) with the
+        # name of a routine (ImportName) in some unmanaged DLL (ImportScope)"; so we calculate and map the MemberForwarded
+        # MethodDef table token to help us later record native import method calls made from CIL
+        token: int = calculate_dotnet_token_value(member_forward_table, member_forward_row)
+
+        # like Kernel32.dll
+        if module and "." in module:
+            module = module.split(".")[0]
+
+        # like kernel32.CreateFileA
+        yield DnUnmanagedMethod(token, module, method)
+
+
+def get_dotnet_table_row(pe: dnfile.dnPE, table_index: int, row_index: int) -> Optional[dnfile.base.MDTableRow]:
+    assert pe.net is not None
+    assert pe.net.mdtables is not None
+
+    if row_index - 1 <= 0:
+        return None
+
+    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(table_index)
+    if table is None:
+        return None
+
+    try:
+        return table[row_index - 1]
+    except IndexError:
+        return None
+
+
+def resolve_nested_typedef_name(
+    nested_class_table: dict, index: int, typedef: dnfile.mdtable.TypeDefRow, pe: dnfile.dnPE
+) -> Tuple[str, Tuple[str, ...]]:
+    """Resolves all nested TypeDef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
+
+    if index in nested_class_table:
+        typedef_name = []
+        name = str(typedef.TypeName)
+
+        # Append the current typedef name
+        typedef_name.append(name)
+
+        while nested_class_table[index] in nested_class_table:
+            # Iterate through the typedef table to resolve the nested name
+            table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeDef.number, nested_class_table[index])
+            if table_row is None:
+                return str(typedef.TypeNamespace), tuple(typedef_name[::-1])
+
+            name = str(table_row.TypeName)
+            typedef_name.append(name)
+            index = nested_class_table[index]
+
+        # Document the root enclosing details
+        table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeDef.number, nested_class_table[index])
+        if table_row is None:
+            return str(typedef.TypeNamespace), tuple(typedef_name[::-1])
+
+        enclosing_name = str(table_row.TypeName)
+        typedef_name.append(enclosing_name)
+
+        return str(table_row.TypeNamespace), tuple(typedef_name[::-1])
+
+    else:
+        return str(typedef.TypeNamespace), (str(typedef.TypeName),)
+
+
+def resolve_nested_typeref_name(
+    index: int, typeref: dnfile.mdtable.TypeRefRow, pe: dnfile.dnPE
+) -> Tuple[str, Tuple[str, ...]]:
+    """Resolves all nested TypeRef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
+    # If the ResolutionScope decodes to a typeRef type then it is nested
+    if isinstance(typeref.ResolutionScope.table, dnfile.mdtable.TypeRef):
+        typeref_name = []
+        name = str(typeref.TypeName)
+        # Not appending the current typeref name to avoid potential duplicate
+
+        # Validate index
+        table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeRef.number, index)
+        if table_row is None:
+            return str(typeref.TypeNamespace), (str(typeref.TypeName),)
+
+        while isinstance(table_row.ResolutionScope.table, dnfile.mdtable.TypeRef):
+            # Iterate through the typeref table to resolve the nested name
+            typeref_name.append(name)
+            name = str(table_row.TypeName)
+            table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeRef.number, table_row.ResolutionScope.row_index)
+            if table_row is None:
+                return str(typeref.TypeNamespace), tuple(typeref_name[::-1])
+
+        # Document the root enclosing details
+        typeref_name.append(str(table_row.TypeName))
+
+        return str(table_row.TypeNamespace), tuple(typeref_name[::-1])
+
+    else:
+        return str(typeref.TypeNamespace), (str(typeref.TypeName),)
+
+
+def get_dotnet_nested_class_table_index(pe: dnfile.dnPE) -> Dict[int, int]:
+    """Build index for EnclosingClass based off the NestedClass row index in the nestedclass table"""
+    nested_class_table = {}
+
+    # Used to find nested classes in typedef
+    for _, nestedclass in iter_dotnet_table(pe, dnfile.mdtable.NestedClass.number):
+        assert isinstance(nestedclass, dnfile.mdtable.NestedClassRow)
+        nested_class_table[nestedclass.NestedClass.row_index] = nestedclass.EnclosingClass.row_index
+
+    return nested_class_table
+
+
+def get_dotnet_types(pe: dnfile.dnPE) -> Iterator[DnType]:
+    """get .NET types from TypeDef and TypeRef tables"""
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
+        typedef_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
+        yield DnType(typedef_token, typedefname, namespace=typedefnamespace)
+
+    for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
+        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
+
+        typerefnamespace, typerefname = resolve_nested_typeref_name(typeref.ResolutionScope.row_index, typeref, pe)
+
+        typeref_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
+        yield DnType(typeref_token, typerefname, namespace=typerefnamespace)
+
+
+def calculate_dotnet_token_value(table: int, rid: int) -> int:
+    return ((table & 0xFF) << Token.TABLE_SHIFT) | (rid & Token.RID_MASK)
+
+
+def is_dotnet_mixed_mode(pe: dnfile.dnPE) -> bool:
+    assert pe.net is not None
+    assert pe.net.Flags is not None
+
+    return not bool(pe.net.Flags.CLR_ILONLY)
+
+
+def iter_dotnet_table(pe: dnfile.dnPE, table_index: int) -> Iterator[Tuple[int, dnfile.base.MDTableRow]]:
+    assert pe.net is not None
+    assert pe.net.mdtables is not None
+
+    for rid, row in enumerate(pe.net.mdtables.tables.get(table_index, [])):
+        # .NET tables are 1-indexed
+        yield rid + 1, row

capa/features/extractors/dnfile/insn.py

@@ -0,0 +1,227 @@
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Tuple, Union, Iterator, Optional
+
+if TYPE_CHECKING:
+    from capa.features.extractors.dnfile.extractor import DnFileFeatureExtractorCache
+
+import dnfile
+from dncil.clr.token import Token, StringToken, InvalidToken
+from dncil.cil.opcode import OpCodes
+
+import capa.features.extractors.helpers
+from capa.features.insn import API, Number, Property
+from capa.features.common import Class, String, Feature, Namespace, FeatureAccess, Characteristic
+from capa.features.address import Address
+from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+from capa.features.extractors.dnfile.helpers import (
+    resolve_dotnet_token,
+    read_dotnet_user_string,
+    calculate_dotnet_token_value,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def get_callee(
+    pe: dnfile.dnPE, cache: DnFileFeatureExtractorCache, token: Token
+) -> Optional[Union[DnType, DnUnmanagedMethod]]:
+    """map .NET token to un/managed (generic) method"""
+    token_: int
+    if token.table == dnfile.mdtable.MethodSpec.number:
+        # map MethodSpec to MethodDef or MemberRef
+        row: Union[dnfile.base.MDTableRow, InvalidToken, str] = resolve_dotnet_token(pe, token)
+        assert isinstance(row, dnfile.mdtable.MethodSpecRow)
+
+        if row.Method.table is None:
+            logger.debug("MethodSpec[0x%X] Method table is None", token.rid)
+            return None
+
+        token_ = calculate_dotnet_token_value(row.Method.table.number, row.Method.row_index)
+    else:
+        token_ = token.value
+
+    callee: Optional[Union[DnType, DnUnmanagedMethod]] = cache.get_import(token_)
+    if callee is None:
+        # we must check unmanaged imports before managed methods because we map forwarded managed methods
+        # to their unmanaged imports; we prefer a forwarded managed method be mapped to its unmanaged import for analysis
+        callee = cache.get_native_import(token_)
+        if callee is None:
+            callee = cache.get_method(token_)
+    return callee
+
+
+def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction API features"""
+    if ih.inner.opcode not in (
+        OpCodes.Call,
+        OpCodes.Callvirt,
+        OpCodes.Jmp,
+        OpCodes.Newobj,
+    ):
+        return
+
+    callee: Optional[Union[DnType, DnUnmanagedMethod]] = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
+    if isinstance(callee, DnType):
+        # ignore methods used to access properties
+        if callee.access is None:
+            # like System.IO.File::Delete
+            yield API(str(callee)), ih.address
+    elif isinstance(callee, DnUnmanagedMethod):
+        # like kernel32.CreateFileA
+        for name in capa.features.extractors.helpers.generate_symbols(callee.module, callee.method):
+            yield API(name), ih.address
+
+
+def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction property features"""
+    name: Optional[str] = None
+    access: Optional[str] = None
+
+    if ih.inner.opcode in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
+        # property access via MethodDef or MemberRef
+        callee: Optional[Union[DnType, DnUnmanagedMethod]] = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
+        if isinstance(callee, DnType):
+            if callee.access is not None:
+                name = str(callee)
+                access = callee.access
+
+    elif ih.inner.opcode in (OpCodes.Ldfld, OpCodes.Ldflda, OpCodes.Ldsfld, OpCodes.Ldsflda):
+        # property read via Field
+        read_field: Optional[Union[DnType, DnUnmanagedMethod]] = fh.ctx["cache"].get_field(ih.inner.operand.value)
+        if read_field is not None:
+            name = str(read_field)
+            access = FeatureAccess.READ
+
+    elif ih.inner.opcode in (OpCodes.Stfld, OpCodes.Stsfld):
+        # property write via Field
+        write_field: Optional[Union[DnType, DnUnmanagedMethod]] = fh.ctx["cache"].get_field(ih.inner.operand.value)
+        if write_field is not None:
+            name = str(write_field)
+            access = FeatureAccess.WRITE
+
+    if name is not None:
+        if access is not None:
+            yield Property(name, access=access), ih.address
+        yield Property(name), ih.address
+
+
+def extract_insn_namespace_class_features(
+    fh: FunctionHandle, bh, ih: InsnHandle
+) -> Iterator[Tuple[Union[Namespace, Class], Address]]:
+    """parse instruction namespace and class features"""
+    type_: Optional[Union[DnType, DnUnmanagedMethod]] = None
+
+    if ih.inner.opcode in (
+        OpCodes.Call,
+        OpCodes.Callvirt,
+        OpCodes.Jmp,
+        OpCodes.Ldvirtftn,
+        OpCodes.Ldftn,
+        OpCodes.Newobj,
+    ):
+        # method call - includes managed methods (MethodDef, TypeRef) and properties (MethodSemantics, TypeRef)
+        type_ = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
+
+    elif ih.inner.opcode in (
+        OpCodes.Ldfld,
+        OpCodes.Ldflda,
+        OpCodes.Ldsfld,
+        OpCodes.Ldsflda,
+        OpCodes.Stfld,
+        OpCodes.Stsfld,
+    ):
+        # field access
+        type_ = fh.ctx["cache"].get_field(ih.inner.operand.value)
+
+    # ECMA 335 VI.C.4.10
+    elif ih.inner.opcode in (
+        OpCodes.Initobj,
+        OpCodes.Box,
+        OpCodes.Castclass,
+        OpCodes.Cpobj,
+        OpCodes.Isinst,
+        OpCodes.Ldelem,
+        OpCodes.Ldelema,
+        OpCodes.Ldobj,
+        OpCodes.Mkrefany,
+        OpCodes.Newarr,
+        OpCodes.Refanyval,
+        OpCodes.Sizeof,
+        OpCodes.Stobj,
+        OpCodes.Unbox,
+        OpCodes.Constrained,
+        OpCodes.Stelem,
+        OpCodes.Unbox_Any,
+    ):
+        # type access
+        type_ = fh.ctx["cache"].get_type(ih.inner.operand.value)
+
+    if isinstance(type_, DnType):
+        yield Class(DnType.format_name(type_.class_, namespace=type_.namespace)), ih.address
+        if type_.namespace:
+            yield Namespace(type_.namespace), ih.address
+
+
+def extract_insn_number_features(fh, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction number features"""
+    if ih.inner.is_ldc():
+        yield Number(ih.inner.get_ldc()), ih.address
+
+
+def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction string features"""
+    if not ih.inner.is_ldstr():
+        return
+
+    if not isinstance(ih.inner.operand, StringToken):
+        return
+
+    user_string: Optional[str] = read_dotnet_user_string(fh.ctx["pe"], ih.inner.operand)
+    if user_string is None:
+        return
+
+    if len(user_string) >= 4:
+        yield String(user_string), ih.address
+
+
+def extract_unmanaged_call_characteristic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Characteristic, Address]]:
+    if ih.inner.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
+        return
+
+    row: Union[str, InvalidToken, dnfile.base.MDTableRow] = resolve_dotnet_token(fh.ctx["pe"], ih.inner.operand)
+    if not isinstance(row, dnfile.mdtable.MethodDefRow):
+        return
+
+    if any((row.Flags.mdPinvokeImpl, row.ImplFlags.miUnmanaged, row.ImplFlags.miNative)):
+        yield Characteristic("unmanaged call"), ih.address
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract instruction features"""
+    for inst_handler in INSTRUCTION_HANDLERS:
+        for feature, addr in inst_handler(fh, bbh, ih):
+            assert isinstance(addr, Address)
+            yield feature, addr
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_property_features,
+    extract_insn_number_features,
+    extract_insn_string_features,
+    extract_insn_namespace_class_features,
+    extract_unmanaged_call_characteristic_features,
+)

capa/features/extractors/dnfile/types.py

@@ -0,0 +1,80 @@
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from typing import Tuple, Optional
+
+
+class DnType:
+    def __init__(
+        self, token: int, class_: Tuple[str, ...], namespace: str = "", member: str = "", access: Optional[str] = None
+    ):
+        self.token: int = token
+        self.access: Optional[str] = access
+        self.namespace: str = namespace
+        self.class_: Tuple[str, ...] = class_
+
+        if member == ".ctor":
+            member = "ctor"
+        if member == ".cctor":
+            member = "cctor"
+
+        self.member: str = member
+
+    def __hash__(self):
+        return hash((self.token, self.access, self.namespace, self.class_, self.member))
+
+    def __eq__(self, other):
+        return (
+            self.token == other.token
+            and self.access == other.access
+            and self.namespace == other.namespace
+            and self.class_ == other.class_
+            and self.member == other.member
+        )
+
+    def __str__(self):
+        return DnType.format_name(self.class_, namespace=self.namespace, member=self.member)
+
+    def __repr__(self):
+        return str(self)
+
+    @staticmethod
+    def format_name(class_: Tuple[str, ...], namespace: str = "", member: str = ""):
+        if len(class_) > 1:
+            class_str = "/".join(class_)  # Concat items in tuple, separated by a "/"
+        else:
+            class_str = "".join(class_)  # Convert tuple to str
+        # like File::OpenRead
+        name: str = f"{class_str}::{member}" if member else class_str
+        if namespace:
+            # like System.IO.File::OpenRead
+            name = f"{namespace}.{name}"
+        return name
+
+
+class DnUnmanagedMethod:
+    def __init__(self, token: int, module: str, method: str):
+        self.token: int = token
+        self.module: str = module
+        self.method: str = method
+
+    def __hash__(self):
+        return hash((self.token, self.module, self.method))
+
+    def __eq__(self, other):
+        return self.token == other.token and self.module == other.module and self.method == other.method
+
+    def __str__(self):
+        return DnUnmanagedMethod.format_name(self.module, self.method)
+
+    def __repr__(self):
+        return str(self)
+
+    @staticmethod
+    def format_name(module, method):
+        return f"{module}.{method}"

capa/features/extractors/dotnetfile.py

@@ -0,0 +1,248 @@
+# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import Tuple, Iterator
+from pathlib import Path
+
+import dnfile
+import pefile
+
+import capa.features.extractors.helpers
+from capa.features.file import Import, FunctionName
+from capa.features.common import (
+    OS,
+    OS_ANY,
+    ARCH_ANY,
+    ARCH_I386,
+    FORMAT_PE,
+    ARCH_AMD64,
+    FORMAT_DOTNET,
+    Arch,
+    Class,
+    Format,
+    String,
+    Feature,
+    Namespace,
+    Characteristic,
+)
+from capa.features.address import NO_ADDRESS, Address, DNTokenAddress
+from capa.features.extractors.dnfile.types import DnType
+from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
+from capa.features.extractors.dnfile.helpers import (
+    iter_dotnet_table,
+    is_dotnet_mixed_mode,
+    get_dotnet_managed_imports,
+    get_dotnet_managed_methods,
+    resolve_nested_typedef_name,
+    resolve_nested_typeref_name,
+    calculate_dotnet_token_value,
+    get_dotnet_unmanaged_imports,
+    get_dotnet_nested_class_table_index,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def extract_file_format(**kwargs) -> Iterator[Tuple[Format, Address]]:
+    yield Format(FORMAT_DOTNET), NO_ADDRESS
+    yield Format(FORMAT_PE), NO_ADDRESS
+
+
+def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Import, Address]]:
+    for method in get_dotnet_managed_imports(pe):
+        # like System.IO.File::OpenRead
+        yield Import(str(method)), DNTokenAddress(method.token)
+
+    for imp in get_dotnet_unmanaged_imports(pe):
+        # like kernel32.CreateFileA
+        for name in capa.features.extractors.helpers.generate_symbols(imp.module, imp.method, include_dll=True):
+            yield Import(name), DNTokenAddress(imp.token)
+
+
+def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[FunctionName, Address]]:
+    for method in get_dotnet_managed_methods(pe):
+        yield FunctionName(str(method)), DNTokenAddress(method.token)
+
+
+def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Namespace, Address]]:
+    """emit namespace features from TypeRef and TypeDef tables"""
+
+    # namespaces may be referenced multiple times, so we need to filter
+    namespaces = set()
+
+    for _, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        # emit internal .NET namespaces
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+        namespaces.add(str(typedef.TypeNamespace))
+
+    for _, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
+        # emit external .NET namespaces
+        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
+        namespaces.add(str(typeref.TypeNamespace))
+
+    # namespaces may be empty, discard
+    namespaces.discard("")
+
+    for namespace in namespaces:
+        # namespace do not have an associated token, so we yield 0x0
+        yield Namespace(namespace), NO_ADDRESS
+
+
+def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Class, Address]]:
+    """emit class features from TypeRef and TypeDef tables"""
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
+    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
+        # emit internal .NET classes
+        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
+
+        typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
+        token = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
+        yield Class(DnType.format_name(typedefname, namespace=typedefnamespace)), DNTokenAddress(token)
+
+    for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
+        # emit external .NET classes
+        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
+
+        typerefnamespace, typerefname = resolve_nested_typeref_name(typeref.ResolutionScope.row_index, typeref, pe)
+
+        token = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
+        yield Class(DnType.format_name(typerefname, namespace=typerefnamespace)), DNTokenAddress(token)
+
+
+def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
+    yield OS(OS_ANY), NO_ADDRESS
+
+
+def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Arch, Address]]:
+    # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
+    # .NET 4.5 added option: any CPU, 32-bit preferred
+    assert pe.net is not None
+    assert pe.net.Flags is not None
+
+    if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
+        yield Arch(ARCH_I386), NO_ADDRESS
+    elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+    else:
+        yield Arch(ARCH_ANY), NO_ADDRESS
+
+
+def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[String, Address]]:
+    yield from capa.features.extractors.common.extract_file_strings(pe.__data__)
+
+
+def extract_file_mixed_mode_characteristic_features(
+    pe: dnfile.dnPE, **kwargs
+) -> Iterator[Tuple[Characteristic, Address]]:
+    if is_dotnet_mixed_mode(pe):
+        yield Characteristic("mixed mode"), NO_ADDRESS
+
+
+def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
+    for file_handler in FILE_HANDLERS:
+        for feature, addr in file_handler(pe=pe):  # type: ignore
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_file_import_names,
+    extract_file_function_names,
+    extract_file_strings,
+    extract_file_format,
+    extract_file_mixed_mode_characteristic_features,
+    extract_file_namespace_features,
+    extract_file_class_features,
+)
+
+
+def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
+    for handler in GLOBAL_HANDLERS:
+        for feature, va in handler(pe=pe):  # type: ignore
+            yield feature, va
+
+
+GLOBAL_HANDLERS = (
+    extract_file_os,
+    extract_file_arch,
+)
+
+
+class DotnetFileFeatureExtractor(StaticFeatureExtractor):
+    def __init__(self, path: Path):
+        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
+        self.path: Path = path
+        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+
+    def get_base_address(self):
+        return NO_ADDRESS
+
+    def get_entry_point(self) -> int:
+        # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
+        #  True: native EP: Token
+        #  False: managed EP: RVA
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+
+        return self.pe.net.struct.EntryPointTokenOrRva
+
+    def extract_global_features(self):
+        yield from extract_global_features(self.pe)
+
+    def extract_file_features(self):
+        yield from extract_file_features(self.pe)
+
+    def is_dotnet_file(self) -> bool:
+        return bool(self.pe.net)
+
+    def is_mixed_mode(self) -> bool:
+        return is_dotnet_mixed_mode(self.pe)
+
+    def get_runtime_version(self) -> Tuple[int, int]:
+        assert self.pe.net is not None
+        assert self.pe.net.struct is not None
+        assert self.pe.net.struct.MajorRuntimeVersion is not None
+        assert self.pe.net.struct.MinorRuntimeVersion is not None
+
+        return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
+
+    def get_meta_version_string(self) -> str:
+        assert self.pe.net is not None
+        assert self.pe.net.metadata is not None
+        assert self.pe.net.metadata.struct is not None
+        assert self.pe.net.metadata.struct.Version is not None
+
+        vbuf = self.pe.net.metadata.struct.Version
+        assert isinstance(vbuf, bytes)
+
+        return vbuf.rstrip(b"\x00").decode("utf-8")
+
+    def get_functions(self):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def extract_function_features(self, f):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def get_basic_blocks(self, f):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def extract_basic_block_features(self, f, bb):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def get_instructions(self, f, bb):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def extract_insn_features(self, f, bb, insn):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def is_library_function(self, va):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
+
+    def get_function_name(self, va):
+        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")

capa/features/extractors/drakvuf/call.py

@@ -0,0 +1,56 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.insn import API, Number
+from capa.features.common import String, Feature
+from capa.features.address import Address
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
+from capa.features.extractors.drakvuf.models import Call
+
+logger = logging.getLogger(__name__)
+
+
+def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    This method extracts the given call's features (such as API name and arguments),
+    and returns them as API, Number, and String features.
+
+    args:
+      ph: process handle (for defining the extraction scope)
+      th: thread handle (for defining the extraction scope)
+      ch: call handle (for defining the extraction scope)
+
+    yields:
+      Feature, address; where Feature is either: API, Number, or String.
+    """
+    call: Call = ch.inner
+
+    # list similar to disassembly: arguments right-to-left, call
+    for arg_value in reversed(call.arguments.values()):
+        try:
+            yield Number(int(arg_value, 0)), ch.address
+        except ValueError:
+            # DRAKVUF automatically resolves the contents of memory addresses, (e.g. Arg1="0xc6f217efe0:\"ntdll.dll\"").
+            # For those cases we yield the entire string as it, since yielding the address only would
+            # likely not provide any matches, and yielding just the memory contentswould probably be misleading,
+            # but yielding the entire string would be helpful for an analyst looking at the verbose output
+            yield String(arg_value), ch.address
+
+    yield API(call.name), ch.address
+
+
+def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
+    for handler in CALL_HANDLERS:
+        for feature, addr in handler(ph, th, ch):
+            yield feature, addr
+
+
+CALL_HANDLERS = (extract_call_features,)

capa/features/extractors/drakvuf/extractor.py

@@ -0,0 +1,96 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Dict, List, Tuple, Union, Iterator
+
+import capa.features.extractors.drakvuf.call
+import capa.features.extractors.drakvuf.file
+import capa.features.extractors.drakvuf.thread
+import capa.features.extractors.drakvuf.global_
+import capa.features.extractors.drakvuf.process
+from capa.features.common import Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, ThreadAddress, ProcessAddress, AbsoluteVirtualAddress, _NoAddress
+from capa.features.extractors.base_extractor import (
+    CallHandle,
+    SampleHashes,
+    ThreadHandle,
+    ProcessHandle,
+    DynamicFeatureExtractor,
+)
+from capa.features.extractors.drakvuf.models import Call, DrakvufReport
+from capa.features.extractors.drakvuf.helpers import index_calls
+
+logger = logging.getLogger(__name__)
+
+
+class DrakvufExtractor(DynamicFeatureExtractor):
+    def __init__(self, report: DrakvufReport):
+        super().__init__(
+            # DRAKVUF currently does not yield hash information about the sample in its output
+            hashes=SampleHashes(md5="", sha1="", sha256="")
+        )
+
+        self.report: DrakvufReport = report
+
+        # sort the api calls to prevent going through the entire list each time
+        self.sorted_calls: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]] = index_calls(report)
+
+        # pre-compute these because we'll yield them at *every* scope.
+        self.global_features = list(capa.features.extractors.drakvuf.global_.extract_features(self.report))
+
+    def get_base_address(self) -> Union[AbsoluteVirtualAddress, _NoAddress, None]:
+        # DRAKVUF currently does not yield information about the PE's address
+        return NO_ADDRESS
+
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from self.global_features
+
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.drakvuf.file.extract_features(self.report)
+
+    def get_processes(self) -> Iterator[ProcessHandle]:
+        yield from capa.features.extractors.drakvuf.file.get_processes(self.sorted_calls)
+
+    def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.drakvuf.process.extract_features(ph)
+
+    def get_process_name(self, ph: ProcessHandle) -> str:
+        return ph.inner["process_name"]
+
+    def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
+        yield from capa.features.extractors.drakvuf.process.get_threads(self.sorted_calls, ph)
+
+    def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
+        if False:
+            # force this routine to be a generator,
+            # but we don't actually have any elements to generate.
+            yield Characteristic("never"), NO_ADDRESS
+        return
+
+    def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+        yield from capa.features.extractors.drakvuf.thread.get_calls(self.sorted_calls, ph, th)
+
+    def get_call_name(self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> str:
+        call: Call = ch.inner
+        call_name = "{}({}){}".format(
+            call.name,
+            ", ".join(f"{arg_name}={arg_value}" for arg_name, arg_value in call.arguments.items()),
+            (f" -> {getattr(call, 'return_value', '')}"),  # SysCalls don't have a return value, while WinApi calls do
+        )
+        return call_name
+
+    def extract_call_features(
+        self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.drakvuf.call.extract_features(ph, th, ch)
+
+    @classmethod
+    def from_report(cls, report: Iterator[Dict]) -> "DrakvufExtractor":
+        dr = DrakvufReport.from_raw_report(report)
+        return DrakvufExtractor(report=dr)

capa/features/extractors/drakvuf/file.py

@@ -0,0 +1,56 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Dict, List, Tuple, Iterator
+
+from capa.features.file import Import
+from capa.features.common import Feature
+from capa.features.address import Address, ThreadAddress, ProcessAddress, AbsoluteVirtualAddress
+from capa.features.extractors.helpers import generate_symbols
+from capa.features.extractors.base_extractor import ProcessHandle
+from capa.features.extractors.drakvuf.models import Call, DrakvufReport
+
+logger = logging.getLogger(__name__)
+
+
+def get_processes(calls: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]]) -> Iterator[ProcessHandle]:
+    """
+    Get all the created processes for a sample.
+    """
+    for proc_addr, calls_per_thread in calls.items():
+        sample_call = next(iter(calls_per_thread.values()))[0]  # get process name
+        yield ProcessHandle(proc_addr, inner={"process_name": sample_call.process_name})
+
+
+def extract_import_names(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
+    """
+    Extract imported function names.
+    """
+    if report.loaded_dlls is None:
+        return
+    dlls = report.loaded_dlls
+
+    for dll in dlls:
+        dll_base_name = dll.name.split("\\")[-1]
+        for function_name, function_address in dll.imports.items():
+            for name in generate_symbols(dll_base_name, function_name, include_dll=True):
+                yield Import(name), AbsoluteVirtualAddress(function_address)
+
+
+def extract_features(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
+    for handler in FILE_HANDLERS:
+        for feature, addr in handler(report):
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    # TODO(yelhamer): extract more file features from other DRAKVUF plugins
+    # https://github.com/mandiant/capa/issues/2169
+    extract_import_names,
+)

capa/features/extractors/drakvuf/global_.py

@@ -0,0 +1,44 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.common import OS, FORMAT_PE, ARCH_AMD64, OS_WINDOWS, Arch, Format, Feature
+from capa.features.address import NO_ADDRESS, Address
+from capa.features.extractors.drakvuf.models import DrakvufReport
+
+logger = logging.getLogger(__name__)
+
+
+def extract_format(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
+    # DRAKVUF sandbox currently supports only Windows as the guest: https://drakvuf-sandbox.readthedocs.io/en/latest/usage/getting_started.html
+    yield Format(FORMAT_PE), NO_ADDRESS
+
+
+def extract_os(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
+    # DRAKVUF sandbox currently supports only PE files: https://drakvuf-sandbox.readthedocs.io/en/latest/usage/getting_started.html
+    yield OS(OS_WINDOWS), NO_ADDRESS
+
+
+def extract_arch(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
+    # DRAKVUF sandbox currently supports only x64 Windows as the guest: https://drakvuf-sandbox.readthedocs.io/en/latest/usage/getting_started.html
+    yield Arch(ARCH_AMD64), NO_ADDRESS
+
+
+def extract_features(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
+    for global_handler in GLOBAL_HANDLER:
+        for feature, addr in global_handler(report):
+            yield feature, addr
+
+
+GLOBAL_HANDLER = (
+    extract_format,
+    extract_os,
+    extract_arch,
+)

capa/features/extractors/drakvuf/helpers.py

@@ -0,0 +1,39 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import itertools
+from typing import Dict, List
+
+from capa.features.address import ThreadAddress, ProcessAddress
+from capa.features.extractors.drakvuf.models import Call, DrakvufReport
+
+
+def index_calls(report: DrakvufReport) -> Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]]:
+    # this method organizes calls into processes and threads, and then sorts them based on
+    # timestamp so that we can address individual calls per index (CallAddress requires call index)
+    result: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]] = {}
+    for call in itertools.chain(report.syscalls, report.apicalls):
+        if call.pid == 0:
+            # DRAKVUF captures api/native calls from all processes running on the system.
+            # we ignore the pid 0 since it's a system process and it's unlikely for it to
+            # be hijacked or so on, in addition to capa addresses not supporting null pids
+            continue
+        proc_addr = ProcessAddress(pid=call.pid, ppid=call.ppid)
+        thread_addr = ThreadAddress(process=proc_addr, tid=call.tid)
+        if proc_addr not in result:
+            result[proc_addr] = {}
+        if thread_addr not in result[proc_addr]:
+            result[proc_addr][thread_addr] = []
+
+        result[proc_addr][thread_addr].append(call)
+
+    for proc, threads in result.items():
+        for thread in threads:
+            result[proc][thread].sort(key=lambda call: call.timestamp)
+
+    return result

capa/features/extractors/drakvuf/models.py

@@ -0,0 +1,137 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import Any, Dict, List, Iterator
+
+from pydantic import Field, BaseModel, ConfigDict, model_validator
+
+logger = logging.getLogger(__name__)
+
+
+REQUIRED_SYSCALL_FIELD_NAMES = {
+    "Plugin",
+    "TimeStamp",
+    "PID",
+    "PPID",
+    "TID",
+    "UserName",
+    "UserId",
+    "ProcessName",
+    "Method",
+    "EventUID",
+    "Module",
+    "vCPU",
+    "CR3",
+    "Syscall",
+    "NArgs",
+}
+
+
+class ConciseModel(BaseModel):
+    ConfigDict(extra="ignore")
+
+
+class DiscoveredDLL(ConciseModel):
+    plugin_name: str = Field(alias="Plugin")
+    event: str = Field(alias="Event")
+    name: str = Field(alias="DllName")
+    pid: int = Field(alias="PID")
+
+
+class LoadedDLL(ConciseModel):
+    plugin_name: str = Field(alias="Plugin")
+    event: str = Field(alias="Event")
+    name: str = Field(alias="DllName")
+    imports: Dict[str, int] = Field(alias="Rva")
+
+
+class Call(ConciseModel):
+    plugin_name: str = Field(alias="Plugin")
+    timestamp: str = Field(alias="TimeStamp")
+    process_name: str = Field(alias="ProcessName")
+    ppid: int = Field(alias="PPID")
+    pid: int = Field(alias="PID")
+    tid: int = Field(alias="TID")
+    name: str = Field(alias="Method")
+    arguments: Dict[str, str]
+
+
+class WinApiCall(Call):
+    # This class models Windows API calls captured by DRAKVUF (DLLs, etc.).
+    arguments: Dict[str, str] = Field(alias="Arguments")
+    event: str = Field(alias="Event")
+    return_value: str = Field(alias="ReturnValue")
+
+    @model_validator(mode="before")
+    @classmethod
+    def build_arguments(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        args = values["Arguments"]
+        values["Arguments"] = dict(arg.split("=", 1) for arg in args)
+        return values
+
+
+class SystemCall(Call):
+    # This class models native Windows API calls captured by DRAKVUF.
+    # Schema: {
+    #   "Plugin": "syscall",
+    #   "TimeStamp": "1716999134.582553",
+    #   "PID": 3888, "PPID": 2852, "TID": 368, "UserName": "SessionID", "UserId": 2,
+    #   "ProcessName": "\\Device\\HarddiskVolume2\\Windows\\explorer.exe",
+    #   "Method": "NtSetIoCompletionEx",
+    #   "EventUID": "0x27",
+    #   "Module": "nt",
+    #   "vCPU": 0,
+    #   "CR3": "0x119b1002",
+    #   "Syscall": 419,
+    #   "NArgs": 6,
+    #   "IoCompletionHandle": "0xffffffff80001ac0", "IoCompletionReserveHandle": "0xffffffff8000188c",
+    #   "KeyContext": "0x0", "ApcContext": "0x2", "IoStatus": "0x7ffb00000000", "IoStatusInformation": "0x0"
+    # }
+    # The keys up until "NArgs" are common to all the native calls that DRAKVUF reports, with
+    # the remaining keys representing the call's specific arguments.
+    syscall_number: int = Field(alias="Syscall")
+    module: str = Field(alias="Module")
+    nargs: int = Field(alias="NArgs")
+
+    @model_validator(mode="before")
+    @classmethod
+    def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        # DRAKVUF stores argument names and values as entries in the syscall's entry.
+        # This model validator collects those arguments into a list in the model.
+        values["arguments"] = {
+            name: value for name, value in values.items() if name not in REQUIRED_SYSCALL_FIELD_NAMES
+        }
+        return values
+
+
+class DrakvufReport(ConciseModel):
+    syscalls: List[SystemCall] = []
+    apicalls: List[WinApiCall] = []
+    discovered_dlls: List[DiscoveredDLL] = []
+    loaded_dlls: List[LoadedDLL] = []
+
+    @classmethod
+    def from_raw_report(cls, entries: Iterator[Dict]) -> "DrakvufReport":
+        report = cls()
+
+        for entry in entries:
+            plugin = entry.get("Plugin")
+            # TODO(yelhamer): add support for more DRAKVUF plugins
+            # https://github.com/mandiant/capa/issues/2181
+            if plugin == "syscall":
+                report.syscalls.append(SystemCall(**entry))
+            elif plugin == "apimon":
+                event = entry.get("Event")
+                if event == "api_called":
+                    report.apicalls.append(WinApiCall(**entry))
+                elif event == "dll_loaded":
+                    report.loaded_dlls.append(LoadedDLL(**entry))
+                elif event == "dll_discovered":
+                    report.discovered_dlls.append(DiscoveredDLL(**entry))
+
+        return report

capa/features/extractors/drakvuf/process.py

@@ -0,0 +1,40 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Dict, List, Tuple, Iterator
+
+from capa.features.common import String, Feature
+from capa.features.address import Address, ThreadAddress, ProcessAddress
+from capa.features.extractors.base_extractor import ThreadHandle, ProcessHandle
+from capa.features.extractors.drakvuf.models import Call
+
+logger = logging.getLogger(__name__)
+
+
+def get_threads(
+    calls: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]], ph: ProcessHandle
+) -> Iterator[ThreadHandle]:
+    """
+    Get the threads associated with a given process.
+    """
+    for thread_addr in calls[ph.address]:
+        yield ThreadHandle(address=thread_addr, inner={})
+
+
+def extract_process_name(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+    yield String(ph.inner["process_name"]), ph.address
+
+
+def extract_features(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+    for handler in PROCESS_HANDLERS:
+        for feature, addr in handler(ph):
+            yield feature, addr
+
+
+PROCESS_HANDLERS = (extract_process_name,)

capa/features/extractors/drakvuf/thread.py

@@ -0,0 +1,24 @@
+# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Dict, List, Iterator
+
+from capa.features.address import ThreadAddress, ProcessAddress, DynamicCallAddress
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
+from capa.features.extractors.drakvuf.models import Call
+
+logger = logging.getLogger(__name__)
+
+
+def get_calls(
+    sorted_calls: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]], ph: ProcessHandle, th: ThreadHandle
+) -> Iterator[CallHandle]:
+    for i, call in enumerate(sorted_calls[ph.address][th.address]):
+        call_addr = DynamicCallAddress(thread=th.address, id=i)
+        yield CallHandle(address=call_addr, inner=call)

capa/features/extractors/elffile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,86 +7,173 @@
 # See the License for the specific language governing permissions and limitations under the License.
 import io
 import logging
-import contextlib
 from typing import Tuple, Iterator
+from pathlib import Path
 
-from elftools.elf.elffile import ELFFile, SymbolTableSection
+from elftools.elf.elffile import ELFFile, DynamicSegment, SymbolTableSection
 
 import capa.features.extractors.common
-from capa.features.file import Import, Section
+from capa.features.file import Export, Import, Section
 from capa.features.common import OS, FORMAT_ELF, Arch, Format, Feature
-from capa.features.extractors.elf import Arch as ElfArch
-from capa.features.extractors.base_extractor import FeatureExtractor
+from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
 
 logger = logging.getLogger(__name__)
 
 
-def extract_file_import_names(elf, **kwargs):
-    # see https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/scripts/readelf.py#L372
-    symbol_tables = [(idx, s) for idx, s in enumerate(elf.iter_sections()) if isinstance(s, SymbolTableSection)]
-
-    for section_index, section in symbol_tables:
+def extract_file_export_names(elf: ELFFile, **kwargs):
+    for section in elf.iter_sections():
         if not isinstance(section, SymbolTableSection):
             continue
 
         if section["sh_entsize"] == 0:
-            logger.debug("Symbol table '%s' has a sh_entsize of zero!" % (section.name))
+            logger.debug("Symbol table '%s' has a sh_entsize of zero!", section.name)
             continue
 
-        logger.debug("Symbol table '%s' contains %s entries:" % (section.name, section.num_symbols()))
+        logger.debug("Symbol table '%s' contains %s entries:", section.name, section.num_symbols())
 
-        for nsym, symbol in enumerate(section.iter_symbols()):
-            if symbol.name and symbol.entry.st_info.type == "STT_FUNC":
-                # TODO symbol address
-                # TODO symbol version info?
-                yield Import(symbol.name), 0x0
+        for symbol in section.iter_symbols():
+            # The following conditions are based on the following article
+            # http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
+            if not symbol.name:
+                continue
+            if symbol.entry.st_info.type not in ["STT_FUNC", "STT_OBJECT", "STT_IFUNC"]:
+                continue
+            if symbol.entry.st_value == 0:
+                continue
+            if symbol.entry.st_shndx == "SHN_UNDEF":
+                continue
+
+            yield Export(symbol.name), AbsoluteVirtualAddress(symbol.entry.st_value)
+
+    for segment in elf.iter_segments():
+        if not isinstance(segment, DynamicSegment):
+            continue
+
+        tab_ptr, tab_offset = segment.get_table_offset("DT_SYMTAB")
+        if tab_ptr is None or tab_offset is None:
+            logger.debug("Dynamic segment doesn't contain DT_SYMTAB")
+            continue
+
+        logger.debug("Dynamic segment contains %s symbols: ", segment.num_symbols())
+
+        for symbol in segment.iter_symbols():
+            # The following conditions are based on the following article
+            # http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
+            if not symbol.name:
+                continue
+            if symbol.entry.st_info.type not in ["STT_FUNC", "STT_OBJECT", "STT_IFUNC"]:
+                continue
+            if symbol.entry.st_value == 0:
+                continue
+            if symbol.entry.st_shndx == "SHN_UNDEF":
+                continue
+
+            yield Export(symbol.name), AbsoluteVirtualAddress(symbol.entry.st_value)
 
 
-def extract_file_section_names(elf, **kwargs):
+def extract_file_import_names(elf: ELFFile, **kwargs):
+    # Create a dictionary to store symbol names by their index
+    symbol_names = {}
+
+    # Extract symbol names and store them in the dictionary
+    for segment in elf.iter_segments():
+        if not isinstance(segment, DynamicSegment):
+            continue
+
+        tab_ptr, tab_offset = segment.get_table_offset("DT_SYMTAB")
+        if tab_ptr is None or tab_offset is None:
+            logger.debug("Dynamic segment doesn't contain DT_SYMTAB")
+            continue
+
+        for _, symbol in enumerate(segment.iter_symbols()):
+            # The following conditions are based on the following article
+            # http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
+            if not symbol.name:
+                continue
+            if symbol.entry.st_info.type not in ["STT_FUNC", "STT_OBJECT", "STT_IFUNC"]:
+                continue
+            if symbol.entry.st_value != 0:
+                continue
+            if symbol.entry.st_shndx != "SHN_UNDEF":
+                continue
+            if symbol.entry.st_name == 0:
+                continue
+
+            symbol_names[_] = symbol.name
+
+    for segment in elf.iter_segments():
+        if not isinstance(segment, DynamicSegment):
+            continue
+
+        relocation_tables = segment.get_relocation_tables()
+        logger.debug("Dynamic Segment contains %s relocation tables:", len(relocation_tables))
+
+        for relocation_table in relocation_tables.values():
+            relocations = []
+            for i in range(relocation_table.num_relocations()):
+                try:
+                    relocations.append(relocation_table.get_relocation(i))
+                except TypeError:
+                    # ELF is corrupt and the relocation table is invalid,
+                    # so stop processing it.
+                    break
+
+            for relocation in relocations:
+                # Extract the symbol name from the symbol table using the symbol index in the relocation
+                if relocation["r_info_sym"] not in symbol_names:
+                    continue
+                yield Import(symbol_names[relocation["r_info_sym"]]), FileOffsetAddress(relocation["r_offset"])
+
+
+def extract_file_section_names(elf: ELFFile, **kwargs):
     for section in elf.iter_sections():
         if section.name:
-            yield Section(section.name), section.header.sh_addr
+            yield Section(section.name), AbsoluteVirtualAddress(section.header.sh_addr)
         elif section.is_null():
-            yield Section("NULL"), section.header.sh_addr
+            yield Section("NULL"), AbsoluteVirtualAddress(section.header.sh_addr)
 
 
 def extract_file_strings(buf, **kwargs):
     yield from capa.features.extractors.common.extract_file_strings(buf)
 
 
-def extract_file_os(elf, buf, **kwargs):
+def extract_file_os(elf: ELFFile, buf, **kwargs):
     # our current approach does not always get an OS value, e.g. for packed samples
     # for file limitation purposes, we're more lax here
     try:
-        os = next(capa.features.extractors.common.extract_os(buf))
-        yield os
+        os_tuple = next(capa.features.extractors.common.extract_os(buf))
+        yield os_tuple
     except StopIteration:
-        yield OS("unknown"), 0x0
+        yield OS("unknown"), NO_ADDRESS
 
 
 def extract_file_format(**kwargs):
-    yield Format(FORMAT_ELF), 0x0
+    yield Format(FORMAT_ELF), NO_ADDRESS
 
 
-def extract_file_arch(elf, **kwargs):
-    # TODO merge with capa.features.extractors.elf.detect_elf_arch()
+def extract_file_arch(elf: ELFFile, **kwargs):
     arch = elf.get_machine_arch()
     if arch == "x86":
-        yield Arch(ElfArch.I386), 0x0
+        yield Arch("i386"), NO_ADDRESS
     elif arch == "x64":
-        yield Arch(ElfArch.AMD64), 0x0
+        yield Arch("amd64"), NO_ADDRESS
+    elif arch == "ARM":
+        yield Arch("arm"), NO_ADDRESS
+    elif arch == "AArch64":
+        yield Arch("aarch64"), NO_ADDRESS
     else:
         logger.warning("unsupported architecture: %s", arch)
 
 
 def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
     for file_handler in FILE_HANDLERS:
-        for feature, va in file_handler(elf=elf, buf=buf):  # type: ignore
-            yield feature, va
+        for feature, addr in file_handler(elf=elf, buf=buf):  # type: ignore
+            yield feature, addr
 
 
 FILE_HANDLERS = (
-    # TODO extract_file_export_names,
+    extract_file_export_names,
     extract_file_import_names,
     extract_file_section_names,
     extract_file_strings,
@@ -97,8 +184,8 @@ FILE_HANDLERS = (
 
 def extract_global_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
     for global_handler in GLOBAL_HANDLERS:
-        for feature, va in global_handler(elf=elf, buf=buf):  # type: ignore
-            yield feature, va
+        for feature, addr in global_handler(elf=elf, buf=buf):  # type: ignore
+            yield feature, addr
 
 
 GLOBAL_HANDLERS = (
@@ -107,32 +194,29 @@ GLOBAL_HANDLERS = (
 )
 
 
-class ElfFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
-        super(ElfFeatureExtractor, self).__init__()
-        self.path = path
-        with open(self.path, "rb") as f:
-            self.elf = ELFFile(io.BytesIO(f.read()))
+class ElfFeatureExtractor(StaticFeatureExtractor):
+    def __init__(self, path: Path):
+        super().__init__(SampleHashes.from_bytes(path.read_bytes()))
+        self.path: Path = path
+        self.elf = ELFFile(io.BytesIO(path.read_bytes()))
 
     def get_base_address(self):
         # virtual address of the first segment with type LOAD
         for segment in self.elf.iter_segments():
             if segment.header.p_type == "PT_LOAD":
-                return segment.header.p_vaddr
+                return AbsoluteVirtualAddress(segment.header.p_vaddr)
 
     def extract_global_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = self.path.read_bytes()
 
-        for feature, va in extract_global_features(self.elf, buf):
-            yield feature, va
+        for feature, addr in extract_global_features(self.elf, buf):
+            yield feature, addr
 
     def extract_file_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = self.path.read_bytes()
 
-        for feature, va in extract_file_features(self.elf, buf):
-            yield feature, va
+        for feature, addr in extract_file_features(self.elf, buf):
+            yield feature, addr
 
     def get_functions(self):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
@@ -152,8 +236,8 @@ class ElfFeatureExtractor(FeatureExtractor):
     def extract_insn_features(self, f, bb, insn):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
 
-    def is_library_function(self, va):
+    def is_library_function(self, addr):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
 
-    def get_function_name(self, va):
+    def get_function_name(self, addr):
         raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")

capa/features/extractors/ghidra/basicblock.py

@@ -0,0 +1,152 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import string
+import struct
+from typing import Tuple, Iterator
+
+import ghidra
+from ghidra.program.model.lang import OperandType
+
+import capa.features.extractors.ghidra.helpers
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
+
+
+def get_printable_len(op: ghidra.program.model.scalar.Scalar) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    op_bit_len = op.bitLength()
+    op_byte_len = op_bit_len // 8
+    op_val = op.getValue()
+
+    if op_bit_len == 8:
+        chars = struct.pack("<B", op_val & 0xFF)
+    elif op_bit_len == 16:
+        chars = struct.pack("<H", op_val & 0xFFFF)
+    elif op_bit_len == 32:
+        chars = struct.pack("<I", op_val & 0xFFFFFFFF)
+    elif op_bit_len == 64:
+        chars = struct.pack("<Q", op_val & 0xFFFFFFFFFFFFFFFF)
+    else:
+        raise ValueError(f"Unhandled operand data type 0x{op_bit_len:x}.")
+
+    def is_printable_ascii(chars_: bytes):
+        return all(c < 127 and chr(c) in string.printable for c in chars_)
+
+    def is_printable_utf16le(chars_: bytes):
+        if all(c == 0x00 for c in chars_[1::2]):
+            return is_printable_ascii(chars_[::2])
+
+    if is_printable_ascii(chars):
+        return op_byte_len
+
+    if is_printable_utf16le(chars):
+        return op_byte_len
+
+    return 0
+
+
+def is_mov_imm_to_stack(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    """verify instruction moves immediate onto stack"""
+
+    # Ghidra will Bitwise OR the OperandTypes to assign multiple
+    # i.e., the first operand is a stackvar (dynamically allocated),
+    # and the second is a scalar value (single int/char/float/etc.)
+    mov_its_ops = [(OperandType.ADDRESS | OperandType.DYNAMIC), OperandType.SCALAR]
+    found = False
+
+    # MOV dword ptr [EBP + local_*], 0x65
+    if insn.getMnemonicString().startswith("MOV"):
+        found = all(insn.getOperandType(i) == mov_its_ops[i] for i in range(2))
+
+    return found
+
+
+def bb_contains_stackstring(bb: ghidra.program.model.block.CodeBlock) -> bool:
+    """check basic block for stackstring indicators
+
+    true if basic block contains enough moves of constant bytes to the stack
+    """
+    count = 0
+    for insn in currentProgram().getListing().getInstructions(bb, True):  # type: ignore [name-defined] # noqa: F821
+        if is_mov_imm_to_stack(insn):
+            count += get_printable_len(insn.getScalar(1))
+        if count > MIN_STACKSTRING_LEN:
+            return True
+    return False
+
+
+def _bb_has_tight_loop(bb: ghidra.program.model.block.CodeBlock):
+    """
+    parse tight loops, true if last instruction in basic block branches to bb start
+    """
+    # Reverse Ordered, first InstructionDB
+    last_insn = currentProgram().getListing().getInstructions(bb, False).next()  # type: ignore [name-defined] # noqa: F821
+
+    if last_insn.getFlowType().isJump():
+        return last_insn.getAddress(0) == bb.getMinAddress()
+
+    return False
+
+
+def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract stackstring indicators from basic block"""
+    bb: ghidra.program.model.block.CodeBlock = bbh.inner
+
+    if bb_contains_stackstring(bb):
+        yield Characteristic("stack string"), bbh.address
+
+
+def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """check basic block for tight loop indicators"""
+    bb: ghidra.program.model.block.CodeBlock = bbh.inner
+
+    if _bb_has_tight_loop(bb):
+        yield Characteristic("tight loop"), bbh.address
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    extract_bb_stackstring,
+)
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract features from the given basic block.
+
+    args:
+        bb: the basic block to process.
+
+    yields:
+      Tuple[Feature, int]: the features and their location found in this basic block.
+    """
+    yield BasicBlock(), bbh.address
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, addr in bb_handler(fh, bbh):
+            yield feature, addr
+
+
+def main():
+    features = []
+    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
+
+    for fh in GhidraFeatureExtractor().get_functions():
+        for bbh in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
+            features.extend(list(extract_features(fh, bbh)))
+
+    import pprint
+
+    pprint.pprint(features)  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ghidra/extractor.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Tuple, Iterator
+
+import capa.features.extractors.ghidra.file
+import capa.features.extractors.ghidra.insn
+import capa.features.extractors.ghidra.global_
+import capa.features.extractors.ghidra.function
+import capa.features.extractors.ghidra.basicblock
+from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
+
+
+class GhidraFeatureExtractor(StaticFeatureExtractor):
+    def __init__(self):
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        super().__init__(
+            SampleHashes(
+                md5=capa.ghidra.helpers.get_file_md5(),
+                # ghidra doesn't expose this hash.
+                # https://ghidra.re/ghidra_docs/api/ghidra/program/model/listing/Program.html
+                #
+                # the hashes are stored in the database, not computed on the fly,
+                # so it's probably not trivial to add SHA1.
+                sha1="",
+                sha256=capa.ghidra.helpers.get_file_sha256(),
+            )
+        )
+
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.ghidra.file.extract_file_format())
+        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_os())
+        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_arch())
+        self.imports = ghidra_helpers.get_file_imports()
+        self.externs = ghidra_helpers.get_file_externs()
+        self.fakes = ghidra_helpers.map_fake_import_addrs()
+
+    def get_base_address(self):
+        return AbsoluteVirtualAddress(currentProgram().getImageBase().getOffset())  # type: ignore [name-defined] # noqa: F821
+
+    def extract_global_features(self):
+        yield from self.global_features
+
+    def extract_file_features(self):
+        yield from capa.features.extractors.ghidra.file.extract_features()
+
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        for fhandle in ghidra_helpers.get_function_symbols():
+            fh: FunctionHandle = FunctionHandle(
+                address=AbsoluteVirtualAddress(fhandle.getEntryPoint().getOffset()),
+                inner=fhandle,
+                ctx={"imports_cache": self.imports, "externs_cache": self.externs, "fakes_cache": self.fakes},
+            )
+            yield fh
+
+    @staticmethod
+    def get_function(addr: int) -> FunctionHandle:
+        func = getFunctionContaining(toAddr(addr))  # type: ignore [name-defined] # noqa: F821
+        return FunctionHandle(address=AbsoluteVirtualAddress(func.getEntryPoint().getOffset()), inner=func)
+
+    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.ghidra.function.extract_features(fh)
+
+    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        yield from ghidra_helpers.get_function_blocks(fh)
+
+    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.ghidra.basicblock.extract_features(fh, bbh)
+
+    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        yield from ghidra_helpers.get_insn_in_range(bbh)
+
+    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
+        yield from capa.features.extractors.ghidra.insn.extract_features(fh, bbh, ih)

capa/features/extractors/ghidra/file.py

@@ -0,0 +1,204 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import re
+import struct
+from typing import List, Tuple, Iterator
+
+from ghidra.program.model.symbol import SourceType, SymbolType
+
+import capa.features.extractors.common
+import capa.features.extractors.helpers
+import capa.features.extractors.strings
+import capa.features.extractors.ghidra.helpers
+from capa.features.file import Export, Import, Section, FunctionName
+from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
+
+MAX_OFFSET_PE_AFTER_MZ = 0x200
+
+
+def find_embedded_pe(block_bytez: bytes, mz_xor: List[Tuple[bytes, bytes, int]]) -> Iterator[Tuple[int, int]]:
+    """check segment for embedded PE
+
+    adapted for Ghidra from:
+    https://github.com/vivisect/vivisect/blob/91e8419a861f4977https://github.com/vivisect/vivisect/blob/91e8419a861f49779f18316f155311967e696836/PE/carve.py#L259f18316f155311967e696836/PE/carve.py#L25
+    """
+    todo = []
+
+    for mzx, pex, i in mz_xor:
+        for match in re.finditer(re.escape(mzx), block_bytez):
+            todo.append((match.start(), mzx, pex, i))
+
+    seg_max = len(block_bytez)  # noqa: F821
+    while len(todo):
+        off, mzx, pex, i = todo.pop()
+
+        # MZ header has one field we will check e_lfanew is at 0x3c
+        e_lfanew = off + 0x3C
+
+        if seg_max < e_lfanew + 4:
+            continue
+
+        e_lfanew_bytes = block_bytez[e_lfanew : e_lfanew + 4]
+        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(e_lfanew_bytes, i))[0]
+
+        # assume XOR'd "PE" bytes exist within threshold
+        if newoff > MAX_OFFSET_PE_AFTER_MZ:
+            continue
+
+        peoff = off + newoff
+        if seg_max < peoff + 2:
+            continue
+
+        pe_bytes = block_bytez[peoff : peoff + 2]
+        if pe_bytes == pex:
+            yield off, i
+
+
+def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
+    """extract embedded PE features"""
+
+    # pre-compute XOR pairs
+    mz_xor: List[Tuple[bytes, bytes, int]] = [
+        (
+            capa.features.extractors.helpers.xor_static(b"MZ", i),
+            capa.features.extractors.helpers.xor_static(b"PE", i),
+            i,
+        )
+        for i in range(256)
+    ]
+
+    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
+        if not all((block.isLoaded(), block.isInitialized(), "Headers" not in block.getName())):
+            continue
+
+        for off, _ in find_embedded_pe(capa.features.extractors.ghidra.helpers.get_block_bytes(block), mz_xor):
+            # add offset back to block start
+            ea: int = block.getStart().add(off).getOffset()
+
+            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
+
+
+def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
+    """extract function exports"""
+    st = currentProgram().getSymbolTable()  # type: ignore [name-defined] # noqa: F821
+    for addr in st.getExternalEntryPointIterator():
+        yield Export(st.getPrimarySymbol(addr).getName()), AbsoluteVirtualAddress(addr.getOffset())
+
+
+def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
+    """extract function imports
+
+    1. imports by ordinal:
+     - modulename.#ordinal
+
+    2. imports by name, results in two features to support importname-only
+       matching:
+     - modulename.importname
+     - importname
+    """
+
+    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
+        for r in f.getSymbol().getReferences():
+            if r.getReferenceType().isData():
+                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
+
+        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_*
+        if "Ordinal_" in fstr[1]:
+            fstr[1] = f"#{fstr[1].split('_')[1]}"
+
+        for name in capa.features.extractors.helpers.generate_symbols(fstr[0][:-4], fstr[1], include_dll=True):
+            yield Import(name), AbsoluteVirtualAddress(addr)
+
+
+def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
+    """extract section names"""
+
+    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
+        yield Section(block.getName()), AbsoluteVirtualAddress(block.getStart().getOffset())
+
+
+def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
+    """extract ASCII and UTF-16 LE strings"""
+
+    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
+        if not block.isInitialized():
+            continue
+
+        p_bytes = capa.features.extractors.ghidra.helpers.get_block_bytes(block)
+
+        for s in capa.features.extractors.strings.extract_ascii_strings(p_bytes):
+            offset = block.getStart().getOffset() + s.offset
+            yield String(s.s), FileOffsetAddress(offset)
+
+        for s in capa.features.extractors.strings.extract_unicode_strings(p_bytes):
+            offset = block.getStart().getOffset() + s.offset
+            yield String(s.s), FileOffsetAddress(offset)
+
+
+def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract the names of statically-linked library functions.
+    """
+
+    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
+        # .isExternal() misses more than this config for the function symbols
+        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
+            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
+            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
+                name = name[13:]
+            addr = AbsoluteVirtualAddress(sym.getAddress().getOffset())
+            yield FunctionName(name), addr
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                yield FunctionName(name[1:]), addr
+
+
+def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
+    ef = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+    if "PE" in ef:
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif "ELF" in ef:
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    elif "Raw" in ef:
+        # no file type to return when processing a binary file, but we want to continue processing
+        return
+    else:
+        raise NotImplementedError(f"unexpected file format: {ef}")
+
+
+def extract_features() -> Iterator[Tuple[Feature, Address]]:
+    """extract file features"""
+    for file_handler in FILE_HANDLERS:
+        for feature, addr in file_handler():
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_file_embedded_pe,
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_section_names,
+    extract_file_strings,
+    extract_file_function_names,
+    extract_file_format,
+)
+
+
+def main():
+    """ """
+    import pprint
+
+    pprint.pprint(list(extract_features()))  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ghidra/function.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Tuple, Iterator
+
+import ghidra
+from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
+
+import capa.features.extractors.ghidra.helpers
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors import loops
+from capa.features.extractors.base_extractor import FunctionHandle
+
+
+def extract_function_calls_to(fh: FunctionHandle):
+    """extract callers to a function"""
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+    for ref in f.getSymbol().getReferences():
+        if ref.getReferenceType().isCall():
+            yield Characteristic("calls to"), AbsoluteVirtualAddress(ref.getFromAddress().getOffset())
+
+
+def extract_function_loop(fh: FunctionHandle):
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+
+    edges = []
+    for block in SimpleBlockIterator(BasicBlockModel(currentProgram()), f.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
+        dests = block.getDestinations(monitor())  # type: ignore [name-defined] # noqa: F821
+        s_addrs = block.getStartAddresses()
+
+        while dests.hasNext():  # For loop throws Python TypeError
+            for addr in s_addrs:
+                edges.append((addr.getOffset(), dests.next().getDestinationAddress().getOffset()))
+
+    if loops.has_loop(edges):
+        yield Characteristic("loop"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
+
+
+def extract_recursive_call(fh: FunctionHandle):
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+
+    for func in f.getCalledFunctions(monitor()):  # type: ignore [name-defined] # noqa: F821
+        if func.getEntryPoint().getOffset() == f.getEntryPoint().getOffset():
+            yield Characteristic("recursive call"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
+
+
+def main():
+    """ """
+    features = []
+    for fhandle in capa.features.extractors.ghidra.helpers.get_function_symbols():
+        features.extend(list(extract_features(fhandle)))
+
+    import pprint
+
+    pprint.pprint(features)  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ghidra/global_.py

@@ -0,0 +1,67 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+import contextlib
+from typing import Tuple, Iterator
+
+import capa.ghidra.helpers
+import capa.features.extractors.elf
+import capa.features.extractors.ghidra.helpers
+from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
+from capa.features.address import NO_ADDRESS, Address
+
+logger = logging.getLogger(__name__)
+
+
+def extract_os() -> Iterator[Tuple[Feature, Address]]:
+    format_name: str = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+
+    if "PE" in format_name:
+        yield OS(OS_WINDOWS), NO_ADDRESS
+
+    elif "ELF" in format_name:
+        with contextlib.closing(capa.ghidra.helpers.GHIDRAIO()) as f:
+            os = capa.features.extractors.elf.detect_elf_os(f)
+
+        yield OS(os), NO_ADDRESS
+
+    else:
+        # we likely end up here:
+        #  1. handling shellcode, or
+        #  2. handling a new file format (e.g. macho)
+        #
+        # for (1) we can't do much - its shellcode and all bets are off.
+        # we could maybe accept a further CLI argument to specify the OS,
+        # but i think this would be rarely used.
+        # rules that rely on OS conditions will fail to match on shellcode.
+        #
+        # for (2), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported file format: %s, will not guess OS", format_name)
+        return
+
+
+def extract_arch() -> Iterator[Tuple[Feature, Address]]:
+    lang_id = currentProgram().getMetadata().get("Language ID")  # type: ignore [name-defined] # noqa: F821
+
+    if "x86" in lang_id and "64" in lang_id:
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+
+    elif "x86" in lang_id and "32" in lang_id:
+        yield Arch(ARCH_I386), NO_ADDRESS
+
+    elif "x86" not in lang_id:
+        logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel")
+        return
+
+    else:
+        # we likely end up here:
+        #  1. handling a new architecture (e.g. aarch64)
+        #
+        # for (1), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported architecture: %s", lang_id)
+        return

capa/features/extractors/ghidra/helpers.py

@@ -0,0 +1,301 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Dict, List, Iterator
+
+import ghidra
+import java.lang
+from ghidra.program.model.lang import OperandType
+from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
+from ghidra.program.model.symbol import SourceType, SymbolType
+from ghidra.program.model.address import AddressSpace
+
+import capa.features.extractors.helpers
+from capa.features.common import THUNK_CHAIN_DEPTH_DELTA
+from capa.features.address import AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+
+def ints_to_bytes(bytez: List[int]) -> bytes:
+    """convert Java signed ints to Python bytes
+
+    args:
+        bytez: list of Java signed ints
+    """
+    return bytes([b & 0xFF for b in bytez])
+
+
+def find_byte_sequence(addr: ghidra.program.model.address.Address, seq: bytes) -> Iterator[int]:
+    """yield all ea of a given byte sequence
+
+    args:
+        addr: start address
+        seq: bytes to search e.g. b"\x01\x03"
+    """
+    seqstr = "".join([f"\\x{b:02x}" for b in seq])
+    eas = findBytes(addr, seqstr, java.lang.Integer.MAX_VALUE, 1)  # type: ignore [name-defined] # noqa: F821
+
+    yield from eas
+
+
+def get_bytes(addr: ghidra.program.model.address.Address, length: int) -> bytes:
+    """yield length bytes at addr
+
+    args:
+        addr: Address to begin pull from
+        length: length of bytes to pull
+    """
+    try:
+        return ints_to_bytes(getBytes(addr, length))  # type: ignore [name-defined] # noqa: F821
+    except RuntimeError:
+        return b""
+
+
+def get_block_bytes(block: ghidra.program.model.mem.MemoryBlock) -> bytes:
+    """yield all bytes in a given block
+
+    args:
+        block: MemoryBlock to pull from
+    """
+    return get_bytes(block.getStart(), block.getSize())
+
+
+def get_function_symbols():
+    """yield all non-external function symbols"""
+    yield from currentProgram().getFunctionManager().getFunctionsNoStubs(True)  # type: ignore [name-defined] # noqa: F821
+
+
+def get_function_blocks(fh: FunctionHandle) -> Iterator[BBHandle]:
+    """yield BBHandle for each bb in a given function"""
+
+    func: ghidra.program.database.function.FunctionDB = fh.inner
+    for bb in SimpleBlockIterator(BasicBlockModel(currentProgram()), func.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
+        yield BBHandle(address=AbsoluteVirtualAddress(bb.getMinAddress().getOffset()), inner=bb)
+
+
+def get_insn_in_range(bbh: BBHandle) -> Iterator[InsnHandle]:
+    """yield InshHandle for each insn in a given basicblock"""
+    for insn in currentProgram().getListing().getInstructions(bbh.inner, True):  # type: ignore [name-defined] # noqa: F821
+        yield InsnHandle(address=AbsoluteVirtualAddress(insn.getAddress().getOffset()), inner=insn)
+
+
+def get_file_imports() -> Dict[int, List[str]]:
+    """get all import names & addrs"""
+
+    import_dict: Dict[int, List[str]] = {}
+
+    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
+        for r in f.getSymbol().getReferences():
+            if r.getReferenceType().isData():
+                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
+
+        ex_loc = f.getExternalLocation().getAddress()  # map external locations as well (offset into module files)
+
+        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_* / <EXTERNAL>::import
+        if "Ordinal_" in fstr[1]:
+            fstr[1] = f"#{fstr[1].split('_')[1]}"
+
+        # <EXTERNAL> mostly shows up in ELF files, otherwise, strip '.dll' w/ [:-4]
+        fstr[0] = "*" if "<EXTERNAL>" in fstr[0] else fstr[0][:-4]
+
+        for name in capa.features.extractors.helpers.generate_symbols(fstr[0], fstr[1]):
+            import_dict.setdefault(addr, []).append(name)
+            if ex_loc:
+                import_dict.setdefault(ex_loc.getOffset(), []).append(name)
+
+    return import_dict
+
+
+def get_file_externs() -> Dict[int, List[str]]:
+    """
+    Gets function names & addresses of statically-linked library functions
+
+    Ghidra's external namespace is mostly reserved for dynamically-linked
+    imports. Statically-linked functions are part of the global namespace.
+    Filtering on the type, source, and namespace of the symbols yield more
+    statically-linked library functions.
+
+    Example: (PMA Lab 16-01.exe_) 7faafc7e4a5c736ebfee6abbbc812d80:0x407490
+    - __aulldiv
+    - Note: See Symbol Table labels
+    """
+
+    extern_dict: Dict[int, List[str]] = {}
+
+    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
+        # .isExternal() misses more than this config for the function symbols
+        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
+            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
+            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
+                name = name[13:]
+            extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name)
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name[1:])
+
+    return extern_dict
+
+
+def map_fake_import_addrs() -> Dict[int, List[int]]:
+    """
+    Map ghidra's fake import entrypoints to their
+    real addresses
+
+    Helps as many Ghidra Scripting API calls end up returning
+    these external (fake) addresses.
+
+    Undocumented but intended Ghidra behavior:
+     - Import entryPoint fields are stored in the 'EXTERNAL:' AddressSpace.
+       'getEntryPoint()' returns the entryPoint field, which is an offset
+       from the beginning of the assigned AddressSpace. In the case of externals,
+       they start from 1 and increment.
+    https://github.com/NationalSecurityAgency/ghidra/blob/26d4bd9104809747c21f2528cab8aba9aef9acd5/Ghidra/Features/Base/src/test.slow/java/ghidra/program/database/function/ExternalFunctionDBTest.java#L90
+
+    Example: (mimikatz.exe_) 5f66b82558ca92e54e77f216ef4c066c:0x473090
+    - 0x473090 -> PTR_CreateServiceW_00473090
+    - 'EXTERNAL:00000025' -> External Address (ghidra.program.model.address.SpecialAddress)
+    """
+    fake_dict: Dict[int, List[int]] = {}
+
+    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
+        for r in f.getSymbol().getReferences():
+            if r.getReferenceType().isData():
+                fake_dict.setdefault(f.getEntryPoint().getOffset(), []).append(r.getFromAddress().getOffset())
+
+    return fake_dict
+
+
+def check_addr_for_api(
+    addr: ghidra.program.model.address.Address,
+    fakes: Dict[int, List[int]],
+    imports: Dict[int, List[str]],
+    externs: Dict[int, List[str]],
+) -> bool:
+    offset = addr.getOffset()
+
+    fake = fakes.get(offset)
+    if fake:
+        return True
+
+    imp = imports.get(offset)
+    if imp:
+        return True
+
+    extern = externs.get(offset)
+    if extern:
+        return True
+
+    return False
+
+
+def is_call_or_jmp(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    return any(mnem in insn.getMnemonicString() for mnem in ["CALL", "J"])  # JMP, JNE, JNZ, etc
+
+
+def is_sp_modified(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == OperandType.REGISTER:
+            return "SP" in insn.getRegister(i).getName() and insn.getOperandRefType(i).isWrite()
+    return False
+
+
+def is_stack_referenced(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    """generic catch-all for stack references"""
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == OperandType.REGISTER:
+            if "BP" in insn.getRegister(i).getName():
+                return True
+            else:
+                continue
+
+    return any(ref.isStackReference() for ref in insn.getReferencesFrom())
+
+
+def is_zxor(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    # assume XOR insn
+    # XOR's against the same operand zero out
+    ops = []
+    operands = []
+    for i in range(insn.getNumOperands()):
+        ops.append(insn.getOpObjects(i))
+
+    # Operands stored in a 2D array
+    for j in range(len(ops)):
+        for k in range(len(ops[j])):
+            operands.append(ops[j][k])
+
+    return all(n == operands[0] for n in operands)
+
+
+def handle_thunk(addr: ghidra.program.model.address.Address):
+    """Follow thunk chains down to a reasonable depth"""
+    ref = addr
+    for _ in range(THUNK_CHAIN_DEPTH_DELTA):
+        thunk_jmp = getInstructionAt(ref)  # type: ignore [name-defined] # noqa: F821
+        if thunk_jmp and is_call_or_jmp(thunk_jmp):
+            if OperandType.isAddress(thunk_jmp.getOperandType(0)):
+                ref = thunk_jmp.getAddress(0)
+        else:
+            thunk_dat = getDataContaining(ref)  # type: ignore [name-defined] # noqa: F821
+            if thunk_dat and thunk_dat.isDefined() and thunk_dat.isPointer():
+                ref = thunk_dat.getValue()
+                break  # end of thunk chain reached
+    return ref
+
+
+def dereference_ptr(insn: ghidra.program.database.code.InstructionDB):
+    addr_code = OperandType.ADDRESS | OperandType.CODE
+    to_deref = insn.getAddress(0)
+    dat = getDataContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
+
+    if insn.getOperandType(0) == addr_code:
+        thfunc = getFunctionContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
+        if thfunc and thfunc.isThunk():
+            return handle_thunk(to_deref)
+        else:
+            # if it doesn't point to a thunk, it's usually a jmp to a label
+            return to_deref
+    if not dat:
+        return to_deref
+    if dat.isDefined() and dat.isPointer():
+        addr = dat.getValue()
+        # now we need to check the addr space to see if it is truly resolvable
+        # ghidra sometimes likes to hand us direct RAM addrs, which typically point
+        # to api calls that we can't actually resolve as such
+        if addr.getAddressSpace().getType() == AddressSpace.TYPE_RAM:
+            return to_deref
+        else:
+            return addr
+    else:
+        return to_deref
+
+
+def find_data_references_from_insn(insn, max_depth: int = 10):
+    """yield data references from given instruction"""
+    for reference in insn.getReferencesFrom():
+        if not reference.getReferenceType().isData():
+            # only care about data references
+            continue
+
+        to_addr = reference.getToAddress()
+
+        for _ in range(max_depth - 1):
+            data = getDataAt(to_addr)  # type: ignore [name-defined] # noqa: F821
+            if data and data.isPointer():
+                ptr_value = data.getValue()
+
+                if ptr_value is None:
+                    break
+
+                to_addr = ptr_value
+            else:
+                break
+
+        yield to_addr

capa/features/extractors/ghidra/insn.py

@@ -0,0 +1,503 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Any, Dict, Tuple, Iterator
+
+import ghidra
+from ghidra.program.model.lang import OperandType
+from ghidra.program.model.block import SimpleBlockModel
+
+import capa.features.extractors.helpers
+import capa.features.extractors.ghidra.helpers
+from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
+from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+# security cookie checks may perform non-zeroing XORs, these are expected within a certain
+# byte range within the first and returning basic blocks, this helps to reduce FP features
+SECURITY_COOKIE_BYTES_DELTA = 0x40
+
+
+OPERAND_TYPE_DYNAMIC_ADDRESS = OperandType.DYNAMIC | OperandType.ADDRESS
+
+
+def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    """Populate the import cache for this context"""
+    if "imports_cache" not in ctx:
+        ctx["imports_cache"] = capa.features.extractors.ghidra.helpers.get_file_imports()
+    return ctx["imports_cache"]
+
+
+def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    """Populate the externs cache for this context"""
+    if "externs_cache" not in ctx:
+        ctx["externs_cache"] = capa.features.extractors.ghidra.helpers.get_file_externs()
+    return ctx["externs_cache"]
+
+
+def get_fakes(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    """Populate the fake import addrs cache for this context"""
+    if "fakes_cache" not in ctx:
+        ctx["fakes_cache"] = capa.features.extractors.ghidra.helpers.map_fake_import_addrs()
+    return ctx["fakes_cache"]
+
+
+def check_for_api_call(
+    insn, externs: Dict[int, Any], fakes: Dict[int, Any], imports: Dict[int, Any], imp_or_ex: bool
+) -> Iterator[Any]:
+    """check instruction for API call
+
+    params:
+        externs - external library functions cache
+        fakes - mapped fake import addresses cache
+        imports - imported functions cache
+        imp_or_ex - flag to check imports or externs
+
+    yields:
+        matched api calls
+    """
+    info = ()
+    funcs = imports if imp_or_ex else externs
+
+    # assume only CALLs or JMPs are passed
+    ref_type = insn.getOperandType(0)
+    addr_data = OperandType.ADDRESS | OperandType.DATA  # needs dereferencing
+    addr_code = OperandType.ADDRESS | OperandType.CODE  # needs dereferencing
+
+    if OperandType.isRegister(ref_type):
+        if OperandType.isAddress(ref_type):
+            # If it's an address in a register, check the mapped fake addrs
+            # since they're dereferenced to their fake addrs
+            op_ref = insn.getAddress(0).getOffset()
+            ref = fakes.get(op_ref)  # obtain the real addr
+            if not ref:
+                return
+        else:
+            return
+    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
+        # we must dereference and check if the addr is a pointer to an api function
+        addr_ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
+        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
+            return
+        ref = addr_ref.getOffset()
+    elif ref_type == OPERAND_TYPE_DYNAMIC_ADDRESS or ref_type == OperandType.DYNAMIC:
+        return  # cannot resolve dynamics statically
+    else:
+        # pure address does not need to get dereferenced/ handled
+        addr_ref = insn.getAddress(0)
+        if not addr_ref:
+            # If it returned null, it was an indirect
+            # that had no address reference.
+            # This check is faster than checking for (indirect and not address)
+            return
+        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
+            return
+        ref = addr_ref.getOffset()
+
+    if isinstance(ref, list):  # ref from REG | ADDR
+        for r in ref:
+            info = funcs.get(r)  # type: ignore
+            if info:
+                yield info
+    else:
+        info = funcs.get(ref)  # type: ignore
+        if info:
+            yield info
+
+
+def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    externs = get_externs(fh.ctx)
+    fakes = get_fakes(fh.ctx)
+    imports = get_imports(fh.ctx)
+
+    # check calls to imported functions
+    for api in check_for_api_call(insn, externs, fakes, imports, True):
+        for imp in api:
+            yield API(imp), ih.address
+
+    # check calls to extern functions
+    for api in check_for_api_call(insn, externs, fakes, imports, False):
+        for ext in api:
+            yield API(ext), ih.address
+
+
+def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction number features
+    example:
+        push    3136B0h         ; dwControlCode
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("RET"):
+        # skip things like:
+        #   .text:0042250E retn 8
+        return
+
+    if capa.features.extractors.ghidra.helpers.is_sp_modified(insn):
+        # skip things like:
+        #   .text:00401145 add esp, 0Ch
+        return
+
+    for i in range(insn.getNumOperands()):
+        # Exceptions for LEA insn:
+        # invalid operand encoding, considered numbers instead of offsets
+        # see: mimikatz.exe_:0x4018C0
+        if insn.getOperandType(i) == OperandType.DYNAMIC and insn.getMnemonicString().startswith("LEA"):
+            # Additional check, avoid yielding "wide" values (ex. mimikatz.exe:0x471EE6 LEA EBX, [ECX + EAX*0x4])
+            op_objs = insn.getOpObjects(i)
+            if len(op_objs) == 3:  # ECX, EAX, 0x4
+                continue
+
+            if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
+                const = op_objs[-1].getUnsignedValue()
+                addr = ih.address
+
+                yield Number(const), addr
+                yield OperandNumber(i, const), addr
+        elif not OperandType.isScalar(insn.getOperandType(i)):
+            # skip things like:
+            #   references, void types
+            continue
+        else:
+            const = insn.getScalar(i).getUnsignedValue()
+            addr = ih.address
+
+            yield Number(const), addr
+            yield OperandNumber(i, const), addr
+
+            if insn.getMnemonicString().startswith("ADD") and 0 < const < MAX_STRUCTURE_SIZE:
+                # for pattern like:
+                #
+                #     add eax, 0x10
+                #
+                # assume 0x10 is also an offset (imagine eax is a pointer).
+                yield Offset(const), addr
+                yield OperandOffset(i, const), addr
+
+
+def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction structure offset features
+
+    example:
+        .text:0040112F cmp [esi+4], ebx
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("LEA"):
+        return
+
+    if capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
+        # ignore stack references
+        return
+
+    # Ghidra stores operands in 2D arrays if they contain offsets
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == OperandType.DYNAMIC:  # e.g. [esi + 4]
+            # manual extraction, since the default api calls only work on the 1st dimension of the array
+            op_objs = insn.getOpObjects(i)
+            if not op_objs:
+                continue
+
+            if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
+                op_off = op_objs[-1].getValue()
+            else:
+                op_off = 0
+
+            yield Offset(op_off), ih.address
+            yield OperandOffset(i, op_off), ih.address
+
+
+def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse referenced byte sequences
+
+    example:
+        push    offset iid_004118d4_IShellLinkA ; riid
+    """
+    for addr in capa.features.extractors.ghidra.helpers.find_data_references_from_insn(ih.inner):
+        data = getDataAt(addr)  # type: ignore [name-defined] # noqa: F821
+        if data and not data.hasStringValue():
+            extracted_bytes = capa.features.extractors.ghidra.helpers.get_bytes(addr, MAX_BYTES_FEATURE_SIZE)
+            if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
+                yield Bytes(extracted_bytes), ih.address
+
+
+def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction string features
+
+    example:
+        push offset aAcr     ; "ACR  > "
+    """
+    for addr in capa.features.extractors.ghidra.helpers.find_data_references_from_insn(ih.inner):
+        data = getDataAt(addr)  # type: ignore [name-defined] # noqa: F821
+        if data and data.hasStringValue():
+            yield String(data.getValue()), ih.address
+
+
+def extract_insn_mnemonic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction mnemonic features"""
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    yield Mnemonic(insn.getMnemonicString().lower()), ih.address
+
+
+def extract_insn_obfs_call_plus_5_characteristic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse call $+5 instruction from the given instruction.
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    code_ref = OperandType.ADDRESS | OperandType.CODE
+    ref = insn.getAddress()
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == code_ref:
+            ref = insn.getAddress(i)
+
+    if insn.getAddress().add(5) == ref:
+        yield Characteristic("call $+5"), ih.address
+
+
+def extract_insn_segment_access_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction fs or gs access"""
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    insn_str = insn.toString()
+
+    if "FS:" in insn_str:
+        yield Characteristic("fs access"), ih.address
+
+    if "GS:" in insn_str:
+        yield Characteristic("gs access"), ih.address
+
+
+def extract_insn_peb_access_characteristic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction peb access
+
+    fs:[0x30] on x86, gs:[0x60] on x64
+
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    insn_str = insn.toString()
+    if insn_str.startswith(("PUSH", "MOV")):
+        if "FS:[0x30]" in insn_str or "GS:[0x60]" in insn_str:
+            yield Characteristic("peb access"), ih.address
+
+
+def extract_insn_cross_section_cflow(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    externs = get_externs(fh.ctx)
+    fakes = get_fakes(fh.ctx)
+    imports = get_imports(fh.ctx)
+
+    # OperandType to dereference
+    addr_data = OperandType.ADDRESS | OperandType.DATA
+    addr_code = OperandType.ADDRESS | OperandType.CODE
+
+    ref_type = insn.getOperandType(0)
+
+    # both OperandType flags must be present
+    # bail on REGISTER alone
+    if OperandType.isRegister(ref_type):
+        if OperandType.isAddress(ref_type):
+            ref = insn.getAddress(0)  # Ghidra dereferences REG | ADDR
+            if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
+                return
+        else:
+            return
+    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
+        # we must dereference and check if the addr is a pointer to an api function
+        ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
+        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
+            return
+    elif ref_type == OPERAND_TYPE_DYNAMIC_ADDRESS or ref_type == OperandType.DYNAMIC:
+        return  # cannot resolve dynamics statically
+    else:
+        # pure address does not need to get dereferenced/ handled
+        ref = insn.getAddress(0)
+        if not ref:
+            # If it returned null, it was an indirect
+            # that had no address reference.
+            # This check is faster than checking for (indirect and not address)
+            return
+        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
+            return
+
+    this_mem_block = getMemoryBlock(insn.getAddress())  # type: ignore [name-defined] # noqa: F821
+    ref_block = getMemoryBlock(ref)  # type: ignore [name-defined] # noqa: F821
+    if ref_block != this_mem_block:
+        yield Characteristic("cross section flow"), ih.address
+
+
+def extract_function_calls_from(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    ih: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    """extract functions calls from features
+
+    most relevant at the function scope, however, its most efficient to extract at the instruction scope
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("CALL"):
+        # This method of "dereferencing" addresses/ pointers
+        # is not as robust as methods in other functions,
+        # but works just fine for this one
+        reference = 0
+        for ref in insn.getReferencesFrom():
+            addr = ref.getToAddress()
+
+            # avoid returning fake addrs
+            if not addr.isExternalAddress():
+                reference = addr.getOffset()
+
+            # if a reference is < 0, then ghidra pulled an offset from a DYNAMIC | ADDR (usually a stackvar)
+            # these cannot be resolved to actual addrs
+            if reference > 0:
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(reference)
+
+
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    ih: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
+    does not include calls like => call ds:dword_ABD4974
+
+    most relevant at the function or basic block scope;
+    however, its most efficient to extract at the instruction scope
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("CALL"):
+        if OperandType.isRegister(insn.getOperandType(0)):
+            yield Characteristic("indirect call"), ih.address
+        if OperandType.isIndirect(insn.getOperandType(0)):
+            yield Characteristic("indirect call"), ih.address
+
+
+def check_nzxor_security_cookie_delta(
+    fh: ghidra.program.database.function.FunctionDB, insn: ghidra.program.database.code.InstructionDB
+):
+    """Get the function containing the insn
+    Get the last block of the function that contains the insn
+
+    Check the bb containing the insn
+    Check the last bb of the function containing the insn
+    """
+
+    model = SimpleBlockModel(currentProgram())  # type: ignore [name-defined] # noqa: F821
+    insn_addr = insn.getAddress()
+    func_asv = fh.getBody()
+    first_addr = func_asv.getMinAddress()
+    last_addr = func_asv.getMaxAddress()
+
+    if model.getFirstCodeBlockContaining(
+        first_addr, monitor()  # type: ignore [name-defined] # noqa: F821
+    ) == model.getFirstCodeBlockContaining(
+        last_addr, monitor()  # type: ignore [name-defined] # noqa: F821
+    ):
+        if insn_addr < first_addr.add(SECURITY_COOKIE_BYTES_DELTA):
+            return True
+        else:
+            return insn_addr > last_addr.add(SECURITY_COOKIE_BYTES_DELTA * -1)
+    else:
+        return False
+
+
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    ih: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if "XOR" not in insn.getMnemonicString():
+        return
+    if capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
+        return
+    if capa.features.extractors.ghidra.helpers.is_zxor(insn):
+        return
+    if check_nzxor_security_cookie_delta(f, insn):
+        return
+    yield Characteristic("nzxor"), ih.address
+
+
+def extract_features(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    insn: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    for insn_handler in INSTRUCTION_HANDLERS:
+        for feature, addr in insn_handler(fh, bb, insn):
+            yield feature, addr
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_bytes_features,
+    extract_insn_string_features,
+    extract_insn_offset_features,
+    extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    extract_insn_obfs_call_plus_5_characteristic_features,
+    extract_insn_peb_access_characteristic_features,
+    extract_insn_cross_section_cflow,
+    extract_insn_segment_access_features,
+    extract_function_calls_from,
+    extract_function_indirect_call_characteristic_features,
+)
+
+
+def main():
+    """ """
+    features = []
+    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
+
+    for fh in GhidraFeatureExtractor().get_functions():
+        for bb in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
+            for insn in capa.features.extractors.ghidra.helpers.get_insn_in_range(bb):
+                features.extend(list(extract_features(fh, bb, insn)))
+
+    import pprint
+
+    pprint.pprint(features)  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()