move sigs to capa directory

build(deps-dev): bump pytest from 7.4.3 to 7.4.4

.github/mypy/mypy.ini

@@ -86,3 +86,6 @@ ignore_missing_imports = True
 
 [mypy-netnode.*]
 ignore_missing_imports = True
+
+[mypy-ghidra.*]
+ignore_missing_imports = True

.github/pyinstaller/pyinstaller.spec

@@ -18,7 +18,7 @@ a = Analysis(
         # this gets invoked from the directory of the spec file,
         # i.e. ./.github/pyinstaller
         ("../../rules", "rules"),
-        ("../../sigs", "sigs"),
+        ("../../capa/sigs", "sigs"),
         ("../../cache", "cache"),
         # capa.render.default uses tabulate that depends on wcwidth.
         # it seems wcwidth uses a json file `version.json`
@@ -79,7 +79,7 @@ exe = EXE(
     name="capa",
     icon="logo.ico",
     debug=False,
-    strip=None,
+    strip=False,
     upx=True,
     console=True,
 )

.github/workflows/build.yml

@@ -11,34 +11,41 @@ permissions:
 
 jobs:
   build:
-    name: PyInstaller for ${{ matrix.os }}
+    name: PyInstaller for ${{ matrix.os }} / Py ${{ matrix.python_version }}
     runs-on: ${{ matrix.os }}
     strategy:
       # set to false for debugging
       fail-fast: true
       matrix:
+        # using Python 3.8 to support running across multiple operating systems including Windows 7
         include:
           - os: ubuntu-20.04
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
+            python_version: 3.8
+          - os: ubuntu-20.04
+            artifact_name: capa
+            asset_name: linux-py311
+            python_version: 3.11
           - os: windows-2019
             artifact_name: capa.exe
             asset_name: windows
+            python_version: 3.8
           - os: macos-11
             # use older macOS for assumed better portability
             artifact_name: capa
             asset_name: macos
+            python_version: 3.8
     steps:
       - name: Checkout capa
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: true
-      # using Python 3.8 to support running across multiple operating systems including Windows 7
-      - name: Set up Python 3.8
+      - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
-          python-version: 3.8
+          python-version: ${{ matrix.python_version }}
       - if: matrix.os == 'ubuntu-20.04'
         run: sudo apt-get install -y libyaml-dev
       - name: Upgrade pip, setuptools
@@ -55,13 +62,17 @@ jobs:
         run: dist/capa "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
       - name: Does it run (ELF)?
         run: dist/capa "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
+      - name: Does it run (CAPE)?
+        run: |
+          7z e "tests/data/dynamic/cape/v2.2/d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json.gz"
+          dist/capa "d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json"
       - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: ${{ matrix.asset_name }}
           path: dist/${{ matrix.artifact_name }}
 
   test_run:
-    name: Test run on ${{ matrix.os }}
+    name: Test run on ${{ matrix.os }} / ${{ matrix.asset_name }}
     runs-on: ${{ matrix.os }}
     needs: [build]
     strategy:
@@ -71,6 +82,9 @@ jobs:
           - os: ubuntu-22.04
             artifact_name: capa
             asset_name: linux
+          - os: ubuntu-22.04
+            artifact_name: capa
+            asset_name: linux-py311
           - os: windows-2022
             artifact_name: capa.exe
             asset_name: windows
@@ -96,6 +110,8 @@ jobs:
         include:
           - asset_name: linux
             artifact_name: capa
+          - asset_name: linux-py311
+            artifact_name: capa
           - asset_name: windows
             artifact_name: capa.exe
           - asset_name: macos

.github/workflows/pip-audit.yml

@@ -0,0 +1,21 @@
+name: PIP audit
+
+on:
+  schedule:
+  - cron: '0 8 * * 1'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - uses: pypa/gh-action-pip-audit@v1.0.8
+        with:
+          inputs: .

.github/workflows/tests.yml

@@ -39,13 +39,13 @@ jobs:
     - name: Lint with ruff
       run: pre-commit run ruff
     - name: Lint with isort
-      run: pre-commit run isort
+      run: pre-commit run isort --show-diff-on-failure
     - name: Lint with black
-      run: pre-commit run black
+      run: pre-commit run black --show-diff-on-failure
     - name: Lint with flake8
-      run: pre-commit run flake8
+      run: pre-commit run flake8 --hook-stage manual
     - name: Check types with mypy
-      run:  pre-commit run mypy
+      run:  pre-commit run mypy --hook-stage manual
 
   rule_linter:
     runs-on: ubuntu-20.04
@@ -95,6 +95,10 @@ jobs:
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
       run: pip install -e .[dev]
+    - name: Run tests (fast)
+      # this set of tests runs about 80% of the cases in 20% of the time,
+      # and should catch most errors quickly.
+      run:  pre-commit run pytest-fast --all-files --hook-stage manual
     - name: Run tests
       run: pytest -v tests/
 
@@ -103,7 +107,7 @@ jobs:
     env:
       BN_SERIAL: ${{ secrets.BN_SERIAL }}
     runs-on: ubuntu-20.04
-    needs: [code_style, rule_linter]
+    needs: [tests]
     strategy:
       fail-fast: false
       matrix:
@@ -139,3 +143,62 @@ jobs:
       env:
         BN_LICENSE: ${{ secrets.BN_LICENSE }}
       run: pytest -v tests/test_binja_features.py  # explicitly refer to the binja tests for performance. other tests run above.
+
+  ghidra-tests:
+    name: Ghidra tests for ${{ matrix.python-version }}
+    runs-on: ubuntu-20.04
+    needs: [tests]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.11"]
+        java-version: ["17"]
+        gradle-version: ["7.3"]
+        ghidra-version: ["10.3"]
+        public-version: ["PUBLIC_20230510"] # for ghidra releases
+        jep-version: ["4.1.1"]
+        ghidrathon-version: ["3.0.0"]
+    steps:
+    - name: Checkout capa with submodules
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      with:
+        submodules: true
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Set up Java ${{ matrix.java-version }}
+      uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java-version }}
+    - name: Set up Gradle ${{ matrix.gradle-version }} 
+      uses: gradle/gradle-build-action@40b6781dcdec2762ad36556682ac74e31030cfe2 # v2.5.1
+      with:
+        gradle-version: ${{ matrix.gradle-version }}
+    - name: Install Jep ${{ matrix.jep-version }} 
+      run : pip install jep==${{ matrix.jep-version }}
+    - name: Install Ghidra ${{ matrix.ghidra-version }} 
+      run: |
+        mkdir ./.github/ghidra
+        wget "https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_${{ matrix.ghidra-version }}_build/ghidra_${{ matrix.ghidra-version }}_${{ matrix.public-version }}.zip" -O ./.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip
+        unzip .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip -d .github/ghidra/
+    - name: Install Ghidrathon
+      run : |
+        mkdir ./.github/ghidrathon
+        curl -o ./.github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip "https://codeload.github.com/mandiant/Ghidrathon/zip/refs/tags/v${{ matrix.ghidrathon-version }}"
+        unzip .github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/
+        gradle -p ./.github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/ -PGHIDRA_INSTALL_DIR=$(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC
+        unzip .github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/dist/*.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions
+    - name: Install pyyaml
+      run: sudo apt-get install -y libyaml-dev
+    - name: Install capa
+      run: pip install -e .[dev] 
+    - name: Run tests
+      run: | 
+        mkdir ./.github/ghidra/project
+        .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/support/analyzeHeadless .github/ghidra/project ghidra_test -Import ./tests/data/mimikatz.exe_ -ScriptPath ./tests/ -PostScript test_ghidra_features.py > ../output.log
+        cat ../output.log
+        exit_code=$(cat ../output.log | grep exit | awk '{print $NF}')
+        exit $exit_code
+ 

.gitmodules

@@ -1,6 +1,8 @@
 [submodule "rules"]
 	path = rules
 	url = ../capa-rules.git
+	branch = dynamic-syntax
 [submodule "tests/data"]
 	path = tests/data
 	url = ../capa-testfiles.git
+	branch = dynamic-feature-extractor

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
     hooks:
     -   id: isort
         name: isort
-        stages: [commit, push]
+        stages: [commit, push, manual]
         language: system
         entry: isort
         args: 
@@ -45,7 +45,7 @@ repos:
     hooks:
     -   id: black
         name: black
-        stages: [commit, push]
+        stages: [commit, push, manual]
         language: system
         entry: black
         args: 
@@ -62,7 +62,7 @@ repos:
     hooks:
     -   id: ruff
         name: ruff
-        stages: [commit, push]
+        stages: [commit, push, manual]
         language: system
         entry: ruff
         args: 
@@ -79,7 +79,7 @@ repos:
     hooks:
     -   id: flake8
         name: flake8
-        stages: [commit, push]
+        stages: [push, manual]
         language: system
         entry: flake8
         args: 
@@ -97,7 +97,7 @@ repos:
     hooks:
     -   id: mypy
         name: mypy
-        stages: [commit, push]
+        stages: [push, manual]
         language: system
         entry: mypy
         args: 
@@ -109,3 +109,21 @@ repos:
         -   "tests/"
         always_run: true
         pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: pytest-fast
+        name: pytest (fast)
+        stages: [manual]
+        language: system
+        entry: pytest
+        args: 
+        -   "tests/"
+        -   "--ignore=tests/test_binja_features.py"
+        -   "--ignore=tests/test_ghidra_features.py"
+        -   "--ignore=tests/test_ida_features.py"
+        -   "--ignore=tests/test_viv_features.py"
+        -   "--ignore=tests/test_main.py"
+        -   "--ignore=tests/test_scripts.py"
+        always_run: true
+        pass_filenames: false

CHANGELOG.md

@@ -3,18 +3,79 @@
 ## master (unreleased)
 
 ### New Features
+- add Ghidra backend #1770 #1767 @colton-gabertan @mike-hunhoff
+- add dynamic analysis via CAPE sandbox reports #48 #1535 @yelhamer
+  - add call scope #771 @yelhamer
+  - add thread scope #1517 @yelhamer
+  - add process scope #1517 @yelhamer
+  - rules: change `meta.scope` to `meta.scopes` @yelhamer
+  - protobuf: add `Metadata.flavor` @williballenthin
+- binja: add support for forwarded exports #1646 @xusheng6
+- binja: add support for symtab names #1504 @xusheng6
+- add com class/interface features #322 @Aayush-goel-04
+- dotnet: emit enclosing class information for nested classes #1780 #1913 @bkojusner @mike-hunhoff
 
 ### Breaking Changes
 
-### New Rules (0)
+- remove the `SCOPE_*` constants in favor of the `Scope` enum #1764 @williballenthin
+- protobuf: deprecate `RuleMetadata.scope` in favor of `RuleMetadata.scopes` @williballenthin
+- protobuf: deprecate `Metadata.analysis` in favor of `Metadata.analysis2` that is dynamic analysis aware @williballenthin
+- update freeze format to v3, adding support for dynamic analysis @williballenthin
+- extractor: ignore DLL name for api features #1815 @mr-tz
 
+### New Rules (39)
+
+- nursery/get-ntoskrnl-base-address @mr-tz
+- host-interaction/network/connectivity/set-tcp-connection-state @johnk3r
+- nursery/capture-process-snapshot-data @mr-tz
+- collection/network/capture-packets-using-sharppcap jakub.jozwiak@mandiant.com
+- nursery/communicate-with-kernel-module-via-netlink-socket-on-linux michael.hunhoff@mandiant.com
+- nursery/get-current-pid-on-linux michael.hunhoff@mandiant.com
+- nursery/get-file-system-information-on-linux michael.hunhoff@mandiant.com
+- nursery/get-password-database-entry-on-linux michael.hunhoff@mandiant.com
+- nursery/mark-thread-detached-on-linux michael.hunhoff@mandiant.com
+- nursery/persist-via-gnome-autostart-on-linux michael.hunhoff@mandiant.com
+- nursery/set-thread-name-on-linux michael.hunhoff@mandiant.com
+- load-code/dotnet/load-windows-common-language-runtime michael.hunhoff@mandiant.com blas.kojusner@mandiant.com jakub.jozwiak@mandiant.com
+- nursery/log-keystrokes-via-input-method-manager @mr-tz
+- nursery/encrypt-data-using-rc4-via-systemfunction032 richard.weiss@mandiant.com
+- nursery/add-value-to-global-atom-table @mr-tz
+- nursery/enumerate-processes-that-use-resource @Ana06
+- host-interaction/process/inject/allocate-or-change-rwx-memory @mr-tz
+- lib/allocate-or-change-rw-memory 0x534a@mailbox.org @mr-tz
+- lib/change-memory-protection @mr-tz
+- anti-analysis/anti-av/patch-antimalware-scan-interface-function jakub.jozwiak@mandiant.com
+- executable/dotnet-singlefile/bundled-with-dotnet-single-file-deployment sara.rincon@mandiant.com
+- internal/limitation/file/internal-dotnet-single-file-deployment-limitation sara.rincon@mandiant.com
+- data-manipulation/encoding/encode-data-using-add-xor-sub-operations jakub.jozwiak@mandiant.com
+- nursery/access-camera-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/capture-microphone-audio-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/capture-screenshot-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/check-for-incoming-call-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/check-for-outgoing-call-in-dotnet-on-android michael.hunhoff@mandiant.com
+- nursery/compiled-with-xamarin michael.hunhoff@mandiant.com
+- nursery/get-os-version-in-dotnet-on-android michael.hunhoff@mandiant.com
+- data-manipulation/compression/create-cabinet-on-windows michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com
+- data-manipulation/compression/extract-cabinet-on-windows jakub.jozwiak@mandiant.com
+- lib/create-file-decompression-interface-context-on-windows jakub.jozwiak@mandiant.com
+- nursery/enumerate-files-in-dotnet moritz.raabe@mandiant.com anushka.virgaonkar@mandiant.com
+- nursery/get-mac-address-in-dotnet moritz.raabe@mandiant.com michael.hunhoff@mandiant.com echernofsky@google.com
+- nursery/get-current-process-command-line william.ballenthin@mandiant.com
+- nursery/get-current-process-file-path william.ballenthin@mandiant.com
+- nursery/hook-routines-via-dlsym-rtld_next william.ballenthin@mandiant.com
 -
 
 ### Bug Fixes
+- ghidra: fix `ints_to_bytes` performance #1761 @mike-hunhoff
+- binja: improve function call site detection @xusheng6
+- binja: use `binaryninja.load` to open files @xusheng6
+- binja: bump binja version to 3.5 #1789 @xusheng6
+- elf: better detect ELF OS via GCC .ident directives #1928 @williballenthin
 
 ### capa explorer IDA Pro plugin
 
 ### Development
+- update ATT&CK/MBC data for linting #1932 @mr-tz
 
 ### Raw diffs
 - [capa v6.1.0...master](https://github.com/mandiant/capa/compare/v6.1.0...master)

README.md

@@ -2,13 +2,13 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-831-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-864-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
 capa detects capabilities in executable files.
-You run it against a PE, ELF, .NET module, or shellcode file and it tells you what it thinks the program can do.
+You run it against a PE, ELF, .NET module, shellcode file, or a sandbox report and it tells you what it thinks the program can do.
 For example, it might suggest that the file is a backdoor, is capable of installing services, or relies on HTTP to communicate.
 
 Check out:
@@ -125,6 +125,96 @@ function @ 0x4011C0
 ...
 ```
 
+Additionally, capa also supports analyzing [CAPE](https://github.com/kevoreilly/CAPEv2) sandbox reports for dynamic capabilty extraction.
+In order to use this, you first submit your sample to CAPE for analysis, and then run capa against the generated report (JSON).
+
+Here's an example of running capa against a packed binary, and then running capa against the CAPE report of that binary:
+
+```yaml
+$ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.exe
+WARNING:capa.capabilities.common:--------------------------------------------------------------------------------
+WARNING:capa.capabilities.common: This sample appears to be packed.
+WARNING:capa.capabilities.common: 
+WARNING:capa.capabilities.common: Packed samples have often been obfuscated to hide their logic.
+WARNING:capa.capabilities.common: capa cannot handle obfuscation well using static analysis. This means the results may be misleading or incomplete.
+WARNING:capa.capabilities.common: If possible, you should try to unpack this input file before analyzing it with capa.
+WARNING:capa.capabilities.common: Alternatively, run the sample in a supported sandbox and invoke capa against the report to obtain dynamic analysis results.
+WARNING:capa.capabilities.common: 
+WARNING:capa.capabilities.common: Identified via rule: (internal) packer file limitation
+WARNING:capa.capabilities.common: 
+WARNING:capa.capabilities.common: Use -v or -vv if you really want to see the capabilities identified by capa.
+WARNING:capa.capabilities.common:--------------------------------------------------------------------------------
+
+$ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.json
+
+┍━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
+│ ATT&CK Tactic          │ ATT&CK Technique                                                                   │
+┝━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
+│ CREDENTIAL ACCESS      │ Credentials from Password Stores T1555                                             │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ DEFENSE EVASION        │ File and Directory Permissions Modification T1222                                  │
+│                        │ Modify Registry T1112                                                              │
+│                        │ Obfuscated Files or Information T1027                                              │
+│                        │ Virtualization/Sandbox Evasion::User Activity Based Checks T1497.002               │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ DISCOVERY              │ Account Discovery T1087                                                            │
+│                        │ Application Window Discovery T1010                                                 │
+│                        │ File and Directory Discovery T1083                                                 │
+│                        │ Query Registry T1012                                                               │
+│                        │ System Information Discovery T1082                                                 │
+│                        │ System Location Discovery::System Language Discovery T1614.001                     │
+│                        │ System Owner/User Discovery T1033                                                  │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ EXECUTION              │ System Services::Service Execution T1569.002                                       │
+├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
+│ PERSISTENCE            │ Boot or Logon Autostart Execution::Registry Run Keys / Startup Folder T1547.001    │
+│                        │ Boot or Logon Autostart Execution::Winlogon Helper DLL T1547.004                   │
+│                        │ Create or Modify System Process::Windows Service T1543.003                         │
+┕━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
+
+┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
+│ Capability                                           │ Namespace                                            │
+┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
+│ check for unmoving mouse cursor (3 matches)          │ anti-analysis/anti-vm/vm-detection                   │
+│ gather bitkinex information                          │ collection/file-managers                             │
+│ gather classicftp information                        │ collection/file-managers                             │
+│ gather filezilla information                         │ collection/file-managers                             │
+│ gather total-commander information                   │ collection/file-managers                             │
+│ gather ultrafxp information                          │ collection/file-managers                             │
+│ resolve DNS (23 matches)                             │ communication/dns                                    │
+│ initialize Winsock library (7 matches)               │ communication/socket                                 │
+│ act as TCP client (3 matches)                        │ communication/tcp/client                             │
+│ create new key via CryptAcquireContext               │ data-manipulation/encryption                         │
+│ encrypt or decrypt via WinCrypt                      │ data-manipulation/encryption                         │
+│ hash data via WinCrypt                               │ data-manipulation/hashing                            │
+│ initialize hashing via WinCrypt                      │ data-manipulation/hashing                            │
+│ hash data with MD5                                   │ data-manipulation/hashing/md5                        │
+│ generate random numbers via WinAPI                   │ data-manipulation/prng                               │
+│ extract resource via kernel32 functions (2 matches)  │ executable/resource                                  │
+│ interact with driver via control codes (2 matches)   │ host-interaction/driver                              │
+│ get Program Files directory (18 matches)             │ host-interaction/file-system                         │
+│ get common file path (575 matches)                   │ host-interaction/file-system                         │
+│ create directory (2 matches)                         │ host-interaction/file-system/create                  │
+│ delete file                                          │ host-interaction/file-system/delete                  │
+│ get file attributes (122 matches)                    │ host-interaction/file-system/meta                    │
+│ set file attributes (8 matches)                      │ host-interaction/file-system/meta                    │
+│ move file                                            │ host-interaction/file-system/move                    │
+│ find taskbar (3 matches)                             │ host-interaction/gui/taskbar/find                    │
+│ get keyboard layout (12 matches)                     │ host-interaction/hardware/keyboard                   │
+│ get disk size                                        │ host-interaction/hardware/storage                    │
+│ get hostname (4 matches)                             │ host-interaction/os/hostname                         │
+│ allocate or change RWX memory (3 matches)            │ host-interaction/process/inject                      │
+│ query or enumerate registry key (3 matches)          │ host-interaction/registry                            │
+│ query or enumerate registry value (8 matches)        │ host-interaction/registry                            │
+│ delete registry key                                  │ host-interaction/registry/delete                     │
+│ start service                                        │ host-interaction/service/start                       │
+│ get session user name                                │ host-interaction/session                             │
+│ persist via Run registry key                         │ persistence/registry/run                             │
+│ persist via Winlogon Helper DLL registry key         │ persistence/registry/winlogon-helper                 │
+│ persist via Windows service (2 matches)              │ persistence/service                                  │
+┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
+```
+
 capa uses a collection of rules to identify capabilities within a program.
 These rules are easy to write, even for those new to reverse engineering.
 By authoring rules, you can extend the capabilities that capa recognizes.
@@ -135,31 +225,30 @@ Here's an example rule used by capa:
 ```yaml
 rule:
   meta:
-    name: hash data with CRC32
-    namespace: data-manipulation/checksum/crc32
+    name: create TCP socket
+    namespace: communication/socket/tcp
     authors:
-      - moritz.raabe@mandiant.com
-    scope: function
+      - william.ballenthin@mandiant.com
+      - joakim@intezer.com
+      - anushka.virgaonkar@mandiant.com
+    scopes:
+      static: basic block
+      dynamic: call
     mbc:
-      - Data::Checksum::CRC32 [C0032.001]
+      - Communication::Socket Communication::Create TCP Socket [C0001.011]
     examples:
-      - 2D3EDC218A90F03089CC01715A9F047F:0x403CBD
-      - 7D28CB106CB54876B2A5C111724A07CD:0x402350  # RtlComputeCrc32
-      - 7EFF498DE13CC734262F87E6B3EF38AB:0x100084A6
+      - Practical Malware Analysis Lab 01-01.dll_:0x10001010
   features:
     - or:
       - and:
-        - mnemonic: shr
+        - number: 6 = IPPROTO_TCP
+        - number: 1 = SOCK_STREAM
+        - number: 2 = AF_INET
         - or:
-          - number: 0xEDB88320
-          - bytes: 00 00 00 00 96 30 07 77 2C 61 0E EE BA 51 09 99 19 C4 6D 07 8F F4 6A 70 35 A5 63 E9 A3 95 64 9E = crc32_tab
-        - number: 8
-        - characteristic: nzxor
-      - and:
-        - number: 0x8320
-        - number: 0xEDB8
-        - characteristic: nzxor
-      - api: RtlComputeCrc32
+          - api: ws2_32.socket
+          - api: ws2_32.WSASocket
+          - api: socket
+      - property/read: System.Net.Sockets.TcpClient::Client
 ```
 
 The [github.com/mandiant/capa-rules](https://github.com/mandiant/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
@@ -170,6 +259,8 @@ capa explorer helps you identify interesting areas of a program and build new ca
 
 ![capa + IDA Pro integration](https://github.com/mandiant/capa/blob/master/doc/img/explorer_expanded.png)
 
+If you use Ghidra, you can use the Python 3 [Ghidra feature extractor](/capa/ghidra/). This integration enables capa to extract features directly from your Ghidra database, which can help you identify capabilities in programs that you analyze using Ghidra.
+
 # further information
 ## capa
 - [Installation](https://github.com/mandiant/capa/blob/master/doc/installation.md)

capa/capabilities/common.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+import itertools
+import collections
+from typing import Any, Tuple
+
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.features.address import NO_ADDRESS
+from capa.features.extractors.base_extractor import FeatureExtractor, StaticFeatureExtractor, DynamicFeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def find_file_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, function_features: FeatureSet):
+    file_features: FeatureSet = collections.defaultdict(set)
+
+    for feature, va in itertools.chain(extractor.extract_file_features(), extractor.extract_global_features()):
+        # not all file features may have virtual addresses.
+        # if not, then at least ensure the feature shows up in the index.
+        # the set of addresses will still be empty.
+        if va:
+            file_features[feature].add(va)
+        else:
+            if feature not in file_features:
+                file_features[feature] = set()
+
+    logger.debug("analyzed file and extracted %d features", len(file_features))
+
+    file_features.update(function_features)
+
+    _, matches = ruleset.match(Scope.FILE, file_features, NO_ADDRESS)
+    return matches, len(file_features)
+
+
+def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalone=True) -> bool:
+    file_limitation_rules = list(filter(lambda r: r.is_file_limitation_rule(), rules.rules.values()))
+
+    for file_limitation_rule in file_limitation_rules:
+        if file_limitation_rule.name not in capabilities:
+            continue
+
+        logger.warning("-" * 80)
+        for line in file_limitation_rule.meta.get("description", "").split("\n"):
+            logger.warning(" %s", line)
+        logger.warning(" Identified via rule: %s", file_limitation_rule.name)
+        if is_standalone:
+            logger.warning(" ")
+            logger.warning(" Use -v or -vv if you really want to see the capabilities identified by capa.")
+        logger.warning("-" * 80)
+
+        # bail on first file limitation
+        return True
+
+    return False
+
+
+def find_capabilities(
+    ruleset: RuleSet, extractor: FeatureExtractor, disable_progress=None, **kwargs
+) -> Tuple[MatchResults, Any]:
+    from capa.capabilities.static import find_static_capabilities
+    from capa.capabilities.dynamic import find_dynamic_capabilities
+
+    if isinstance(extractor, StaticFeatureExtractor):
+        # for the time being, extractors are either static or dynamic.
+        # Remove this assertion once that has changed
+        assert not isinstance(extractor, DynamicFeatureExtractor)
+        return find_static_capabilities(ruleset, extractor, disable_progress=disable_progress, **kwargs)
+    if isinstance(extractor, DynamicFeatureExtractor):
+        return find_dynamic_capabilities(ruleset, extractor, disable_progress=disable_progress, **kwargs)
+
+    raise ValueError(f"unexpected extractor type: {extractor.__class__.__name__}")

capa/capabilities/dynamic.py

@@ -0,0 +1,198 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+import itertools
+import collections
+from typing import Any, Tuple
+
+import tqdm
+
+import capa.perf
+import capa.features.freeze as frz
+import capa.render.result_document as rdoc
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.helpers import redirecting_print_to_tqdm
+from capa.capabilities.common import find_file_capabilities
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle, DynamicFeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def find_call_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+) -> Tuple[FeatureSet, MatchResults]:
+    """
+    find matches for the given rules for the given call.
+
+    returns: tuple containing (features for call, match results for call)
+    """
+    # all features found for the call.
+    features: FeatureSet = collections.defaultdict(set)
+
+    for feature, addr in itertools.chain(
+        extractor.extract_call_features(ph, th, ch), extractor.extract_global_features()
+    ):
+        features[feature].add(addr)
+
+    # matches found at this thread.
+    _, matches = ruleset.match(Scope.CALL, features, ch.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for addr, _ in res:
+            capa.engine.index_rule_matches(features, rule, [addr])
+
+    return features, matches
+
+
+def find_thread_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle
+) -> Tuple[FeatureSet, MatchResults, MatchResults]:
+    """
+    find matches for the given rules within the given thread.
+
+    returns: tuple containing (features for thread, match results for thread, match results for calls)
+    """
+    # all features found within this thread,
+    # includes features found within calls.
+    features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the call scope.
+    # might be found at different calls, thats ok.
+    call_matches: MatchResults = collections.defaultdict(list)
+
+    for ch in extractor.get_calls(ph, th):
+        ifeatures, imatches = find_call_capabilities(ruleset, extractor, ph, th, ch)
+        for feature, vas in ifeatures.items():
+            features[feature].update(vas)
+
+        for rule_name, res in imatches.items():
+            call_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(extractor.extract_thread_features(ph, th), extractor.extract_global_features()):
+        features[feature].add(va)
+
+    # matches found within this thread.
+    _, matches = ruleset.match(Scope.THREAD, features, th.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for va, _ in res:
+            capa.engine.index_rule_matches(features, rule, [va])
+
+    return features, matches, call_matches
+
+
+def find_process_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle
+) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
+    """
+    find matches for the given rules within the given process.
+
+    returns: tuple containing (match results for process, match results for threads, match results for calls, number of features)
+    """
+    # all features found within this process,
+    # includes features found within threads (and calls).
+    process_features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the basic threads.
+    # might be found at different threads, thats ok.
+    thread_matches: MatchResults = collections.defaultdict(list)
+
+    # matches found at the call scope.
+    # might be found at different calls, thats ok.
+    call_matches: MatchResults = collections.defaultdict(list)
+
+    for th in extractor.get_threads(ph):
+        features, tmatches, cmatches = find_thread_capabilities(ruleset, extractor, ph, th)
+        for feature, vas in features.items():
+            process_features[feature].update(vas)
+
+        for rule_name, res in tmatches.items():
+            thread_matches[rule_name].extend(res)
+
+        for rule_name, res in cmatches.items():
+            call_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(extractor.extract_process_features(ph), extractor.extract_global_features()):
+        process_features[feature].add(va)
+
+    _, process_matches = ruleset.match(Scope.PROCESS, process_features, ph.address)
+    return process_matches, thread_matches, call_matches, len(process_features)
+
+
+def find_dynamic_capabilities(
+    ruleset: RuleSet, extractor: DynamicFeatureExtractor, disable_progress=None
+) -> Tuple[MatchResults, Any]:
+    all_process_matches: MatchResults = collections.defaultdict(list)
+    all_thread_matches: MatchResults = collections.defaultdict(list)
+    all_call_matches: MatchResults = collections.defaultdict(list)
+
+    feature_counts = rdoc.DynamicFeatureCounts(file=0, processes=())
+
+    assert isinstance(extractor, DynamicFeatureExtractor)
+    with redirecting_print_to_tqdm(disable_progress):
+        with tqdm.contrib.logging.logging_redirect_tqdm():
+            pbar = tqdm.tqdm
+            if disable_progress:
+                # do not use tqdm to avoid unnecessary side effects when caller intends
+                # to disable progress completely
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            processes = list(extractor.get_processes())
+
+            pb = pbar(processes, desc="matching", unit=" processes", leave=False)
+            for p in pb:
+                process_matches, thread_matches, call_matches, feature_count = find_process_capabilities(
+                    ruleset, extractor, p
+                )
+                feature_counts.processes += (
+                    rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count),
+                )
+                logger.debug("analyzed %s and extracted %d features", p.address, feature_count)
+
+                for rule_name, res in process_matches.items():
+                    all_process_matches[rule_name].extend(res)
+                for rule_name, res in thread_matches.items():
+                    all_thread_matches[rule_name].extend(res)
+                for rule_name, res in call_matches.items():
+                    all_call_matches[rule_name].extend(res)
+
+    # collection of features that captures the rule matches within process and thread scopes.
+    # mapping from feature (matched rule) to set of addresses at which it matched.
+    process_and_lower_features: FeatureSet = collections.defaultdict(set)
+    for rule_name, results in itertools.chain(
+        all_process_matches.items(), all_thread_matches.items(), all_call_matches.items()
+    ):
+        locations = {p[0] for p in results}
+        rule = ruleset[rule_name]
+        capa.engine.index_rule_matches(process_and_lower_features, rule, locations)
+
+    all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, process_and_lower_features)
+    feature_counts.file = feature_count
+
+    matches = dict(
+        itertools.chain(
+            # each rule exists in exactly one scope,
+            # so there won't be any overlap among these following MatchResults,
+            # and we can merge the dictionaries naively.
+            all_thread_matches.items(),
+            all_process_matches.items(),
+            all_call_matches.items(),
+            all_file_matches.items(),
+        )
+    )
+
+    meta = {
+        "feature_counts": feature_counts,
+    }
+
+    return matches, meta

capa/capabilities/static.py

@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import time
+import logging
+import itertools
+import collections
+from typing import Any, Tuple
+
+import tqdm.contrib.logging
+
+import capa.perf
+import capa.features.freeze as frz
+import capa.render.result_document as rdoc
+from capa.rules import Scope, RuleSet
+from capa.engine import FeatureSet, MatchResults
+from capa.helpers import redirecting_print_to_tqdm
+from capa.capabilities.common import find_file_capabilities
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, StaticFeatureExtractor
+
+logger = logging.getLogger(__name__)
+
+
+def find_instruction_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
+) -> Tuple[FeatureSet, MatchResults]:
+    """
+    find matches for the given rules for the given instruction.
+
+    returns: tuple containing (features for instruction, match results for instruction)
+    """
+    # all features found for the instruction.
+    features: FeatureSet = collections.defaultdict(set)
+
+    for feature, addr in itertools.chain(
+        extractor.extract_insn_features(f, bb, insn), extractor.extract_global_features()
+    ):
+        features[feature].add(addr)
+
+    # matches found at this instruction.
+    _, matches = ruleset.match(Scope.INSTRUCTION, features, insn.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for addr, _ in res:
+            capa.engine.index_rule_matches(features, rule, [addr])
+
+    return features, matches
+
+
+def find_basic_block_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle
+) -> Tuple[FeatureSet, MatchResults, MatchResults]:
+    """
+    find matches for the given rules within the given basic block.
+
+    returns: tuple containing (features for basic block, match results for basic block, match results for instructions)
+    """
+    # all features found within this basic block,
+    # includes features found within instructions.
+    features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the instruction scope.
+    # might be found at different instructions, thats ok.
+    insn_matches: MatchResults = collections.defaultdict(list)
+
+    for insn in extractor.get_instructions(f, bb):
+        ifeatures, imatches = find_instruction_capabilities(ruleset, extractor, f, bb, insn)
+        for feature, vas in ifeatures.items():
+            features[feature].update(vas)
+
+        for rule_name, res in imatches.items():
+            insn_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(
+        extractor.extract_basic_block_features(f, bb), extractor.extract_global_features()
+    ):
+        features[feature].add(va)
+
+    # matches found within this basic block.
+    _, matches = ruleset.match(Scope.BASIC_BLOCK, features, bb.address)
+
+    for rule_name, res in matches.items():
+        rule = ruleset[rule_name]
+        for va, _ in res:
+            capa.engine.index_rule_matches(features, rule, [va])
+
+    return features, matches, insn_matches
+
+
+def find_code_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, fh: FunctionHandle
+) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
+    """
+    find matches for the given rules within the given function.
+
+    returns: tuple containing (match results for function, match results for basic blocks, match results for instructions, number of features)
+    """
+    # all features found within this function,
+    # includes features found within basic blocks (and instructions).
+    function_features: FeatureSet = collections.defaultdict(set)
+
+    # matches found at the basic block scope.
+    # might be found at different basic blocks, thats ok.
+    bb_matches: MatchResults = collections.defaultdict(list)
+
+    # matches found at the instruction scope.
+    # might be found at different instructions, thats ok.
+    insn_matches: MatchResults = collections.defaultdict(list)
+
+    for bb in extractor.get_basic_blocks(fh):
+        features, bmatches, imatches = find_basic_block_capabilities(ruleset, extractor, fh, bb)
+        for feature, vas in features.items():
+            function_features[feature].update(vas)
+
+        for rule_name, res in bmatches.items():
+            bb_matches[rule_name].extend(res)
+
+        for rule_name, res in imatches.items():
+            insn_matches[rule_name].extend(res)
+
+    for feature, va in itertools.chain(extractor.extract_function_features(fh), extractor.extract_global_features()):
+        function_features[feature].add(va)
+
+    _, function_matches = ruleset.match(Scope.FUNCTION, function_features, fh.address)
+    return function_matches, bb_matches, insn_matches, len(function_features)
+
+
+def find_static_capabilities(
+    ruleset: RuleSet, extractor: StaticFeatureExtractor, disable_progress=None
+) -> Tuple[MatchResults, Any]:
+    all_function_matches: MatchResults = collections.defaultdict(list)
+    all_bb_matches: MatchResults = collections.defaultdict(list)
+    all_insn_matches: MatchResults = collections.defaultdict(list)
+
+    feature_counts = rdoc.StaticFeatureCounts(file=0, functions=())
+    library_functions: Tuple[rdoc.LibraryFunction, ...] = ()
+
+    assert isinstance(extractor, StaticFeatureExtractor)
+    with redirecting_print_to_tqdm(disable_progress):
+        with tqdm.contrib.logging.logging_redirect_tqdm():
+            pbar = tqdm.tqdm
+            if capa.helpers.is_runtime_ghidra():
+                # Ghidrathon interpreter cannot properly handle
+                # the TMonitor thread that is created via a monitor_interval
+                # > 0
+                pbar.monitor_interval = 0
+            if disable_progress:
+                # do not use tqdm to avoid unnecessary side effects when caller intends
+                # to disable progress completely
+                def pbar(s, *args, **kwargs):
+                    return s
+
+            functions = list(extractor.get_functions())
+            n_funcs = len(functions)
+
+            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False)
+            for f in pb:
+                t0 = time.time()
+                if extractor.is_library_function(f.address):
+                    function_name = extractor.get_function_name(f.address)
+                    logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
+                    library_functions += (
+                        rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name),
+                    )
+                    n_libs = len(library_functions)
+                    percentage = round(100 * (n_libs / n_funcs))
+                    if isinstance(pb, tqdm.tqdm):
+                        pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
+                    continue
+
+                function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(
+                    ruleset, extractor, f
+                )
+                feature_counts.functions += (
+                    rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count),
+                )
+                t1 = time.time()
+
+                match_count = sum(len(res) for res in function_matches.values())
+                match_count += sum(len(res) for res in bb_matches.values())
+                match_count += sum(len(res) for res in insn_matches.values())
+                logger.debug(
+                    "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
+                    f.address,
+                    feature_count,
+                    match_count,
+                    t1 - t0,
+                )
+
+                for rule_name, res in function_matches.items():
+                    all_function_matches[rule_name].extend(res)
+                for rule_name, res in bb_matches.items():
+                    all_bb_matches[rule_name].extend(res)
+                for rule_name, res in insn_matches.items():
+                    all_insn_matches[rule_name].extend(res)
+
+    # collection of features that captures the rule matches within function, BB, and instruction scopes.
+    # mapping from feature (matched rule) to set of addresses at which it matched.
+    function_and_lower_features: FeatureSet = collections.defaultdict(set)
+    for rule_name, results in itertools.chain(
+        all_function_matches.items(), all_bb_matches.items(), all_insn_matches.items()
+    ):
+        locations = {p[0] for p in results}
+        rule = ruleset[rule_name]
+        capa.engine.index_rule_matches(function_and_lower_features, rule, locations)
+
+    all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, function_and_lower_features)
+    feature_counts.file = feature_count
+
+    matches = dict(
+        itertools.chain(
+            # each rule exists in exactly one scope,
+            # so there won't be any overlap among these following MatchResults,
+            # and we can merge the dictionaries naively.
+            all_insn_matches.items(),
+            all_bb_matches.items(),
+            all_function_matches.items(),
+            all_file_matches.items(),
+        )
+    )
+
+    meta = {
+        "feature_counts": feature_counts,
+        "library_functions": library_functions,
+    }
+
+    return matches, meta

capa/engine.py

@@ -304,7 +304,7 @@ def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -
     other strategies can be imagined that match differently; implement these elsewhere.
     specifically, this routine does "top down" matching of the given rules against the feature set.
     """
-    results = collections.defaultdict(list)  # type: MatchResults
+    results: MatchResults = collections.defaultdict(list)
 
     # copy features so that we can modify it
     # without affecting the caller (keep this function pure)

capa/exceptions.py

@@ -19,3 +19,7 @@ class UnsupportedArchError(ValueError):
 
 class UnsupportedOSError(ValueError):
     pass
+
+
+class EmptyReportError(ValueError):
+    pass

capa/features/address.py

@@ -43,6 +43,79 @@ class AbsoluteVirtualAddress(int, Address):
         return int.__hash__(self)
 
 
+class ProcessAddress(Address):
+    """an address of a process in a dynamic execution trace"""
+
+    def __init__(self, pid: int, ppid: int = 0):
+        assert ppid >= 0
+        assert pid > 0
+        self.ppid = ppid
+        self.pid = pid
+
+    def __repr__(self):
+        return "process(%s%s)" % (
+            f"ppid: {self.ppid}, " if self.ppid > 0 else "",
+            f"pid: {self.pid}",
+        )
+
+    def __hash__(self):
+        return hash((self.ppid, self.pid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) == (other.ppid, other.pid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ProcessAddress)
+        return (self.ppid, self.pid) < (other.ppid, other.pid)
+
+
+class ThreadAddress(Address):
+    """addresses a thread in a dynamic execution trace"""
+
+    def __init__(self, process: ProcessAddress, tid: int):
+        assert tid >= 0
+        self.process = process
+        self.tid = tid
+
+    def __repr__(self):
+        return f"{self.process}, thread(tid: {self.tid})"
+
+    def __hash__(self):
+        return hash((self.process, self.tid))
+
+    def __eq__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) == (other.process, other.tid)
+
+    def __lt__(self, other):
+        assert isinstance(other, ThreadAddress)
+        return (self.process, self.tid) < (other.process, other.tid)
+
+
+class DynamicCallAddress(Address):
+    """addesses a call in a dynamic execution trace"""
+
+    def __init__(self, thread: ThreadAddress, id: int):
+        assert id >= 0
+        self.thread = thread
+        self.id = id
+
+    def __repr__(self):
+        return f"{self.thread}, call(id: {self.id})"
+
+    def __hash__(self):
+        return hash((self.thread, self.id))
+
+    def __eq__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) == (other.thread, other.id)
+
+    def __lt__(self, other):
+        assert isinstance(other, DynamicCallAddress)
+        return (self.thread, self.id) < (other.thread, other.id)
+
+
 class RelativeVirtualAddress(int, Address):
     """a memory address relative to a base address"""
 

capa/features/com/__init__.py

@@ -0,0 +1,36 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from enum import Enum
+from typing import Dict, List
+
+from capa.helpers import assert_never
+
+
+class ComType(Enum):
+    CLASS = "class"
+    INTERFACE = "interface"
+
+
+COM_PREFIXES = {
+    ComType.CLASS: "CLSID_",
+    ComType.INTERFACE: "IID_",
+}
+
+
+def load_com_database(com_type: ComType) -> Dict[str, List[str]]:
+    # lazy load these python files since they are so large.
+    # that is, don't load them unless a COM feature is being handled.
+    import capa.features.com.classes
+    import capa.features.com.interfaces
+
+    if com_type == ComType.CLASS:
+        return capa.features.com.classes.COM_CLASSES
+    elif com_type == ComType.INTERFACE:
+        return capa.features.com.interfaces.COM_INTERFACES
+    else:
+        assert_never(com_type)

capa/features/common.py

@@ -457,6 +457,17 @@ VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET)
 FORMAT_AUTO = "auto"
 FORMAT_SC32 = "sc32"
 FORMAT_SC64 = "sc64"
+FORMAT_CAPE = "cape"
+STATIC_FORMATS = {
+    FORMAT_SC32,
+    FORMAT_SC64,
+    FORMAT_PE,
+    FORMAT_ELF,
+    FORMAT_DOTNET,
+}
+DYNAMIC_FORMATS = {
+    FORMAT_CAPE,
+}
 FORMAT_FREEZE = "freeze"
 FORMAT_RESULT = "result"
 FORMAT_UNKNOWN = "unknown"

capa/features/extractors/base_extractor.py

@@ -7,13 +7,18 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import abc
+import hashlib
 import dataclasses
 from typing import Any, Dict, Tuple, Union, Iterator
 from dataclasses import dataclass
 
+# TODO(williballenthin): use typing.TypeAlias directly when Python 3.9 is deprecated
+# https://github.com/mandiant/capa/issues/1699
+from typing_extensions import TypeAlias
+
 import capa.features.address
 from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.address import Address, ThreadAddress, ProcessAddress, DynamicCallAddress, AbsoluteVirtualAddress
 
 # feature extractors may reference functions, BBs, insns by opaque handle values.
 # you can use the `.address` property to get and render the address of the feature.
@@ -22,6 +27,24 @@ from capa.features.address import Address, AbsoluteVirtualAddress
 # the feature extractor from which they were created.
 
 
+@dataclass
+class SampleHashes:
+    md5: str
+    sha1: str
+    sha256: str
+
+    @classmethod
+    def from_bytes(cls, buf: bytes) -> "SampleHashes":
+        md5 = hashlib.md5()
+        sha1 = hashlib.sha1()
+        sha256 = hashlib.sha256()
+        md5.update(buf)
+        sha1.update(buf)
+        sha256.update(buf)
+
+        return cls(md5=md5.hexdigest(), sha1=sha1.hexdigest(), sha256=sha256.hexdigest())
+
+
 @dataclass
 class FunctionHandle:
     """reference to a function recognized by a feature extractor.
@@ -63,16 +86,18 @@ class InsnHandle:
     inner: Any
 
 
-class FeatureExtractor:
+class StaticFeatureExtractor:
     """
-    FeatureExtractor defines the interface for fetching features from a sample.
+    StaticFeatureExtractor defines the interface for fetching features from a
+    sample without running it; extractors that rely on the execution trace of
+    a sample must implement the other sibling class, DynamicFeatureExtracor.
 
     There may be multiple backends that support fetching features for capa.
     For example, we use vivisect by default, but also want to support saving
      and restoring features from a JSON file.
     When we restore the features, we'd like to use exactly the same matching logic
      to find matching rules.
-    Therefore, we can define a FeatureExtractor that provides features from the
+    Therefore, we can define a StaticFeatureExtractor that provides features from the
      serialized JSON file and do matching without a binary analysis pass.
     Also, this provides a way to hook in an IDA backend.
 
@@ -81,13 +106,14 @@ class FeatureExtractor:
 
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self):
+    def __init__(self, hashes: SampleHashes):
         #
         # note: a subclass should define ctor parameters for its own use.
         #  for example, the Vivisect feature extract might require the vw and/or path.
         # this base class doesn't know what to do with that info, though.
         #
         super().__init__()
+        self._sample_hashes = hashes
 
     @abc.abstractmethod
     def get_base_address(self) -> Union[AbsoluteVirtualAddress, capa.features.address._NoAddress]:
@@ -100,6 +126,12 @@ class FeatureExtractor:
         """
         raise NotImplementedError()
 
+    def get_sample_hashes(self) -> SampleHashes:
+        """
+        fetch the hashes for the sample contained within the extractor.
+        """
+        return self._sample_hashes
+
     @abc.abstractmethod
     def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
         """
@@ -262,3 +294,177 @@ class FeatureExtractor:
           Tuple[Feature, Address]: feature and its location
         """
         raise NotImplementedError()
+
+
+@dataclass
+class ProcessHandle:
+    """
+    reference to a process extracted by the sandbox.
+
+    Attributes:
+        address: process's address (pid)
+        inner: sandbox-specific data
+    """
+
+    address: ProcessAddress
+    inner: Any
+
+
+@dataclass
+class ThreadHandle:
+    """
+    reference to a thread extracted by the sandbox.
+
+    Attributes:
+        address: thread's address (tid)
+        inner: sandbox-specific data
+    """
+
+    address: ThreadAddress
+    inner: Any
+
+
+@dataclass
+class CallHandle:
+    """
+    reference to an api call extracted by the sandbox.
+
+    Attributes:
+        address: call's address, such as event index or id
+        inner: sandbox-specific data
+    """
+
+    address: DynamicCallAddress
+    inner: Any
+
+
+class DynamicFeatureExtractor:
+    """
+    DynamicFeatureExtractor defines the interface for fetching features from a
+    sandbox' analysis of a sample; extractors that rely on statically analyzing
+    a sample must implement the sibling extractor, StaticFeatureExtractor.
+
+    Features are grouped mainly into threads that alongside their meta-features are also grouped into
+    processes (that also have their own features). Other scopes (such as function and file) may also apply
+    for a specific sandbox.
+
+    This class is not instantiated directly; it is the base class for other implementations.
+    """
+
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, hashes: SampleHashes):
+        #
+        # note: a subclass should define ctor parameters for its own use.
+        #  for example, the Vivisect feature extract might require the vw and/or path.
+        # this base class doesn't know what to do with that info, though.
+        #
+        super().__init__()
+        self._sample_hashes = hashes
+
+    def get_sample_hashes(self) -> SampleHashes:
+        """
+        fetch the hashes for the sample contained within the extractor.
+        """
+        return self._sample_hashes
+
+    @abc.abstractmethod
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
+        """
+        extract features found at every scope ("global").
+
+        example::
+
+            extractor = CapeFeatureExtractor.from_report(json.loads(buf))
+            for feature, addr in extractor.get_global_features():
+                print(addr, feature)
+
+        yields:
+          Tuple[Feature, Address]: feature and its location
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
+        """
+        extract file-scope features.
+
+        example::
+
+            extractor = CapeFeatureExtractor.from_report(json.loads(buf))
+            for feature, addr in extractor.get_file_features():
+                print(addr, feature)
+
+        yields:
+          Tuple[Feature, Address]: feature and its location
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_processes(self) -> Iterator[ProcessHandle]:
+        """
+        Enumerate processes in the trace.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+        """
+        Yields all the features of a process. These include:
+        - file features of the process' image
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_process_name(self, ph: ProcessHandle) -> str:
+        """
+        Returns the human-readable name for the given process,
+        such as the filename.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
+        """
+        Enumerate threads in the given process.
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
+        """
+        Yields all the features of a thread. These include:
+        - sequenced api traces
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+        """
+        Enumerate calls in the given thread
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def extract_call_features(
+        self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
+        """
+        Yields all features of a call. These include:
+        - api name
+        - bytes/strings/numbers extracted from arguments
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def get_call_name(self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> str:
+        """
+        Returns the human-readable name for the given call,
+        such as as rendered API log entry, like:
+
+            Foo(1, "two", b"\x00\x11") -> -1
+        """
+        raise NotImplementedError()
+
+
+FeatureExtractor: TypeAlias = Union[StaticFeatureExtractor, DynamicFeatureExtractor]

capa/features/extractors/binja/extractor.py

@@ -17,12 +17,18 @@ import capa.features.extractors.binja.function
 import capa.features.extractors.binja.basicblock
 from capa.features.common import Feature
 from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
 
 
-class BinjaFeatureExtractor(FeatureExtractor):
+class BinjaFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, bv: binja.BinaryView):
-        super().__init__()
+        super().__init__(hashes=SampleHashes.from_bytes(bv.file.raw.read(0, len(bv.file.raw))))
         self.bv = bv
         self.global_features: List[Tuple[Feature, Address]] = []
         self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv))

capa/features/extractors/binja/file.py

@@ -17,7 +17,7 @@ import capa.features.extractors.strings
 from capa.features.file import Export, Import, Section, FunctionName
 from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
 from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.binja.helpers import unmangle_c_name
+from capa.features.extractors.binja.helpers import read_c_string, unmangle_c_name
 
 
 def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[int, int]]:
@@ -82,6 +82,24 @@ def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address
             if name != unmangled_name:
                 yield Export(unmangled_name), AbsoluteVirtualAddress(sym.address)
 
+    for sym in bv.get_symbols_of_type(SymbolType.DataSymbol):
+        if sym.binding not in [SymbolBinding.GlobalBinding]:
+            continue
+
+        name = sym.short_name
+        if not name.startswith("__forwarder_name"):
+            continue
+
+        # Due to https://github.com/Vector35/binaryninja-api/issues/4641, in binja version 3.5, the symbol's name
+        # does not contain the DLL name. As a workaround, we read the C string at the symbol's address, which contains
+        # both the DLL name and the function name.
+        # Once the above issue is closed in the next binjs stable release, we can update the code here to use the
+        # symbol name directly.
+        name = read_c_string(bv, sym.address, 1024)
+        forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(name)
+        yield Export(forwarded_name), AbsoluteVirtualAddress(sym.address)
+        yield Characteristic("forwarded export"), AbsoluteVirtualAddress(sym.address)
+
 
 def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
     """extract function imports
@@ -97,13 +115,13 @@ def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address
     for sym in bv.get_symbols_of_type(SymbolType.ImportAddressSymbol):
         lib_name = str(sym.namespace)
         addr = AbsoluteVirtualAddress(sym.address)
-        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym.short_name):
+        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym.short_name, include_dll=True):
             yield Import(name), addr
 
         ordinal = sym.ordinal
         if ordinal != 0 and (lib_name != ""):
             ordinal_name = f"#{ordinal}"
-            for name in capa.features.extractors.helpers.generate_symbols(lib_name, ordinal_name):
+            for name in capa.features.extractors.helpers.generate_symbols(lib_name, ordinal_name, include_dll=True):
                 yield Import(name), addr
 
 
@@ -125,15 +143,17 @@ def extract_file_function_names(bv: BinaryView) -> Iterator[Tuple[Feature, Addre
     """
     for sym_name in bv.symbols:
         for sym in bv.symbols[sym_name]:
-            if sym.type == SymbolType.LibraryFunctionSymbol:
-                name = sym.short_name
-                yield FunctionName(name), sym.address
-                if name.startswith("_"):
-                    # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                    # extract features for both the mangled and un-mangled representations.
-                    # e.g. `_fwrite` -> `fwrite`
-                    # see: https://stackoverflow.com/a/2628384/87207
-                    yield FunctionName(name[1:]), sym.address
+            if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
+                continue
+
+            name = sym.short_name
+            yield FunctionName(name), sym.address
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                yield FunctionName(name[1:]), sym.address
 
 
 def extract_file_format(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:

capa/features/extractors/binja/function.py

@@ -7,8 +7,9 @@
 # See the License for the specific language governing permissions and limitations under the License.
 from typing import Tuple, Iterator
 
-from binaryninja import Function, BinaryView, LowLevelILOperation
+from binaryninja import Function, BinaryView, SymbolType, RegisterValueType, LowLevelILOperation
 
+from capa.features.file import FunctionName
 from capa.features.common import Feature, Characteristic
 from capa.features.address import Address, AbsoluteVirtualAddress
 from capa.features.extractors import loops
@@ -23,13 +24,27 @@ def extract_function_calls_to(fh: FunctionHandle):
         # Everything that is a code reference to the current function is considered a caller, which actually includes
         # many other references that are NOT a caller. For example, an instruction `push function_start` will also be
         # considered a caller to the function
-        if caller.llil is not None and caller.llil.operation in [
+        llil = caller.llil
+        if (llil is None) or llil.operation not in [
             LowLevelILOperation.LLIL_CALL,
             LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
             LowLevelILOperation.LLIL_JUMP,
             LowLevelILOperation.LLIL_TAILCALL,
         ]:
-            yield Characteristic("calls to"), AbsoluteVirtualAddress(caller.address)
+            continue
+
+        if llil.dest.value.type not in [
+            RegisterValueType.ImportedAddressValue,
+            RegisterValueType.ConstantValue,
+            RegisterValueType.ConstantPointerValue,
+        ]:
+            continue
+
+        address = llil.dest.value.value
+        if address != func.start:
+            continue
+
+        yield Characteristic("calls to"), AbsoluteVirtualAddress(caller.address)
 
 
 def extract_function_loop(fh: FunctionHandle):
@@ -59,10 +74,31 @@ def extract_recursive_call(fh: FunctionHandle):
             yield Characteristic("recursive call"), fh.address
 
 
+def extract_function_name(fh: FunctionHandle):
+    """extract function names (e.g., symtab names)"""
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+    if bv is None:
+        return
+
+    for sym in bv.get_symbols(func.start):
+        if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
+            continue
+
+        name = sym.short_name
+        yield FunctionName(name), sym.address
+        if name.startswith("_"):
+            # some linkers may prefix linked routines with a `_` to avoid name collisions.
+            # extract features for both the mangled and un-mangled representations.
+            # e.g. `_fwrite` -> `fwrite`
+            # see: https://stackoverflow.com/a/2628384/87207
+            yield FunctionName(name[1:]), sym.address
+
+
 def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
     for func_handler in FUNCTION_HANDLERS:
         for feature, addr in func_handler(fh):
             yield feature, addr
 
 
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call, extract_function_name)

capa/features/extractors/binja/helpers.py

@@ -9,7 +9,7 @@ import re
 from typing import List, Callable
 from dataclasses import dataclass
 
-from binaryninja import LowLevelILInstruction
+from binaryninja import BinaryView, LowLevelILInstruction
 from binaryninja.architecture import InstructionTextToken
 
 
@@ -51,3 +51,19 @@ def unmangle_c_name(name: str) -> str:
             return match.group(1)
 
     return name
+
+
+def read_c_string(bv: BinaryView, offset: int, max_len: int) -> str:
+    s: List[str] = []
+    while len(s) < max_len:
+        try:
+            c = bv.read(offset + len(s), 1)[0]
+        except Exception:
+            break
+
+        if c == 0:
+            break
+
+        s.append(chr(c))
+
+    return "".join(s)

capa/features/extractors/binja/insn.py

@@ -94,28 +94,32 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
                 candidate_addrs.append(stub_addr)
 
             for address in candidate_addrs:
-                sym = func.view.get_symbol_at(address)
-                if sym is None or sym.type not in [SymbolType.ImportAddressSymbol, SymbolType.ImportedFunctionSymbol]:
-                    continue
+                for sym in func.view.get_symbols(address):
+                    if sym is None or sym.type not in [
+                        SymbolType.ImportAddressSymbol,
+                        SymbolType.ImportedFunctionSymbol,
+                        SymbolType.FunctionSymbol,
+                    ]:
+                        continue
 
-                sym_name = sym.short_name
+                    sym_name = sym.short_name
 
-                lib_name = ""
-                import_lib = bv.lookup_imported_object_library(sym.address)
-                if import_lib is not None:
-                    lib_name = import_lib[0].name
-                    if lib_name.endswith(".dll"):
-                        lib_name = lib_name[:-4]
-                    elif lib_name.endswith(".so"):
-                        lib_name = lib_name[:-3]
+                    lib_name = ""
+                    import_lib = bv.lookup_imported_object_library(sym.address)
+                    if import_lib is not None:
+                        lib_name = import_lib[0].name
+                        if lib_name.endswith(".dll"):
+                            lib_name = lib_name[:-4]
+                        elif lib_name.endswith(".so"):
+                            lib_name = lib_name[:-3]
 
-                for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name):
-                    yield API(name), ih.address
-
-                if sym_name.startswith("_"):
-                    for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name[1:]):
+                    for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name):
                         yield API(name), ih.address
 
+                    if sym_name.startswith("_"):
+                        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name[1:]):
+                            yield API(name), ih.address
+
 
 def extract_insn_number_features(
     fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle

capa/features/extractors/cape/call.py

@@ -0,0 +1,62 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.helpers import assert_never
+from capa.features.insn import API, Number
+from capa.features.common import String, Feature
+from capa.features.address import Address
+from capa.features.extractors.cape.models import Call
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    this method extracts the given call's features (such as API name and arguments),
+    and returns them as API, Number, and String features.
+
+    args:
+      ph: process handle (for defining the extraction scope)
+      th: thread handle (for defining the extraction scope)
+      ch: call handle (for defining the extraction scope)
+
+    yields:
+      Feature, address; where Feature is either: API, Number, or String.
+    """
+    call: Call = ch.inner
+
+    # list similar to disassembly: arguments right-to-left, call
+    for arg in reversed(call.arguments):
+        value = arg.value
+        if isinstance(value, list) and len(value) == 0:
+            # unsure why CAPE captures arguments as empty lists?
+            continue
+
+        elif isinstance(value, str):
+            yield String(value), ch.address
+
+        elif isinstance(value, int):
+            yield Number(value), ch.address
+
+        else:
+            assert_never(value)
+
+    yield API(call.api), ch.address
+
+
+def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
+    for handler in CALL_HANDLERS:
+        for feature, addr in handler(ph, th, ch):
+            yield feature, addr
+
+
+CALL_HANDLERS = (extract_call_features,)

capa/features/extractors/cape/extractor.py

@@ -0,0 +1,145 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Dict, Tuple, Union, Iterator
+
+import capa.features.extractors.cape.call
+import capa.features.extractors.cape.file
+import capa.features.extractors.cape.thread
+import capa.features.extractors.cape.global_
+import capa.features.extractors.cape.process
+from capa.exceptions import EmptyReportError, UnsupportedFormatError
+from capa.features.common import Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress, _NoAddress
+from capa.features.extractors.cape.models import Call, Static, Process, CapeReport
+from capa.features.extractors.base_extractor import (
+    CallHandle,
+    SampleHashes,
+    ThreadHandle,
+    ProcessHandle,
+    DynamicFeatureExtractor,
+)
+
+logger = logging.getLogger(__name__)
+
+TESTED_VERSIONS = {"2.2-CAPE", "2.4-CAPE"}
+
+
+class CapeExtractor(DynamicFeatureExtractor):
+    def __init__(self, report: CapeReport):
+        super().__init__(
+            hashes=SampleHashes(
+                md5=report.target.file.md5.lower(),
+                sha1=report.target.file.sha1.lower(),
+                sha256=report.target.file.sha256.lower(),
+            )
+        )
+        self.report: CapeReport = report
+
+        # pre-compute these because we'll yield them at *every* scope.
+        self.global_features = list(capa.features.extractors.cape.global_.extract_features(self.report))
+
+    def get_base_address(self) -> Union[AbsoluteVirtualAddress, _NoAddress, None]:
+        # value according to the PE header, the actual trace may use a different imagebase
+        assert self.report.static is not None and self.report.static.pe is not None
+        return AbsoluteVirtualAddress(self.report.static.pe.imagebase)
+
+    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from self.global_features
+
+    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.cape.file.extract_features(self.report)
+
+    def get_processes(self) -> Iterator[ProcessHandle]:
+        yield from capa.features.extractors.cape.file.get_processes(self.report)
+
+    def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.cape.process.extract_features(ph)
+
+    def get_process_name(self, ph) -> str:
+        process: Process = ph.inner
+        return process.process_name
+
+    def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
+        yield from capa.features.extractors.cape.process.get_threads(ph)
+
+    def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
+        if False:
+            # force this routine to be a generator,
+            # but we don't actually have any elements to generate.
+            yield Characteristic("never"), NO_ADDRESS
+        return
+
+    def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+        yield from capa.features.extractors.cape.thread.get_calls(ph, th)
+
+    def extract_call_features(
+        self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
+    ) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.cape.call.extract_features(ph, th, ch)
+
+    def get_call_name(self, ph, th, ch) -> str:
+        call: Call = ch.inner
+
+        parts = []
+        parts.append(call.api)
+        parts.append("(")
+        for argument in call.arguments:
+            parts.append(argument.name)
+            parts.append("=")
+
+            if argument.pretty_value:
+                parts.append(argument.pretty_value)
+            else:
+                if isinstance(argument.value, int):
+                    parts.append(hex(argument.value))
+                elif isinstance(argument.value, str):
+                    parts.append('"')
+                    parts.append(argument.value)
+                    parts.append('"')
+                elif isinstance(argument.value, list):
+                    pass
+                else:
+                    capa.helpers.assert_never(argument.value)
+
+            parts.append(", ")
+        if call.arguments:
+            # remove the trailing comma
+            parts.pop()
+        parts.append(")")
+        parts.append(" -> ")
+        if call.pretty_return:
+            parts.append(call.pretty_return)
+        else:
+            parts.append(hex(call.return_))
+
+        return "".join(parts)
+
+    @classmethod
+    def from_report(cls, report: Dict) -> "CapeExtractor":
+        cr = CapeReport.model_validate(report)
+
+        if cr.info.version not in TESTED_VERSIONS:
+            logger.warning("CAPE version '%s' not tested/supported yet", cr.info.version)
+
+        # observed in 2.4-CAPE reports from capesandbox.com
+        if cr.static is None and cr.target.file.pe is not None:
+            cr.static = Static()
+            cr.static.pe = cr.target.file.pe
+
+        if cr.static is None:
+            raise UnsupportedFormatError("CAPE report missing static analysis")
+
+        if cr.static.pe is None:
+            raise UnsupportedFormatError("CAPE report missing PE analysis")
+
+        if len(cr.behavior.processes) == 0:
+            raise EmptyReportError("CAPE did not capture any processes")
+
+        return cls(cr)

capa/features/extractors/cape/file.py

@@ -0,0 +1,132 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.file import Export, Import, Section
+from capa.features.common import String, Feature
+from capa.features.address import NO_ADDRESS, Address, ProcessAddress, AbsoluteVirtualAddress
+from capa.features.extractors.helpers import generate_symbols
+from capa.features.extractors.cape.models import CapeReport
+from capa.features.extractors.base_extractor import ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def get_processes(report: CapeReport) -> Iterator[ProcessHandle]:
+    """
+    get all the created processes for a sample
+    """
+    seen_processes = {}
+    for process in report.behavior.processes:
+        addr = ProcessAddress(pid=process.process_id, ppid=process.parent_id)
+        yield ProcessHandle(address=addr, inner=process)
+
+        # check for pid and ppid reuse
+        if addr not in seen_processes:
+            seen_processes[addr] = [process]
+        else:
+            logger.warning(
+                "pid and ppid reuse detected between process %s and process%s: %s",
+                process,
+                "es" if len(seen_processes[addr]) > 1 else "",
+                seen_processes[addr],
+            )
+            seen_processes[addr].append(process)
+
+
+def extract_import_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract imported function names
+    """
+    assert report.static is not None and report.static.pe is not None
+    imports = report.static.pe.imports
+
+    if isinstance(imports, dict):
+        imports = list(imports.values())
+
+    assert isinstance(imports, list)
+
+    for library in imports:
+        for function in library.imports:
+            if not function.name:
+                continue
+
+            for name in generate_symbols(library.dll, function.name, include_dll=True):
+                yield Import(name), AbsoluteVirtualAddress(function.address)
+
+
+def extract_export_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    assert report.static is not None and report.static.pe is not None
+    for function in report.static.pe.exports:
+        yield Export(function.name), AbsoluteVirtualAddress(function.address)
+
+
+def extract_section_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    assert report.static is not None and report.static.pe is not None
+    for section in report.static.pe.sections:
+        yield Section(section.name), AbsoluteVirtualAddress(section.virtual_address)
+
+
+def extract_file_strings(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    if report.strings is not None:
+        for string in report.strings:
+            yield String(string), NO_ADDRESS
+
+
+def extract_used_regkeys(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for regkey in report.behavior.summary.keys:
+        yield String(regkey), NO_ADDRESS
+
+
+def extract_used_files(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for file in report.behavior.summary.files:
+        yield String(file), NO_ADDRESS
+
+
+def extract_used_mutexes(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for mutex in report.behavior.summary.mutexes:
+        yield String(mutex), NO_ADDRESS
+
+
+def extract_used_commands(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for cmd in report.behavior.summary.executed_commands:
+        yield String(cmd), NO_ADDRESS
+
+
+def extract_used_apis(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for symbol in report.behavior.summary.resolved_apis:
+        yield String(symbol), NO_ADDRESS
+
+
+def extract_used_services(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for svc in report.behavior.summary.created_services:
+        yield String(svc), NO_ADDRESS
+    for svc in report.behavior.summary.started_services:
+        yield String(svc), NO_ADDRESS
+
+
+def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for handler in FILE_HANDLERS:
+        for feature, addr in handler(report):
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_import_names,
+    extract_export_names,
+    extract_section_names,
+    extract_file_strings,
+    extract_used_regkeys,
+    extract_used_files,
+    extract_used_mutexes,
+    extract_used_commands,
+    extract_used_apis,
+    extract_used_services,
+)

capa/features/extractors/cape/global_.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Tuple, Iterator
+
+from capa.features.common import (
+    OS,
+    OS_ANY,
+    OS_LINUX,
+    ARCH_I386,
+    FORMAT_PE,
+    ARCH_AMD64,
+    FORMAT_ELF,
+    OS_WINDOWS,
+    Arch,
+    Format,
+    Feature,
+)
+from capa.features.address import NO_ADDRESS, Address
+from capa.features.extractors.cape.models import CapeReport
+
+logger = logging.getLogger(__name__)
+
+
+def extract_arch(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    if "Intel 80386" in report.target.file.type:
+        yield Arch(ARCH_I386), NO_ADDRESS
+    elif "x86-64" in report.target.file.type:
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+    else:
+        logger.warning("unrecognized Architecture: %s", report.target.file.type)
+        raise ValueError(
+            f"unrecognized Architecture from the CAPE report; output of file command: {report.target.file.type}"
+        )
+
+
+def extract_format(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    if "PE" in report.target.file.type:
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif "ELF" in report.target.file.type:
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    else:
+        logger.warning("unknown file format, file command output: %s", report.target.file.type)
+        raise ValueError(
+            "unrecognized file format from the CAPE report; output of file command: {report.target.file.type}"
+        )
+
+
+def extract_os(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    # this variable contains the output of the file command
+    file_output = report.target.file.type
+
+    if "windows" in file_output.lower():
+        yield OS(OS_WINDOWS), NO_ADDRESS
+    elif "elf" in file_output.lower():
+        # operating systems recognized by the file command: https://github.com/file/file/blob/master/src/readelf.c#L609
+        if "Linux" in file_output:
+            yield OS(OS_LINUX), NO_ADDRESS
+        elif "Hurd" in file_output:
+            yield OS("hurd"), NO_ADDRESS
+        elif "Solaris" in file_output:
+            yield OS("solaris"), NO_ADDRESS
+        elif "kFreeBSD" in file_output:
+            yield OS("freebsd"), NO_ADDRESS
+        elif "kNetBSD" in file_output:
+            yield OS("netbsd"), NO_ADDRESS
+        else:
+            # if the operating system information is missing from the cape report, it's likely a bug
+            logger.warning("unrecognized OS: %s", file_output)
+            raise ValueError("unrecognized OS from the CAPE report; output of file command: {file_output}")
+    else:
+        # the sample is shellcode
+        logger.debug("unsupported file format, file command output: %s", file_output)
+        yield OS(OS_ANY), NO_ADDRESS
+
+
+def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
+    for global_handler in GLOBAL_HANDLER:
+        for feature, addr in global_handler(report):
+            yield feature, addr
+
+
+GLOBAL_HANDLER = (
+    extract_format,
+    extract_os,
+    extract_arch,
+)

capa/features/extractors/cape/helpers.py

@@ -0,0 +1,29 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from typing import Any, Dict, List
+
+from capa.features.extractors.base_extractor import ProcessHandle
+
+
+def find_process(processes: List[Dict[str, Any]], ph: ProcessHandle) -> Dict[str, Any]:
+    """
+    find a specific process identified by a process handler.
+
+    args:
+      processes: a list of processes extracted by CAPE
+      ph: handle of the sought process
+
+    return:
+      a CAPE-defined dictionary for the sought process' information
+    """
+
+    for process in processes:
+        if ph.address.ppid == process["parent_id"] and ph.address.pid == process["process_id"]:
+            return process
+    return {}

capa/features/extractors/cape/models.py

@@ -0,0 +1,446 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import binascii
+from typing import Any, Dict, List, Union, Literal, Optional
+
+from pydantic import Field, BaseModel, ConfigDict
+from typing_extensions import Annotated, TypeAlias
+from pydantic.functional_validators import BeforeValidator
+
+
+def validate_hex_int(value):
+    if isinstance(value, str):
+        return int(value, 16) if value.startswith("0x") else int(value, 10)
+    else:
+        return value
+
+
+def validate_hex_bytes(value):
+    return binascii.unhexlify(value) if isinstance(value, str) else value
+
+
+HexInt = Annotated[int, BeforeValidator(validate_hex_int)]
+HexBytes = Annotated[bytes, BeforeValidator(validate_hex_bytes)]
+
+
+# a model that *cannot* have extra fields
+# if they do, pydantic raises an exception.
+# use this for models we rely upon and cannot change.
+#
+# for things that may be extended and we don't care,
+# use FlexibleModel.
+class ExactModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+# a model that can have extra fields that we ignore.
+# use this if we don't want to raise an exception for extra
+# data fields that we didn't expect.
+class FlexibleModel(BaseModel):
+    pass
+
+
+# use this type to indicate that we won't model this data.
+# because its not relevant to our use in capa.
+#
+# while its nice to have full coverage of the data shape,
+# it can easily change and break our parsing.
+# so we really only want to describe what we'll use.
+Skip: TypeAlias = Optional[Any]
+
+
+# mark fields that we haven't seen yet and need to model.
+# pydantic should raise an error when encountering data
+# in a field with this type.
+# then we can update the model with the discovered shape.
+TODO: TypeAlias = None
+ListTODO: TypeAlias = List[None]
+DictTODO: TypeAlias = ExactModel
+
+EmptyDict: TypeAlias = BaseModel
+EmptyList: TypeAlias = List[Any]
+
+
+class Info(FlexibleModel):
+    version: str
+
+
+class ImportedSymbol(ExactModel):
+    address: HexInt
+    name: Optional[str] = None
+
+
+class ImportedDll(ExactModel):
+    dll: str
+    imports: List[ImportedSymbol]
+
+
+class DirectoryEntry(ExactModel):
+    name: str
+    virtual_address: HexInt
+    size: HexInt
+
+
+class Section(ExactModel):
+    name: str
+    raw_address: HexInt
+    virtual_address: HexInt
+    virtual_size: HexInt
+    size_of_data: HexInt
+    characteristics: str
+    characteristics_raw: HexInt
+    entropy: float
+
+
+class Resource(ExactModel):
+    name: str
+    language: Optional[str] = None
+    sublanguage: str
+    filetype: Optional[str]
+    offset: HexInt
+    size: HexInt
+    entropy: float
+
+
+class DigitalSigner(FlexibleModel):
+    md5_fingerprint: str
+    not_after: str
+    not_before: str
+    serial_number: str
+    sha1_fingerprint: str
+    sha256_fingerprint: str
+
+    issuer_commonName: Optional[str] = None
+    issuer_countryName: Optional[str] = None
+    issuer_localityName: Optional[str] = None
+    issuer_organizationName: Optional[str] = None
+    issuer_stateOrProvinceName: Optional[str] = None
+
+    subject_commonName: Optional[str] = None
+    subject_countryName: Optional[str] = None
+    subject_localityName: Optional[str] = None
+    subject_organizationName: Optional[str] = None
+    subject_stateOrProvinceName: Optional[str] = None
+
+    extensions_authorityInfoAccess_caIssuers: Optional[str] = None
+    extensions_authorityKeyIdentifier: Optional[str] = None
+    extensions_cRLDistributionPoints_0: Optional[str] = None
+    extensions_certificatePolicies_0: Optional[str] = None
+    extensions_subjectAltName_0: Optional[str] = None
+    extensions_subjectKeyIdentifier: Optional[str] = None
+
+
+class AuxSigner(ExactModel):
+    name: str
+    issued_to: str = Field(alias="Issued to")
+    issued_by: str = Field(alias="Issued by")
+    expires: str = Field(alias="Expires")
+    sha1_hash: str = Field(alias="SHA1 hash")
+
+
+class Signer(ExactModel):
+    aux_sha1: Optional[str] = None
+    aux_timestamp: Optional[str] = None
+    aux_valid: Optional[bool] = None
+    aux_error: Optional[bool] = None
+    aux_error_desc: Optional[str] = None
+    aux_signers: Optional[List[AuxSigner]] = None
+
+
+class Overlay(ExactModel):
+    offset: HexInt
+    size: HexInt
+
+
+class KV(ExactModel):
+    name: str
+    value: str
+
+
+class ExportedSymbol(ExactModel):
+    address: HexInt
+    name: str
+    ordinal: int
+
+
+class PE(ExactModel):
+    peid_signatures: TODO
+    imagebase: HexInt
+    entrypoint: HexInt
+    reported_checksum: HexInt
+    actual_checksum: HexInt
+    osversion: str
+    pdbpath: Optional[str] = None
+    timestamp: str
+
+    # List[ImportedDll], or Dict[basename(dll), ImportedDll]
+    imports: Union[List[ImportedDll], Dict[str, ImportedDll]]
+    imported_dll_count: Optional[int] = None
+    imphash: str
+
+    exported_dll_name: Optional[str] = None
+    exports: List[ExportedSymbol]
+
+    dirents: List[DirectoryEntry]
+    sections: List[Section]
+
+    ep_bytes: Optional[HexBytes] = None
+
+    overlay: Optional[Overlay] = None
+    resources: List[Resource]
+    versioninfo: List[KV]
+
+    # base64 encoded data
+    icon: Optional[str] = None
+    # MD5-like hash
+    icon_hash: Optional[str] = None
+    # MD5-like hash
+    icon_fuzzy: Optional[str] = None
+    # short hex string
+    icon_dhash: Optional[str] = None
+
+    digital_signers: List[DigitalSigner]
+    guest_signers: Signer
+
+
+# TODO(mr-tz): target.file.dotnet, target.file.extracted_files, target.file.extracted_files_tool,
+#  target.file.extracted_files_time
+# https://github.com/mandiant/capa/issues/1814
+class File(FlexibleModel):
+    type: str
+    cape_type_code: Optional[int] = None
+    cape_type: Optional[str] = None
+
+    pid: Optional[Union[int, Literal[""]]] = None
+    name: Union[List[str], str]
+    path: str
+    guest_paths: Union[List[str], str, None]
+    timestamp: Optional[str] = None
+
+    #
+    # hashes
+    #
+    crc32: str
+    md5: str
+    sha1: str
+    sha256: str
+    sha512: str
+    sha3_384: str
+    ssdeep: str
+    # unsure why this would ever be "False"
+    tlsh: Optional[Union[str, bool]] = None
+    rh_hash: Optional[str] = None
+
+    #
+    # other metadata, static analysis
+    #
+    size: int
+    pe: Optional[PE] = None
+    ep_bytes: Optional[HexBytes] = None
+    entrypoint: Optional[int] = None
+    data: Optional[str] = None
+    strings: Optional[List[str]] = None
+
+    #
+    # detections (skip)
+    #
+    yara: Skip = None
+    cape_yara: Skip = None
+    clamav: Skip = None
+    virustotal: Skip = None
+
+
+class ProcessFile(File):
+    #
+    # like a File, but also has dynamic analysis results
+    #
+    pid: Optional[int] = None
+    process_path: Optional[str] = None
+    process_name: Optional[str] = None
+    module_path: Optional[str] = None
+    virtual_address: Optional[HexInt] = None
+    target_pid: Optional[Union[int, str]] = None
+    target_path: Optional[str] = None
+    target_process: Optional[str] = None
+
+
+class Argument(ExactModel):
+    name: str
+    # unsure why empty list is provided here
+    value: Union[HexInt, int, str, EmptyList]
+    pretty_value: Optional[str] = None
+
+
+class Call(ExactModel):
+    timestamp: str
+    thread_id: int
+    category: str
+
+    api: str
+
+    arguments: List[Argument]
+    status: bool
+    return_: HexInt = Field(alias="return")
+    pretty_return: Optional[str] = None
+
+    repeated: int
+
+    # virtual addresses
+    caller: HexInt
+    parentcaller: HexInt
+
+    # index into calls array
+    id: int
+
+
+class Process(ExactModel):
+    process_id: int
+    process_name: str
+    parent_id: int
+    module_path: str
+    first_seen: str
+    calls: List[Call]
+    threads: List[int]
+    environ: Dict[str, str]
+
+
+class ProcessTree(ExactModel):
+    name: str
+    pid: int
+    parent_id: int
+    module_path: str
+    threads: List[int]
+    environ: Dict[str, str]
+    children: List["ProcessTree"]
+
+
+class Summary(ExactModel):
+    files: List[str]
+    read_files: List[str]
+    write_files: List[str]
+    delete_files: List[str]
+    keys: List[str]
+    read_keys: List[str]
+    write_keys: List[str]
+    delete_keys: List[str]
+    executed_commands: List[str]
+    resolved_apis: List[str]
+    mutexes: List[str]
+    created_services: List[str]
+    started_services: List[str]
+
+
+class EncryptedBuffer(ExactModel):
+    process_name: str
+    pid: int
+
+    api_call: str
+    buffer: str
+    buffer_size: Optional[int] = None
+    crypt_key: Optional[Union[HexInt, str]] = None
+
+
+class Behavior(ExactModel):
+    summary: Summary
+
+    # list of processes, of threads, of calls
+    processes: List[Process]
+    # tree of processes
+    processtree: List[ProcessTree]
+
+    anomaly: List[str]
+    encryptedbuffers: List[EncryptedBuffer]
+    # these are small objects that describe atomic events,
+    # like file move, registery access.
+    # we'll detect the same with our API call analyis.
+    enhanced: Skip = None
+
+
+class Target(ExactModel):
+    category: str
+    file: File
+    pe: Optional[PE] = None
+
+
+class Static(ExactModel):
+    pe: Optional[PE] = None
+    flare_capa: Skip = None
+
+
+class Cape(ExactModel):
+    payloads: List[ProcessFile]
+    configs: Skip = None
+
+
+# flexible because there may be more sorts of analysis
+# but we only care about the ones described here.
+class CapeReport(FlexibleModel):
+    # the input file, I think
+    target: Target
+    # info about the processing job, like machine and distributed metadata.
+    info: Info
+
+    #
+    # static analysis results
+    #
+    static: Optional[Static] = None
+    strings: Optional[List[str]] = None
+
+    #
+    # dynamic analysis results
+    #
+    # post-processed results: process tree, anomalies, etc
+    behavior: Behavior
+
+    # post-processed results: payloads and extracted configs
+    CAPE: Optional[Cape] = None
+    dropped: Optional[List[File]] = None
+    procdump: Optional[List[ProcessFile]] = None
+    procmemory: ListTODO
+
+    # =========================================================================
+    # information we won't use in capa
+    #
+
+    #
+    # NBIs and HBIs
+    # these are super interesting, but they don't enable use to detect behaviors.
+    # they take a lot of code to model and details to maintain.
+    #
+    # if we come up with a future use for this, go ahead and re-enable!
+    #
+    network: Skip = None
+    suricata: Skip = None
+    curtain: Skip = None
+    sysmon: Skip = None
+    url_analysis: Skip = None
+
+    # screenshot hash values
+    deduplicated_shots: Skip = None
+    # k-v pairs describing the time it took to run each stage.
+    statistics: Skip = None
+    # k-v pairs of ATT&CK ID to signature name or similar.
+    ttps: Skip = None
+    # debug log messages
+    debug: Skip = None
+
+    # various signature matches
+    # we could potentially extend capa to use this info one day,
+    # though it would be quite sandbox-specific,
+    # and more detection-oriented than capability detection.
+    signatures: Skip = None
+    malfamily_tag: Optional[str] = None
+    malscore: float
+    detections: Skip = None
+    detections2pid: Optional[Dict[int, List[str]]] = None
+    # AV detections for the sample.
+    virustotal: Skip = None
+
+    @classmethod
+    def from_buf(cls, buf: bytes) -> "CapeReport":
+        return cls.model_validate_json(buf)

capa/features/extractors/cape/process.py

@@ -0,0 +1,48 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import List, Tuple, Iterator
+
+from capa.features.common import String, Feature
+from capa.features.address import Address, ThreadAddress
+from capa.features.extractors.cape.models import Process
+from capa.features.extractors.base_extractor import ThreadHandle, ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def get_threads(ph: ProcessHandle) -> Iterator[ThreadHandle]:
+    """
+    get the threads associated with a given process
+    """
+    process: Process = ph.inner
+    threads: List[int] = process.threads
+
+    for thread in threads:
+        address: ThreadAddress = ThreadAddress(process=ph.address, tid=thread)
+        yield ThreadHandle(address=address, inner={})
+
+
+def extract_environ_strings(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract strings from a process' provided environment variables.
+    """
+    process: Process = ph.inner
+
+    for value in (value for value in process.environ.values() if value):
+        yield String(value), ph.address
+
+
+def extract_features(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
+    for handler in PROCESS_HANDLERS:
+        for feature, addr in handler(ph):
+            yield feature, addr
+
+
+PROCESS_HANDLERS = (extract_environ_strings,)

capa/features/extractors/cape/thread.py

@@ -0,0 +1,32 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+from typing import Iterator
+
+from capa.features.address import DynamicCallAddress
+from capa.features.extractors.helpers import generate_symbols
+from capa.features.extractors.cape.models import Process
+from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
+
+logger = logging.getLogger(__name__)
+
+
+def get_calls(ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
+    process: Process = ph.inner
+
+    tid = th.address.tid
+    for call_index, call in enumerate(process.calls):
+        if call.thread_id != tid:
+            continue
+
+        for symbol in generate_symbols("", call.api):
+            call.api = symbol
+
+            addr = DynamicCallAddress(thread=th.address, id=call_index)
+            yield CallHandle(address=addr, inner=call)

capa/features/extractors/common.py

@@ -6,6 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import io
+import re
 import logging
 import binascii
 import contextlib
@@ -41,6 +42,7 @@ logger = logging.getLogger(__name__)
 MATCH_PE = b"MZ"
 MATCH_ELF = b"\x7fELF"
 MATCH_RESULT = b'{"meta":'
+MATCH_JSON_OBJECT = b'{"'
 
 
 def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[String, Address]]:
@@ -63,6 +65,11 @@ def extract_format(buf) -> Iterator[Tuple[Feature, Address]]:
         yield Format(FORMAT_FREEZE), NO_ADDRESS
     elif buf.startswith(MATCH_RESULT):
         yield Format(FORMAT_RESULT), NO_ADDRESS
+    elif re.sub(rb"\s", b"", buf[:20]).startswith(MATCH_JSON_OBJECT):
+        # potential start of JSON object data without whitespace
+        # we don't know what it is exactly, but may support it (e.g. a dynamic CAPE sandbox report)
+        # skip verdict here and let subsequent code analyze this further
+        return
     else:
         # we likely end up here:
         #  1. handling a file format (e.g. macho)

capa/features/extractors/dnfile/extractor.py

@@ -22,7 +22,13 @@ import capa.features.extractors.dnfile.function
 from capa.features.common import Feature
 from capa.features.address import NO_ADDRESS, Address, DNTokenAddress, DNTokenOffsetAddress
 from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
 from capa.features.extractors.dnfile.helpers import (
     get_dotnet_types,
     get_dotnet_fields,
@@ -68,10 +74,10 @@ class DnFileFeatureExtractorCache:
         return self.types.get(token)
 
 
-class DnfileFeatureExtractor(FeatureExtractor):
+class DnfileFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, path: Path):
-        super().__init__()
         self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
 
         # pre-compute .NET token lookup tables; each .NET method has access to this cache for feature extraction
         # most relevant at instruction scope

capa/features/extractors/dnfile/helpers.py

@@ -131,10 +131,14 @@ def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[DnType]:
             # remove get_/set_ from MemberRef name
             member_ref_name = member_ref_name[4:]
 
+        typerefnamespace, typerefname = resolve_nested_typeref_name(
+            member_ref.Class.row_index, member_ref.Class.row, pe
+        )
+
         yield DnType(
             token,
-            member_ref.Class.row.TypeName,
-            namespace=member_ref.Class.row.TypeNamespace,
+            typerefname,
+            namespace=typerefnamespace,
             member=member_ref_name,
             access=access,
         )
@@ -188,6 +192,8 @@ def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
             TypeNamespace (index into String heap)
             MethodList (index into MethodDef table; it marks the first of a contiguous run of Methods owned by this Type)
     """
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
     accessor_map: Dict[int, str] = {}
     for methoddef, methoddef_access in get_dotnet_methoddef_property_accessors(pe):
         accessor_map[methoddef] = methoddef_access
@@ -211,7 +217,9 @@ def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
                 # remove get_/set_
                 method_name = method_name[4:]
 
-            yield DnType(token, typedef.TypeName, namespace=typedef.TypeNamespace, member=method_name, access=access)
+            typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
+            yield DnType(token, typedefname, namespace=typedefnamespace, member=method_name, access=access)
 
 
 def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
@@ -225,6 +233,8 @@ def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
             TypeNamespace (index into String heap)
             FieldList (index into Field table; it marks the first of a contiguous run of Fields owned by this Type)
     """
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
     for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
         assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
 
@@ -235,8 +245,11 @@ def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
             if field.row is None:
                 logger.debug("TypeDef[0x%X] FieldList[0x%X] row is None", rid, idx)
                 continue
+
+            typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
             token: int = calculate_dotnet_token_value(field.table.number, field.row_index)
-            yield DnType(token, typedef.TypeName, namespace=typedef.TypeNamespace, member=field.row.Name)
+            yield DnType(token, typedefname, namespace=typedefnamespace, member=field.row.Name)
 
 
 def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
@@ -300,19 +313,119 @@ def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[DnUnmanagedMethod]
         yield DnUnmanagedMethod(token, module, method)
 
 
+def get_dotnet_table_row(pe: dnfile.dnPE, table_index: int, row_index: int) -> Optional[dnfile.base.MDTableRow]:
+    assert pe.net is not None
+    assert pe.net.mdtables is not None
+
+    if row_index - 1 <= 0:
+        return None
+
+    try:
+        table = pe.net.mdtables.tables.get(table_index, [])
+        return table[row_index - 1]
+    except IndexError:
+        return None
+
+
+def resolve_nested_typedef_name(
+    nested_class_table: dict, index: int, typedef: dnfile.mdtable.TypeDefRow, pe: dnfile.dnPE
+) -> Tuple[str, Tuple[str, ...]]:
+    """Resolves all nested TypeDef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
+
+    if index in nested_class_table:
+        typedef_name = []
+        name = typedef.TypeName
+
+        # Append the current typedef name
+        typedef_name.append(name)
+
+        while nested_class_table[index] in nested_class_table:
+            # Iterate through the typedef table to resolve the nested name
+            table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeDef.number, nested_class_table[index])
+            if table_row is None:
+                return typedef.TypeNamespace, tuple(typedef_name[::-1])
+
+            name = table_row.TypeName
+            typedef_name.append(name)
+            index = nested_class_table[index]
+
+        # Document the root enclosing details
+        table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeDef.number, nested_class_table[index])
+        if table_row is None:
+            return typedef.TypeNamespace, tuple(typedef_name[::-1])
+
+        enclosing_name = table_row.TypeName
+        typedef_name.append(enclosing_name)
+
+        return table_row.TypeNamespace, tuple(typedef_name[::-1])
+
+    else:
+        return typedef.TypeNamespace, (typedef.TypeName,)
+
+
+def resolve_nested_typeref_name(
+    index: int, typeref: dnfile.mdtable.TypeRefRow, pe: dnfile.dnPE
+) -> Tuple[str, Tuple[str, ...]]:
+    """Resolves all nested TypeRef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
+    # If the ResolutionScope decodes to a typeRef type then it is nested
+    if isinstance(typeref.ResolutionScope.table, dnfile.mdtable.TypeRef):
+        typeref_name = []
+        name = typeref.TypeName
+        # Not appending the current typeref name to avoid potential duplicate
+
+        # Validate index
+        table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeRef.number, index)
+        if table_row is None:
+            return typeref.TypeNamespace, (typeref.TypeName,)
+
+        while isinstance(table_row.ResolutionScope.table, dnfile.mdtable.TypeRef):
+            # Iterate through the typeref table to resolve the nested name
+            typeref_name.append(name)
+            name = table_row.TypeName
+            table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeRef.number, table_row.ResolutionScope.row_index)
+            if table_row is None:
+                return typeref.TypeNamespace, tuple(typeref_name[::-1])
+
+        # Document the root enclosing details
+        typeref_name.append(table_row.TypeName)
+
+        return table_row.TypeNamespace, tuple(typeref_name[::-1])
+
+    else:
+        return typeref.TypeNamespace, (typeref.TypeName,)
+
+
+def get_dotnet_nested_class_table_index(pe: dnfile.dnPE) -> Dict[int, int]:
+    """Build index for EnclosingClass based off the NestedClass row index in the nestedclass table"""
+    nested_class_table = {}
+
+    # Used to find nested classes in typedef
+    for _, nestedclass in iter_dotnet_table(pe, dnfile.mdtable.NestedClass.number):
+        assert isinstance(nestedclass, dnfile.mdtable.NestedClassRow)
+        nested_class_table[nestedclass.NestedClass.row_index] = nestedclass.EnclosingClass.row_index
+
+    return nested_class_table
+
+
 def get_dotnet_types(pe: dnfile.dnPE) -> Iterator[DnType]:
     """get .NET types from TypeDef and TypeRef tables"""
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
     for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
         assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
 
+        typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
         typedef_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
-        yield DnType(typedef_token, typedef.TypeName, namespace=typedef.TypeNamespace)
+        yield DnType(typedef_token, typedefname, namespace=typedefnamespace)
 
     for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
         assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
 
+        typerefnamespace, typerefname = resolve_nested_typeref_name(typeref.ResolutionScope.row_index, typeref, pe)
+
         typeref_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
-        yield DnType(typeref_token, typeref.TypeName, namespace=typeref.TypeNamespace)
+        yield DnType(typeref_token, typerefname, namespace=typerefnamespace)
 
 
 def calculate_dotnet_token_value(table: int, rid: int) -> int:

capa/features/extractors/dnfile/types.py

@@ -6,15 +6,17 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from typing import Optional
+from typing import Tuple, Optional
 
 
 class DnType:
-    def __init__(self, token: int, class_: str, namespace: str = "", member: str = "", access: Optional[str] = None):
+    def __init__(
+        self, token: int, class_: Tuple[str, ...], namespace: str = "", member: str = "", access: Optional[str] = None
+    ):
         self.token: int = token
         self.access: Optional[str] = access
         self.namespace: str = namespace
-        self.class_: str = class_
+        self.class_: Tuple[str, ...] = class_
 
         if member == ".ctor":
             member = "ctor"
@@ -42,9 +44,13 @@ class DnType:
         return str(self)
 
     @staticmethod
-    def format_name(class_: str, namespace: str = "", member: str = ""):
+    def format_name(class_: Tuple[str, ...], namespace: str = "", member: str = ""):
+        if len(class_) > 1:
+            class_str = "/".join(class_)  # Concat items in tuple, separated by a "/"
+        else:
+            class_str = "".join(class_)  # Convert tuple to str
         # like File::OpenRead
-        name: str = f"{class_}::{member}" if member else class_
+        name: str = f"{class_str}::{member}" if member else class_str
         if namespace:
             # like System.IO.File::OpenRead
             name = f"{namespace}.{name}"

capa/features/extractors/dnfile_.py

@@ -1,158 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-from typing import Tuple, Iterator
-from pathlib import Path
-
-import dnfile
-import pefile
-
-from capa.features.common import (
-    OS,
-    OS_ANY,
-    ARCH_ANY,
-    ARCH_I386,
-    FORMAT_PE,
-    ARCH_AMD64,
-    FORMAT_DOTNET,
-    Arch,
-    Format,
-    Feature,
-)
-from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_format(**kwargs) -> Iterator[Tuple[Feature, Address]]:
-    yield Format(FORMAT_PE), NO_ADDRESS
-    yield Format(FORMAT_DOTNET), NO_ADDRESS
-
-
-def extract_file_os(**kwargs) -> Iterator[Tuple[Feature, Address]]:
-    yield OS(OS_ANY), NO_ADDRESS
-
-
-def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Feature, Address]]:
-    # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
-    # .NET 4.5 added option: any CPU, 32-bit preferred
-    assert pe.net is not None
-    assert pe.net.Flags is not None
-
-    if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        yield Arch(ARCH_ANY), NO_ADDRESS
-
-
-def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for file_handler in FILE_HANDLERS:
-        for feature, address in file_handler(pe=pe):  # type: ignore
-            yield feature, address
-
-
-FILE_HANDLERS = (
-    # extract_file_export_names,
-    # extract_file_import_names,
-    # extract_file_section_names,
-    # extract_file_strings,
-    # extract_file_function_names,
-    extract_file_format,
-)
-
-
-def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for handler in GLOBAL_HANDLERS:
-        for feature, addr in handler(pe=pe):  # type: ignore
-            yield feature, addr
-
-
-GLOBAL_HANDLERS = (
-    extract_file_os,
-    extract_file_arch,
-)
-
-
-class DnfileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: Path):
-        super().__init__()
-        self.path: Path = path
-        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
-
-    def get_base_address(self) -> AbsoluteVirtualAddress:
-        return AbsoluteVirtualAddress(0x0)
-
-    def get_entry_point(self) -> int:
-        # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
-        #  True: native EP: Token
-        #  False: managed EP: RVA
-        assert self.pe.net is not None
-        assert self.pe.net.struct is not None
-
-        return self.pe.net.struct.EntryPointTokenOrRva
-
-    def extract_global_features(self):
-        yield from extract_global_features(self.pe)
-
-    def extract_file_features(self):
-        yield from extract_file_features(self.pe)
-
-    def is_dotnet_file(self) -> bool:
-        return bool(self.pe.net)
-
-    def is_mixed_mode(self) -> bool:
-        assert self.pe is not None
-        assert self.pe.net is not None
-        assert self.pe.net.Flags is not None
-
-        return not bool(self.pe.net.Flags.CLR_ILONLY)
-
-    def get_runtime_version(self) -> Tuple[int, int]:
-        assert self.pe is not None
-        assert self.pe.net is not None
-        assert self.pe.net.struct is not None
-
-        return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
-
-    def get_meta_version_string(self) -> str:
-        assert self.pe.net is not None
-        assert self.pe.net.metadata is not None
-        assert self.pe.net.metadata.struct is not None
-        assert self.pe.net.metadata.struct.Version is not None
-
-        vbuf = self.pe.net.metadata.struct.Version
-        assert isinstance(vbuf, bytes)
-
-        return vbuf.rstrip(b"\x00").decode("utf-8")
-
-    def get_functions(self):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def extract_function_features(self, f):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def get_basic_blocks(self, f):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def extract_basic_block_features(self, f, bb):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def get_instructions(self, f, bb):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def extract_insn_features(self, f, bb, insn):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def is_library_function(self, va):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def get_function_name(self, va):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")

capa/features/extractors/dotnetfile.py

@@ -31,15 +31,18 @@ from capa.features.common import (
     Characteristic,
 )
 from capa.features.address import NO_ADDRESS, Address, DNTokenAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
+from capa.features.extractors.dnfile.types import DnType
+from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
 from capa.features.extractors.dnfile.helpers import (
-    DnType,
     iter_dotnet_table,
     is_dotnet_mixed_mode,
     get_dotnet_managed_imports,
     get_dotnet_managed_methods,
+    resolve_nested_typedef_name,
+    resolve_nested_typeref_name,
     calculate_dotnet_token_value,
     get_dotnet_unmanaged_imports,
+    get_dotnet_nested_class_table_index,
 )
 
 logger = logging.getLogger(__name__)
@@ -57,7 +60,7 @@ def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Impor
 
     for imp in get_dotnet_unmanaged_imports(pe):
         # like kernel32.CreateFileA
-        for name in capa.features.extractors.helpers.generate_symbols(imp.module, imp.method):
+        for name in capa.features.extractors.helpers.generate_symbols(imp.module, imp.method, include_dll=True):
             yield Import(name), DNTokenAddress(imp.token)
 
 
@@ -92,19 +95,25 @@ def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple
 
 def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Class, Address]]:
     """emit class features from TypeRef and TypeDef tables"""
+    nested_class_table = get_dotnet_nested_class_table_index(pe)
+
     for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
         # emit internal .NET classes
         assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
 
+        typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
+
         token = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
-        yield Class(DnType.format_name(typedef.TypeName, namespace=typedef.TypeNamespace)), DNTokenAddress(token)
+        yield Class(DnType.format_name(typedefname, namespace=typedefnamespace)), DNTokenAddress(token)
 
     for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
         # emit external .NET classes
         assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
 
+        typerefnamespace, typerefname = resolve_nested_typeref_name(typeref.ResolutionScope.row_index, typeref, pe)
+
         token = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
-        yield Class(DnType.format_name(typeref.TypeName, namespace=typeref.TypeNamespace)), DNTokenAddress(token)
+        yield Class(DnType.format_name(typerefname, namespace=typerefnamespace)), DNTokenAddress(token)
 
 
 def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
@@ -165,9 +174,9 @@ GLOBAL_HANDLERS = (
 )
 
 
-class DotnetFileFeatureExtractor(FeatureExtractor):
+class DotnetFileFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, path: Path):
-        super().__init__()
+        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
         self.path: Path = path
         self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
 

capa/features/extractors/elf.py

@@ -108,6 +108,9 @@ class Shdr:
             buf,
         )
 
+    def get_name(self, elf: "ELF") -> str:
+        return elf.shstrtab.buf[self.name :].partition(b"\x00")[0].decode("ascii")
+
 
 class ELF:
     def __init__(self, f: BinaryIO):
@@ -120,6 +123,7 @@ class ELF:
         self.e_phnum: int
         self.e_shentsize: int
         self.e_shnum: int
+        self.e_shstrndx: int
         self.phbuf: bytes
         self.shbuf: bytes
 
@@ -151,11 +155,15 @@ class ELF:
         if self.bitness == 32:
             e_phoff, e_shoff = struct.unpack_from(self.endian + "II", self.file_header, 0x1C)
             self.e_phentsize, self.e_phnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x2A)
-            self.e_shentsize, self.e_shnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x2E)
+            self.e_shentsize, self.e_shnum, self.e_shstrndx = struct.unpack_from(
+                self.endian + "HHH", self.file_header, 0x2E
+            )
         elif self.bitness == 64:
             e_phoff, e_shoff = struct.unpack_from(self.endian + "QQ", self.file_header, 0x20)
             self.e_phentsize, self.e_phnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x36)
-            self.e_shentsize, self.e_shnum = struct.unpack_from(self.endian + "HH", self.file_header, 0x3A)
+            self.e_shentsize, self.e_shnum, self.e_shstrndx = struct.unpack_from(
+                self.endian + "HHH", self.file_header, 0x3A
+            )
         else:
             raise NotImplementedError()
 
@@ -365,6 +373,10 @@ class ELF:
             except ValueError:
                 continue
 
+    @property
+    def shstrtab(self) -> Shdr:
+        return self.parse_section_header(self.e_shstrndx)
+
     @property
     def linker(self):
         PT_INTERP = 0x3
@@ -816,6 +828,48 @@ def guess_os_from_sh_notes(elf: ELF) -> Optional[OS]:
     return None
 
 
+def guess_os_from_ident_directive(elf: ELF) -> Optional[OS]:
+    # GCC inserts the GNU version via an .ident directive
+    # that gets stored in a section named ".comment".
+    # look at the version and recognize common OSes.
+    #
+    # assume the GCC version matches the target OS version,
+    # which I guess could be wrong during cross-compilation?
+    # therefore, don't rely on this if possible.
+    #
+    # https://stackoverflow.com/q/6263425
+    # https://gcc.gnu.org/onlinedocs/cpp/Other-Directives.html
+
+    SHT_PROGBITS = 0x1
+    for shdr in elf.section_headers:
+        if shdr.type != SHT_PROGBITS:
+            continue
+
+        if shdr.get_name(elf) != ".comment":
+            continue
+
+        try:
+            comment = shdr.buf.decode("utf-8")
+        except ValueError:
+            continue
+
+        if "GCC:" not in comment:
+            continue
+
+        logger.debug(".ident: %s", comment)
+
+        # these values come from our testfiles, like:
+        # rg -a "GCC: " tests/data/
+        if "Debian" in comment:
+            return OS.LINUX
+        elif "Ubuntu" in comment:
+            return OS.LINUX
+        elif "Red Hat" in comment:
+            return OS.LINUX
+
+    return None
+
+
 def guess_os_from_linker(elf: ELF) -> Optional[OS]:
     # search for recognizable dynamic linkers (interpreters)
     # for example, on linux, we see file paths like: /lib64/ld-linux-x86-64.so.2
@@ -851,8 +905,10 @@ def guess_os_from_abi_versions_needed(elf: ELF) -> Optional[OS]:
                 return OS.HURD
 
             else:
-                # we don't have any good guesses based on versions needed
-                pass
+                # in practice, Hurd isn't a common/viable OS,
+                # so this is almost certain to be Linux,
+                # so lets just make that guess.
+                return OS.LINUX
 
     return None
 
@@ -898,7 +954,7 @@ def guess_os_from_symtab(elf: ELF) -> Optional[OS]:
 
 def detect_elf_os(f) -> str:
     """
-    f: type Union[BinaryIO, IDAIO]
+    f: type Union[BinaryIO, IDAIO, GHIDRAIO]
     """
     try:
         elf = ELF(f)
@@ -927,6 +983,13 @@ def detect_elf_os(f) -> str:
         logger.warning("Error guessing OS from section header notes: %s", e)
         sh_notes_guess = None
 
+    try:
+        ident_guess = guess_os_from_ident_directive(elf)
+        logger.debug("guess: .ident: %s", ident_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from .ident directive: %s", e)
+        ident_guess = None
+
     try:
         linker_guess = guess_os_from_linker(elf)
         logger.debug("guess: linker: %s", linker_guess)
@@ -960,6 +1023,10 @@ def detect_elf_os(f) -> str:
     if osabi_guess:
         ret = osabi_guess
 
+    elif ident_guess:
+        # we don't trust this too much due to non-cross-compilation assumptions
+        ret = ident_guess
+
     elif ph_notes_guess:
         ret = ph_notes_guess
 

capa/features/extractors/elffile.py

@@ -17,7 +17,7 @@ import capa.features.extractors.common
 from capa.features.file import Export, Import, Section
 from capa.features.common import OS, FORMAT_ELF, Arch, Format, Feature
 from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
+from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
 
 logger = logging.getLogger(__name__)
 
@@ -154,9 +154,9 @@ GLOBAL_HANDLERS = (
 )
 
 
-class ElfFeatureExtractor(FeatureExtractor):
+class ElfFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, path: Path):
-        super().__init__()
+        super().__init__(SampleHashes.from_bytes(path.read_bytes()))
         self.path: Path = path
         self.elf = ELFFile(io.BytesIO(path.read_bytes()))
 

capa/features/extractors/ghidra/basicblock.py

@@ -0,0 +1,152 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import string
+import struct
+from typing import Tuple, Iterator
+
+import ghidra
+from ghidra.program.model.lang import OperandType
+
+import capa.features.extractors.ghidra.helpers
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
+
+
+def get_printable_len(op: ghidra.program.model.scalar.Scalar) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    op_bit_len = op.bitLength()
+    op_byte_len = op_bit_len // 8
+    op_val = op.getValue()
+
+    if op_bit_len == 8:
+        chars = struct.pack("<B", op_val & 0xFF)
+    elif op_bit_len == 16:
+        chars = struct.pack("<H", op_val & 0xFFFF)
+    elif op_bit_len == 32:
+        chars = struct.pack("<I", op_val & 0xFFFFFFFF)
+    elif op_bit_len == 64:
+        chars = struct.pack("<Q", op_val & 0xFFFFFFFFFFFFFFFF)
+    else:
+        raise ValueError(f"Unhandled operand data type 0x{op_bit_len:x}.")
+
+    def is_printable_ascii(chars_: bytes):
+        return all(c < 127 and chr(c) in string.printable for c in chars_)
+
+    def is_printable_utf16le(chars_: bytes):
+        if all(c == 0x00 for c in chars_[1::2]):
+            return is_printable_ascii(chars_[::2])
+
+    if is_printable_ascii(chars):
+        return op_byte_len
+
+    if is_printable_utf16le(chars):
+        return op_byte_len
+
+    return 0
+
+
+def is_mov_imm_to_stack(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    """verify instruction moves immediate onto stack"""
+
+    # Ghidra will Bitwise OR the OperandTypes to assign multiple
+    # i.e., the first operand is a stackvar (dynamically allocated),
+    # and the second is a scalar value (single int/char/float/etc.)
+    mov_its_ops = [(OperandType.ADDRESS | OperandType.DYNAMIC), OperandType.SCALAR]
+    found = False
+
+    # MOV dword ptr [EBP + local_*], 0x65
+    if insn.getMnemonicString().startswith("MOV"):
+        found = all(insn.getOperandType(i) == mov_its_ops[i] for i in range(2))
+
+    return found
+
+
+def bb_contains_stackstring(bb: ghidra.program.model.block.CodeBlock) -> bool:
+    """check basic block for stackstring indicators
+
+    true if basic block contains enough moves of constant bytes to the stack
+    """
+    count = 0
+    for insn in currentProgram().getListing().getInstructions(bb, True):  # type: ignore [name-defined] # noqa: F821
+        if is_mov_imm_to_stack(insn):
+            count += get_printable_len(insn.getScalar(1))
+        if count > MIN_STACKSTRING_LEN:
+            return True
+    return False
+
+
+def _bb_has_tight_loop(bb: ghidra.program.model.block.CodeBlock):
+    """
+    parse tight loops, true if last instruction in basic block branches to bb start
+    """
+    # Reverse Ordered, first InstructionDB
+    last_insn = currentProgram().getListing().getInstructions(bb, False).next()  # type: ignore [name-defined] # noqa: F821
+
+    if last_insn.getFlowType().isJump():
+        return last_insn.getAddress(0) == bb.getMinAddress()
+
+    return False
+
+
+def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract stackstring indicators from basic block"""
+    bb: ghidra.program.model.block.CodeBlock = bbh.inner
+
+    if bb_contains_stackstring(bb):
+        yield Characteristic("stack string"), bbh.address
+
+
+def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """check basic block for tight loop indicators"""
+    bb: ghidra.program.model.block.CodeBlock = bbh.inner
+
+    if _bb_has_tight_loop(bb):
+        yield Characteristic("tight loop"), bbh.address
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    extract_bb_stackstring,
+)
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract features from the given basic block.
+
+    args:
+        bb: the basic block to process.
+
+    yields:
+      Tuple[Feature, int]: the features and their location found in this basic block.
+    """
+    yield BasicBlock(), bbh.address
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, addr in bb_handler(fh, bbh):
+            yield feature, addr
+
+
+def main():
+    features = []
+    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
+
+    for fh in GhidraFeatureExtractor().get_functions():
+        for bbh in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
+            features.extend(list(extract_features(fh, bbh)))
+
+    import pprint
+
+    pprint.pprint(features)  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ghidra/extractor.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Tuple, Iterator
+
+import capa.features.extractors.ghidra.file
+import capa.features.extractors.ghidra.insn
+import capa.features.extractors.ghidra.global_
+import capa.features.extractors.ghidra.function
+import capa.features.extractors.ghidra.basicblock
+from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
+
+
+class GhidraFeatureExtractor(StaticFeatureExtractor):
+    def __init__(self):
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        super().__init__(
+            SampleHashes(
+                md5=capa.ghidra.helpers.get_file_md5(),
+                # ghidra doesn't expose this hash.
+                # https://ghidra.re/ghidra_docs/api/ghidra/program/model/listing/Program.html
+                #
+                # the hashes are stored in the database, not computed on the fly,
+                # so its probably not trivial to add SHA1.
+                sha1="",
+                sha256=capa.ghidra.helpers.get_file_sha256(),
+            )
+        )
+
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.ghidra.file.extract_file_format())
+        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_os())
+        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_arch())
+        self.imports = ghidra_helpers.get_file_imports()
+        self.externs = ghidra_helpers.get_file_externs()
+        self.fakes = ghidra_helpers.map_fake_import_addrs()
+
+    def get_base_address(self):
+        return AbsoluteVirtualAddress(currentProgram().getImageBase().getOffset())  # type: ignore [name-defined] # noqa: F821
+
+    def extract_global_features(self):
+        yield from self.global_features
+
+    def extract_file_features(self):
+        yield from capa.features.extractors.ghidra.file.extract_features()
+
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        for fhandle in ghidra_helpers.get_function_symbols():
+            fh: FunctionHandle = FunctionHandle(
+                address=AbsoluteVirtualAddress(fhandle.getEntryPoint().getOffset()),
+                inner=fhandle,
+                ctx={"imports_cache": self.imports, "externs_cache": self.externs, "fakes_cache": self.fakes},
+            )
+            yield fh
+
+    @staticmethod
+    def get_function(addr: int) -> FunctionHandle:
+        func = getFunctionContaining(toAddr(addr))  # type: ignore [name-defined] # noqa: F821
+        return FunctionHandle(address=AbsoluteVirtualAddress(func.getEntryPoint().getOffset()), inner=func)
+
+    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.ghidra.function.extract_features(fh)
+
+    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        yield from ghidra_helpers.get_function_blocks(fh)
+
+    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.ghidra.basicblock.extract_features(fh, bbh)
+
+    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        yield from ghidra_helpers.get_insn_in_range(bbh)
+
+    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
+        yield from capa.features.extractors.ghidra.insn.extract_features(fh, bbh, ih)

capa/features/extractors/ghidra/file.py

@@ -0,0 +1,204 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import re
+import struct
+from typing import List, Tuple, Iterator
+
+from ghidra.program.model.symbol import SourceType, SymbolType
+
+import capa.features.extractors.common
+import capa.features.extractors.helpers
+import capa.features.extractors.strings
+import capa.features.extractors.ghidra.helpers
+from capa.features.file import Export, Import, Section, FunctionName
+from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
+
+MAX_OFFSET_PE_AFTER_MZ = 0x200
+
+
+def find_embedded_pe(block_bytez: bytes, mz_xor: List[Tuple[bytes, bytes, int]]) -> Iterator[Tuple[int, int]]:
+    """check segment for embedded PE
+
+    adapted for Ghidra from:
+    https://github.com/vivisect/vivisect/blob/91e8419a861f4977https://github.com/vivisect/vivisect/blob/91e8419a861f49779f18316f155311967e696836/PE/carve.py#L259f18316f155311967e696836/PE/carve.py#L25
+    """
+    todo = []
+
+    for mzx, pex, i in mz_xor:
+        for match in re.finditer(re.escape(mzx), block_bytez):
+            todo.append((match.start(), mzx, pex, i))
+
+    seg_max = len(block_bytez)  # noqa: F821
+    while len(todo):
+        off, mzx, pex, i = todo.pop()
+
+        # MZ header has one field we will check e_lfanew is at 0x3c
+        e_lfanew = off + 0x3C
+
+        if seg_max < e_lfanew + 4:
+            continue
+
+        e_lfanew_bytes = block_bytez[e_lfanew : e_lfanew + 4]
+        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(e_lfanew_bytes, i))[0]
+
+        # assume XOR'd "PE" bytes exist within threshold
+        if newoff > MAX_OFFSET_PE_AFTER_MZ:
+            continue
+
+        peoff = off + newoff
+        if seg_max < peoff + 2:
+            continue
+
+        pe_bytes = block_bytez[peoff : peoff + 2]
+        if pe_bytes == pex:
+            yield off, i
+
+
+def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
+    """extract embedded PE features"""
+
+    # pre-compute XOR pairs
+    mz_xor: List[Tuple[bytes, bytes, int]] = [
+        (
+            capa.features.extractors.helpers.xor_static(b"MZ", i),
+            capa.features.extractors.helpers.xor_static(b"PE", i),
+            i,
+        )
+        for i in range(256)
+    ]
+
+    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
+        if not all((block.isLoaded(), block.isInitialized(), "Headers" not in block.getName())):
+            continue
+
+        for off, _ in find_embedded_pe(capa.features.extractors.ghidra.helpers.get_block_bytes(block), mz_xor):
+            # add offset back to block start
+            ea: int = block.getStart().add(off).getOffset()
+
+            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
+
+
+def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
+    """extract function exports"""
+    st = currentProgram().getSymbolTable()  # type: ignore [name-defined] # noqa: F821
+    for addr in st.getExternalEntryPointIterator():
+        yield Export(st.getPrimarySymbol(addr).getName()), AbsoluteVirtualAddress(addr.getOffset())
+
+
+def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
+    """extract function imports
+
+    1. imports by ordinal:
+     - modulename.#ordinal
+
+    2. imports by name, results in two features to support importname-only
+       matching:
+     - modulename.importname
+     - importname
+    """
+
+    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
+        for r in f.getSymbol().getReferences():
+            if r.getReferenceType().isData():
+                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
+
+        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_*
+        if "Ordinal_" in fstr[1]:
+            fstr[1] = f"#{fstr[1].split('_')[1]}"
+
+        for name in capa.features.extractors.helpers.generate_symbols(fstr[0][:-4], fstr[1], include_dll=True):
+            yield Import(name), AbsoluteVirtualAddress(addr)
+
+
+def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
+    """extract section names"""
+
+    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
+        yield Section(block.getName()), AbsoluteVirtualAddress(block.getStart().getOffset())
+
+
+def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
+    """extract ASCII and UTF-16 LE strings"""
+
+    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
+        if not block.isInitialized():
+            continue
+
+        p_bytes = capa.features.extractors.ghidra.helpers.get_block_bytes(block)
+
+        for s in capa.features.extractors.strings.extract_ascii_strings(p_bytes):
+            offset = block.getStart().getOffset() + s.offset
+            yield String(s.s), FileOffsetAddress(offset)
+
+        for s in capa.features.extractors.strings.extract_unicode_strings(p_bytes):
+            offset = block.getStart().getOffset() + s.offset
+            yield String(s.s), FileOffsetAddress(offset)
+
+
+def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract the names of statically-linked library functions.
+    """
+
+    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
+        # .isExternal() misses more than this config for the function symbols
+        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
+            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
+            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
+                name = name[13:]
+            addr = AbsoluteVirtualAddress(sym.getAddress().getOffset())
+            yield FunctionName(name), addr
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                yield FunctionName(name[1:]), addr
+
+
+def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
+    ef = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+    if "PE" in ef:
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif "ELF" in ef:
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    elif "Raw" in ef:
+        # no file type to return when processing a binary file, but we want to continue processing
+        return
+    else:
+        raise NotImplementedError(f"unexpected file format: {ef}")
+
+
+def extract_features() -> Iterator[Tuple[Feature, Address]]:
+    """extract file features"""
+    for file_handler in FILE_HANDLERS:
+        for feature, addr in file_handler():
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_file_embedded_pe,
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_section_names,
+    extract_file_strings,
+    extract_file_function_names,
+    extract_file_format,
+)
+
+
+def main():
+    """ """
+    import pprint
+
+    pprint.pprint(list(extract_features()))  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ghidra/function.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Tuple, Iterator
+
+import ghidra
+from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
+
+import capa.features.extractors.ghidra.helpers
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors import loops
+from capa.features.extractors.base_extractor import FunctionHandle
+
+
+def extract_function_calls_to(fh: FunctionHandle):
+    """extract callers to a function"""
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+    for ref in f.getSymbol().getReferences():
+        if ref.getReferenceType().isCall():
+            yield Characteristic("calls to"), AbsoluteVirtualAddress(ref.getFromAddress().getOffset())
+
+
+def extract_function_loop(fh: FunctionHandle):
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+
+    edges = []
+    for block in SimpleBlockIterator(BasicBlockModel(currentProgram()), f.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
+        dests = block.getDestinations(monitor())  # type: ignore [name-defined] # noqa: F821
+        s_addrs = block.getStartAddresses()
+
+        while dests.hasNext():  # For loop throws Python TypeError
+            for addr in s_addrs:
+                edges.append((addr.getOffset(), dests.next().getDestinationAddress().getOffset()))
+
+    if loops.has_loop(edges):
+        yield Characteristic("loop"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
+
+
+def extract_recursive_call(fh: FunctionHandle):
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+
+    for func in f.getCalledFunctions(monitor()):  # type: ignore [name-defined] # noqa: F821
+        if func.getEntryPoint().getOffset() == f.getEntryPoint().getOffset():
+            yield Characteristic("recursive call"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
+
+
+def main():
+    """ """
+    features = []
+    for fhandle in capa.features.extractors.ghidra.helpers.get_function_symbols():
+        features.extend(list(extract_features(fhandle)))
+
+    import pprint
+
+    pprint.pprint(features)  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ghidra/global_.py

@@ -0,0 +1,67 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+import contextlib
+from typing import Tuple, Iterator
+
+import capa.ghidra.helpers
+import capa.features.extractors.elf
+import capa.features.extractors.ghidra.helpers
+from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
+from capa.features.address import NO_ADDRESS, Address
+
+logger = logging.getLogger(__name__)
+
+
+def extract_os() -> Iterator[Tuple[Feature, Address]]:
+    format_name: str = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+
+    if "PE" in format_name:
+        yield OS(OS_WINDOWS), NO_ADDRESS
+
+    elif "ELF" in format_name:
+        with contextlib.closing(capa.ghidra.helpers.GHIDRAIO()) as f:
+            os = capa.features.extractors.elf.detect_elf_os(f)
+
+        yield OS(os), NO_ADDRESS
+
+    else:
+        # we likely end up here:
+        #  1. handling shellcode, or
+        #  2. handling a new file format (e.g. macho)
+        #
+        # for (1) we can't do much - its shellcode and all bets are off.
+        # we could maybe accept a further CLI argument to specify the OS,
+        # but i think this would be rarely used.
+        # rules that rely on OS conditions will fail to match on shellcode.
+        #
+        # for (2), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported file format: %s, will not guess OS", format_name)
+        return
+
+
+def extract_arch() -> Iterator[Tuple[Feature, Address]]:
+    lang_id = currentProgram().getMetadata().get("Language ID")  # type: ignore [name-defined] # noqa: F821
+
+    if "x86" in lang_id and "64" in lang_id:
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+
+    elif "x86" in lang_id and "32" in lang_id:
+        yield Arch(ARCH_I386), NO_ADDRESS
+
+    elif "x86" not in lang_id:
+        logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel")
+        return
+
+    else:
+        # we likely end up here:
+        #  1. handling a new architecture (e.g. aarch64)
+        #
+        # for (1), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported architecture: %s", lang_id)
+        return

capa/features/extractors/ghidra/helpers.py

@@ -0,0 +1,301 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Dict, List, Iterator
+
+import ghidra
+import java.lang
+from ghidra.program.model.lang import OperandType
+from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
+from ghidra.program.model.symbol import SourceType, SymbolType
+from ghidra.program.model.address import AddressSpace
+
+import capa.features.extractors.helpers
+from capa.features.common import THUNK_CHAIN_DEPTH_DELTA
+from capa.features.address import AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+
+def ints_to_bytes(bytez: List[int]) -> bytes:
+    """convert Java signed ints to Python bytes
+
+    args:
+        bytez: list of Java signed ints
+    """
+    return bytes([b & 0xFF for b in bytez])
+
+
+def find_byte_sequence(addr: ghidra.program.model.address.Address, seq: bytes) -> Iterator[int]:
+    """yield all ea of a given byte sequence
+
+    args:
+        addr: start address
+        seq: bytes to search e.g. b"\x01\x03"
+    """
+    seqstr = "".join([f"\\x{b:02x}" for b in seq])
+    eas = findBytes(addr, seqstr, java.lang.Integer.MAX_VALUE, 1)  # type: ignore [name-defined] # noqa: F821
+
+    yield from eas
+
+
+def get_bytes(addr: ghidra.program.model.address.Address, length: int) -> bytes:
+    """yield length bytes at addr
+
+    args:
+        addr: Address to begin pull from
+        length: length of bytes to pull
+    """
+    try:
+        return ints_to_bytes(getBytes(addr, length))  # type: ignore [name-defined] # noqa: F821
+    except RuntimeError:
+        return b""
+
+
+def get_block_bytes(block: ghidra.program.model.mem.MemoryBlock) -> bytes:
+    """yield all bytes in a given block
+
+    args:
+        block: MemoryBlock to pull from
+    """
+    return get_bytes(block.getStart(), block.getSize())
+
+
+def get_function_symbols():
+    """yield all non-external function symbols"""
+    yield from currentProgram().getFunctionManager().getFunctionsNoStubs(True)  # type: ignore [name-defined] # noqa: F821
+
+
+def get_function_blocks(fh: FunctionHandle) -> Iterator[BBHandle]:
+    """yield BBHandle for each bb in a given function"""
+
+    func: ghidra.program.database.function.FunctionDB = fh.inner
+    for bb in SimpleBlockIterator(BasicBlockModel(currentProgram()), func.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
+        yield BBHandle(address=AbsoluteVirtualAddress(bb.getMinAddress().getOffset()), inner=bb)
+
+
+def get_insn_in_range(bbh: BBHandle) -> Iterator[InsnHandle]:
+    """yield InshHandle for each insn in a given basicblock"""
+    for insn in currentProgram().getListing().getInstructions(bbh.inner, True):  # type: ignore [name-defined] # noqa: F821
+        yield InsnHandle(address=AbsoluteVirtualAddress(insn.getAddress().getOffset()), inner=insn)
+
+
+def get_file_imports() -> Dict[int, List[str]]:
+    """get all import names & addrs"""
+
+    import_dict: Dict[int, List[str]] = {}
+
+    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
+        for r in f.getSymbol().getReferences():
+            if r.getReferenceType().isData():
+                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
+
+        ex_loc = f.getExternalLocation().getAddress()  # map external locations as well (offset into module files)
+
+        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_* / <EXTERNAL>::import
+        if "Ordinal_" in fstr[1]:
+            fstr[1] = f"#{fstr[1].split('_')[1]}"
+
+        # <EXTERNAL> mostly shows up in ELF files, otherwise, strip '.dll' w/ [:-4]
+        fstr[0] = "*" if "<EXTERNAL>" in fstr[0] else fstr[0][:-4]
+
+        for name in capa.features.extractors.helpers.generate_symbols(fstr[0], fstr[1]):
+            import_dict.setdefault(addr, []).append(name)
+            if ex_loc:
+                import_dict.setdefault(ex_loc.getOffset(), []).append(name)
+
+    return import_dict
+
+
+def get_file_externs() -> Dict[int, List[str]]:
+    """
+    Gets function names & addresses of statically-linked library functions
+
+    Ghidra's external namespace is mostly reserved for dynamically-linked
+    imports. Statically-linked functions are part of the global namespace.
+    Filtering on the type, source, and namespace of the symbols yield more
+    statically-linked library functions.
+
+    Example: (PMA Lab 16-01.exe_) 7faafc7e4a5c736ebfee6abbbc812d80:0x407490
+    - __aulldiv
+    - Note: See Symbol Table labels
+    """
+
+    extern_dict: Dict[int, List[str]] = {}
+
+    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
+        # .isExternal() misses more than this config for the function symbols
+        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
+            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
+            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
+                name = name[13:]
+            extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name)
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name[1:])
+
+    return extern_dict
+
+
+def map_fake_import_addrs() -> Dict[int, List[int]]:
+    """
+    Map ghidra's fake import entrypoints to their
+    real addresses
+
+    Helps as many Ghidra Scripting API calls end up returning
+    these external (fake) addresses.
+
+    Undocumented but intended Ghidra behavior:
+     - Import entryPoint fields are stored in the 'EXTERNAL:' AddressSpace.
+       'getEntryPoint()' returns the entryPoint field, which is an offset
+       from the beginning of the assigned AddressSpace. In the case of externals,
+       they start from 1 and increment.
+    https://github.com/NationalSecurityAgency/ghidra/blob/26d4bd9104809747c21f2528cab8aba9aef9acd5/Ghidra/Features/Base/src/test.slow/java/ghidra/program/database/function/ExternalFunctionDBTest.java#L90
+
+    Example: (mimikatz.exe_) 5f66b82558ca92e54e77f216ef4c066c:0x473090
+    - 0x473090 -> PTR_CreateServiceW_00473090
+    - 'EXTERNAL:00000025' -> External Address (ghidra.program.model.address.SpecialAddress)
+    """
+    fake_dict: Dict[int, List[int]] = {}
+
+    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
+        for r in f.getSymbol().getReferences():
+            if r.getReferenceType().isData():
+                fake_dict.setdefault(f.getEntryPoint().getOffset(), []).append(r.getFromAddress().getOffset())
+
+    return fake_dict
+
+
+def check_addr_for_api(
+    addr: ghidra.program.model.address.Address,
+    fakes: Dict[int, List[int]],
+    imports: Dict[int, List[str]],
+    externs: Dict[int, List[str]],
+) -> bool:
+    offset = addr.getOffset()
+
+    fake = fakes.get(offset)
+    if fake:
+        return True
+
+    imp = imports.get(offset)
+    if imp:
+        return True
+
+    extern = externs.get(offset)
+    if extern:
+        return True
+
+    return False
+
+
+def is_call_or_jmp(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    return any(mnem in insn.getMnemonicString() for mnem in ["CALL", "J"])  # JMP, JNE, JNZ, etc
+
+
+def is_sp_modified(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == OperandType.REGISTER:
+            return "SP" in insn.getRegister(i).getName() and insn.getOperandRefType(i).isWrite()
+    return False
+
+
+def is_stack_referenced(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    """generic catch-all for stack references"""
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == OperandType.REGISTER:
+            if "BP" in insn.getRegister(i).getName():
+                return True
+            else:
+                continue
+
+    return any(ref.isStackReference() for ref in insn.getReferencesFrom())
+
+
+def is_zxor(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    # assume XOR insn
+    # XOR's against the same operand zero out
+    ops = []
+    operands = []
+    for i in range(insn.getNumOperands()):
+        ops.append(insn.getOpObjects(i))
+
+    # Operands stored in a 2D array
+    for j in range(len(ops)):
+        for k in range(len(ops[j])):
+            operands.append(ops[j][k])
+
+    return all(n == operands[0] for n in operands)
+
+
+def handle_thunk(addr: ghidra.program.model.address.Address):
+    """Follow thunk chains down to a reasonable depth"""
+    ref = addr
+    for _ in range(THUNK_CHAIN_DEPTH_DELTA):
+        thunk_jmp = getInstructionAt(ref)  # type: ignore [name-defined] # noqa: F821
+        if thunk_jmp and is_call_or_jmp(thunk_jmp):
+            if OperandType.isAddress(thunk_jmp.getOperandType(0)):
+                ref = thunk_jmp.getAddress(0)
+        else:
+            thunk_dat = getDataContaining(ref)  # type: ignore [name-defined] # noqa: F821
+            if thunk_dat and thunk_dat.isDefined() and thunk_dat.isPointer():
+                ref = thunk_dat.getValue()
+                break  # end of thunk chain reached
+    return ref
+
+
+def dereference_ptr(insn: ghidra.program.database.code.InstructionDB):
+    addr_code = OperandType.ADDRESS | OperandType.CODE
+    to_deref = insn.getAddress(0)
+    dat = getDataContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
+
+    if insn.getOperandType(0) == addr_code:
+        thfunc = getFunctionContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
+        if thfunc and thfunc.isThunk():
+            return handle_thunk(to_deref)
+        else:
+            # if it doesn't poin to a thunk, it's usually a jmp to a label
+            return to_deref
+    if not dat:
+        return to_deref
+    if dat.isDefined() and dat.isPointer():
+        addr = dat.getValue()
+        # now we need to check the addr space to see if it is truly resolvable
+        # ghidra sometimes likes to hand us direct RAM addrs, which typically point
+        # to api calls that we can't actually resolve as such
+        if addr.getAddressSpace().getType() == AddressSpace.TYPE_RAM:
+            return to_deref
+        else:
+            return addr
+    else:
+        return to_deref
+
+
+def find_data_references_from_insn(insn, max_depth: int = 10):
+    """yield data references from given instruction"""
+    for reference in insn.getReferencesFrom():
+        if not reference.getReferenceType().isData():
+            # only care about data references
+            continue
+
+        to_addr = reference.getToAddress()
+
+        for _ in range(max_depth - 1):
+            data = getDataAt(to_addr)  # type: ignore [name-defined] # noqa: F821
+            if data and data.isPointer():
+                ptr_value = data.getValue()
+
+                if ptr_value is None:
+                    break
+
+                to_addr = ptr_value
+            else:
+                break
+
+        yield to_addr

capa/features/extractors/ghidra/insn.py

@@ -0,0 +1,503 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Any, Dict, Tuple, Iterator
+
+import ghidra
+from ghidra.program.model.lang import OperandType
+from ghidra.program.model.block import SimpleBlockModel
+
+import capa.features.extractors.helpers
+import capa.features.extractors.ghidra.helpers
+from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
+from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+# security cookie checks may perform non-zeroing XORs, these are expected within a certain
+# byte range within the first and returning basic blocks, this helps to reduce FP features
+SECURITY_COOKIE_BYTES_DELTA = 0x40
+
+
+OPERAND_TYPE_DYNAMIC_ADDRESS = OperandType.DYNAMIC | OperandType.ADDRESS
+
+
+def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    """Populate the import cache for this context"""
+    if "imports_cache" not in ctx:
+        ctx["imports_cache"] = capa.features.extractors.ghidra.helpers.get_file_imports()
+    return ctx["imports_cache"]
+
+
+def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    """Populate the externs cache for this context"""
+    if "externs_cache" not in ctx:
+        ctx["externs_cache"] = capa.features.extractors.ghidra.helpers.get_file_externs()
+    return ctx["externs_cache"]
+
+
+def get_fakes(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    """Populate the fake import addrs cache for this context"""
+    if "fakes_cache" not in ctx:
+        ctx["fakes_cache"] = capa.features.extractors.ghidra.helpers.map_fake_import_addrs()
+    return ctx["fakes_cache"]
+
+
+def check_for_api_call(
+    insn, externs: Dict[int, Any], fakes: Dict[int, Any], imports: Dict[int, Any], imp_or_ex: bool
+) -> Iterator[Any]:
+    """check instruction for API call
+
+    params:
+        externs - external library functions cache
+        fakes - mapped fake import addresses cache
+        imports - imported functions cache
+        imp_or_ex - flag to check imports or externs
+
+    yields:
+        matched api calls
+    """
+    info = ()
+    funcs = imports if imp_or_ex else externs
+
+    # assume only CALLs or JMPs are passed
+    ref_type = insn.getOperandType(0)
+    addr_data = OperandType.ADDRESS | OperandType.DATA  # needs dereferencing
+    addr_code = OperandType.ADDRESS | OperandType.CODE  # needs dereferencing
+
+    if OperandType.isRegister(ref_type):
+        if OperandType.isAddress(ref_type):
+            # If it's an address in a register, check the mapped fake addrs
+            # since they're dereferenced to their fake addrs
+            op_ref = insn.getAddress(0).getOffset()
+            ref = fakes.get(op_ref)  # obtain the real addr
+            if not ref:
+                return
+        else:
+            return
+    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
+        # we must dereference and check if the addr is a pointer to an api function
+        addr_ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
+        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
+            return
+        ref = addr_ref.getOffset()
+    elif ref_type == OPERAND_TYPE_DYNAMIC_ADDRESS or ref_type == OperandType.DYNAMIC:
+        return  # cannot resolve dynamics statically
+    else:
+        # pure address does not need to get dereferenced/ handled
+        addr_ref = insn.getAddress(0)
+        if not addr_ref:
+            # If it returned null, it was an indirect
+            # that had no address reference.
+            # This check is faster than checking for (indirect and not address)
+            return
+        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
+            return
+        ref = addr_ref.getOffset()
+
+    if isinstance(ref, list):  # ref from REG | ADDR
+        for r in ref:
+            info = funcs.get(r)  # type: ignore
+            if info:
+                yield info
+    else:
+        info = funcs.get(ref)  # type: ignore
+        if info:
+            yield info
+
+
+def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    externs = get_externs(fh.ctx)
+    fakes = get_fakes(fh.ctx)
+    imports = get_imports(fh.ctx)
+
+    # check calls to imported functions
+    for api in check_for_api_call(insn, externs, fakes, imports, True):
+        for imp in api:
+            yield API(imp), ih.address
+
+    # check calls to extern functions
+    for api in check_for_api_call(insn, externs, fakes, imports, False):
+        for ext in api:
+            yield API(ext), ih.address
+
+
+def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction number features
+    example:
+        push    3136B0h         ; dwControlCode
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("RET"):
+        # skip things like:
+        #   .text:0042250E retn 8
+        return
+
+    if capa.features.extractors.ghidra.helpers.is_sp_modified(insn):
+        # skip things like:
+        #   .text:00401145 add esp, 0Ch
+        return
+
+    for i in range(insn.getNumOperands()):
+        # Exceptions for LEA insn:
+        # invalid operand encoding, considered numbers instead of offsets
+        # see: mimikatz.exe_:0x4018C0
+        if insn.getOperandType(i) == OperandType.DYNAMIC and insn.getMnemonicString().startswith("LEA"):
+            # Additional check, avoid yielding "wide" values (ex. mimikatz.exe:0x471EE6 LEA EBX, [ECX + EAX*0x4])
+            op_objs = insn.getOpObjects(i)
+            if len(op_objs) == 3:  # ECX, EAX, 0x4
+                continue
+
+            if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
+                const = op_objs[-1].getUnsignedValue()
+                addr = ih.address
+
+                yield Number(const), addr
+                yield OperandNumber(i, const), addr
+        elif not OperandType.isScalar(insn.getOperandType(i)):
+            # skip things like:
+            #   references, void types
+            continue
+        else:
+            const = insn.getScalar(i).getUnsignedValue()
+            addr = ih.address
+
+            yield Number(const), addr
+            yield OperandNumber(i, const), addr
+
+            if insn.getMnemonicString().startswith("ADD") and 0 < const < MAX_STRUCTURE_SIZE:
+                # for pattern like:
+                #
+                #     add eax, 0x10
+                #
+                # assume 0x10 is also an offset (imagine eax is a pointer).
+                yield Offset(const), addr
+                yield OperandOffset(i, const), addr
+
+
+def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction structure offset features
+
+    example:
+        .text:0040112F cmp [esi+4], ebx
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("LEA"):
+        return
+
+    if capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
+        # ignore stack references
+        return
+
+    # Ghidra stores operands in 2D arrays if they contain offsets
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == OperandType.DYNAMIC:  # e.g. [esi + 4]
+            # manual extraction, since the default api calls only work on the 1st dimension of the array
+            op_objs = insn.getOpObjects(i)
+            if not op_objs:
+                continue
+
+            if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
+                op_off = op_objs[-1].getValue()
+            else:
+                op_off = 0
+
+            yield Offset(op_off), ih.address
+            yield OperandOffset(i, op_off), ih.address
+
+
+def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse referenced byte sequences
+
+    example:
+        push    offset iid_004118d4_IShellLinkA ; riid
+    """
+    for addr in capa.features.extractors.ghidra.helpers.find_data_references_from_insn(ih.inner):
+        data = getDataAt(addr)  # type: ignore [name-defined] # noqa: F821
+        if data and not data.hasStringValue():
+            extracted_bytes = capa.features.extractors.ghidra.helpers.get_bytes(addr, MAX_BYTES_FEATURE_SIZE)
+            if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
+                yield Bytes(extracted_bytes), ih.address
+
+
+def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction string features
+
+    example:
+        push offset aAcr     ; "ACR  > "
+    """
+    for addr in capa.features.extractors.ghidra.helpers.find_data_references_from_insn(ih.inner):
+        data = getDataAt(addr)  # type: ignore [name-defined] # noqa: F821
+        if data and data.hasStringValue():
+            yield String(data.getValue()), ih.address
+
+
+def extract_insn_mnemonic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction mnemonic features"""
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    yield Mnemonic(insn.getMnemonicString().lower()), ih.address
+
+
+def extract_insn_obfs_call_plus_5_characteristic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse call $+5 instruction from the given instruction.
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    code_ref = OperandType.ADDRESS | OperandType.CODE
+    ref = insn.getAddress()
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == code_ref:
+            ref = insn.getAddress(i)
+
+    if insn.getAddress().add(5) == ref:
+        yield Characteristic("call $+5"), ih.address
+
+
+def extract_insn_segment_access_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction fs or gs access"""
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    insn_str = insn.toString()
+
+    if "FS:" in insn_str:
+        yield Characteristic("fs access"), ih.address
+
+    if "GS:" in insn_str:
+        yield Characteristic("gs access"), ih.address
+
+
+def extract_insn_peb_access_characteristic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction peb access
+
+    fs:[0x30] on x86, gs:[0x60] on x64
+
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    insn_str = insn.toString()
+    if insn_str.startswith(("PUSH", "MOV")):
+        if "FS:[0x30]" in insn_str or "GS:[0x60]" in insn_str:
+            yield Characteristic("peb access"), ih.address
+
+
+def extract_insn_cross_section_cflow(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    externs = get_externs(fh.ctx)
+    fakes = get_fakes(fh.ctx)
+    imports = get_imports(fh.ctx)
+
+    # OperandType to dereference
+    addr_data = OperandType.ADDRESS | OperandType.DATA
+    addr_code = OperandType.ADDRESS | OperandType.CODE
+
+    ref_type = insn.getOperandType(0)
+
+    # both OperandType flags must be present
+    # bail on REGISTER alone
+    if OperandType.isRegister(ref_type):
+        if OperandType.isAddress(ref_type):
+            ref = insn.getAddress(0)  # Ghidra dereferences REG | ADDR
+            if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
+                return
+        else:
+            return
+    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
+        # we must dereference and check if the addr is a pointer to an api function
+        ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
+        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
+            return
+    elif ref_type == OPERAND_TYPE_DYNAMIC_ADDRESS or ref_type == OperandType.DYNAMIC:
+        return  # cannot resolve dynamics statically
+    else:
+        # pure address does not need to get dereferenced/ handled
+        ref = insn.getAddress(0)
+        if not ref:
+            # If it returned null, it was an indirect
+            # that had no address reference.
+            # This check is faster than checking for (indirect and not address)
+            return
+        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
+            return
+
+    this_mem_block = getMemoryBlock(insn.getAddress())  # type: ignore [name-defined] # noqa: F821
+    ref_block = getMemoryBlock(ref)  # type: ignore [name-defined] # noqa: F821
+    if ref_block != this_mem_block:
+        yield Characteristic("cross section flow"), ih.address
+
+
+def extract_function_calls_from(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    ih: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    """extract functions calls from features
+
+    most relevant at the function scope, however, its most efficient to extract at the instruction scope
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("CALL"):
+        # This method of "dereferencing" addresses/ pointers
+        # is not as robust as methods in other functions,
+        # but works just fine for this one
+        reference = 0
+        for ref in insn.getReferencesFrom():
+            addr = ref.getToAddress()
+
+            # avoid returning fake addrs
+            if not addr.isExternalAddress():
+                reference = addr.getOffset()
+
+            # if a reference is < 0, then ghidra pulled an offset from a DYNAMIC | ADDR (usually a stackvar)
+            # these cannot be resolved to actual addrs
+            if reference > 0:
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(reference)
+
+
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    ih: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
+    does not include calls like => call ds:dword_ABD4974
+
+    most relevant at the function or basic block scope;
+    however, its most efficient to extract at the instruction scope
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("CALL"):
+        if OperandType.isRegister(insn.getOperandType(0)):
+            yield Characteristic("indirect call"), ih.address
+        if OperandType.isIndirect(insn.getOperandType(0)):
+            yield Characteristic("indirect call"), ih.address
+
+
+def check_nzxor_security_cookie_delta(
+    fh: ghidra.program.database.function.FunctionDB, insn: ghidra.program.database.code.InstructionDB
+):
+    """Get the function containing the insn
+    Get the last block of the function that contains the insn
+
+    Check the bb containing the insn
+    Check the last bb of the function containing the insn
+    """
+
+    model = SimpleBlockModel(currentProgram())  # type: ignore [name-defined] # noqa: F821
+    insn_addr = insn.getAddress()
+    func_asv = fh.getBody()
+    first_addr = func_asv.getMinAddress()
+    last_addr = func_asv.getMaxAddress()
+
+    if model.getFirstCodeBlockContaining(
+        first_addr, monitor()  # type: ignore [name-defined] # noqa: F821
+    ) == model.getFirstCodeBlockContaining(
+        last_addr, monitor()  # type: ignore [name-defined] # noqa: F821
+    ):
+        if insn_addr < first_addr.add(SECURITY_COOKIE_BYTES_DELTA):
+            return True
+        else:
+            return insn_addr > last_addr.add(SECURITY_COOKIE_BYTES_DELTA * -1)
+    else:
+        return False
+
+
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    ih: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if "XOR" not in insn.getMnemonicString():
+        return
+    if capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
+        return
+    if capa.features.extractors.ghidra.helpers.is_zxor(insn):
+        return
+    if check_nzxor_security_cookie_delta(f, insn):
+        return
+    yield Characteristic("nzxor"), ih.address
+
+
+def extract_features(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    insn: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    for insn_handler in INSTRUCTION_HANDLERS:
+        for feature, addr in insn_handler(fh, bb, insn):
+            yield feature, addr
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_bytes_features,
+    extract_insn_string_features,
+    extract_insn_offset_features,
+    extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    extract_insn_obfs_call_plus_5_characteristic_features,
+    extract_insn_peb_access_characteristic_features,
+    extract_insn_cross_section_cflow,
+    extract_insn_segment_access_features,
+    extract_function_calls_from,
+    extract_function_indirect_call_characteristic_features,
+)
+
+
+def main():
+    """ """
+    features = []
+    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
+
+    for fh in GhidraFeatureExtractor().get_functions():
+        for bb in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
+            for insn in capa.features.extractors.ghidra.helpers.get_insn_in_range(bb):
+                features.extend(list(extract_features(fh, bb, insn)))
+
+    import pprint
+
+    pprint.pprint(features)  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/helpers.py

@@ -41,38 +41,50 @@ def is_ordinal(symbol: str) -> bool:
     return False
 
 
-def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
+def generate_symbols(dll: str, symbol: str, include_dll=False) -> Iterator[str]:
     """
     for a given dll and symbol name, generate variants.
     we over-generate features to make matching easier.
     these include:
-      - kernel32.CreateFileA
-      - kernel32.CreateFile
       - CreateFileA
       - CreateFile
+      - ws2_32.#1
+
+    note that since capa v7 only `import` features and APIs called via ordinal include DLL names:
+      - kernel32.CreateFileA
+      - kernel32.CreateFile
+      - ws2_32.#1
+
+    for `api` features dll names are good for documentation but not used during matching
     """
     # normalize dll name
     dll = dll.lower()
 
-    # kernel32.CreateFileA
-    yield f"{dll}.{symbol}"
+    # trim extensions observed in dynamic traces
+    dll = dll[0:-4] if dll.endswith(".dll") else dll
+    dll = dll[0:-4] if dll.endswith(".drv") else dll
+
+    if include_dll or is_ordinal(symbol):
+        # ws2_32.#1
+        # kernel32.CreateFileA
+        yield f"{dll}.{symbol}"
 
     if not is_ordinal(symbol):
         # CreateFileA
         yield symbol
 
-    if is_aw_function(symbol):
-        # kernel32.CreateFile
-        yield f"{dll}.{symbol[:-1]}"
+        if is_aw_function(symbol):
+            if include_dll:
+                # kernel32.CreateFile
+                yield f"{dll}.{symbol[:-1]}"
 
-        if not is_ordinal(symbol):
             # CreateFile
             yield symbol[:-1]
 
 
 def reformat_forwarded_export_name(forwarded_name: str) -> str:
     """
-    a forwarded export has a DLL name/path an symbol name.
+    a forwarded export has a DLL name/path and symbol name.
     we want the former to be lowercase, and the latter to be verbatim.
     """
 

capa/features/extractors/ida/extractor.py

@@ -8,6 +8,7 @@
 from typing import List, Tuple, Iterator
 
 import idaapi
+import ida_nalt
 
 import capa.ida.helpers
 import capa.features.extractors.elf
@@ -18,12 +19,22 @@ import capa.features.extractors.ida.function
 import capa.features.extractors.ida.basicblock
 from capa.features.common import Feature
 from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
 
 
-class IdaFeatureExtractor(FeatureExtractor):
+class IdaFeatureExtractor(StaticFeatureExtractor):
     def __init__(self):
-        super().__init__()
+        super().__init__(
+            hashes=SampleHashes(
+                md5=ida_nalt.retrieve_input_file_md5(), sha1="(unknown)", sha256=ida_nalt.retrieve_input_file_sha256()
+            )
+        )
         self.global_features: List[Tuple[Feature, Address]] = []
         self.global_features.extend(capa.features.extractors.ida.file.extract_file_format())
         self.global_features.extend(capa.features.extractors.ida.global_.extract_os())

capa/features/extractors/ida/file.py

@@ -110,7 +110,7 @@ def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
         if info[1] and info[2]:
             # e.g. in mimikatz: ('cabinet', 'FCIAddFile', 11L)
             # extract by name here and by ordinal below
-            for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1]):
+            for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1], include_dll=True):
                 yield Import(name), addr
             dll = info[0]
             symbol = f"#{info[2]}"
@@ -123,7 +123,7 @@ def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
         else:
             continue
 
-        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol, include_dll=True):
             yield Import(name), addr
 
     for ea, info in capa.features.extractors.ida.helpers.get_file_externs().items():

capa/features/extractors/null.py

@@ -5,12 +5,24 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 from dataclasses import dataclass
 
+from typing_extensions import TypeAlias
+
 from capa.features.common import Feature
-from capa.features.address import NO_ADDRESS, Address
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.address import NO_ADDRESS, Address, ThreadAddress, ProcessAddress, DynamicCallAddress
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    CallHandle,
+    InsnHandle,
+    SampleHashes,
+    ThreadHandle,
+    ProcessHandle,
+    FunctionHandle,
+    StaticFeatureExtractor,
+    DynamicFeatureExtractor,
+)
 
 
 @dataclass
@@ -31,7 +43,7 @@ class FunctionFeatures:
 
 
 @dataclass
-class NullFeatureExtractor(FeatureExtractor):
+class NullStaticFeatureExtractor(StaticFeatureExtractor):
     """
     An extractor that extracts some user-provided features.
 
@@ -39,6 +51,7 @@ class NullFeatureExtractor(FeatureExtractor):
     """
 
     base_address: Address
+    sample_hashes: SampleHashes
     global_features: List[Feature]
     file_features: List[Tuple[Address, Feature]]
     functions: Dict[Address, FunctionFeatures]
@@ -46,6 +59,9 @@ class NullFeatureExtractor(FeatureExtractor):
     def get_base_address(self):
         return self.base_address
 
+    def get_sample_hashes(self) -> SampleHashes:
+        return self.sample_hashes
+
     def extract_global_features(self):
         for feature in self.global_features:
             yield feature, NO_ADDRESS
@@ -77,3 +93,78 @@ class NullFeatureExtractor(FeatureExtractor):
     def extract_insn_features(self, f, bb, insn):
         for address, feature in self.functions[f.address].basic_blocks[bb.address].instructions[insn.address].features:
             yield feature, address
+
+
+@dataclass
+class CallFeatures:
+    name: str
+    features: List[Tuple[Address, Feature]]
+
+
+@dataclass
+class ThreadFeatures:
+    features: List[Tuple[Address, Feature]]
+    calls: Dict[Address, CallFeatures]
+
+
+@dataclass
+class ProcessFeatures:
+    features: List[Tuple[Address, Feature]]
+    threads: Dict[Address, ThreadFeatures]
+    name: str
+
+
+@dataclass
+class NullDynamicFeatureExtractor(DynamicFeatureExtractor):
+    base_address: Address
+    sample_hashes: SampleHashes
+    global_features: List[Feature]
+    file_features: List[Tuple[Address, Feature]]
+    processes: Dict[Address, ProcessFeatures]
+
+    def extract_global_features(self):
+        for feature in self.global_features:
+            yield feature, NO_ADDRESS
+
+    def get_sample_hashes(self) -> SampleHashes:
+        return self.sample_hashes
+
+    def extract_file_features(self):
+        for address, feature in self.file_features:
+            yield feature, address
+
+    def get_processes(self):
+        for address in sorted(self.processes.keys()):
+            assert isinstance(address, ProcessAddress)
+            yield ProcessHandle(address=address, inner={})
+
+    def extract_process_features(self, ph):
+        for addr, feature in self.processes[ph.address].features:
+            yield feature, addr
+
+    def get_process_name(self, ph) -> str:
+        return self.processes[ph.address].name
+
+    def get_threads(self, ph):
+        for address in sorted(self.processes[ph.address].threads.keys()):
+            assert isinstance(address, ThreadAddress)
+            yield ThreadHandle(address=address, inner={})
+
+    def extract_thread_features(self, ph, th):
+        for addr, feature in self.processes[ph.address].threads[th.address].features:
+            yield feature, addr
+
+    def get_calls(self, ph, th):
+        for address in sorted(self.processes[ph.address].threads[th.address].calls.keys()):
+            assert isinstance(address, DynamicCallAddress)
+            yield CallHandle(address=address, inner={})
+
+    def extract_call_features(self, ph, th, ch):
+        for address, feature in self.processes[ph.address].threads[th.address].calls[ch.address].features:
+            yield feature, address
+
+    def get_call_name(self, ph, th, ch) -> str:
+        return self.processes[ph.address].threads[th.address].calls[ch.address].name
+
+
+NullFeatureExtractor: TypeAlias = Union[NullStaticFeatureExtractor, NullDynamicFeatureExtractor]

capa/features/extractors/pefile.py

@@ -19,7 +19,7 @@ import capa.features.extractors.strings
 from capa.features.file import Export, Import, Section
 from capa.features.common import OS, ARCH_I386, FORMAT_PE, ARCH_AMD64, OS_WINDOWS, Arch, Format, Characteristic
 from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
+from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
 
 logger = logging.getLogger(__name__)
 
@@ -84,7 +84,7 @@ def extract_file_import_names(pe, **kwargs):
                     except UnicodeDecodeError:
                         continue
 
-                for name in capa.features.extractors.helpers.generate_symbols(modname, impname):
+                for name in capa.features.extractors.helpers.generate_symbols(modname, impname, include_dll=True):
                     yield Import(name), AbsoluteVirtualAddress(imp.address)
 
 
@@ -185,9 +185,9 @@ GLOBAL_HANDLERS = (
 )
 
 
-class PefileFeatureExtractor(FeatureExtractor):
+class PefileFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, path: Path):
-        super().__init__()
+        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
         self.path: Path = path
         self.pe = pefile.PE(str(path))
 

capa/features/extractors/viv/basicblock.py

@@ -140,7 +140,7 @@ def is_printable_ascii(chars: bytes) -> bool:
 
 
 def is_printable_utf16le(chars: bytes) -> bool:
-    if all(c == b"\x00" for c in chars[1::2]):
+    if all(c == 0x0 for c in chars[1::2]):
         return is_printable_ascii(chars[::2])
     return False
 

capa/features/extractors/viv/extractor.py

@@ -20,17 +20,23 @@ import capa.features.extractors.viv.function
 import capa.features.extractors.viv.basicblock
 from capa.features.common import Feature
 from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.extractors.base_extractor import (
+    BBHandle,
+    InsnHandle,
+    SampleHashes,
+    FunctionHandle,
+    StaticFeatureExtractor,
+)
 
 logger = logging.getLogger(__name__)
 
 
-class VivisectFeatureExtractor(FeatureExtractor):
+class VivisectFeatureExtractor(StaticFeatureExtractor):
     def __init__(self, vw, path: Path, os):
-        super().__init__()
         self.vw = vw
         self.path = path
         self.buf = path.read_bytes()
+        super().__init__(hashes=SampleHashes.from_bytes(self.buf))
 
         # pre-compute these because we'll yield them at *every* scope.
         self.global_features: List[Tuple[Feature, Address]] = []

capa/features/extractors/viv/file.py

@@ -73,7 +73,7 @@ def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]
             impname = "#" + impname[len("ord") :]
 
         addr = AbsoluteVirtualAddress(va)
-        for name in capa.features.extractors.helpers.generate_symbols(modname, impname):
+        for name in capa.features.extractors.helpers.generate_symbols(modname, impname, include_dll=True):
             yield Import(name), addr
 
 

capa/features/freeze/__init__.py

@@ -9,13 +9,18 @@ Unless required by applicable law or agreed to in writing, software distributed
  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
+import json
 import zlib
 import logging
 from enum import Enum
-from typing import List, Tuple, Union
+from typing import List, Tuple, Union, Literal
 
 from pydantic import Field, BaseModel, ConfigDict
 
+# TODO(williballenthin): use typing.TypeAlias directly in Python 3.10+
+# https://github.com/mandiant/capa/issues/1699
+from typing_extensions import TypeAlias
+
 import capa.helpers
 import capa.version
 import capa.features.file
@@ -23,12 +28,20 @@ import capa.features.insn
 import capa.features.common
 import capa.features.address
 import capa.features.basicblock
-import capa.features.extractors.base_extractor
+import capa.features.extractors.null as null
 from capa.helpers import assert_never
 from capa.features.freeze.features import Feature, feature_from_capa
+from capa.features.extractors.base_extractor import (
+    SampleHashes,
+    FeatureExtractor,
+    StaticFeatureExtractor,
+    DynamicFeatureExtractor,
+)
 
 logger = logging.getLogger(__name__)
 
+CURRENT_VERSION = 3
+
 
 class HashableModel(BaseModel):
     model_config = ConfigDict(frozen=True)
@@ -40,12 +53,15 @@ class AddressType(str, Enum):
     FILE = "file"
     DN_TOKEN = "dn token"
     DN_TOKEN_OFFSET = "dn token offset"
+    PROCESS = "process"
+    THREAD = "thread"
+    CALL = "call"
     NO_ADDRESS = "no address"
 
 
 class Address(HashableModel):
     type: AddressType
-    value: Union[int, Tuple[int, int], None] = None  # None default value to support deserialization of NO_ADDRESS
+    value: Union[int, Tuple[int, ...], None] = None  # None default value to support deserialization of NO_ADDRESS
 
     @classmethod
     def from_capa(cls, a: capa.features.address.Address) -> "Address":
@@ -64,6 +80,15 @@ class Address(HashableModel):
         elif isinstance(a, capa.features.address.DNTokenOffsetAddress):
             return cls(type=AddressType.DN_TOKEN_OFFSET, value=(a.token, a.offset))
 
+        elif isinstance(a, capa.features.address.ProcessAddress):
+            return cls(type=AddressType.PROCESS, value=(a.ppid, a.pid))
+
+        elif isinstance(a, capa.features.address.ThreadAddress):
+            return cls(type=AddressType.THREAD, value=(a.process.ppid, a.process.pid, a.tid))
+
+        elif isinstance(a, capa.features.address.DynamicCallAddress):
+            return cls(type=AddressType.CALL, value=(a.thread.process.ppid, a.thread.process.pid, a.thread.tid, a.id))
+
         elif a == capa.features.address.NO_ADDRESS or isinstance(a, capa.features.address._NoAddress):
             return cls(type=AddressType.NO_ADDRESS, value=None)
 
@@ -100,6 +125,33 @@ class Address(HashableModel):
             assert isinstance(offset, int)
             return capa.features.address.DNTokenOffsetAddress(token, offset)
 
+        elif self.type is AddressType.PROCESS:
+            assert isinstance(self.value, tuple)
+            ppid, pid = self.value
+            assert isinstance(ppid, int)
+            assert isinstance(pid, int)
+            return capa.features.address.ProcessAddress(ppid=ppid, pid=pid)
+
+        elif self.type is AddressType.THREAD:
+            assert isinstance(self.value, tuple)
+            ppid, pid, tid = self.value
+            assert isinstance(ppid, int)
+            assert isinstance(pid, int)
+            assert isinstance(tid, int)
+            return capa.features.address.ThreadAddress(
+                process=capa.features.address.ProcessAddress(ppid=ppid, pid=pid), tid=tid
+            )
+
+        elif self.type is AddressType.CALL:
+            assert isinstance(self.value, tuple)
+            ppid, pid, tid, id_ = self.value
+            return capa.features.address.DynamicCallAddress(
+                thread=capa.features.address.ThreadAddress(
+                    process=capa.features.address.ProcessAddress(ppid=ppid, pid=pid), tid=tid
+                ),
+                id=id_,
+            )
+
         elif self.type is AddressType.NO_ADDRESS:
             return capa.features.address.NO_ADDRESS
 
@@ -130,6 +182,48 @@ class FileFeature(HashableModel):
     feature: Feature
 
 
+class ProcessFeature(HashableModel):
+    """
+    args:
+        process: the address of the process to which this feature belongs.
+        address: the address at which this feature is found.
+
+    process != address because, e.g., the feature may be found *within* the scope (process).
+    """
+
+    process: Address
+    address: Address
+    feature: Feature
+
+
+class ThreadFeature(HashableModel):
+    """
+    args:
+        thread: the address of the thread to which this feature belongs.
+        address: the address at which this feature is found.
+
+    thread != address because, e.g., the feature may be found *within* the scope (thread).
+    """
+
+    thread: Address
+    address: Address
+    feature: Feature
+
+
+class CallFeature(HashableModel):
+    """
+    args:
+        call: the address of the call to which this feature belongs.
+        address: the address at which this feature is found.
+
+    call != address for consistency with Process and Thread.
+    """
+
+    call: Address
+    address: Address
+    feature: Feature
+
+
 class FunctionFeature(HashableModel):
     """
     args:
@@ -167,8 +261,7 @@ class InstructionFeature(HashableModel):
         instruction: the address of the instruction to which this feature belongs.
         address: the address at which this feature is found.
 
-    instruction != address because, e.g., the feature may be found *within* the scope (basic block),
-    versus right at its starting address.
+    instruction != address because, for consistency with Function and BasicBlock.
     """
 
     instruction: Address
@@ -194,13 +287,42 @@ class FunctionFeatures(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
 
-class Features(BaseModel):
+class CallFeatures(BaseModel):
+    address: Address
+    name: str
+    features: Tuple[CallFeature, ...]
+
+
+class ThreadFeatures(BaseModel):
+    address: Address
+    features: Tuple[ThreadFeature, ...]
+    calls: Tuple[CallFeatures, ...]
+
+
+class ProcessFeatures(BaseModel):
+    address: Address
+    name: str
+    features: Tuple[ProcessFeature, ...]
+    threads: Tuple[ThreadFeatures, ...]
+
+
+class StaticFeatures(BaseModel):
     global_: Tuple[GlobalFeature, ...] = Field(alias="global")
     file: Tuple[FileFeature, ...]
     functions: Tuple[FunctionFeatures, ...]
     model_config = ConfigDict(populate_by_name=True)
 
 
+class DynamicFeatures(BaseModel):
+    global_: Tuple[GlobalFeature, ...] = Field(alias="global")
+    file: Tuple[FileFeature, ...]
+    processes: Tuple[ProcessFeatures, ...]
+    model_config = ConfigDict(populate_by_name=True)
+
+
+Features: TypeAlias = Union[StaticFeatures, DynamicFeatures]
+
+
 class Extractor(BaseModel):
     name: str
     version: str = capa.version.__version__
@@ -208,18 +330,19 @@ class Extractor(BaseModel):
 
 
 class Freeze(BaseModel):
-    version: int = 2
+    version: int = CURRENT_VERSION
     base_address: Address = Field(alias="base address")
+    sample_hashes: SampleHashes
+    flavor: Literal["static", "dynamic"]
     extractor: Extractor
     features: Features
     model_config = ConfigDict(populate_by_name=True)
 
 
-def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> str:
+def dumps_static(extractor: StaticFeatureExtractor) -> str:
     """
     serialize the given extractor to a string
     """
-
     global_features: List[GlobalFeature] = []
     for feature, _ in extractor.extract_global_features():
         global_features.append(
@@ -298,7 +421,7 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
             # Mypy is unable to recognise `basic_blocks` as a argument due to alias
         )
 
-    features = Features(
+    features = StaticFeatures(
         global_=global_features,
         file=tuple(file_features),
         functions=tuple(function_features),
@@ -306,8 +429,10 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
     # Mypy is unable to recognise `global_` as a argument due to alias
 
     freeze = Freeze(
-        version=2,
+        version=CURRENT_VERSION,
         base_address=Address.from_capa(extractor.get_base_address()),
+        sample_hashes=extractor.get_sample_hashes(),
+        flavor="static",
         extractor=Extractor(name=extractor.__class__.__name__),
         features=features,
     )  # type: ignore
@@ -316,16 +441,127 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
     return freeze.model_dump_json()
 
 
-def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
-    """deserialize a set of features (as a NullFeatureExtractor) from a string."""
-    import capa.features.extractors.null as null
+def dumps_dynamic(extractor: DynamicFeatureExtractor) -> str:
+    """
+    serialize the given extractor to a string
+    """
+    global_features: List[GlobalFeature] = []
+    for feature, _ in extractor.extract_global_features():
+        global_features.append(
+            GlobalFeature(
+                feature=feature_from_capa(feature),
+            )
+        )
 
+    file_features: List[FileFeature] = []
+    for feature, address in extractor.extract_file_features():
+        file_features.append(
+            FileFeature(
+                feature=feature_from_capa(feature),
+                address=Address.from_capa(address),
+            )
+        )
+
+    process_features: List[ProcessFeatures] = []
+    for p in extractor.get_processes():
+        paddr = Address.from_capa(p.address)
+        pname = extractor.get_process_name(p)
+        pfeatures = [
+            ProcessFeature(
+                process=paddr,
+                address=Address.from_capa(addr),
+                feature=feature_from_capa(feature),
+            )
+            for feature, addr in extractor.extract_process_features(p)
+        ]
+
+        threads = []
+        for t in extractor.get_threads(p):
+            taddr = Address.from_capa(t.address)
+            tfeatures = [
+                ThreadFeature(
+                    basic_block=taddr,
+                    address=Address.from_capa(addr),
+                    feature=feature_from_capa(feature),
+                )  # type: ignore
+                # Mypy is unable to recognise `basic_block` as a argument due to alias
+                for feature, addr in extractor.extract_thread_features(p, t)
+            ]
+
+            calls = []
+            for call in extractor.get_calls(p, t):
+                caddr = Address.from_capa(call.address)
+                cname = extractor.get_call_name(p, t, call)
+                cfeatures = [
+                    CallFeature(
+                        call=caddr,
+                        address=Address.from_capa(addr),
+                        feature=feature_from_capa(feature),
+                    )
+                    for feature, addr in extractor.extract_call_features(p, t, call)
+                ]
+
+                calls.append(
+                    CallFeatures(
+                        address=caddr,
+                        name=cname,
+                        features=tuple(cfeatures),
+                    )
+                )
+
+            threads.append(
+                ThreadFeatures(
+                    address=taddr,
+                    features=tuple(tfeatures),
+                    calls=tuple(calls),
+                )
+            )
+
+        process_features.append(
+            ProcessFeatures(
+                address=paddr,
+                name=pname,
+                features=tuple(pfeatures),
+                threads=tuple(threads),
+            )
+        )
+
+    features = DynamicFeatures(
+        global_=global_features,
+        file=tuple(file_features),
+        processes=tuple(process_features),
+    )  # type: ignore
+    # Mypy is unable to recognise `global_` as a argument due to alias
+
+    # workaround around mypy issue: https://github.com/python/mypy/issues/1424
+    get_base_addr = getattr(extractor, "get_base_addr", None)
+    base_addr = get_base_addr() if get_base_addr else capa.features.address.NO_ADDRESS
+
+    freeze = Freeze(
+        version=CURRENT_VERSION,
+        base_address=Address.from_capa(base_addr),
+        sample_hashes=extractor.get_sample_hashes(),
+        flavor="dynamic",
+        extractor=Extractor(name=extractor.__class__.__name__),
+        features=features,
+    )  # type: ignore
+    # Mypy is unable to recognise `base_address` as a argument due to alias
+
+    return freeze.model_dump_json()
+
+
+def loads_static(s: str) -> StaticFeatureExtractor:
+    """deserialize a set of features (as a NullStaticFeatureExtractor) from a string."""
     freeze = Freeze.model_validate_json(s)
-    if freeze.version != 2:
+    if freeze.version != CURRENT_VERSION:
         raise ValueError(f"unsupported freeze format version: {freeze.version}")
 
-    return null.NullFeatureExtractor(
+    assert freeze.flavor == "static"
+    assert isinstance(freeze.features, StaticFeatures)
+
+    return null.NullStaticFeatureExtractor(
         base_address=freeze.base_address.to_capa(),
+        sample_hashes=freeze.sample_hashes,
         global_features=[f.feature.to_capa() for f in freeze.features.global_],
         file_features=[(f.address.to_capa(), f.feature.to_capa()) for f in freeze.features.file],
         functions={
@@ -349,10 +585,59 @@ def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
     )
 
 
+def loads_dynamic(s: str) -> DynamicFeatureExtractor:
+    """deserialize a set of features (as a NullDynamicFeatureExtractor) from a string."""
+    freeze = Freeze.model_validate_json(s)
+    if freeze.version != CURRENT_VERSION:
+        raise ValueError(f"unsupported freeze format version: {freeze.version}")
+
+    assert freeze.flavor == "dynamic"
+    assert isinstance(freeze.features, DynamicFeatures)
+
+    return null.NullDynamicFeatureExtractor(
+        base_address=freeze.base_address.to_capa(),
+        sample_hashes=freeze.sample_hashes,
+        global_features=[f.feature.to_capa() for f in freeze.features.global_],
+        file_features=[(f.address.to_capa(), f.feature.to_capa()) for f in freeze.features.file],
+        processes={
+            p.address.to_capa(): null.ProcessFeatures(
+                name=p.name,
+                features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in p.features],
+                threads={
+                    t.address.to_capa(): null.ThreadFeatures(
+                        features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in t.features],
+                        calls={
+                            c.address.to_capa(): null.CallFeatures(
+                                name=c.name,
+                                features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in c.features],
+                            )
+                            for c in t.calls
+                        },
+                    )
+                    for t in p.threads
+                },
+            )
+            for p in freeze.features.processes
+        },
+    )
+
+
 MAGIC = "capa0000".encode("ascii")
 
 
-def dump(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> bytes:
+def dumps(extractor: FeatureExtractor) -> str:
+    """serialize the given extractor to a string."""
+    if isinstance(extractor, StaticFeatureExtractor):
+        doc = dumps_static(extractor)
+    elif isinstance(extractor, DynamicFeatureExtractor):
+        doc = dumps_dynamic(extractor)
+    else:
+        raise ValueError("Invalid feature extractor")
+
+    return doc
+
+
+def dump(extractor: FeatureExtractor) -> bytes:
     """serialize the given extractor to a byte array."""
     return MAGIC + zlib.compress(dumps(extractor).encode("utf-8"))
 
@@ -361,11 +646,28 @@ def is_freeze(buf: bytes) -> bool:
     return buf[: len(MAGIC)] == MAGIC
 
 
-def load(buf: bytes) -> capa.features.extractors.base_extractor.FeatureExtractor:
+def loads(s: str):
+    doc = json.loads(s)
+
+    if doc["version"] != CURRENT_VERSION:
+        raise ValueError(f"unsupported freeze format version: {doc['version']}")
+
+    if doc["flavor"] == "static":
+        return loads_static(s)
+    elif doc["flavor"] == "dynamic":
+        return loads_dynamic(s)
+    else:
+        raise ValueError(f"unsupported freeze format flavor: {doc['flavor']}")
+
+
+def load(buf: bytes):
     """deserialize a set of features (as a NullFeatureExtractor) from a byte array."""
     if not is_freeze(buf):
         raise ValueError("missing magic header")
-    return loads(zlib.decompress(buf[len(MAGIC) :]).decode("utf-8"))
+
+    s = zlib.decompress(buf[len(MAGIC) :]).decode("utf-8")
+
+    return loads(s)
 
 
 def main(argv=None):

capa/ghidra/README.md

@@ -0,0 +1,172 @@
+<div align="center">
+    <img src="/doc/img/ghidra_backend_logo.png" width=300 height=175>
+</div>
+
+The Ghidra feature extractor is an application of the FLARE team's open-source project, Ghidrathon, to integrate capa with Ghidra using Python 3. capa is a framework that uses a well-defined collection of rules to identify capabilities in a program. You can run capa against a PE file, ELF file, or shellcode and it tells you what it thinks the program can do. For example, it might suggest that the program is a backdoor, can install services, or relies on HTTP to communicate. The Ghidra feature extractor can be used to run capa analysis on your Ghidra databases without needing access to the original binary file.
+
+<img src="/doc/img/ghidra_script_mngr_output.png">
+
+## Getting Started
+
+### Installation
+
+Please ensure that you have the following dependencies installed before continuing:
+
+| Dependency | Version | Source |
+|------------|---------|--------|
+| Ghidrathon | `>= 3.0.0` | https://github.com/mandiant/Ghidrathon |
+| Python | `>= 3.8` | https://www.python.org/downloads |
+| Ghidra | `>= 10.2` | https://ghidra-sre.org |
+
+In order to run capa using using Ghidra, you must install capa as a library, obtain the official capa rules that match the capa version you have installed, and configure the Python 3 script [capa_ghidra.py](/capa/ghidra/capa_ghidra.py). You can do this by completing the following steps using the Python 3 interpreter that you have configured for your Ghidrathon installation:
+
+1. Install capa and its dependencies from PyPI using the following command:
+```bash
+$ pip install flare-capa
+```
+
+2. Download and extract the [official capa rules](https://github.com/mandiant/capa-rules/releases) that match the capa version you have installed. Use the following command to view the version of capa you have installed:
+```bash
+$ pip show flare-capa
+OR
+$ capa --version
+```
+
+3. Copy [capa_ghidra.py](/capa/ghidra/capa_ghidra.py) to your `$USER_HOME/ghidra_scripts` directory or manually add `</path/to/ghidra_capa.py/>` to the Ghidra Script Manager.
+
+## Usage
+
+After completing the installation steps you can execute `capa_ghidra.py` using the Ghidra Script Manager or Headless Analyzer.
+
+### Ghidra Script Manager
+
+To execute `capa_ghidra.py` using the Ghidra Script Manager, first open the Ghidra Script Manager by navigating to `Window > Script Manager` in the Ghidra Code Browser. Next, locate `capa_ghidra.py` by selecting the `Python 3 > capa` category or using the Ghidra Script Manager search funtionality. Finally, double-click `capa_ghidra.py` to execute the script. If you don't see `capa_ghidra.py`, make sure you have copied the script to your `$USER_HOME/ghidra_scripts` directory or manually added `</path/to/ghidra_capa.py/>` to the Ghidra Script Manager
+
+When executed, `capa_ghidra.py` asks you to provide your capa rules directory and preferred output format. `capa_ghidra.py` supports `default`, `verbose`, and `vverbose` output formats when executed from the Ghidra Script Manager. `capa_ghidra.py` writes output to the Ghidra Console Window.
+
+#### Example
+
+The following is an example of running `capa_ghidra.py` using the Ghidra Script Manager:
+
+Selecting capa rules:
+<img src="/doc/img/ghidra_script_mngr_rules.png">
+
+Choosing output format:
+<img src="/doc/img/ghidra_script_mngr_verbosity.png">
+
+Viewing results in Ghidra Console Window:
+<img src="/doc/img/ghidra_script_mngr_output.png">
+
+### Ghidra Headless Analyzer
+
+To execute `capa_ghidra.py` using the Ghidra Headless Analyzer, you can use the Ghidra `analyzeHeadless` script located in your `$GHIDRA_HOME/support` directory. You will need to provide the following arguments to the Ghidra `analyzeHeadless` script:
+
+1. `</path/to/ghidra/project/>`: path to Ghidra project
+2. `<ghidra_project_name>`: name of Ghidra Project
+3. `-process <sample_name>`: name of sample `<sample_name>`
+4. `-ScriptPath </path/to/capa_ghidra/>`: OPTIONAL argument specifying path `</path/to/capa_ghidra/>` to `capa_ghidra.py`
+5. `-PostScript capa_ghidra.py`: executes `capa_ghidra.py` as post-analysis script
+6. `"<capa_args>"`: single, quoted string containing capa arguments that must specify capa rules directory and output format, e.g. `"<path/to/capa/rules> --verbose"`. `capa_ghidra.py` supports `default`, `verbose`, `vverbose` and `json` formats when executed using the Ghidra Headless Analyzer. `capa_ghidra.py` writes output to the console window used to execute the Ghidra `analyzeHeadless` script.
+7. `-processor <languageID>`: required ONLY if sample `<sample_name>` is shellcode. More information on specifying the `<languageID>` can be found in the `$GHIDRA_HOME/support/analyzeHeadlessREADME.html` documentation.
+
+The following is an example of combining these arguments into a single `analyzeHeadless` script command:
+
+```
+$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -process <sample_name> -PostScript capa_ghidra.py "/path/to/capa/rules/ --verbose"
+```
+
+You may also want to run capa against a sample that you have not yet imported into your Ghidra project. The following is an example of importing a sample and running `capa_ghidra.py` using a single `analyzeHeadless` script command:
+
+```
+$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -Import </path/to/sample> -PostScript capa_ghidra.py "/path/to/capa/rules/ --verbose"
+```
+
+You can also provide `capa_ghidra.py` the single argument `"help"` to view supported arguments when running the script using the Ghidra Headless Analyzer:
+```
+$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -process <sample_name> -PostScript capa_ghidra.py "help"
+```
+
+#### Example
+
+The following is an example of running `capa_ghidra.py` against a shellcode sample using the Ghidra `analyzeHeadless` script:
+```
+$ analyzeHeadless /home/wumbo/Desktop/ghidra_projects/ capa_test -process 499c2a85f6e8142c3f48d4251c9c7cd6.raw32 -processor x86:LE:32:default -PostScript capa_ghidra.py "/home/wumbo/capa/rules -vv"
+[...]
+
+INFO  REPORT: Analysis succeeded for file: /499c2a85f6e8142c3f48d4251c9c7cd6.raw32 (HeadlessAnalyzer)  
+INFO  SCRIPT: /home/wumbo/ghidra_scripts/capa_ghidra.py (HeadlessAnalyzer)  
+md5                     499c2a85f6e8142c3f48d4251c9c7cd6                                                                                                                                                                                                    
+sha1
+sha256                  e8e02191c1b38c808d27a899ac164b3675eb5cadd3a8907b0ffa863714000e72
+path                    /home/wumbo/capa/tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32
+timestamp               2023-08-29 17:57:00.946588
+capa version            6.1.0
+os                      unknown os
+format                  Raw Binary
+arch                    x86
+extractor               ghidra
+base address            global
+rules                   /home/wumbo/capa/rules
+function count          42
+library function count  0
+total feature count     1970
+
+contain loop (24 matches, only showing first match of library rule)
+author  moritz.raabe@mandiant.com
+scope   function
+function @ 0x0
+  or:
+    characteristic: loop @ 0x0
+    characteristic: tight loop @ 0x278
+
+contain obfuscated stackstrings
+namespace  anti-analysis/obfuscation/string/stackstring
+author     moritz.raabe@mandiant.com
+scope      basic block
+att&ck     Defense Evasion::Obfuscated Files or Information::Indicator Removal from Tools [T1027.005]
+mbc        Anti-Static Analysis::Executable Code Obfuscation::Argument Obfuscation [B0032.020], Anti-Static Analysis::Executable Code Obfuscation::Stack Strings [B0032.017]
+basic block @ 0x0 in function 0x0
+  characteristic: stack string @ 0x0
+
+encode data using XOR
+namespace  data-manipulation/encoding/xor
+author     moritz.raabe@mandiant.com
+scope      basic block
+att&ck     Defense Evasion::Obfuscated Files or Information [T1027]
+mbc        Defense Evasion::Obfuscated Files or Information::Encoding-Standard Algorithm [E1027.m02], Data::Encode Data::XOR [C0026.002]
+basic block @ 0x8AF in function 0x8A1
+  and:
+    characteristic: tight loop @ 0x8AF
+    characteristic: nzxor @ 0x8C0
+    not: = filter for potential false positives
+      or:
+        or: = unsigned bitwise negation operation (~i)
+          number: 0xFFFFFFFF = bitwise negation for unsigned 32 bits
+          number: 0xFFFFFFFFFFFFFFFF = bitwise negation for unsigned 64 bits
+        or: = signed bitwise negation operation (~i)
+          number: 0xFFFFFFF = bitwise negation for signed 32 bits
+          number: 0xFFFFFFFFFFFFFFF = bitwise negation for signed 64 bits
+        or: = Magic constants used in the implementation of strings functions.
+          number: 0x7EFEFEFF = optimized string constant for 32 bits
+          number: 0x81010101 = -0x81010101 = 0x7EFEFEFF
+          number: 0x81010100 = 0x81010100 = ~0x7EFEFEFF
+          number: 0x7EFEFEFEFEFEFEFF = optimized string constant for 64 bits
+          number: 0x8101010101010101 = -0x8101010101010101 = 0x7EFEFEFEFEFEFEFF
+          number: 0x8101010101010100 = 0x8101010101010100 = ~0x7EFEFEFEFEFEFEFF
+
+get OS information via KUSER_SHARED_DATA
+namespace   host-interaction/os/version
+author      @mr-tz
+scope       function
+att&ck      Discovery::System Information Discovery [T1082]
+references  https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/ntexapi_x/kuser_shared_data/index.htm
+function @ 0x1CA6
+  or:
+    number: 0x7FFE026C = NtMajorVersion @ 0x1D18
+
+
+
+Script /home/wumbo/ghidra_scripts/capa_ghidra.py called exit with code 0
+
+[...]
+```

capa/ghidra/capa_ghidra.py

@@ -0,0 +1,167 @@
+# Run capa against loaded Ghidra database
+# @author Mike Hunhoff (mehunhoff@google.com)
+# @category Python 3.capa
+
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import sys
+import logging
+import pathlib
+import argparse
+
+import capa
+import capa.main
+import capa.rules
+import capa.ghidra.helpers
+import capa.render.default
+import capa.capabilities.common
+import capa.features.extractors.ghidra.extractor
+
+logger = logging.getLogger("capa_ghidra")
+
+
+def run_headless():
+    parser = argparse.ArgumentParser(description="The FLARE team's open-source tool to integrate capa with Ghidra.")
+
+    parser.add_argument(
+        "rules",
+        type=str,
+        help="path to rule file or directory",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", help="enable verbose result document (no effect with --json)"
+    )
+    parser.add_argument(
+        "-vv", "--vverbose", action="store_true", help="enable very verbose result document (no effect with --json)"
+    )
+    parser.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR")
+    parser.add_argument("-q", "--quiet", action="store_true", help="disable all output but errors")
+    parser.add_argument("-j", "--json", action="store_true", help="emit JSON instead of text")
+
+    script_args = list(getScriptArgs())  # type: ignore [name-defined] # noqa: F821
+    if not script_args or len(script_args) > 1:
+        script_args = []
+    else:
+        script_args = script_args[0].split()
+        for idx, arg in enumerate(script_args):
+            if arg.lower() == "help":
+                script_args[idx] = "--help"
+
+    args = parser.parse_args(args=script_args)
+
+    if args.quiet:
+        logging.basicConfig(level=logging.WARNING)
+        logging.getLogger().setLevel(logging.WARNING)
+    elif args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+        logging.getLogger().setLevel(logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+        logging.getLogger().setLevel(logging.INFO)
+
+    logger.debug("running in Ghidra headless mode")
+
+    rules_path = pathlib.Path(args.rules)
+
+    logger.debug("rule path: %s", rules_path)
+    rules = capa.main.get_rules([rules_path])
+
+    meta = capa.ghidra.helpers.collect_metadata([rules_path])
+    extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
+
+    capabilities, counts = capa.capabilities.common.find_capabilities(rules, extractor, False)
+
+    meta.analysis.feature_counts = counts["feature_counts"]
+    meta.analysis.library_functions = counts["library_functions"]
+    meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
+
+    if capa.capabilities.common.has_file_limitation(rules, capabilities, is_standalone=True):
+        logger.info("capa encountered warnings during analysis")
+
+    if args.json:
+        print(capa.render.json.render(meta, rules, capabilities))  # noqa: T201
+    elif args.vverbose:
+        print(capa.render.vverbose.render(meta, rules, capabilities))  # noqa: T201
+    elif args.verbose:
+        print(capa.render.verbose.render(meta, rules, capabilities))  # noqa: T201
+    else:
+        print(capa.render.default.render(meta, rules, capabilities))  # noqa: T201
+
+    return 0
+
+
+def run_ui():
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger().setLevel(logging.INFO)
+
+    rules_dir: str = ""
+    try:
+        selected_dir = askDirectory("Choose capa rules directory", "Ok")  # type: ignore [name-defined] # noqa: F821
+        if selected_dir:
+            rules_dir = selected_dir.getPath()
+    except RuntimeError:
+        # RuntimeError thrown when user selects "Cancel"
+        pass
+
+    if not rules_dir:
+        logger.info("You must choose a capa rules directory before running capa.")
+        return capa.main.E_MISSING_RULES
+
+    verbose = askChoice(  # type: ignore [name-defined] # noqa: F821
+        "capa output verbosity", "Choose capa output verbosity", ["default", "verbose", "vverbose"], "default"
+    )
+
+    rules_path: pathlib.Path = pathlib.Path(rules_dir)
+    logger.info("running capa using rules from %s", str(rules_path))
+
+    rules = capa.main.get_rules([rules_path])
+
+    meta = capa.ghidra.helpers.collect_metadata([rules_path])
+    extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
+
+    capabilities, counts = capa.capabilities.common.find_capabilities(rules, extractor, True)
+
+    meta.analysis.feature_counts = counts["feature_counts"]
+    meta.analysis.library_functions = counts["library_functions"]
+    meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
+
+    if capa.capabilities.common.has_file_limitation(rules, capabilities, is_standalone=False):
+        logger.info("capa encountered warnings during analysis")
+
+    if verbose == "vverbose":
+        print(capa.render.vverbose.render(meta, rules, capabilities))  # noqa: T201
+    elif verbose == "verbose":
+        print(capa.render.verbose.render(meta, rules, capabilities))  # noqa: T201
+    else:
+        print(capa.render.default.render(meta, rules, capabilities))  # noqa: T201
+
+    return 0
+
+
+def main():
+    if not capa.ghidra.helpers.is_supported_ghidra_version():
+        return capa.main.E_UNSUPPORTED_GHIDRA_VERSION
+
+    if not capa.ghidra.helpers.is_supported_file_type():
+        return capa.main.E_INVALID_FILE_TYPE
+
+    if not capa.ghidra.helpers.is_supported_arch_type():
+        return capa.main.E_INVALID_FILE_ARCH
+
+    if isRunningHeadless():  # type: ignore [name-defined] # noqa: F821
+        return run_headless()
+    else:
+        return run_ui()
+
+
+if __name__ == "__main__":
+    if sys.version_info < (3, 8):
+        from capa.exceptions import UnsupportedRuntimeError
+
+        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+")
+    sys.exit(main())

capa/ghidra/helpers.py

@@ -0,0 +1,160 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+import datetime
+import contextlib
+from typing import List
+from pathlib import Path
+
+import capa
+import capa.version
+import capa.features.common
+import capa.features.freeze
+import capa.render.result_document as rdoc
+import capa.features.extractors.ghidra.helpers
+
+logger = logging.getLogger("capa")
+
+# file type as returned by Ghidra
+SUPPORTED_FILE_TYPES = ("Executable and Linking Format (ELF)", "Portable Executable (PE)", "Raw Binary")
+
+
+class GHIDRAIO:
+    """
+    An object that acts as a file-like object,
+    using bytes from the current Ghidra listing.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+        self.offset = 0
+        self.bytes_ = self.get_bytes()
+
+    def seek(self, offset, whence=0):
+        assert whence == 0
+        self.offset = offset
+
+    def read(self, size):
+        logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, currentProgram().getImageBase().add(self.offset).getOffset())  # type: ignore [name-defined] # noqa: F821
+
+        if size > len(self.bytes_) - self.offset:
+            logger.debug("cannot read 0x%x bytes at 0x%x (ea: BADADDR)", size, self.offset)
+            return b""
+        else:
+            return self.bytes_[self.offset : self.offset + size]
+
+    def close(self):
+        return
+
+    def get_bytes(self):
+        file_bytes = currentProgram().getMemory().getAllFileBytes()[0]  # type: ignore [name-defined] # noqa: F821
+
+        # getOriginalByte() allows for raw file parsing on the Ghidra side
+        # other functions will fail as Ghidra will think that it's reading uninitialized memory
+        bytes_ = [file_bytes.getOriginalByte(i) for i in range(file_bytes.getSize())]
+
+        return capa.features.extractors.ghidra.helpers.ints_to_bytes(bytes_)
+
+
+def is_supported_ghidra_version():
+    version = float(getGhidraVersion()[:4])  # type: ignore [name-defined] # noqa: F821
+    if version < 10.2:
+        warning_msg = "capa does not support this Ghidra version"
+        logger.warning(warning_msg)
+        logger.warning("Your Ghidra version is: %s. Supported versions are: Ghidra >= 10.2", version)
+        return False
+    return True
+
+
+def is_running_headless():
+    return isRunningHeadless()  # type: ignore [name-defined] # noqa: F821
+
+
+def is_supported_file_type():
+    file_info = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+    if file_info not in SUPPORTED_FILE_TYPES:
+        logger.error("-" * 80)
+        logger.error(" Input file does not appear to be a supported file type.")
+        logger.error(" ")
+        logger.error(
+            " capa currently only supports analyzing PE, ELF, or binary files containing x86 (32- and 64-bit) shellcode."
+        )
+        logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
+        logger.error("-" * 80)
+        return False
+    return True
+
+
+def is_supported_arch_type():
+    lang_id = str(currentProgram().getLanguageID()).lower()  # type: ignore [name-defined] # noqa: F821
+
+    if not all((lang_id.startswith("x86"), any(arch in lang_id for arch in ("32", "64")))):
+        logger.error("-" * 80)
+        logger.error(" Input file does not appear to target a supported architecture.")
+        logger.error(" ")
+        logger.error(" capa currently only supports analyzing x86 (32- and 64-bit).")
+        logger.error("-" * 80)
+        return False
+    return True
+
+
+def get_file_md5():
+    return currentProgram().getExecutableMD5()  # type: ignore [name-defined] # noqa: F821
+
+
+def get_file_sha256():
+    return currentProgram().getExecutableSHA256()  # type: ignore [name-defined] # noqa: F821
+
+
+def collect_metadata(rules: List[Path]):
+    md5 = get_file_md5()
+    sha256 = get_file_sha256()
+
+    info = currentProgram().getLanguageID().toString()  # type: ignore [name-defined] # noqa: F821
+    if "x86" in info and "64" in info:
+        arch = "x86_64"
+    elif "x86" in info and "32" in info:
+        arch = "x86"
+    else:
+        arch = "unknown arch"
+
+    format_name: str = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+    if "PE" in format_name:
+        os = "windows"
+    elif "ELF" in format_name:
+        with contextlib.closing(capa.ghidra.helpers.GHIDRAIO()) as f:
+            os = capa.features.extractors.elf.detect_elf_os(f)
+    else:
+        os = "unknown os"
+
+    return rdoc.Metadata(
+        timestamp=datetime.datetime.now(),
+        version=capa.version.__version__,
+        argv=(),
+        sample=rdoc.Sample(
+            md5=md5,
+            sha1="",
+            sha256=sha256,
+            path=currentProgram().getExecutablePath(),  # type: ignore [name-defined] # noqa: F821
+        ),
+        flavor=rdoc.Flavor.STATIC,
+        analysis=rdoc.StaticAnalysis(
+            format=currentProgram().getExecutableFormat(),  # type: ignore [name-defined] # noqa: F821
+            arch=arch,
+            os=os,
+            extractor="ghidra",
+            rules=tuple(r.resolve().absolute().as_posix() for r in rules),
+            base_address=capa.features.freeze.Address.from_capa(currentProgram().getImageBase().getOffset()),  # type: ignore [name-defined] # noqa: F821
+            layout=rdoc.StaticLayout(
+                functions=(),
+            ),
+            feature_counts=rdoc.StaticFeatureCounts(file=0, functions=()),
+            library_functions=(),
+        ),
+    )

capa/helpers.py

@@ -5,6 +5,7 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import json
 import inspect
 import logging
 import contextlib
@@ -15,10 +16,11 @@ from pathlib import Path
 import tqdm
 
 from capa.exceptions import UnsupportedFormatError
-from capa.features.common import FORMAT_PE, FORMAT_SC32, FORMAT_SC64, FORMAT_DOTNET, FORMAT_UNKNOWN, Format
+from capa.features.common import FORMAT_PE, FORMAT_CAPE, FORMAT_SC32, FORMAT_SC64, FORMAT_DOTNET, FORMAT_UNKNOWN, Format
 
 EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32")
 EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64")
+EXTENSIONS_DYNAMIC = ("json", "json_")
 EXTENSIONS_ELF = "elf_"
 
 logger = logging.getLogger("capa")
@@ -43,20 +45,45 @@ def is_runtime_ida():
     return importlib.util.find_spec("idc") is not None
 
 
+def is_runtime_ghidra():
+    try:
+        currentProgram  # type: ignore [name-defined] # noqa: F821
+    except NameError:
+        return False
+    return True
+
+
 def assert_never(value) -> NoReturn:
     # careful: python -O will remove this assertion.
     # but this is only used for type checking, so it's ok.
     assert False, f"Unhandled value: {value} ({type(value).__name__})"  # noqa: B011
 
 
-def get_format_from_extension(sample: Path) -> str:
-    if sample.name.endswith(EXTENSIONS_SHELLCODE_32):
-        return FORMAT_SC32
-    elif sample.name.endswith(EXTENSIONS_SHELLCODE_64):
-        return FORMAT_SC64
+def get_format_from_report(sample: Path) -> str:
+    report = json.load(sample.open(encoding="utf-8"))
+
+    if "CAPE" in report:
+        return FORMAT_CAPE
+
+    if "target" in report and "info" in report and "behavior" in report:
+        # CAPE report that's missing the "CAPE" key,
+        # which is not going to be much use, but its correct.
+        return FORMAT_CAPE
+
     return FORMAT_UNKNOWN
 
 
+def get_format_from_extension(sample: Path) -> str:
+    format_ = FORMAT_UNKNOWN
+    if sample.name.endswith(EXTENSIONS_SHELLCODE_32):
+        format_ = FORMAT_SC32
+    elif sample.name.endswith(EXTENSIONS_SHELLCODE_64):
+        format_ = FORMAT_SC64
+    elif sample.name.endswith(EXTENSIONS_DYNAMIC):
+        format_ = get_format_from_report(sample)
+    return format_
+
+
 def get_auto_format(path: Path) -> str:
     format_ = get_format(path)
     if format_ == FORMAT_UNKNOWN:
@@ -69,13 +96,13 @@ def get_auto_format(path: Path) -> str:
 def get_format(sample: Path) -> str:
     # imported locally to avoid import cycle
     from capa.features.extractors.common import extract_format
-    from capa.features.extractors.dnfile_ import DnfileFeatureExtractor
+    from capa.features.extractors.dotnetfile import DotnetFileFeatureExtractor
 
     buf = sample.read_bytes()
 
     for feature, _ in extract_format(buf):
         if feature == Format(FORMAT_PE):
-            dnfile_extractor = DnfileFeatureExtractor(sample)
+            dnfile_extractor = DotnetFileFeatureExtractor(sample)
             if dnfile_extractor.is_dotnet_file():
                 feature = Format(FORMAT_DOTNET)
 
@@ -120,15 +147,32 @@ def redirecting_print_to_tqdm(disable_progress):
 
 def log_unsupported_format_error():
     logger.error("-" * 80)
-    logger.error(" Input file does not appear to be a PE or ELF file.")
+    logger.error(" Input file does not appear to be a supported file.")
     logger.error(" ")
-    logger.error(
-        " capa currently only supports analyzing PE and ELF files (or shellcode, when using --format sc32|sc64)."
-    )
+    logger.error(" See all supported file formats via capa's help output (-h).")
     logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
     logger.error("-" * 80)
 
 
+def log_unsupported_cape_report_error(error: str):
+    logger.error("-" * 80)
+    logger.error("Input file is not a valid CAPE report: %s", error)
+    logger.error(" ")
+    logger.error(" capa currently only supports analyzing standard CAPE reports in JSON format.")
+    logger.error(
+        " Please make sure your report file is in the standard format and contains both the static and dynamic sections."
+    )
+    logger.error("-" * 80)
+
+
+def log_empty_cape_report_error(error: str):
+    logger.error("-" * 80)
+    logger.error(" CAPE report is empty or only contains little useful data: %s", error)
+    logger.error(" ")
+    logger.error(" Please make sure the sandbox run captures useful behaviour of your sample.")
+    logger.error("-" * 80)
+
+
 def log_unsupported_os_error():
     logger.error("-" * 80)
     logger.error(" Input file does not appear to target a supported OS.")

capa/ida/helpers.py

@@ -152,14 +152,15 @@ def collect_metadata(rules: List[Path]):
             sha256=sha256,
             path=idaapi.get_input_file_path(),
         ),
-        analysis=rdoc.Analysis(
+        flavor=rdoc.Flavor.STATIC,
+        analysis=rdoc.StaticAnalysis(
             format=idaapi.get_file_type_name(),
             arch=arch,
             os=os,
             extractor="ida",
             rules=tuple(r.resolve().absolute().as_posix() for r in rules),
             base_address=capa.features.freeze.Address.from_capa(idaapi.get_imagebase()),
-            layout=rdoc.Layout(
+            layout=rdoc.StaticLayout(
                 functions=(),
                 # this is updated after capabilities have been collected.
                 # will look like:
@@ -167,7 +168,7 @@ def collect_metadata(rules: List[Path]):
                 # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
             ),
             # ignore these for now - not used by IDA plugin.
-            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
+            feature_counts=rdoc.StaticFeatureCounts(file=0, functions=()),
             library_functions=(),
         ),
     )

capa/ida/plugin/form.py

@@ -25,6 +25,7 @@ import capa.version
 import capa.ida.helpers
 import capa.render.json
 import capa.features.common
+import capa.capabilities.common
 import capa.render.result_document
 import capa.features.extractors.ida.extractor
 from capa.rules import Rule
@@ -768,7 +769,7 @@ class CapaExplorerForm(idaapi.PluginForm):
 
                 try:
                     meta = capa.ida.helpers.collect_metadata([Path(settings.user[CAPA_SETTINGS_RULE_PATH])])
-                    capabilities, counts = capa.main.find_capabilities(
+                    capabilities, counts = capa.capabilities.common.find_capabilities(
                         ruleset, self.feature_extractor, disable_progress=True
                     )
 
@@ -810,7 +811,7 @@ class CapaExplorerForm(idaapi.PluginForm):
 
                         capa.ida.helpers.inform_user_ida_ui("capa encountered file type warnings during analysis")
 
-                    if capa.main.has_file_limitation(ruleset, capabilities, is_standalone=False):
+                    if capa.capabilities.common.has_file_limitation(ruleset, capabilities, is_standalone=False):
                         capa.ida.helpers.inform_user_ida_ui("capa encountered file limitation warnings during analysis")
                 except Exception as e:
                     logger.exception("Failed to check for file limitations (error: %s)", e)
@@ -1192,10 +1193,13 @@ class CapaExplorerForm(idaapi.PluginForm):
             return
 
         is_match: bool = False
-        if self.rulegen_current_function is not None and rule.scope in (
-            capa.rules.Scope.FUNCTION,
-            capa.rules.Scope.BASIC_BLOCK,
-            capa.rules.Scope.INSTRUCTION,
+        if self.rulegen_current_function is not None and any(
+            s in rule.scopes
+            for s in (
+                capa.rules.Scope.FUNCTION,
+                capa.rules.Scope.BASIC_BLOCK,
+                capa.rules.Scope.INSTRUCTION,
+            )
         ):
             try:
                 _, func_matches, bb_matches, insn_matches = self.rulegen_feature_cache.find_code_capabilities(
@@ -1205,13 +1209,13 @@ class CapaExplorerForm(idaapi.PluginForm):
                 self.set_rulegen_status(f"Failed to create function rule matches from rule set ({e})")
                 return
 
-            if rule.scope == capa.rules.Scope.FUNCTION and rule.name in func_matches:
+            if capa.rules.Scope.FUNCTION in rule.scopes and rule.name in func_matches:
                 is_match = True
-            elif rule.scope == capa.rules.Scope.BASIC_BLOCK and rule.name in bb_matches:
+            elif capa.rules.Scope.BASIC_BLOCK in rule.scopes and rule.name in bb_matches:
                 is_match = True
-            elif rule.scope == capa.rules.Scope.INSTRUCTION and rule.name in insn_matches:
+            elif capa.rules.Scope.INSTRUCTION in rule.scopes and rule.name in insn_matches:
                 is_match = True
-        elif rule.scope == capa.rules.Scope.FILE:
+        elif capa.rules.Scope.FILE in rule.scopes:
             try:
                 _, file_matches = self.rulegen_feature_cache.find_file_capabilities(ruleset)
             except Exception as e:

capa/ida/plugin/model.py

@@ -500,16 +500,16 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                 location = location_.to_capa()
 
                 parent2: CapaExplorerDataItem
-                if rule.meta.scope == capa.rules.FILE_SCOPE:
+                if capa.rules.Scope.FILE in rule.meta.scopes:
                     parent2 = parent
-                elif rule.meta.scope == capa.rules.FUNCTION_SCOPE:
+                elif capa.rules.Scope.FUNCTION in rule.meta.scopes:
                     parent2 = CapaExplorerFunctionItem(parent, location)
-                elif rule.meta.scope == capa.rules.BASIC_BLOCK_SCOPE:
+                elif capa.rules.Scope.BASIC_BLOCK in rule.meta.scopes:
                     parent2 = CapaExplorerBlockItem(parent, location)
-                elif rule.meta.scope == capa.rules.INSTRUCTION_SCOPE:
+                elif capa.rules.Scope.INSTRUCTION in rule.meta.scopes:
                     parent2 = CapaExplorerInstructionItem(parent, location)
                 else:
-                    raise RuntimeError("unexpected rule scope: " + str(rule.meta.scope))
+                    raise RuntimeError("unexpected rule scope: " + str(rule.meta.scopes.static))
 
                 self.render_capa_doc_match(parent2, match, doc)
 

capa/main.py

@@ -11,23 +11,21 @@ See the License for the specific language governing permissions and limitations
 import io
 import os
 import sys
+import json
 import time
-import hashlib
 import logging
 import argparse
 import datetime
 import textwrap
-import itertools
 import contextlib
-import collections
-from typing import Any, Dict, List, Tuple, Callable, Optional
+from types import TracebackType
+from typing import Any, Set, Dict, List, Callable, Optional
 from pathlib import Path
 
 import halo
-import tqdm
 import colorama
-import tqdm.contrib.logging
 from pefile import PEFormatError
+from typing_extensions import assert_never
 from elftools.common.exceptions import ELFError
 
 import capa.perf
@@ -47,22 +45,28 @@ import capa.render.result_document
 import capa.render.result_document as rdoc
 import capa.features.extractors.common
 import capa.features.extractors.pefile
-import capa.features.extractors.dnfile_
 import capa.features.extractors.elffile
 import capa.features.extractors.dotnetfile
 import capa.features.extractors.base_extractor
-from capa.rules import Rule, Scope, RuleSet
-from capa.engine import FeatureSet, MatchResults
+import capa.features.extractors.cape.extractor
+from capa.rules import Rule, RuleSet
+from capa.engine import MatchResults
 from capa.helpers import (
-    get_format,
     get_file_taste,
     get_auto_format,
     log_unsupported_os_error,
-    redirecting_print_to_tqdm,
     log_unsupported_arch_error,
+    log_empty_cape_report_error,
     log_unsupported_format_error,
+    log_unsupported_cape_report_error,
+)
+from capa.exceptions import (
+    EmptyReportError,
+    UnsupportedOSError,
+    UnsupportedArchError,
+    UnsupportedFormatError,
+    UnsupportedRuntimeError,
 )
-from capa.exceptions import UnsupportedOSError, UnsupportedArchError, UnsupportedFormatError, UnsupportedRuntimeError
 from capa.features.common import (
     OS_AUTO,
     OS_LINUX,
@@ -71,14 +75,21 @@ from capa.features.common import (
     FORMAT_ELF,
     OS_WINDOWS,
     FORMAT_AUTO,
+    FORMAT_CAPE,
     FORMAT_SC32,
     FORMAT_SC64,
     FORMAT_DOTNET,
     FORMAT_FREEZE,
     FORMAT_RESULT,
 )
-from capa.features.address import NO_ADDRESS, Address
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+from capa.features.address import Address
+from capa.capabilities.common import find_capabilities, has_file_limitation, find_file_capabilities
+from capa.features.extractors.base_extractor import (
+    SampleHashes,
+    FeatureExtractor,
+    StaticFeatureExtractor,
+    DynamicFeatureExtractor,
+)
 
 RULES_PATH_DEFAULT_STRING = "(embedded rules)"
 SIGNATURES_PATH_DEFAULT_STRING = "(embedded signatures)"
@@ -97,6 +108,10 @@ E_INVALID_FILE_TYPE = 16
 E_INVALID_FILE_ARCH = 17
 E_INVALID_FILE_OS = 18
 E_UNSUPPORTED_IDA_VERSION = 19
+E_UNSUPPORTED_GHIDRA_VERSION = 20
+E_MISSING_CAPE_STATIC_ANALYSIS = 21
+E_MISSING_CAPE_DYNAMIC_ANALYSIS = 22
+E_EMPTY_REPORT = 23
 
 logger = logging.getLogger("capa")
 
@@ -119,262 +134,6 @@ def set_vivisect_log_level(level):
     logging.getLogger("Elf").setLevel(level)
 
 
-def find_instruction_capabilities(
-    ruleset: RuleSet, extractor: FeatureExtractor, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
-) -> Tuple[FeatureSet, MatchResults]:
-    """
-    find matches for the given rules for the given instruction.
-
-    returns: tuple containing (features for instruction, match results for instruction)
-    """
-    # all features found for the instruction.
-    features = collections.defaultdict(set)  # type: FeatureSet
-
-    for feature, addr in itertools.chain(
-        extractor.extract_insn_features(f, bb, insn), extractor.extract_global_features()
-    ):
-        features[feature].add(addr)
-
-    # matches found at this instruction.
-    _, matches = ruleset.match(Scope.INSTRUCTION, features, insn.address)
-
-    for rule_name, res in matches.items():
-        rule = ruleset[rule_name]
-        for addr, _ in res:
-            capa.engine.index_rule_matches(features, rule, [addr])
-
-    return features, matches
-
-
-def find_basic_block_capabilities(
-    ruleset: RuleSet, extractor: FeatureExtractor, f: FunctionHandle, bb: BBHandle
-) -> Tuple[FeatureSet, MatchResults, MatchResults]:
-    """
-    find matches for the given rules within the given basic block.
-
-    returns: tuple containing (features for basic block, match results for basic block, match results for instructions)
-    """
-    # all features found within this basic block,
-    # includes features found within instructions.
-    features = collections.defaultdict(set)  # type: FeatureSet
-
-    # matches found at the instruction scope.
-    # might be found at different instructions, thats ok.
-    insn_matches = collections.defaultdict(list)  # type: MatchResults
-
-    for insn in extractor.get_instructions(f, bb):
-        ifeatures, imatches = find_instruction_capabilities(ruleset, extractor, f, bb, insn)
-        for feature, vas in ifeatures.items():
-            features[feature].update(vas)
-
-        for rule_name, res in imatches.items():
-            insn_matches[rule_name].extend(res)
-
-    for feature, va in itertools.chain(
-        extractor.extract_basic_block_features(f, bb), extractor.extract_global_features()
-    ):
-        features[feature].add(va)
-
-    # matches found within this basic block.
-    _, matches = ruleset.match(Scope.BASIC_BLOCK, features, bb.address)
-
-    for rule_name, res in matches.items():
-        rule = ruleset[rule_name]
-        for va, _ in res:
-            capa.engine.index_rule_matches(features, rule, [va])
-
-    return features, matches, insn_matches
-
-
-def find_code_capabilities(
-    ruleset: RuleSet, extractor: FeatureExtractor, fh: FunctionHandle
-) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
-    """
-    find matches for the given rules within the given function.
-
-    returns: tuple containing (match results for function, match results for basic blocks, match results for instructions, number of features)
-    """
-    # all features found within this function,
-    # includes features found within basic blocks (and instructions).
-    function_features = collections.defaultdict(set)  # type: FeatureSet
-
-    # matches found at the basic block scope.
-    # might be found at different basic blocks, thats ok.
-    bb_matches = collections.defaultdict(list)  # type: MatchResults
-
-    # matches found at the instruction scope.
-    # might be found at different instructions, thats ok.
-    insn_matches = collections.defaultdict(list)  # type: MatchResults
-
-    for bb in extractor.get_basic_blocks(fh):
-        features, bmatches, imatches = find_basic_block_capabilities(ruleset, extractor, fh, bb)
-        for feature, vas in features.items():
-            function_features[feature].update(vas)
-
-        for rule_name, res in bmatches.items():
-            bb_matches[rule_name].extend(res)
-
-        for rule_name, res in imatches.items():
-            insn_matches[rule_name].extend(res)
-
-    for feature, va in itertools.chain(extractor.extract_function_features(fh), extractor.extract_global_features()):
-        function_features[feature].add(va)
-
-    _, function_matches = ruleset.match(Scope.FUNCTION, function_features, fh.address)
-    return function_matches, bb_matches, insn_matches, len(function_features)
-
-
-def find_file_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, function_features: FeatureSet):
-    file_features = collections.defaultdict(set)  # type: FeatureSet
-
-    for feature, va in itertools.chain(extractor.extract_file_features(), extractor.extract_global_features()):
-        # not all file features may have virtual addresses.
-        # if not, then at least ensure the feature shows up in the index.
-        # the set of addresses will still be empty.
-        if va:
-            file_features[feature].add(va)
-        else:
-            if feature not in file_features:
-                file_features[feature] = set()
-
-    logger.debug("analyzed file and extracted %d features", len(file_features))
-
-    file_features.update(function_features)
-
-    _, matches = ruleset.match(Scope.FILE, file_features, NO_ADDRESS)
-    return matches, len(file_features)
-
-
-def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_progress=None) -> Tuple[MatchResults, Any]:
-    all_function_matches = collections.defaultdict(list)  # type: MatchResults
-    all_bb_matches = collections.defaultdict(list)  # type: MatchResults
-    all_insn_matches = collections.defaultdict(list)  # type: MatchResults
-
-    feature_counts = rdoc.FeatureCounts(file=0, functions=())
-    library_functions: Tuple[rdoc.LibraryFunction, ...] = ()
-
-    with redirecting_print_to_tqdm(disable_progress):
-        with tqdm.contrib.logging.logging_redirect_tqdm():
-            pbar = tqdm.tqdm
-            if disable_progress:
-                # do not use tqdm to avoid unnecessary side effects when caller intends
-                # to disable progress completely
-                def pbar(s, *args, **kwargs):
-                    return s
-
-            functions = list(extractor.get_functions())
-            n_funcs = len(functions)
-
-            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False)
-            for f in pb:
-                t0 = time.time()
-                if extractor.is_library_function(f.address):
-                    function_name = extractor.get_function_name(f.address)
-                    logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
-                    library_functions += (
-                        rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name),
-                    )
-                    n_libs = len(library_functions)
-                    percentage = round(100 * (n_libs / n_funcs))
-                    if isinstance(pb, tqdm.tqdm):
-                        pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
-                    continue
-
-                function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(
-                    ruleset, extractor, f
-                )
-                feature_counts.functions += (
-                    rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count),
-                )
-                t1 = time.time()
-
-                match_count = sum(len(res) for res in function_matches.values())
-                match_count += sum(len(res) for res in bb_matches.values())
-                match_count += sum(len(res) for res in insn_matches.values())
-                logger.debug(
-                    "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
-                    f.address,
-                    feature_count,
-                    match_count,
-                    t1 - t0,
-                )
-
-                for rule_name, res in function_matches.items():
-                    all_function_matches[rule_name].extend(res)
-                for rule_name, res in bb_matches.items():
-                    all_bb_matches[rule_name].extend(res)
-                for rule_name, res in insn_matches.items():
-                    all_insn_matches[rule_name].extend(res)
-
-    # collection of features that captures the rule matches within function, BB, and instruction scopes.
-    # mapping from feature (matched rule) to set of addresses at which it matched.
-    function_and_lower_features: FeatureSet = collections.defaultdict(set)
-    for rule_name, results in itertools.chain(
-        all_function_matches.items(), all_bb_matches.items(), all_insn_matches.items()
-    ):
-        locations = {p[0] for p in results}
-        rule = ruleset[rule_name]
-        capa.engine.index_rule_matches(function_and_lower_features, rule, locations)
-
-    all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, function_and_lower_features)
-    feature_counts.file = feature_count
-
-    matches = dict(
-        itertools.chain(
-            # each rule exists in exactly one scope,
-            # so there won't be any overlap among these following MatchResults,
-            # and we can merge the dictionaries naively.
-            all_insn_matches.items(),
-            all_bb_matches.items(),
-            all_function_matches.items(),
-            all_file_matches.items(),
-        )
-    )
-
-    meta = {
-        "feature_counts": feature_counts,
-        "library_functions": library_functions,
-    }
-
-    return matches, meta
-
-
-def has_rule_with_namespace(rules: RuleSet, capabilities: MatchResults, namespace: str) -> bool:
-    return any(
-        rules.rules[rule_name].meta.get("namespace", "").startswith(namespace) for rule_name in capabilities.keys()
-    )
-
-
-def is_internal_rule(rule: Rule) -> bool:
-    return rule.meta.get("namespace", "").startswith("internal/")
-
-
-def is_file_limitation_rule(rule: Rule) -> bool:
-    return rule.meta.get("namespace", "") == "internal/limitation/file"
-
-
-def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalone=True) -> bool:
-    file_limitation_rules = list(filter(is_file_limitation_rule, rules.rules.values()))
-
-    for file_limitation_rule in file_limitation_rules:
-        if file_limitation_rule.name not in capabilities:
-            continue
-
-        logger.warning("-" * 80)
-        for line in file_limitation_rule.meta.get("description", "").split("\n"):
-            logger.warning(" %s", line)
-        logger.warning(" Identified via rule: %s", file_limitation_rule.name)
-        if is_standalone:
-            logger.warning(" ")
-            logger.warning(" Use -v or -vv if you really want to see the capabilities identified by capa.")
-        logger.warning("-" * 80)
-
-        # bail on first file limitation
-        return True
-
-    return False
-
-
 def is_supported_format(sample: Path) -> bool:
     """
     Return if this is a supported file based on magic header values
@@ -455,7 +214,7 @@ def get_default_signatures() -> List[Path]:
     """
     compute a list of file system paths to the default FLIRT signatures.
     """
-    sigs_path = get_default_root() / "sigs"
+    sigs_path = get_default_root() / "capa" / "sigs"
     logger.debug("signatures path: %s", sigs_path)
 
     ret = []
@@ -526,7 +285,8 @@ def get_extractor(
       UnsupportedArchError
       UnsupportedOSError
     """
-    if format_ not in (FORMAT_SC32, FORMAT_SC64):
+
+    if format_ not in (FORMAT_SC32, FORMAT_SC64, FORMAT_CAPE):
         if not is_supported_format(path):
             raise UnsupportedFormatError()
 
@@ -536,7 +296,13 @@ def get_extractor(
         if os_ == OS_AUTO and not is_supported_os(path):
             raise UnsupportedOSError()
 
-    if format_ == FORMAT_DOTNET:
+    if format_ == FORMAT_CAPE:
+        import capa.features.extractors.cape.extractor
+
+        report = json.load(Path(path).open(encoding="utf-8"))
+        return capa.features.extractors.cape.extractor.CapeExtractor.from_report(report)
+
+    elif format_ == FORMAT_DOTNET:
         import capa.features.extractors.dnfile.extractor
 
         return capa.features.extractors.dnfile.extractor.DnfileFeatureExtractor(path)
@@ -552,7 +318,8 @@ def get_extractor(
                 sys.path.append(str(bn_api))
 
         try:
-            from binaryninja import BinaryView, BinaryViewType
+            import binaryninja
+            from binaryninja import BinaryView
         except ImportError:
             raise RuntimeError(
                 "Cannot import binaryninja module. Please install the Binary Ninja Python API first: "
@@ -562,7 +329,7 @@ def get_extractor(
         import capa.features.extractors.binja.extractor
 
         with halo.Halo(text="analyzing program", spinner="simpleDots", stream=sys.stderr, enabled=not disable_progress):
-            bv: BinaryView = BinaryViewType.get_view_of_file(str(path))
+            bv: BinaryView = binaryninja.load(str(path))
             if bv is None:
                 raise RuntimeError(f"Binary Ninja cannot open file {path}")
 
@@ -603,11 +370,15 @@ def get_file_extractors(sample: Path, format_: str) -> List[FeatureExtractor]:
 
     elif format_ == FORMAT_DOTNET:
         file_extractors.append(capa.features.extractors.pefile.PefileFeatureExtractor(sample))
-        file_extractors.append(capa.features.extractors.dnfile_.DnfileFeatureExtractor(sample))
+        file_extractors.append(capa.features.extractors.dotnetfile.DotnetFileFeatureExtractor(sample))
 
-    elif format_ == capa.features.extractors.common.FORMAT_ELF:
+    elif format_ == capa.features.common.FORMAT_ELF:
         file_extractors.append(capa.features.extractors.elffile.ElfFeatureExtractor(sample))
 
+    elif format_ == FORMAT_CAPE:
+        report = json.load(Path(sample).open(encoding="utf-8"))
+        file_extractors.append(capa.features.extractors.cape.extractor.CapeExtractor.from_report(report))
+
     return file_extractors
 
 
@@ -689,7 +460,7 @@ def get_rules(
     if ruleset is not None:
         return ruleset
 
-    rules = []  # type: List[Rule]
+    rules: List[Rule] = []
 
     total_rule_count = len(rule_file_paths)
     for i, (path, content) in enumerate(zip(rule_file_paths, rule_contents)):
@@ -704,7 +475,7 @@ def get_rules(
             rule.meta["capa/nursery"] = is_nursery_rule_path(path)
 
             rules.append(rule)
-            logger.debug("loaded rule: '%s' with scope: %s", rule.name, rule.scope)
+            logger.debug("loaded rule: '%s' with scope: %s", rule.name, rule.scopes)
 
     ruleset = capa.rules.RuleSet(rules)
 
@@ -739,60 +510,177 @@ def get_signatures(sigs_path: Path) -> List[Path]:
     return paths
 
 
-def collect_metadata(
-    argv: List[str],
-    sample_path: Path,
-    format_: str,
-    os_: str,
-    rules_path: List[Path],
-    extractor: capa.features.extractors.base_extractor.FeatureExtractor,
-) -> rdoc.Metadata:
-    md5 = hashlib.md5()
-    sha1 = hashlib.sha1()
-    sha256 = hashlib.sha256()
-
-    buf = sample_path.read_bytes()
-
-    md5.update(buf)
-    sha1.update(buf)
-    sha256.update(buf)
-
-    rules = tuple(r.resolve().absolute().as_posix() for r in rules_path)
-    format_ = get_format(sample_path) if format_ == FORMAT_AUTO else format_
-    arch = get_arch(sample_path)
-    os_ = get_os(sample_path) if os_ == OS_AUTO else os_
-
-    return rdoc.Metadata(
-        timestamp=datetime.datetime.now(),
-        version=capa.version.__version__,
-        argv=tuple(argv) if argv else None,
-        sample=rdoc.Sample(
-            md5=md5.hexdigest(),
-            sha1=sha1.hexdigest(),
-            sha256=sha256.hexdigest(),
-            path=sample_path.resolve().absolute().as_posix(),
-        ),
-        analysis=rdoc.Analysis(
+def get_sample_analysis(format_, arch, os_, extractor, rules_path, counts):
+    if isinstance(extractor, StaticFeatureExtractor):
+        return rdoc.StaticAnalysis(
             format=format_,
             arch=arch,
             os=os_,
             extractor=extractor.__class__.__name__,
-            rules=rules,
+            rules=tuple(rules_path),
             base_address=frz.Address.from_capa(extractor.get_base_address()),
-            layout=rdoc.Layout(
+            layout=rdoc.StaticLayout(
                 functions=(),
                 # this is updated after capabilities have been collected.
                 # will look like:
                 #
                 # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
             ),
-            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
-            library_functions=(),
+            feature_counts=counts["feature_counts"],
+            library_functions=counts["library_functions"],
+        )
+    elif isinstance(extractor, DynamicFeatureExtractor):
+        return rdoc.DynamicAnalysis(
+            format=format_,
+            arch=arch,
+            os=os_,
+            extractor=extractor.__class__.__name__,
+            rules=tuple(rules_path),
+            layout=rdoc.DynamicLayout(
+                processes=(),
+            ),
+            feature_counts=counts["feature_counts"],
+        )
+    else:
+        raise ValueError("invalid extractor type")
+
+
+def collect_metadata(
+    argv: List[str],
+    sample_path: Path,
+    format_: str,
+    os_: str,
+    rules_path: List[Path],
+    extractor: FeatureExtractor,
+    counts: dict,
+) -> rdoc.Metadata:
+    # if it's a binary sample we hash it, if it's a report
+    # we fetch the hashes from the report
+    sample_hashes: SampleHashes = extractor.get_sample_hashes()
+    md5, sha1, sha256 = sample_hashes.md5, sample_hashes.sha1, sample_hashes.sha256
+
+    global_feats = list(extractor.extract_global_features())
+    extractor_format = [f.value for (f, _) in global_feats if isinstance(f, capa.features.common.Format)]
+    extractor_arch = [f.value for (f, _) in global_feats if isinstance(f, capa.features.common.Arch)]
+    extractor_os = [f.value for (f, _) in global_feats if isinstance(f, capa.features.common.OS)]
+
+    format_ = str(extractor_format[0]) if extractor_format else "unknown" if format_ == FORMAT_AUTO else format_
+    arch = str(extractor_arch[0]) if extractor_arch else "unknown"
+    os_ = str(extractor_os[0]) if extractor_os else "unknown" if os_ == OS_AUTO else os_
+
+    if isinstance(extractor, StaticFeatureExtractor):
+        meta_class: type = rdoc.StaticMetadata
+    elif isinstance(extractor, DynamicFeatureExtractor):
+        meta_class = rdoc.DynamicMetadata
+    else:
+        assert_never(extractor)
+
+    rules = tuple(r.resolve().absolute().as_posix() for r in rules_path)
+
+    return meta_class(
+        timestamp=datetime.datetime.now(),
+        version=capa.version.__version__,
+        argv=tuple(argv) if argv else None,
+        sample=rdoc.Sample(
+            md5=md5,
+            sha1=sha1,
+            sha256=sha256,
+            path=Path(sample_path).resolve().as_posix(),
+        ),
+        analysis=get_sample_analysis(
+            format_,
+            arch,
+            os_,
+            extractor,
+            rules,
+            counts,
         ),
     )
 
 
-def compute_layout(rules, extractor, capabilities) -> rdoc.Layout:
+def compute_dynamic_layout(rules, extractor: DynamicFeatureExtractor, capabilities: MatchResults) -> rdoc.DynamicLayout:
+    """
+    compute a metadata structure that links threads
+    to the processes in which they're found.
+
+    only collect the threads at which some rule matched.
+    otherwise, we may pollute the json document with
+    a large amount of un-referenced data.
+    """
+    assert isinstance(extractor, DynamicFeatureExtractor)
+
+    matched_calls: Set[Address] = set()
+
+    def result_rec(result: capa.features.common.Result):
+        for loc in result.locations:
+            if isinstance(loc, capa.features.address.DynamicCallAddress):
+                matched_calls.add(loc)
+        for child in result.children:
+            result_rec(child)
+
+    for matches in capabilities.values():
+        for _, result in matches:
+            result_rec(result)
+
+    names_by_process: Dict[Address, str] = {}
+    names_by_call: Dict[Address, str] = {}
+
+    matched_processes: Set[Address] = set()
+    matched_threads: Set[Address] = set()
+
+    threads_by_process: Dict[Address, List[Address]] = {}
+    calls_by_thread: Dict[Address, List[Address]] = {}
+
+    for p in extractor.get_processes():
+        threads_by_process[p.address] = []
+
+        for t in extractor.get_threads(p):
+            calls_by_thread[t.address] = []
+
+            for c in extractor.get_calls(p, t):
+                if c.address in matched_calls:
+                    names_by_call[c.address] = extractor.get_call_name(p, t, c)
+                    calls_by_thread[t.address].append(c.address)
+
+            if calls_by_thread[t.address]:
+                matched_threads.add(t.address)
+                threads_by_process[p.address].append(t.address)
+
+        if threads_by_process[p.address]:
+            matched_processes.add(p.address)
+            names_by_process[p.address] = extractor.get_process_name(p)
+
+    layout = rdoc.DynamicLayout(
+        processes=tuple(
+            rdoc.ProcessLayout(
+                address=frz.Address.from_capa(p),
+                name=names_by_process[p],
+                matched_threads=tuple(
+                    rdoc.ThreadLayout(
+                        address=frz.Address.from_capa(t),
+                        matched_calls=tuple(
+                            rdoc.CallLayout(
+                                address=frz.Address.from_capa(c),
+                                name=names_by_call[c],
+                            )
+                            for c in calls_by_thread[t]
+                            if c in matched_calls
+                        ),
+                    )
+                    for t in threads
+                    if t in matched_threads
+                )  # this object is open to extension in the future,
+                # such as with the function name, etc.
+            )
+            for p, threads in threads_by_process.items()
+            if p in matched_processes
+        )
+    )
+
+    return layout
+
+
+def compute_static_layout(rules, extractor: StaticFeatureExtractor, capabilities) -> rdoc.StaticLayout:
     """
     compute a metadata structure that links basic blocks
     to the functions in which they're found.
@@ -812,12 +700,12 @@ def compute_layout(rules, extractor, capabilities) -> rdoc.Layout:
     matched_bbs = set()
     for rule_name, matches in capabilities.items():
         rule = rules[rule_name]
-        if rule.meta.get("scope") == capa.rules.BASIC_BLOCK_SCOPE:
+        if capa.rules.Scope.BASIC_BLOCK in rule.scopes:
             for addr, _ in matches:
                 assert addr in functions_by_bb
                 matched_bbs.add(addr)
 
-    layout = rdoc.Layout(
+    layout = rdoc.StaticLayout(
         functions=tuple(
             rdoc.FunctionLayout(
                 address=frz.Address.from_capa(f),
@@ -834,6 +722,15 @@ def compute_layout(rules, extractor, capabilities) -> rdoc.Layout:
     return layout
 
 
+def compute_layout(rules, extractor, capabilities) -> rdoc.Layout:
+    if isinstance(extractor, StaticFeatureExtractor):
+        return compute_static_layout(rules, extractor, capabilities)
+    elif isinstance(extractor, DynamicFeatureExtractor):
+        return compute_dynamic_layout(rules, extractor, capabilities)
+    else:
+        raise ValueError("extractor must be either a static or dynamic extracotr")
+
+
 def install_common_args(parser, wanted=None):
     """
     register a common set of command line arguments for re-use by main & scripts.
@@ -902,6 +799,7 @@ def install_common_args(parser, wanted=None):
             (FORMAT_ELF, "Executable and Linkable Format"),
             (FORMAT_SC32, "32-bit shellcode"),
             (FORMAT_SC64, "64-bit shellcode"),
+            (FORMAT_CAPE, "CAPE sandbox report"),
             (FORMAT_FREEZE, "features previously frozen by capa"),
         ]
         format_help = ", ".join([f"{f[0]}: {f[1]}" for f in formats])
@@ -1064,7 +962,7 @@ def handle_common_args(args):
             )
             logger.debug("-" * 80)
 
-            sigs_path = get_default_root() / "sigs"
+            sigs_path = get_default_root() / "capa" / "sigs"
 
             if not sigs_path.exists():
                 logger.error(
@@ -1080,6 +978,27 @@ def handle_common_args(args):
         args.signatures = sigs_path
 
 
+def simple_message_exception_handler(exctype, value: BaseException, traceback: TracebackType):
+    """
+    prints friendly message on unexpected exceptions to regular users (debug mode shows regular stack trace)
+
+    args:
+      # TODO(aaronatp): Once capa drops support for Python 3.8, move the exctype type annotation to
+      # the function parameters and remove the "# type: ignore[assignment]" from the relevant place
+      # in the main function, see (https://github.com/mandiant/capa/issues/1896)
+      exctype (type[BaseException]): exception class
+    """
+
+    if exctype is KeyboardInterrupt:
+        print("KeyboardInterrupt detected, program terminated")
+    else:
+        print(
+            f"Unexpected exception raised: {exctype}. Please run capa in debug mode (-d/--debug) "
+            + "to see the stack trace. Please also report your issue on the capa GitHub page so we "
+            + "can improve the code! (https://github.com/mandiant/capa/issues)"
+        )
+
+
 def main(argv: Optional[List[str]] = None):
     if sys.version_info < (3, 8):
         raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+")
@@ -1122,6 +1041,8 @@ def main(argv: Optional[List[str]] = None):
     install_common_args(parser, {"sample", "format", "backend", "os", "signatures", "rules", "tag"})
     parser.add_argument("-j", "--json", action="store_true", help="emit JSON instead of text")
     args = parser.parse_args(args=argv)
+    if not args.debug:
+        sys.excepthook = simple_message_exception_handler  # type: ignore[assignment]
     ret = handle_common_args(args)
     if ret is not None and ret != 0:
         return ret
@@ -1158,7 +1079,7 @@ def main(argv: Optional[List[str]] = None):
             # during the load of the RuleSet, we extract subscope statements into their own rules
             # that are subsequently `match`ed upon. this inflates the total rule count.
             # so, filter out the subscope rules when reporting total number of loaded rules.
-            len(list(filter(lambda r: not r.is_subscope_rule(), rules.rules.values()))),
+            len(list(filter(lambda r: not (r.is_subscope_rule()), rules.rules.values()))),
         )
         if args.tag:
             rules = rules.filter_rules_by_meta(args.tag)
@@ -1197,8 +1118,26 @@ def main(argv: Optional[List[str]] = None):
     except (ELFError, OverflowError) as e:
         logger.error("Input file '%s' is not a valid ELF file: %s", args.sample, str(e))
         return E_CORRUPT_FILE
+    except UnsupportedFormatError as e:
+        if format_ == FORMAT_CAPE:
+            log_unsupported_cape_report_error(str(e))
+        else:
+            log_unsupported_format_error()
+        return E_INVALID_FILE_TYPE
+    except EmptyReportError as e:
+        if format_ == FORMAT_CAPE:
+            log_empty_cape_report_error(str(e))
+            return E_EMPTY_REPORT
+        else:
+            log_unsupported_format_error()
+            return E_INVALID_FILE_TYPE
 
+    found_file_limitation = False
     for file_extractor in file_extractors:
+        if isinstance(file_extractor, DynamicFeatureExtractor):
+            # Dynamic feature extractors can handle packed samples
+            continue
+
         try:
             pure_file_capabilities, _ = find_file_capabilities(rules, file_extractor, {})
         except PEFormatError as e:
@@ -1210,7 +1149,8 @@ def main(argv: Optional[List[str]] = None):
 
         # file limitations that rely on non-file scope won't be detected here.
         # nor on FunctionName features, because pefile doesn't support this.
-        if has_file_limitation(rules, pure_file_capabilities):
+        found_file_limitation = has_file_limitation(rules, pure_file_capabilities)
+        if found_file_limitation:
             # bail if capa encountered file limitation e.g. a packed binary
             # do show the output in verbose mode, though.
             if not (args.verbose or args.vverbose or args.json):
@@ -1232,7 +1172,7 @@ def main(argv: Optional[List[str]] = None):
 
         if format_ == FORMAT_FREEZE:
             # freeze format deserializes directly into an extractor
-            extractor = frz.load(Path(args.sample).read_bytes())
+            extractor: FeatureExtractor = frz.load(Path(args.sample).read_bytes())
         else:
             # all other formats we must create an extractor,
             # such as viv, binary ninja, etc. workspaces
@@ -1250,6 +1190,9 @@ def main(argv: Optional[List[str]] = None):
 
             should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None)
 
+            # TODO(mr-tz): this should be wrapped and refactored as it's tedious to update everywhere
+            #  see same code and show-features above examples
+            #  https://github.com/mandiant/capa/issues/1813
             try:
                 extractor = get_extractor(
                     args.sample,
@@ -1260,8 +1203,11 @@ def main(argv: Optional[List[str]] = None):
                     should_save_workspace,
                     disable_progress=args.quiet or args.debug,
                 )
-            except UnsupportedFormatError:
-                log_unsupported_format_error()
+            except UnsupportedFormatError as e:
+                if format_ == FORMAT_CAPE:
+                    log_unsupported_cape_report_error(str(e))
+                else:
+                    log_unsupported_format_error()
                 return E_INVALID_FILE_TYPE
             except UnsupportedArchError:
                 log_unsupported_arch_error()
@@ -1270,16 +1216,13 @@ def main(argv: Optional[List[str]] = None):
                 log_unsupported_os_error()
                 return E_INVALID_FILE_OS
 
-        meta = collect_metadata(argv, args.sample, args.format, args.os, args.rules, extractor)
-
         capabilities, counts = find_capabilities(rules, extractor, disable_progress=args.quiet)
 
-        meta.analysis.feature_counts = counts["feature_counts"]
-        meta.analysis.library_functions = counts["library_functions"]
+        meta = collect_metadata(argv, args.sample, args.format, args.os, args.rules, extractor, counts)
         meta.analysis.layout = compute_layout(rules, extractor, capabilities)
 
-        if has_file_limitation(rules, capabilities):
-            # bail if capa encountered file limitation e.g. a packed binary
+        if isinstance(extractor, StaticFeatureExtractor) and found_file_limitation:
+            # bail if capa's static feature extractor encountered file limitation e.g. a packed binary
             # do show the output in verbose mode, though.
             if not (args.verbose or args.vverbose or args.json):
                 return E_FILE_LIMITATION
@@ -1338,8 +1281,47 @@ def ida_main():
     print(capa.render.default.render(meta, rules, capabilities))
 
 
+def ghidra_main():
+    import capa.rules
+    import capa.ghidra.helpers
+    import capa.render.default
+    import capa.features.extractors.ghidra.extractor
+
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger().setLevel(logging.INFO)
+
+    logger.debug("-" * 80)
+    logger.debug(" Using default embedded rules.")
+    logger.debug(" ")
+    logger.debug(" You can see the current default rule set here:")
+    logger.debug("     https://github.com/mandiant/capa-rules")
+    logger.debug("-" * 80)
+
+    rules_path = get_default_root() / "rules"
+    logger.debug("rule path: %s", rules_path)
+    rules = get_rules([rules_path])
+
+    meta = capa.ghidra.helpers.collect_metadata([rules_path])
+
+    capabilities, counts = find_capabilities(
+        rules,
+        capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor(),
+        not capa.ghidra.helpers.is_running_headless(),
+    )
+
+    meta.analysis.feature_counts = counts["feature_counts"]
+    meta.analysis.library_functions = counts["library_functions"]
+
+    if has_file_limitation(rules, capabilities, is_standalone=False):
+        logger.info("capa encountered warnings during analysis")
+
+    print(capa.render.default.render(meta, rules, capabilities))
+
+
 if __name__ == "__main__":
     if capa.helpers.is_runtime_ida():
         ida_main()
+    elif capa.helpers.is_runtime_ghidra():
+        ghidra_main()
     else:
         sys.exit(main())

capa/render/default.py

@@ -33,6 +33,7 @@ def render_meta(doc: rd.ResultDocument, ostream: StringIO):
         (width("md5", 22), width(doc.meta.sample.md5, 82)),
         ("sha1", doc.meta.sample.sha1),
         ("sha256", doc.meta.sample.sha256),
+        ("analysis", doc.meta.flavor.value),
         ("os", doc.meta.analysis.os),
         ("format", doc.meta.analysis.format),
         ("arch", doc.meta.analysis.arch),

capa/render/proto/__init__.py

@@ -38,16 +38,6 @@ from capa.helpers import assert_never
 from capa.features.freeze import AddressType
 
 
-def dict_tuple_to_list_values(d: Dict) -> Dict:
-    o = {}
-    for k, v in d.items():
-        if isinstance(v, tuple):
-            o[k] = list(v)
-        else:
-            o[k] = v
-    return o
-
-
 def int_to_pb2(v: int) -> capa_pb2.Integer:
     if v < -2_147_483_648:
         raise ValueError(f"value underflow: {v}")
@@ -100,6 +90,51 @@ def addr_to_pb2(addr: frz.Address) -> capa_pb2.Address:
             token_offset=capa_pb2.Token_Offset(token=int_to_pb2(token), offset=offset),
         )
 
+    elif addr.type is AddressType.PROCESS:
+        assert isinstance(addr.value, tuple)
+        ppid, pid = addr.value
+        assert isinstance(ppid, int)
+        assert isinstance(pid, int)
+        return capa_pb2.Address(
+            type=capa_pb2.AddressType.ADDRESSTYPE_PROCESS,
+            ppid_pid=capa_pb2.Ppid_Pid(
+                ppid=int_to_pb2(ppid),
+                pid=int_to_pb2(pid),
+            ),
+        )
+
+    elif addr.type is AddressType.THREAD:
+        assert isinstance(addr.value, tuple)
+        ppid, pid, tid = addr.value
+        assert isinstance(ppid, int)
+        assert isinstance(pid, int)
+        assert isinstance(tid, int)
+        return capa_pb2.Address(
+            type=capa_pb2.AddressType.ADDRESSTYPE_THREAD,
+            ppid_pid_tid=capa_pb2.Ppid_Pid_Tid(
+                ppid=int_to_pb2(ppid),
+                pid=int_to_pb2(pid),
+                tid=int_to_pb2(tid),
+            ),
+        )
+
+    elif addr.type is AddressType.CALL:
+        assert isinstance(addr.value, tuple)
+        ppid, pid, tid, id_ = addr.value
+        assert isinstance(ppid, int)
+        assert isinstance(pid, int)
+        assert isinstance(tid, int)
+        assert isinstance(id_, int)
+        return capa_pb2.Address(
+            type=capa_pb2.AddressType.ADDRESSTYPE_CALL,
+            ppid_pid_tid_id=capa_pb2.Ppid_Pid_Tid_Id(
+                ppid=int_to_pb2(ppid),
+                pid=int_to_pb2(pid),
+                tid=int_to_pb2(tid),
+                id=int_to_pb2(id_),
+            ),
+        )
+
     elif addr.type is AddressType.NO_ADDRESS:
         # value == None, so only set type
         return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_NO_ADDRESS)
@@ -117,49 +152,129 @@ def scope_to_pb2(scope: capa.rules.Scope) -> capa_pb2.Scope.ValueType:
         return capa_pb2.Scope.SCOPE_BASIC_BLOCK
     elif scope == capa.rules.Scope.INSTRUCTION:
         return capa_pb2.Scope.SCOPE_INSTRUCTION
+    elif scope == capa.rules.Scope.PROCESS:
+        return capa_pb2.Scope.SCOPE_PROCESS
+    elif scope == capa.rules.Scope.THREAD:
+        return capa_pb2.Scope.SCOPE_THREAD
+    elif scope == capa.rules.Scope.CALL:
+        return capa_pb2.Scope.SCOPE_CALL
     else:
         assert_never(scope)
 
 
-def metadata_to_pb2(meta: rd.Metadata) -> capa_pb2.Metadata:
-    return capa_pb2.Metadata(
-        timestamp=str(meta.timestamp),
-        version=meta.version,
-        argv=meta.argv,
-        sample=google.protobuf.json_format.ParseDict(meta.sample.model_dump(), capa_pb2.Sample()),
-        analysis=capa_pb2.Analysis(
-            format=meta.analysis.format,
-            arch=meta.analysis.arch,
-            os=meta.analysis.os,
-            extractor=meta.analysis.extractor,
-            rules=list(meta.analysis.rules),
-            base_address=addr_to_pb2(meta.analysis.base_address),
-            layout=capa_pb2.Layout(
-                functions=[
-                    capa_pb2.FunctionLayout(
-                        address=addr_to_pb2(f.address),
-                        matched_basic_blocks=[
-                            capa_pb2.BasicBlockLayout(address=addr_to_pb2(bb.address)) for bb in f.matched_basic_blocks
-                        ],
-                    )
-                    for f in meta.analysis.layout.functions
-                ]
-            ),
-            feature_counts=capa_pb2.FeatureCounts(
-                file=meta.analysis.feature_counts.file,
-                functions=[
-                    capa_pb2.FunctionFeatureCount(address=addr_to_pb2(f.address), count=f.count)
-                    for f in meta.analysis.feature_counts.functions
-                ],
-            ),
-            library_functions=[
-                capa_pb2.LibraryFunction(address=addr_to_pb2(lf.address), name=lf.name)
-                for lf in meta.analysis.library_functions
+def scopes_to_pb2(scopes: capa.rules.Scopes) -> capa_pb2.Scopes:
+    doc = {}
+    if scopes.static:
+        doc["static"] = scope_to_pb2(scopes.static)
+    if scopes.dynamic:
+        doc["dynamic"] = scope_to_pb2(scopes.dynamic)
+
+    return google.protobuf.json_format.ParseDict(doc, capa_pb2.Scopes())
+
+
+def flavor_to_pb2(flavor: rd.Flavor) -> capa_pb2.Flavor.ValueType:
+    if flavor == rd.Flavor.STATIC:
+        return capa_pb2.Flavor.FLAVOR_STATIC
+    elif flavor == rd.Flavor.DYNAMIC:
+        return capa_pb2.Flavor.FLAVOR_DYNAMIC
+    else:
+        assert_never(flavor)
+
+
+def static_analysis_to_pb2(analysis: rd.StaticAnalysis) -> capa_pb2.StaticAnalysis:
+    return capa_pb2.StaticAnalysis(
+        format=analysis.format,
+        arch=analysis.arch,
+        os=analysis.os,
+        extractor=analysis.extractor,
+        rules=list(analysis.rules),
+        base_address=addr_to_pb2(analysis.base_address),
+        layout=capa_pb2.StaticLayout(
+            functions=[
+                capa_pb2.FunctionLayout(
+                    address=addr_to_pb2(f.address),
+                    matched_basic_blocks=[
+                        capa_pb2.BasicBlockLayout(address=addr_to_pb2(bb.address)) for bb in f.matched_basic_blocks
+                    ],
+                )
+                for f in analysis.layout.functions
+            ]
+        ),
+        feature_counts=capa_pb2.StaticFeatureCounts(
+            file=analysis.feature_counts.file,
+            functions=[
+                capa_pb2.FunctionFeatureCount(address=addr_to_pb2(f.address), count=f.count)
+                for f in analysis.feature_counts.functions
+            ],
+        ),
+        library_functions=[
+            capa_pb2.LibraryFunction(address=addr_to_pb2(lf.address), name=lf.name) for lf in analysis.library_functions
+        ],
+    )
+
+
+def dynamic_analysis_to_pb2(analysis: rd.DynamicAnalysis) -> capa_pb2.DynamicAnalysis:
+    return capa_pb2.DynamicAnalysis(
+        format=analysis.format,
+        arch=analysis.arch,
+        os=analysis.os,
+        extractor=analysis.extractor,
+        rules=list(analysis.rules),
+        layout=capa_pb2.DynamicLayout(
+            processes=[
+                capa_pb2.ProcessLayout(
+                    address=addr_to_pb2(p.address),
+                    name=p.name,
+                    matched_threads=[
+                        capa_pb2.ThreadLayout(
+                            address=addr_to_pb2(t.address),
+                            matched_calls=[
+                                capa_pb2.CallLayout(
+                                    address=addr_to_pb2(c.address),
+                                    name=c.name,
+                                )
+                                for c in t.matched_calls
+                            ],
+                        )
+                        for t in p.matched_threads
+                    ],
+                )
+                for p in analysis.layout.processes
+            ]
+        ),
+        feature_counts=capa_pb2.DynamicFeatureCounts(
+            file=analysis.feature_counts.file,
+            processes=[
+                capa_pb2.ProcessFeatureCount(address=addr_to_pb2(p.address), count=p.count)
+                for p in analysis.feature_counts.processes
             ],
         ),
     )
 
 
+def metadata_to_pb2(meta: rd.Metadata) -> capa_pb2.Metadata:
+    if isinstance(meta.analysis, rd.StaticAnalysis):
+        return capa_pb2.Metadata(
+            timestamp=str(meta.timestamp),
+            version=meta.version,
+            argv=meta.argv,
+            sample=google.protobuf.json_format.ParseDict(meta.sample.model_dump(), capa_pb2.Sample()),
+            flavor=flavor_to_pb2(meta.flavor),
+            static_analysis=static_analysis_to_pb2(meta.analysis),
+        )
+    elif isinstance(meta.analysis, rd.DynamicAnalysis):
+        return capa_pb2.Metadata(
+            timestamp=str(meta.timestamp),
+            version=meta.version,
+            argv=meta.argv,
+            sample=google.protobuf.json_format.ParseDict(meta.sample.model_dump(), capa_pb2.Sample()),
+            flavor=flavor_to_pb2(meta.flavor),
+            dynamic_analysis=dynamic_analysis_to_pb2(meta.analysis),
+        )
+    else:
+        assert_never(meta.analysis)
+
+
 def statement_to_pb2(statement: rd.Statement) -> capa_pb2.StatementNode:
     if isinstance(statement, rd.RangeStatement):
         return capa_pb2.StatementNode(
@@ -390,15 +505,51 @@ def match_to_pb2(match: rd.Match) -> capa_pb2.Match:
         assert_never(match)
 
 
-def rule_metadata_to_pb2(rule_metadata: rd.RuleMetadata) -> capa_pb2.RuleMetadata:
-    # after manual type conversions to the RuleMetadata, we can rely on the protobuf json parser
-    # conversions include tuple -> list and rd.Enum -> proto.enum
-    meta = dict_tuple_to_list_values(rule_metadata.model_dump())
-    meta["scope"] = scope_to_pb2(meta["scope"])
-    meta["attack"] = list(map(dict_tuple_to_list_values, meta.get("attack", [])))
-    meta["mbc"] = list(map(dict_tuple_to_list_values, meta.get("mbc", [])))
+def attack_to_pb2(attack: rd.AttackSpec) -> capa_pb2.AttackSpec:
+    return capa_pb2.AttackSpec(
+        parts=list(attack.parts),
+        tactic=attack.tactic,
+        technique=attack.technique,
+        subtechnique=attack.subtechnique,
+        id=attack.id,
+    )
 
-    return google.protobuf.json_format.ParseDict(meta, capa_pb2.RuleMetadata())
+
+def mbc_to_pb2(mbc: rd.MBCSpec) -> capa_pb2.MBCSpec:
+    return capa_pb2.MBCSpec(
+        parts=list(mbc.parts),
+        objective=mbc.objective,
+        behavior=mbc.behavior,
+        method=mbc.method,
+        id=mbc.id,
+    )
+
+
+def maec_to_pb2(maec: rd.MaecMetadata) -> capa_pb2.MaecMetadata:
+    return capa_pb2.MaecMetadata(
+        analysis_conclusion=maec.analysis_conclusion or "",
+        analysis_conclusion_ov=maec.analysis_conclusion_ov or "",
+        malware_family=maec.malware_family or "",
+        malware_category=maec.malware_category or "",
+        malware_category_ov=maec.malware_category_ov or "",
+    )
+
+
+def rule_metadata_to_pb2(rule_metadata: rd.RuleMetadata) -> capa_pb2.RuleMetadata:
+    return capa_pb2.RuleMetadata(
+        name=rule_metadata.name,
+        namespace=rule_metadata.namespace or "",
+        authors=rule_metadata.authors,
+        attack=[attack_to_pb2(m) for m in rule_metadata.attack],
+        mbc=[mbc_to_pb2(m) for m in rule_metadata.mbc],
+        references=rule_metadata.references,
+        examples=rule_metadata.examples,
+        description=rule_metadata.description,
+        lib=rule_metadata.lib,
+        maec=maec_to_pb2(rule_metadata.maec),
+        is_subscope_rule=rule_metadata.is_subscope_rule,
+        scopes=scopes_to_pb2(rule_metadata.scopes),
+    )
 
 
 def doc_to_pb2(doc: rd.ResultDocument) -> capa_pb2.ResultDocument:
@@ -459,6 +610,24 @@ def addr_from_pb2(addr: capa_pb2.Address) -> frz.Address:
         offset = addr.token_offset.offset
         return frz.Address(type=frz.AddressType.DN_TOKEN_OFFSET, value=(token, offset))
 
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_PROCESS:
+        ppid = int_from_pb2(addr.ppid_pid.ppid)
+        pid = int_from_pb2(addr.ppid_pid.pid)
+        return frz.Address(type=frz.AddressType.PROCESS, value=(ppid, pid))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_THREAD:
+        ppid = int_from_pb2(addr.ppid_pid_tid.ppid)
+        pid = int_from_pb2(addr.ppid_pid_tid.pid)
+        tid = int_from_pb2(addr.ppid_pid_tid.tid)
+        return frz.Address(type=frz.AddressType.THREAD, value=(ppid, pid, tid))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_CALL:
+        ppid = int_from_pb2(addr.ppid_pid_tid_id.ppid)
+        pid = int_from_pb2(addr.ppid_pid_tid_id.pid)
+        tid = int_from_pb2(addr.ppid_pid_tid_id.tid)
+        id_ = int_from_pb2(addr.ppid_pid_tid_id.id)
+        return frz.Address(type=frz.AddressType.CALL, value=(ppid, pid, tid, id_))
+
     elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_NO_ADDRESS:
         return frz.Address(type=frz.AddressType.NO_ADDRESS, value=None)
 
@@ -475,63 +644,146 @@ def scope_from_pb2(scope: capa_pb2.Scope.ValueType) -> capa.rules.Scope:
         return capa.rules.Scope.BASIC_BLOCK
     elif scope == capa_pb2.Scope.SCOPE_INSTRUCTION:
         return capa.rules.Scope.INSTRUCTION
+    elif scope == capa_pb2.Scope.SCOPE_PROCESS:
+        return capa.rules.Scope.PROCESS
+    elif scope == capa_pb2.Scope.SCOPE_THREAD:
+        return capa.rules.Scope.THREAD
+    elif scope == capa_pb2.Scope.SCOPE_CALL:
+        return capa.rules.Scope.CALL
     else:
         assert_never(scope)
 
 
-def metadata_from_pb2(meta: capa_pb2.Metadata) -> rd.Metadata:
-    return rd.Metadata(
-        timestamp=datetime.datetime.fromisoformat(meta.timestamp),
-        version=meta.version,
-        argv=tuple(meta.argv) if meta.argv else None,
-        sample=rd.Sample(
-            md5=meta.sample.md5,
-            sha1=meta.sample.sha1,
-            sha256=meta.sample.sha256,
-            path=meta.sample.path,
-        ),
-        analysis=rd.Analysis(
-            format=meta.analysis.format,
-            arch=meta.analysis.arch,
-            os=meta.analysis.os,
-            extractor=meta.analysis.extractor,
-            rules=tuple(meta.analysis.rules),
-            base_address=addr_from_pb2(meta.analysis.base_address),
-            layout=rd.Layout(
-                functions=tuple(
-                    [
-                        rd.FunctionLayout(
-                            address=addr_from_pb2(f.address),
-                            matched_basic_blocks=tuple(
-                                [
-                                    rd.BasicBlockLayout(address=addr_from_pb2(bb.address))
-                                    for bb in f.matched_basic_blocks
-                                ]
-                            ),
-                        )
-                        for f in meta.analysis.layout.functions
-                    ]
-                )
-            ),
-            feature_counts=rd.FeatureCounts(
-                file=meta.analysis.feature_counts.file,
-                functions=tuple(
-                    [
-                        rd.FunctionFeatureCount(address=addr_from_pb2(f.address), count=f.count)
-                        for f in meta.analysis.feature_counts.functions
-                    ]
-                ),
-            ),
-            library_functions=tuple(
+def scopes_from_pb2(scopes: capa_pb2.Scopes) -> capa.rules.Scopes:
+    return capa.rules.Scopes(
+        static=scope_from_pb2(scopes.static) if scopes.static else None,
+        dynamic=scope_from_pb2(scopes.dynamic) if scopes.dynamic else None,
+    )
+
+
+def flavor_from_pb2(flavor: capa_pb2.Flavor.ValueType) -> rd.Flavor:
+    if flavor == capa_pb2.Flavor.FLAVOR_STATIC:
+        return rd.Flavor.STATIC
+    elif flavor == capa_pb2.Flavor.FLAVOR_DYNAMIC:
+        return rd.Flavor.DYNAMIC
+    else:
+        assert_never(flavor)
+
+
+def static_analysis_from_pb2(analysis: capa_pb2.StaticAnalysis) -> rd.StaticAnalysis:
+    return rd.StaticAnalysis(
+        format=analysis.format,
+        arch=analysis.arch,
+        os=analysis.os,
+        extractor=analysis.extractor,
+        rules=tuple(analysis.rules),
+        base_address=addr_from_pb2(analysis.base_address),
+        layout=rd.StaticLayout(
+            functions=tuple(
                 [
-                    rd.LibraryFunction(address=addr_from_pb2(lf.address), name=lf.name)
-                    for lf in meta.analysis.library_functions
+                    rd.FunctionLayout(
+                        address=addr_from_pb2(f.address),
+                        matched_basic_blocks=tuple(
+                            [rd.BasicBlockLayout(address=addr_from_pb2(bb.address)) for bb in f.matched_basic_blocks]
+                        ),
+                    )
+                    for f in analysis.layout.functions
+                ]
+            )
+        ),
+        feature_counts=rd.StaticFeatureCounts(
+            file=analysis.feature_counts.file,
+            functions=tuple(
+                [
+                    rd.FunctionFeatureCount(address=addr_from_pb2(f.address), count=f.count)
+                    for f in analysis.feature_counts.functions
+                ]
+            ),
+        ),
+        library_functions=tuple(
+            [rd.LibraryFunction(address=addr_from_pb2(lf.address), name=lf.name) for lf in analysis.library_functions]
+        ),
+    )
+
+
+def dynamic_analysis_from_pb2(analysis: capa_pb2.DynamicAnalysis) -> rd.DynamicAnalysis:
+    return rd.DynamicAnalysis(
+        format=analysis.format,
+        arch=analysis.arch,
+        os=analysis.os,
+        extractor=analysis.extractor,
+        rules=tuple(analysis.rules),
+        layout=rd.DynamicLayout(
+            processes=tuple(
+                [
+                    rd.ProcessLayout(
+                        address=addr_from_pb2(p.address),
+                        name=p.name,
+                        matched_threads=tuple(
+                            [
+                                rd.ThreadLayout(
+                                    address=addr_from_pb2(t.address),
+                                    matched_calls=tuple(
+                                        [
+                                            rd.CallLayout(address=addr_from_pb2(c.address), name=c.name)
+                                            for c in t.matched_calls
+                                        ]
+                                    ),
+                                )
+                                for t in p.matched_threads
+                            ]
+                        ),
+                    )
+                    for p in analysis.layout.processes
+                ]
+            )
+        ),
+        feature_counts=rd.DynamicFeatureCounts(
+            file=analysis.feature_counts.file,
+            processes=tuple(
+                [
+                    rd.ProcessFeatureCount(address=addr_from_pb2(p.address), count=p.count)
+                    for p in analysis.feature_counts.processes
                 ]
             ),
         ),
     )
 
 
+def metadata_from_pb2(meta: capa_pb2.Metadata) -> rd.Metadata:
+    analysis_type = meta.WhichOneof("analysis2")
+    if analysis_type == "static_analysis":
+        return rd.Metadata(
+            timestamp=datetime.datetime.fromisoformat(meta.timestamp),
+            version=meta.version,
+            argv=tuple(meta.argv) if meta.argv else None,
+            sample=rd.Sample(
+                md5=meta.sample.md5,
+                sha1=meta.sample.sha1,
+                sha256=meta.sample.sha256,
+                path=meta.sample.path,
+            ),
+            flavor=flavor_from_pb2(meta.flavor),
+            analysis=static_analysis_from_pb2(meta.static_analysis),
+        )
+    elif analysis_type == "dynamic_analysis":
+        return rd.Metadata(
+            timestamp=datetime.datetime.fromisoformat(meta.timestamp),
+            version=meta.version,
+            argv=tuple(meta.argv) if meta.argv else None,
+            sample=rd.Sample(
+                md5=meta.sample.md5,
+                sha1=meta.sample.sha1,
+                sha256=meta.sample.sha256,
+                path=meta.sample.path,
+            ),
+            flavor=flavor_from_pb2(meta.flavor),
+            analysis=dynamic_analysis_from_pb2(meta.dynamic_analysis),
+        )
+    else:
+        assert_never(analysis_type)
+
+
 def statement_from_pb2(statement: capa_pb2.StatementNode) -> rd.Statement:
     type_ = statement.WhichOneof("statement")
 
@@ -711,7 +963,7 @@ def rule_metadata_from_pb2(pb: capa_pb2.RuleMetadata) -> rd.RuleMetadata:
         name=pb.name,
         namespace=pb.namespace or None,
         authors=tuple(pb.authors),
-        scope=scope_from_pb2(pb.scope),
+        scopes=scopes_from_pb2(pb.scopes),
         attack=tuple([attack_from_pb2(attack) for attack in pb.attack]),
         mbc=tuple([mbc_from_pb2(mbc) for mbc in pb.mbc]),
         references=tuple(pb.references),

capa/render/proto/capa.proto

@@ -11,6 +11,9 @@ message Address {
   oneof value {
     Integer v = 2;
     Token_Offset token_offset = 3;
+    Ppid_Pid ppid_pid = 4;
+    Ppid_Pid_Tid ppid_pid_tid = 5;
+    Ppid_Pid_Tid_Id ppid_pid_tid_id = 6;
   };
 }
 
@@ -22,6 +25,9 @@ enum AddressType {
   ADDRESSTYPE_DN_TOKEN = 4;
   ADDRESSTYPE_DN_TOKEN_OFFSET = 5;
   ADDRESSTYPE_NO_ADDRESS = 6;
+  ADDRESSTYPE_PROCESS = 7;
+  ADDRESSTYPE_THREAD = 8;
+  ADDRESSTYPE_CALL = 9;
 }
 
 message Analysis {
@@ -82,6 +88,25 @@ message CompoundStatement {
   optional string description = 2;
 }
 
+message DynamicAnalysis {
+  string format = 1;
+  string arch = 2;
+  string os = 3;
+  string extractor = 4;
+  repeated string rules = 5;
+  DynamicLayout layout = 6;
+  DynamicFeatureCounts feature_counts = 7;
+}
+
+message DynamicFeatureCounts {
+  uint64 file = 1;
+  repeated ProcessFeatureCount processes = 2;
+}
+
+message DynamicLayout {
+  repeated ProcessLayout processes = 1;
+}
+
 message ExportFeature {
   string type = 1;
   string export = 2;
@@ -192,12 +217,26 @@ message MatchFeature {
   optional string description = 3;
 }
 
+enum Flavor {
+  FLAVOR_UNSPECIFIED = 0;
+  FLAVOR_STATIC = 1;
+  FLAVOR_DYNAMIC = 2;
+}
+
 message Metadata {
   string timestamp = 1;  // iso8601 format, like: 2019-01-01T00:00:00Z 
   string version = 2;
   repeated string argv = 3;
   Sample sample = 4;
-  Analysis analysis = 5;
+  // deprecated in v7.0.
+  // use analysis2 instead.
+  Analysis analysis = 5 [deprecated = true];
+  Flavor flavor = 6;
+  oneof analysis2 {
+    // use analysis2 instead of analysis (deprecated in v7.0).
+    StaticAnalysis static_analysis = 7;
+    DynamicAnalysis dynamic_analysis = 8;
+  };
 }
 
 message MnemonicFeature {
@@ -244,6 +283,17 @@ message OperandOffsetFeature {
   optional string description = 4;
 }
 
+message ProcessFeatureCount {
+  Address address = 1;
+  uint64 count = 2;
+}
+
+message ProcessLayout {
+  Address address = 1;
+  repeated ThreadLayout matched_threads = 2;
+  string name = 3;
+}
+
 message PropertyFeature {
   string type = 1;
   string property_ = 2;  // property is a Python top-level decorator name
@@ -281,7 +331,9 @@ message RuleMetadata {
   string name = 1;
   string namespace = 2;
   repeated string authors = 3;
-  Scope scope = 4;
+  // deprecated in v7.0.
+  // use scopes instead.
+  Scope scope = 4 [deprecated = true];
   repeated AttackSpec attack = 5;
   repeated MBCSpec mbc = 6;
   repeated string references = 7;
@@ -290,6 +342,8 @@ message RuleMetadata {
   bool lib = 10;
   MaecMetadata maec = 11;
   bool is_subscope_rule = 12;
+  // use scopes over scope (deprecated in v7.0).
+  Scopes scopes = 13;
 }
 
 message Sample {
@@ -305,6 +359,14 @@ enum Scope {
   SCOPE_FUNCTION = 2;
   SCOPE_BASIC_BLOCK = 3;
   SCOPE_INSTRUCTION = 4;
+  SCOPE_PROCESS = 5;
+  SCOPE_THREAD = 6;
+  SCOPE_CALL = 7;
+}
+
+message Scopes {
+  optional Scope static = 1;
+  optional Scope dynamic = 2;
 }
 
 message SectionFeature {
@@ -329,6 +391,27 @@ message StatementNode {
   };
 }
 
+message StaticAnalysis {
+  string format = 1;
+  string arch = 2;
+  string os = 3;
+  string extractor = 4;
+  repeated string rules = 5;
+  Address base_address = 6;
+  StaticLayout layout = 7;
+  StaticFeatureCounts feature_counts = 8;
+  repeated LibraryFunction library_functions = 9;
+}
+
+message StaticFeatureCounts {
+  uint64 file = 1;
+  repeated FunctionFeatureCount functions = 2;
+}
+
+message StaticLayout {
+  repeated FunctionLayout functions = 1;
+}
+
 message StringFeature {
   string type = 1;
   string string = 2;
@@ -347,6 +430,16 @@ message SubstringFeature {
   optional string description = 3;
 }
 
+message CallLayout {
+  Address address = 1;
+  string name = 2;
+}
+
+message ThreadLayout {
+  Address address = 1;
+  repeated CallLayout matched_calls = 2;
+}
+
 message Addresses { repeated Address address = 1; }
 
 message Pair_Address_Match {
@@ -359,6 +452,24 @@ message Token_Offset {
   uint64 offset = 2;  // offset is always >= 0
 }
 
+message Ppid_Pid {
+  Integer ppid = 1;
+  Integer pid = 2;
+}
+
+message Ppid_Pid_Tid {
+  Integer ppid = 1;
+  Integer pid = 2;
+  Integer tid = 3;
+}
+
+message Ppid_Pid_Tid_Id {
+  Integer ppid = 1;
+  Integer pid = 2;
+  Integer tid = 3;
+  Integer id = 4;
+}
+
 message Integer { oneof value { uint64 u = 1; sint64 i = 2; } }  // unsigned or signed int
 
 message Number { oneof value { uint64 u = 1; sint64 i = 2; double f = 3; } }

capa/render/proto/capa_pb2.pyi

@@ -31,6 +31,9 @@ class _AddressTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._En
     ADDRESSTYPE_DN_TOKEN: _AddressType.ValueType  # 4
     ADDRESSTYPE_DN_TOKEN_OFFSET: _AddressType.ValueType  # 5
     ADDRESSTYPE_NO_ADDRESS: _AddressType.ValueType  # 6
+    ADDRESSTYPE_PROCESS: _AddressType.ValueType  # 7
+    ADDRESSTYPE_THREAD: _AddressType.ValueType  # 8
+    ADDRESSTYPE_CALL: _AddressType.ValueType  # 9
 
 class AddressType(_AddressType, metaclass=_AddressTypeEnumTypeWrapper): ...
 
@@ -41,8 +44,28 @@ ADDRESSTYPE_FILE: AddressType.ValueType  # 3
 ADDRESSTYPE_DN_TOKEN: AddressType.ValueType  # 4
 ADDRESSTYPE_DN_TOKEN_OFFSET: AddressType.ValueType  # 5
 ADDRESSTYPE_NO_ADDRESS: AddressType.ValueType  # 6
+ADDRESSTYPE_PROCESS: AddressType.ValueType  # 7
+ADDRESSTYPE_THREAD: AddressType.ValueType  # 8
+ADDRESSTYPE_CALL: AddressType.ValueType  # 9
 global___AddressType = AddressType
 
+class _Flavor:
+    ValueType = typing.NewType("ValueType", builtins.int)
+    V: typing_extensions.TypeAlias = ValueType
+
+class _FlavorEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_Flavor.ValueType], builtins.type):
+    DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+    FLAVOR_UNSPECIFIED: _Flavor.ValueType  # 0
+    FLAVOR_STATIC: _Flavor.ValueType  # 1
+    FLAVOR_DYNAMIC: _Flavor.ValueType  # 2
+
+class Flavor(_Flavor, metaclass=_FlavorEnumTypeWrapper): ...
+
+FLAVOR_UNSPECIFIED: Flavor.ValueType  # 0
+FLAVOR_STATIC: Flavor.ValueType  # 1
+FLAVOR_DYNAMIC: Flavor.ValueType  # 2
+global___Flavor = Flavor
+
 class _Scope:
     ValueType = typing.NewType("ValueType", builtins.int)
     V: typing_extensions.TypeAlias = ValueType
@@ -54,6 +77,9 @@ class _ScopeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumType
     SCOPE_FUNCTION: _Scope.ValueType  # 2
     SCOPE_BASIC_BLOCK: _Scope.ValueType  # 3
     SCOPE_INSTRUCTION: _Scope.ValueType  # 4
+    SCOPE_PROCESS: _Scope.ValueType  # 5
+    SCOPE_THREAD: _Scope.ValueType  # 6
+    SCOPE_CALL: _Scope.ValueType  # 7
 
 class Scope(_Scope, metaclass=_ScopeEnumTypeWrapper): ...
 
@@ -62,6 +88,9 @@ SCOPE_FILE: Scope.ValueType  # 1
 SCOPE_FUNCTION: Scope.ValueType  # 2
 SCOPE_BASIC_BLOCK: Scope.ValueType  # 3
 SCOPE_INSTRUCTION: Scope.ValueType  # 4
+SCOPE_PROCESS: Scope.ValueType  # 5
+SCOPE_THREAD: Scope.ValueType  # 6
+SCOPE_CALL: Scope.ValueType  # 7
 global___Scope = Scope
 
 @typing_extensions.final
@@ -94,21 +123,33 @@ class Address(google.protobuf.message.Message):
     TYPE_FIELD_NUMBER: builtins.int
     V_FIELD_NUMBER: builtins.int
     TOKEN_OFFSET_FIELD_NUMBER: builtins.int
+    PPID_PID_FIELD_NUMBER: builtins.int
+    PPID_PID_TID_FIELD_NUMBER: builtins.int
+    PPID_PID_TID_ID_FIELD_NUMBER: builtins.int
     type: global___AddressType.ValueType
     @property
     def v(self) -> global___Integer: ...
     @property
     def token_offset(self) -> global___Token_Offset: ...
+    @property
+    def ppid_pid(self) -> global___Ppid_Pid: ...
+    @property
+    def ppid_pid_tid(self) -> global___Ppid_Pid_Tid: ...
+    @property
+    def ppid_pid_tid_id(self) -> global___Ppid_Pid_Tid_Id: ...
     def __init__(
         self,
         *,
         type: global___AddressType.ValueType = ...,
         v: global___Integer | None = ...,
         token_offset: global___Token_Offset | None = ...,
+        ppid_pid: global___Ppid_Pid | None = ...,
+        ppid_pid_tid: global___Ppid_Pid_Tid | None = ...,
+        ppid_pid_tid_id: global___Ppid_Pid_Tid_Id | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["token_offset", b"token_offset", "v", b"v", "value", b"value"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["token_offset", b"token_offset", "type", b"type", "v", b"v", "value", b"value"]) -> None: ...
-    def WhichOneof(self, oneof_group: typing_extensions.Literal["value", b"value"]) -> typing_extensions.Literal["v", "token_offset"] | None: ...
+    def HasField(self, field_name: typing_extensions.Literal["ppid_pid", b"ppid_pid", "ppid_pid_tid", b"ppid_pid_tid", "ppid_pid_tid_id", b"ppid_pid_tid_id", "token_offset", b"token_offset", "v", b"v", "value", b"value"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["ppid_pid", b"ppid_pid", "ppid_pid_tid", b"ppid_pid_tid", "ppid_pid_tid_id", b"ppid_pid_tid_id", "token_offset", b"token_offset", "type", b"type", "v", b"v", "value", b"value"]) -> None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["value", b"value"]) -> typing_extensions.Literal["v", "token_offset", "ppid_pid", "ppid_pid_tid", "ppid_pid_tid_id"] | None: ...
 
 global___Address = Address
 
@@ -335,6 +376,78 @@ class CompoundStatement(google.protobuf.message.Message):
 
 global___CompoundStatement = CompoundStatement
 
+@typing_extensions.final
+class DynamicAnalysis(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    FORMAT_FIELD_NUMBER: builtins.int
+    ARCH_FIELD_NUMBER: builtins.int
+    OS_FIELD_NUMBER: builtins.int
+    EXTRACTOR_FIELD_NUMBER: builtins.int
+    RULES_FIELD_NUMBER: builtins.int
+    LAYOUT_FIELD_NUMBER: builtins.int
+    FEATURE_COUNTS_FIELD_NUMBER: builtins.int
+    format: builtins.str
+    arch: builtins.str
+    os: builtins.str
+    extractor: builtins.str
+    @property
+    def rules(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+    @property
+    def layout(self) -> global___DynamicLayout: ...
+    @property
+    def feature_counts(self) -> global___DynamicFeatureCounts: ...
+    def __init__(
+        self,
+        *,
+        format: builtins.str = ...,
+        arch: builtins.str = ...,
+        os: builtins.str = ...,
+        extractor: builtins.str = ...,
+        rules: collections.abc.Iterable[builtins.str] | None = ...,
+        layout: global___DynamicLayout | None = ...,
+        feature_counts: global___DynamicFeatureCounts | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["feature_counts", b"feature_counts", "layout", b"layout"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["arch", b"arch", "extractor", b"extractor", "feature_counts", b"feature_counts", "format", b"format", "layout", b"layout", "os", b"os", "rules", b"rules"]) -> None: ...
+
+global___DynamicAnalysis = DynamicAnalysis
+
+@typing_extensions.final
+class DynamicFeatureCounts(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    FILE_FIELD_NUMBER: builtins.int
+    PROCESSES_FIELD_NUMBER: builtins.int
+    file: builtins.int
+    @property
+    def processes(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ProcessFeatureCount]: ...
+    def __init__(
+        self,
+        *,
+        file: builtins.int = ...,
+        processes: collections.abc.Iterable[global___ProcessFeatureCount] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["file", b"file", "processes", b"processes"]) -> None: ...
+
+global___DynamicFeatureCounts = DynamicFeatureCounts
+
+@typing_extensions.final
+class DynamicLayout(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    PROCESSES_FIELD_NUMBER: builtins.int
+    @property
+    def processes(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ProcessLayout]: ...
+    def __init__(
+        self,
+        *,
+        processes: collections.abc.Iterable[global___ProcessLayout] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["processes", b"processes"]) -> None: ...
+
+global___DynamicLayout = DynamicLayout
+
 @typing_extensions.final
 class ExportFeature(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
@@ -776,6 +889,9 @@ class Metadata(google.protobuf.message.Message):
     ARGV_FIELD_NUMBER: builtins.int
     SAMPLE_FIELD_NUMBER: builtins.int
     ANALYSIS_FIELD_NUMBER: builtins.int
+    FLAVOR_FIELD_NUMBER: builtins.int
+    STATIC_ANALYSIS_FIELD_NUMBER: builtins.int
+    DYNAMIC_ANALYSIS_FIELD_NUMBER: builtins.int
     timestamp: builtins.str
     """iso8601 format, like: 2019-01-01T00:00:00Z"""
     version: builtins.str
@@ -784,7 +900,16 @@ class Metadata(google.protobuf.message.Message):
     @property
     def sample(self) -> global___Sample: ...
     @property
-    def analysis(self) -> global___Analysis: ...
+    def analysis(self) -> global___Analysis:
+        """deprecated in v7.0.
+        use analysis2 instead.
+        """
+    flavor: global___Flavor.ValueType
+    @property
+    def static_analysis(self) -> global___StaticAnalysis:
+        """use analysis2 instead of analysis (deprecated in v7.0)."""
+    @property
+    def dynamic_analysis(self) -> global___DynamicAnalysis: ...
     def __init__(
         self,
         *,
@@ -793,9 +918,13 @@ class Metadata(google.protobuf.message.Message):
         argv: collections.abc.Iterable[builtins.str] | None = ...,
         sample: global___Sample | None = ...,
         analysis: global___Analysis | None = ...,
+        flavor: global___Flavor.ValueType = ...,
+        static_analysis: global___StaticAnalysis | None = ...,
+        dynamic_analysis: global___DynamicAnalysis | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["analysis", b"analysis", "sample", b"sample"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["analysis", b"analysis", "argv", b"argv", "sample", b"sample", "timestamp", b"timestamp", "version", b"version"]) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["analysis", b"analysis", "analysis2", b"analysis2", "dynamic_analysis", b"dynamic_analysis", "sample", b"sample", "static_analysis", b"static_analysis"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["analysis", b"analysis", "analysis2", b"analysis2", "argv", b"argv", "dynamic_analysis", b"dynamic_analysis", "flavor", b"flavor", "sample", b"sample", "static_analysis", b"static_analysis", "timestamp", b"timestamp", "version", b"version"]) -> None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["analysis2", b"analysis2"]) -> typing_extensions.Literal["static_analysis", "dynamic_analysis"] | None: ...
 
 global___Metadata = Metadata
 
@@ -973,6 +1102,50 @@ class OperandOffsetFeature(google.protobuf.message.Message):
 
 global___OperandOffsetFeature = OperandOffsetFeature
 
+@typing_extensions.final
+class ProcessFeatureCount(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    ADDRESS_FIELD_NUMBER: builtins.int
+    COUNT_FIELD_NUMBER: builtins.int
+    @property
+    def address(self) -> global___Address: ...
+    count: builtins.int
+    def __init__(
+        self,
+        *,
+        address: global___Address | None = ...,
+        count: builtins.int = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["address", b"address"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "count", b"count"]) -> None: ...
+
+global___ProcessFeatureCount = ProcessFeatureCount
+
+@typing_extensions.final
+class ProcessLayout(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    ADDRESS_FIELD_NUMBER: builtins.int
+    MATCHED_THREADS_FIELD_NUMBER: builtins.int
+    NAME_FIELD_NUMBER: builtins.int
+    @property
+    def address(self) -> global___Address: ...
+    @property
+    def matched_threads(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ThreadLayout]: ...
+    name: builtins.str
+    def __init__(
+        self,
+        *,
+        address: global___Address | None = ...,
+        matched_threads: collections.abc.Iterable[global___ThreadLayout] | None = ...,
+        name: builtins.str = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["address", b"address"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "matched_threads", b"matched_threads", "name", b"name"]) -> None: ...
+
+global___ProcessLayout = ProcessLayout
+
 @typing_extensions.final
 class PropertyFeature(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
@@ -1136,11 +1309,15 @@ class RuleMetadata(google.protobuf.message.Message):
     LIB_FIELD_NUMBER: builtins.int
     MAEC_FIELD_NUMBER: builtins.int
     IS_SUBSCOPE_RULE_FIELD_NUMBER: builtins.int
+    SCOPES_FIELD_NUMBER: builtins.int
     name: builtins.str
     namespace: builtins.str
     @property
     def authors(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
     scope: global___Scope.ValueType
+    """deprecated in v7.0.
+    use scopes instead.
+    """
     @property
     def attack(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___AttackSpec]: ...
     @property
@@ -1154,6 +1331,9 @@ class RuleMetadata(google.protobuf.message.Message):
     @property
     def maec(self) -> global___MaecMetadata: ...
     is_subscope_rule: builtins.bool
+    @property
+    def scopes(self) -> global___Scopes:
+        """use scopes over scope (deprecated in v7.0)."""
     def __init__(
         self,
         *,
@@ -1169,9 +1349,10 @@ class RuleMetadata(google.protobuf.message.Message):
         lib: builtins.bool = ...,
         maec: global___MaecMetadata | None = ...,
         is_subscope_rule: builtins.bool = ...,
+        scopes: global___Scopes | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["maec", b"maec"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["attack", b"attack", "authors", b"authors", "description", b"description", "examples", b"examples", "is_subscope_rule", b"is_subscope_rule", "lib", b"lib", "maec", b"maec", "mbc", b"mbc", "name", b"name", "namespace", b"namespace", "references", b"references", "scope", b"scope"]) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["maec", b"maec", "scopes", b"scopes"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["attack", b"attack", "authors", b"authors", "description", b"description", "examples", b"examples", "is_subscope_rule", b"is_subscope_rule", "lib", b"lib", "maec", b"maec", "mbc", b"mbc", "name", b"name", "namespace", b"namespace", "references", b"references", "scope", b"scope", "scopes", b"scopes"]) -> None: ...
 
 global___RuleMetadata = RuleMetadata
 
@@ -1199,6 +1380,29 @@ class Sample(google.protobuf.message.Message):
 
 global___Sample = Sample
 
+@typing_extensions.final
+class Scopes(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    STATIC_FIELD_NUMBER: builtins.int
+    DYNAMIC_FIELD_NUMBER: builtins.int
+    static: global___Scope.ValueType
+    dynamic: global___Scope.ValueType
+    def __init__(
+        self,
+        *,
+        static: global___Scope.ValueType | None = ...,
+        dynamic: global___Scope.ValueType | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["_dynamic", b"_dynamic", "_static", b"_static", "dynamic", b"dynamic", "static", b"static"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["_dynamic", b"_dynamic", "_static", b"_static", "dynamic", b"dynamic", "static", b"static"]) -> None: ...
+    @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_dynamic", b"_dynamic"]) -> typing_extensions.Literal["dynamic"] | None: ...
+    @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_static", b"_static"]) -> typing_extensions.Literal["static"] | None: ...
+
+global___Scopes = Scopes
+
 @typing_extensions.final
 class SectionFeature(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
@@ -1278,6 +1482,86 @@ class StatementNode(google.protobuf.message.Message):
 
 global___StatementNode = StatementNode
 
+@typing_extensions.final
+class StaticAnalysis(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    FORMAT_FIELD_NUMBER: builtins.int
+    ARCH_FIELD_NUMBER: builtins.int
+    OS_FIELD_NUMBER: builtins.int
+    EXTRACTOR_FIELD_NUMBER: builtins.int
+    RULES_FIELD_NUMBER: builtins.int
+    BASE_ADDRESS_FIELD_NUMBER: builtins.int
+    LAYOUT_FIELD_NUMBER: builtins.int
+    FEATURE_COUNTS_FIELD_NUMBER: builtins.int
+    LIBRARY_FUNCTIONS_FIELD_NUMBER: builtins.int
+    format: builtins.str
+    arch: builtins.str
+    os: builtins.str
+    extractor: builtins.str
+    @property
+    def rules(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+    @property
+    def base_address(self) -> global___Address: ...
+    @property
+    def layout(self) -> global___StaticLayout: ...
+    @property
+    def feature_counts(self) -> global___StaticFeatureCounts: ...
+    @property
+    def library_functions(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___LibraryFunction]: ...
+    def __init__(
+        self,
+        *,
+        format: builtins.str = ...,
+        arch: builtins.str = ...,
+        os: builtins.str = ...,
+        extractor: builtins.str = ...,
+        rules: collections.abc.Iterable[builtins.str] | None = ...,
+        base_address: global___Address | None = ...,
+        layout: global___StaticLayout | None = ...,
+        feature_counts: global___StaticFeatureCounts | None = ...,
+        library_functions: collections.abc.Iterable[global___LibraryFunction] | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["base_address", b"base_address", "feature_counts", b"feature_counts", "layout", b"layout"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["arch", b"arch", "base_address", b"base_address", "extractor", b"extractor", "feature_counts", b"feature_counts", "format", b"format", "layout", b"layout", "library_functions", b"library_functions", "os", b"os", "rules", b"rules"]) -> None: ...
+
+global___StaticAnalysis = StaticAnalysis
+
+@typing_extensions.final
+class StaticFeatureCounts(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    FILE_FIELD_NUMBER: builtins.int
+    FUNCTIONS_FIELD_NUMBER: builtins.int
+    file: builtins.int
+    @property
+    def functions(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FunctionFeatureCount]: ...
+    def __init__(
+        self,
+        *,
+        file: builtins.int = ...,
+        functions: collections.abc.Iterable[global___FunctionFeatureCount] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["file", b"file", "functions", b"functions"]) -> None: ...
+
+global___StaticFeatureCounts = StaticFeatureCounts
+
+@typing_extensions.final
+class StaticLayout(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    FUNCTIONS_FIELD_NUMBER: builtins.int
+    @property
+    def functions(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FunctionLayout]: ...
+    def __init__(
+        self,
+        *,
+        functions: collections.abc.Iterable[global___FunctionLayout] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["functions", b"functions"]) -> None: ...
+
+global___StaticLayout = StaticLayout
+
 @typing_extensions.final
 class StringFeature(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
@@ -1347,6 +1631,47 @@ class SubstringFeature(google.protobuf.message.Message):
 
 global___SubstringFeature = SubstringFeature
 
+@typing_extensions.final
+class CallLayout(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    ADDRESS_FIELD_NUMBER: builtins.int
+    NAME_FIELD_NUMBER: builtins.int
+    @property
+    def address(self) -> global___Address: ...
+    name: builtins.str
+    def __init__(
+        self,
+        *,
+        address: global___Address | None = ...,
+        name: builtins.str = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["address", b"address"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "name", b"name"]) -> None: ...
+
+global___CallLayout = CallLayout
+
+@typing_extensions.final
+class ThreadLayout(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    ADDRESS_FIELD_NUMBER: builtins.int
+    MATCHED_CALLS_FIELD_NUMBER: builtins.int
+    @property
+    def address(self) -> global___Address: ...
+    @property
+    def matched_calls(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___CallLayout]: ...
+    def __init__(
+        self,
+        *,
+        address: global___Address | None = ...,
+        matched_calls: collections.abc.Iterable[global___CallLayout] | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["address", b"address"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "matched_calls", b"matched_calls"]) -> None: ...
+
+global___ThreadLayout = ThreadLayout
+
 @typing_extensions.final
 class Addresses(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
@@ -1405,6 +1730,81 @@ class Token_Offset(google.protobuf.message.Message):
 
 global___Token_Offset = Token_Offset
 
+@typing_extensions.final
+class Ppid_Pid(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    PPID_FIELD_NUMBER: builtins.int
+    PID_FIELD_NUMBER: builtins.int
+    @property
+    def ppid(self) -> global___Integer: ...
+    @property
+    def pid(self) -> global___Integer: ...
+    def __init__(
+        self,
+        *,
+        ppid: global___Integer | None = ...,
+        pid: global___Integer | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["pid", b"pid", "ppid", b"ppid"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["pid", b"pid", "ppid", b"ppid"]) -> None: ...
+
+global___Ppid_Pid = Ppid_Pid
+
+@typing_extensions.final
+class Ppid_Pid_Tid(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    PPID_FIELD_NUMBER: builtins.int
+    PID_FIELD_NUMBER: builtins.int
+    TID_FIELD_NUMBER: builtins.int
+    @property
+    def ppid(self) -> global___Integer: ...
+    @property
+    def pid(self) -> global___Integer: ...
+    @property
+    def tid(self) -> global___Integer: ...
+    def __init__(
+        self,
+        *,
+        ppid: global___Integer | None = ...,
+        pid: global___Integer | None = ...,
+        tid: global___Integer | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["pid", b"pid", "ppid", b"ppid", "tid", b"tid"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["pid", b"pid", "ppid", b"ppid", "tid", b"tid"]) -> None: ...
+
+global___Ppid_Pid_Tid = Ppid_Pid_Tid
+
+@typing_extensions.final
+class Ppid_Pid_Tid_Id(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    PPID_FIELD_NUMBER: builtins.int
+    PID_FIELD_NUMBER: builtins.int
+    TID_FIELD_NUMBER: builtins.int
+    ID_FIELD_NUMBER: builtins.int
+    @property
+    def ppid(self) -> global___Integer: ...
+    @property
+    def pid(self) -> global___Integer: ...
+    @property
+    def tid(self) -> global___Integer: ...
+    @property
+    def id(self) -> global___Integer: ...
+    def __init__(
+        self,
+        *,
+        ppid: global___Integer | None = ...,
+        pid: global___Integer | None = ...,
+        tid: global___Integer | None = ...,
+        id: global___Integer | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["id", b"id", "pid", b"pid", "ppid", b"ppid", "tid", b"tid"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "pid", b"pid", "ppid", b"ppid", "tid", b"tid"]) -> None: ...
+
+global___Ppid_Pid_Tid_Id = Ppid_Pid_Tid_Id
+
 @typing_extensions.final
 class Integer(google.protobuf.message.Message):
     DESCRIPTOR: google.protobuf.descriptor.Descriptor

capa/render/result_document.py

@@ -7,10 +7,12 @@
 # See the License for the specific language governing permissions and limitations under the License.
 import datetime
 import collections
+from enum import Enum
 from typing import Dict, List, Tuple, Union, Literal, Optional
 from pathlib import Path
 
 from pydantic import Field, BaseModel, ConfigDict
+from typing_extensions import TypeAlias
 
 import capa.rules
 import capa.engine
@@ -47,10 +49,33 @@ class FunctionLayout(Model):
     matched_basic_blocks: Tuple[BasicBlockLayout, ...]
 
 
-class Layout(Model):
+class CallLayout(Model):
+    address: frz.Address
+    name: str
+
+
+class ThreadLayout(Model):
+    address: frz.Address
+    matched_calls: Tuple[CallLayout, ...]
+
+
+class ProcessLayout(Model):
+    address: frz.Address
+    name: str
+    matched_threads: Tuple[ThreadLayout, ...]
+
+
+class StaticLayout(Model):
     functions: Tuple[FunctionLayout, ...]
 
 
+class DynamicLayout(Model):
+    processes: Tuple[ProcessLayout, ...]
+
+
+Layout: TypeAlias = Union[StaticLayout, DynamicLayout]
+
+
 class LibraryFunction(Model):
     address: frz.Address
     name: str
@@ -61,31 +86,73 @@ class FunctionFeatureCount(Model):
     count: int
 
 
-class FeatureCounts(Model):
+class ProcessFeatureCount(Model):
+    address: frz.Address
+    count: int
+
+
+class StaticFeatureCounts(Model):
     file: int
     functions: Tuple[FunctionFeatureCount, ...]
 
 
-class Analysis(Model):
+class DynamicFeatureCounts(Model):
+    file: int
+    processes: Tuple[ProcessFeatureCount, ...]
+
+
+FeatureCounts: TypeAlias = Union[StaticFeatureCounts, DynamicFeatureCounts]
+
+
+class StaticAnalysis(Model):
     format: str
     arch: str
     os: str
     extractor: str
     rules: Tuple[str, ...]
     base_address: frz.Address
-    layout: Layout
-    feature_counts: FeatureCounts
+    layout: StaticLayout
+    feature_counts: StaticFeatureCounts
     library_functions: Tuple[LibraryFunction, ...]
 
 
+class DynamicAnalysis(Model):
+    format: str
+    arch: str
+    os: str
+    extractor: str
+    rules: Tuple[str, ...]
+    layout: DynamicLayout
+    feature_counts: DynamicFeatureCounts
+
+
+Analysis: TypeAlias = Union[StaticAnalysis, DynamicAnalysis]
+
+
+class Flavor(str, Enum):
+    STATIC = "static"
+    DYNAMIC = "dynamic"
+
+
 class Metadata(Model):
     timestamp: datetime.datetime
     version: str
     argv: Optional[Tuple[str, ...]]
     sample: Sample
+    flavor: Flavor
     analysis: Analysis
 
 
+class StaticMetadata(Metadata):
+    flavor: Flavor = Flavor.STATIC
+    analysis: StaticAnalysis
+
+
+class DynamicMetadata(Metadata):
+    flavor: Flavor = Flavor.DYNAMIC
+    analysis: DynamicAnalysis
+
+
 class CompoundStatementType:
     AND = "and"
     OR = "or"
@@ -155,7 +222,7 @@ def statement_from_capa(node: capa.engine.Statement) -> Statement:
             description=node.description,
             min=node.min,
             max=node.max,
-            child=frz.feature_from_capa(node.child),
+            child=frzf.feature_from_capa(node.child),
         )
 
     elif isinstance(node, capa.engine.Subscope):
@@ -181,7 +248,7 @@ def node_from_capa(node: Union[capa.engine.Statement, capa.engine.Feature]) -> N
         return StatementNode(statement=statement_from_capa(node))
 
     elif isinstance(node, capa.engine.Feature):
-        return FeatureNode(feature=frz.feature_from_capa(node))
+        return FeatureNode(feature=frzf.feature_from_capa(node))
 
     else:
         assert_never(node)
@@ -308,9 +375,11 @@ class Match(FrozenModel):
                     # e.g. `contain loop/30c4c78e29bf4d54894fc74f664c62e8` -> `basic block`
                     #
                     # note! replace `node`
+                    # subscopes cannot have both a static and dynamic scope set
+                    assert None in (rule.scopes.static, rule.scopes.dynamic)
                     node = StatementNode(
                         statement=SubscopeStatement(
-                            scope=rule.meta["scope"],
+                            scope=rule.scopes.static or rule.scopes.dynamic,
                         )
                     )
 
@@ -505,7 +574,7 @@ class RuleMetadata(FrozenModel):
     name: str
     namespace: Optional[str] = None
     authors: Tuple[str, ...]
-    scope: capa.rules.Scope
+    scopes: capa.rules.Scopes
     attack: Tuple[AttackSpec, ...] = Field(alias="att&ck")
     mbc: Tuple[MBCSpec, ...]
     references: Tuple[str, ...]
@@ -522,7 +591,7 @@ class RuleMetadata(FrozenModel):
             name=rule.meta.get("name"),
             namespace=rule.meta.get("namespace"),
             authors=rule.meta.get("authors"),
-            scope=capa.rules.Scope(rule.meta.get("scope")),
+            scopes=capa.rules.Scopes.from_dict(rule.meta.get("scopes")),
             attack=tuple(map(AttackSpec.from_str, rule.meta.get("att&ck", []))),
             mbc=tuple(map(MBCSpec.from_str, rule.meta.get("mbc", []))),
             references=rule.meta.get("references", []),

capa/render/utils.py

@@ -24,6 +24,11 @@ def bold2(s: str) -> str:
     return termcolor.colored(s, "green")
 
 
+def mute(s: str) -> str:
+    """draw attention away from the given string"""
+    return termcolor.colored(s, "dark_grey")
+
+
 def warn(s: str) -> str:
     return termcolor.colored(s, "yellow")
 

capa/render/verbose.py

@@ -22,7 +22,7 @@ Unless required by applicable law or agreed to in writing, software distributed
  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
-import enum
+from typing import cast
 
 import tabulate
 
@@ -54,13 +54,92 @@ def format_address(address: frz.Address) -> str:
         assert isinstance(token, int)
         assert isinstance(offset, int)
         return f"token({capa.helpers.hex(token)})+{capa.helpers.hex(offset)}"
+    elif address.type == frz.AddressType.PROCESS:
+        assert isinstance(address.value, tuple)
+        ppid, pid = address.value
+        assert isinstance(ppid, int)
+        assert isinstance(pid, int)
+        return f"process{{pid:{pid}}}"
+    elif address.type == frz.AddressType.THREAD:
+        assert isinstance(address.value, tuple)
+        ppid, pid, tid = address.value
+        assert isinstance(ppid, int)
+        assert isinstance(pid, int)
+        assert isinstance(tid, int)
+        return f"process{{pid:{pid},tid:{tid}}}"
+    elif address.type == frz.AddressType.CALL:
+        assert isinstance(address.value, tuple)
+        ppid, pid, tid, id_ = address.value
+        return f"process{{pid:{pid},tid:{tid},call:{id_}}}"
     elif address.type == frz.AddressType.NO_ADDRESS:
         return "global"
     else:
         raise ValueError("unexpected address type")
 
 
-def render_meta(ostream, doc: rd.ResultDocument):
+def _get_process_name(layout: rd.DynamicLayout, addr: frz.Address) -> str:
+    for p in layout.processes:
+        if p.address == addr:
+            return p.name
+
+    raise ValueError("name not found for process", addr)
+
+
+def _get_call_name(layout: rd.DynamicLayout, addr: frz.Address) -> str:
+    call = addr.to_capa()
+    assert isinstance(call, capa.features.address.DynamicCallAddress)
+
+    thread = frz.Address.from_capa(call.thread)
+    process = frz.Address.from_capa(call.thread.process)
+
+    # danger: O(n**3)
+    for p in layout.processes:
+        if p.address == process:
+            for t in p.matched_threads:
+                if t.address == thread:
+                    for c in t.matched_calls:
+                        if c.address == addr:
+                            return c.name
+    raise ValueError("name not found for call", addr)
+
+
+def render_process(layout: rd.DynamicLayout, addr: frz.Address) -> str:
+    process = addr.to_capa()
+    assert isinstance(process, capa.features.address.ProcessAddress)
+    name = _get_process_name(layout, addr)
+    return f"{name}{{pid:{process.pid}}}"
+
+
+def render_thread(layout: rd.DynamicLayout, addr: frz.Address) -> str:
+    thread = addr.to_capa()
+    assert isinstance(thread, capa.features.address.ThreadAddress)
+    name = _get_process_name(layout, frz.Address.from_capa(thread.process))
+    return f"{name}{{pid:{thread.process.pid},tid:{thread.tid}}}"
+
+
+def render_call(layout: rd.DynamicLayout, addr: frz.Address) -> str:
+    call = addr.to_capa()
+    assert isinstance(call, capa.features.address.DynamicCallAddress)
+
+    pname = _get_process_name(layout, frz.Address.from_capa(call.thread.process))
+    cname = _get_call_name(layout, addr)
+
+    fname, _, rest = cname.partition("(")
+    args, _, rest = rest.rpartition(")")
+
+    s = []
+    s.append(f"{fname}(")
+    for arg in args.split(", "):
+        s.append(f"  {arg},")
+    s.append(f"){rest}")
+
+    newline = "\n"
+    return (
+        f"{pname}{{pid:{call.thread.process.pid},tid:{call.thread.tid},call:{call.id}}}\n{rutils.mute(newline.join(s))}"
+    )
+
+
+def render_static_meta(ostream, meta: rd.StaticMetadata):
     """
     like:
 
@@ -73,36 +152,90 @@ def render_meta(ostream, doc: rd.ResultDocument):
         os                   windows
         format               pe
         arch                 amd64
+        analysis             static
         extractor            VivisectFeatureExtractor
         base address         0x10000000
         rules                (embedded rules)
         function count       42
         total feature count  1918
     """
+
     rows = [
-        ("md5", doc.meta.sample.md5),
-        ("sha1", doc.meta.sample.sha1),
-        ("sha256", doc.meta.sample.sha256),
-        ("path", doc.meta.sample.path),
-        ("timestamp", doc.meta.timestamp),
-        ("capa version", doc.meta.version),
-        ("os", doc.meta.analysis.os),
-        ("format", doc.meta.analysis.format),
-        ("arch", doc.meta.analysis.arch),
-        ("extractor", doc.meta.analysis.extractor),
-        ("base address", format_address(doc.meta.analysis.base_address)),
-        ("rules", "\n".join(doc.meta.analysis.rules)),
-        ("function count", len(doc.meta.analysis.feature_counts.functions)),
-        ("library function count", len(doc.meta.analysis.library_functions)),
+        ("md5", meta.sample.md5),
+        ("sha1", meta.sample.sha1),
+        ("sha256", meta.sample.sha256),
+        ("path", meta.sample.path),
+        ("timestamp", meta.timestamp),
+        ("capa version", meta.version),
+        ("os", meta.analysis.os),
+        ("format", meta.analysis.format),
+        ("arch", meta.analysis.arch),
+        ("analysis", meta.flavor.value),
+        ("extractor", meta.analysis.extractor),
+        ("base address", format_address(meta.analysis.base_address)),
+        ("rules", "\n".join(meta.analysis.rules)),
+        ("function count", len(meta.analysis.feature_counts.functions)),
+        ("library function count", len(meta.analysis.library_functions)),
         (
             "total feature count",
-            doc.meta.analysis.feature_counts.file + sum(f.count for f in doc.meta.analysis.feature_counts.functions),
+            meta.analysis.feature_counts.file + sum(f.count for f in meta.analysis.feature_counts.functions),
         ),
     ]
 
     ostream.writeln(tabulate.tabulate(rows, tablefmt="plain"))
 
 
+def render_dynamic_meta(ostream, meta: rd.DynamicMetadata):
+    """
+    like:
+
+        md5                  84882c9d43e23d63b82004fae74ebb61
+        sha1                 c6fb3b50d946bec6f391aefa4e54478cf8607211
+        sha256               5eced7367ed63354b4ed5c556e2363514293f614c2c2eb187273381b2ef5f0f9
+        path                 /tmp/packed-report,jspn
+        timestamp            2023-07-17T10:17:05.796933
+        capa version         0.0.0
+        os                   windows
+        format               pe
+        arch                 amd64
+        extractor            CAPEFeatureExtractor
+        rules                (embedded rules)
+        process count        42
+        total feature count  1918
+    """
+
+    rows = [
+        ("md5", meta.sample.md5),
+        ("sha1", meta.sample.sha1),
+        ("sha256", meta.sample.sha256),
+        ("path", meta.sample.path),
+        ("timestamp", meta.timestamp),
+        ("capa version", meta.version),
+        ("os", meta.analysis.os),
+        ("format", meta.analysis.format),
+        ("arch", meta.analysis.arch),
+        ("analysis", meta.flavor.value),
+        ("extractor", meta.analysis.extractor),
+        ("rules", "\n".join(meta.analysis.rules)),
+        ("process count", len(meta.analysis.feature_counts.processes)),
+        (
+            "total feature count",
+            meta.analysis.feature_counts.file + sum(p.count for p in meta.analysis.feature_counts.processes),
+        ),
+    ]
+
+    ostream.writeln(tabulate.tabulate(rows, tablefmt="plain"))
+
+
+def render_meta(osstream, doc: rd.ResultDocument):
+    if doc.meta.flavor == rd.Flavor.STATIC:
+        render_static_meta(osstream, cast(rd.StaticMetadata, doc.meta))
+    elif doc.meta.flavor == rd.Flavor.DYNAMIC:
+        render_dynamic_meta(osstream, cast(rd.DynamicMetadata, doc.meta))
+    else:
+        raise ValueError("invalid meta analysis")
+
+
 def render_rules(ostream, doc: rd.ResultDocument):
     """
     like:
@@ -126,22 +259,55 @@ def render_rules(ostream, doc: rd.ResultDocument):
         had_match = True
 
         rows = []
-        for key in ("namespace", "description", "scope"):
-            v = getattr(rule.meta, key)
-            if not v:
-                continue
 
-            if isinstance(v, list) and len(v) == 1:
-                v = v[0]
+        ns = rule.meta.namespace
+        if ns:
+            rows.append(("namespace", ns))
 
-            if isinstance(v, enum.Enum):
-                v = v.value
+        desc = rule.meta.description
+        if desc:
+            rows.append(("description", desc))
 
-            rows.append((key, v))
+        if doc.meta.flavor == rd.Flavor.STATIC:
+            scope = rule.meta.scopes.static
+        elif doc.meta.flavor == rd.Flavor.DYNAMIC:
+            scope = rule.meta.scopes.dynamic
+        else:
+            raise ValueError("invalid meta analysis")
+        if scope:
+            rows.append(("scope", scope.value))
 
-        if rule.meta.scope != capa.rules.FILE_SCOPE:
+        if capa.rules.Scope.FILE not in rule.meta.scopes:
             locations = [m[0] for m in doc.rules[rule.meta.name].matches]
-            rows.append(("matches", "\n".join(map(format_address, locations))))
+            lines = []
+
+            if doc.meta.flavor == rd.Flavor.STATIC:
+                lines = [format_address(loc) for loc in locations]
+            elif doc.meta.flavor == rd.Flavor.DYNAMIC:
+                assert rule.meta.scopes.dynamic is not None
+                assert isinstance(doc.meta.analysis.layout, rd.DynamicLayout)
+
+                if rule.meta.scopes.dynamic == capa.rules.Scope.PROCESS:
+                    lines = [render_process(doc.meta.analysis.layout, loc) for loc in locations]
+                elif rule.meta.scopes.dynamic == capa.rules.Scope.THREAD:
+                    lines = [render_thread(doc.meta.analysis.layout, loc) for loc in locations]
+                elif rule.meta.scopes.dynamic == capa.rules.Scope.CALL:
+                    # because we're only in verbose mode, we won't show the full call details (name, args, retval)
+                    # we'll only show the details of the thread in which the calls are found.
+                    # so select the thread locations and render those.
+                    thread_locations = set()
+                    for loc in locations:
+                        cloc = loc.to_capa()
+                        assert isinstance(cloc, capa.features.address.DynamicCallAddress)
+                        thread_locations.add(frz.Address.from_capa(cloc.thread))
+
+                    lines = [render_thread(doc.meta.analysis.layout, loc) for loc in thread_locations]
+                else:
+                    capa.helpers.assert_never(rule.meta.scopes.dynamic)
+            else:
+                capa.helpers.assert_never(doc.meta.flavor)
+
+            rows.append(("matches", "\n".join(lines)))
 
         ostream.writeln(tabulate.tabulate(rows, tablefmt="plain"))
         ostream.write("\n")

capa/render/vverbose.py

@@ -5,7 +5,8 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-
+import logging
+import textwrap
 from typing import Dict, Iterable, Optional
 
 import tabulate
@@ -22,8 +23,29 @@ import capa.features.freeze.features as frzf
 from capa.rules import RuleSet
 from capa.engine import MatchResults
 
+logger = logging.getLogger(__name__)
 
-def render_locations(ostream, locations: Iterable[frz.Address]):
+
+def hanging_indent(s: str, indent: int) -> str:
+    """
+    indent the given string, except the first line,
+    such as if the string finishes an existing line.
+
+    e.g.,
+
+        EXISTINGSTUFFHERE + hanging_indent("xxxx...", 1)
+
+    becomes:
+
+        EXISTINGSTUFFHERExxxxx
+          xxxxxx
+          xxxxxx
+    """
+    prefix = "  " * indent
+    return textwrap.indent(s, prefix=prefix)[len(prefix) :]
+
+
+def render_locations(ostream, layout: rd.Layout, locations: Iterable[frz.Address], indent: int):
     import capa.render.verbose as v
 
     # its possible to have an empty locations array here,
@@ -35,9 +57,23 @@ def render_locations(ostream, locations: Iterable[frz.Address]):
         return
 
     ostream.write(" @ ")
+    location0 = locations[0]
 
     if len(locations) == 1:
-        ostream.write(v.format_address(locations[0]))
+        location = locations[0]
+
+        if location.type == frz.AddressType.CALL:
+            assert isinstance(layout, rd.DynamicLayout)
+            ostream.write(hanging_indent(v.render_call(layout, location), indent + 1))
+        else:
+            ostream.write(v.format_address(locations[0]))
+
+    elif location0.type == frz.AddressType.CALL and len(locations) > 1:
+        location = locations[0]
+
+        assert isinstance(layout, rd.DynamicLayout)
+        s = f"{v.render_call(layout, location)}\nand {(len(locations) - 1)} more..."
+        ostream.write(hanging_indent(s, indent + 1))
 
     elif len(locations) > 4:
         # don't display too many locations, because it becomes very noisy.
@@ -52,7 +88,7 @@ def render_locations(ostream, locations: Iterable[frz.Address]):
         raise RuntimeError("unreachable")
 
 
-def render_statement(ostream, match: rd.Match, statement: rd.Statement, indent=0):
+def render_statement(ostream, layout: rd.Layout, match: rd.Match, statement: rd.Statement, indent: int):
     ostream.write("  " * indent)
 
     if isinstance(statement, rd.SubscopeStatement):
@@ -114,7 +150,7 @@ def render_statement(ostream, match: rd.Match, statement: rd.Statement, indent=0
 
         if statement.description:
             ostream.write(f" = {statement.description}")
-        render_locations(ostream, match.locations)
+        render_locations(ostream, layout, match.locations, indent)
         ostream.writeln("")
 
     else:
@@ -125,7 +161,9 @@ def render_string_value(s: str) -> str:
     return f'"{capa.features.common.escape_string(s)}"'
 
 
-def render_feature(ostream, match: rd.Match, feature: frzf.Feature, indent=0):
+def render_feature(
+    ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, feature: frzf.Feature, indent: int
+):
     ostream.write("  " * indent)
 
     key = feature.type
@@ -176,8 +214,17 @@ def render_feature(ostream, match: rd.Match, feature: frzf.Feature, indent=0):
                 ostream.write(capa.rules.DESCRIPTION_SEPARATOR)
                 ostream.write(feature.description)
 
-        if not isinstance(feature, (frzf.OSFeature, frzf.ArchFeature, frzf.FormatFeature)):
-            render_locations(ostream, match.locations)
+        if isinstance(feature, (frzf.OSFeature, frzf.ArchFeature, frzf.FormatFeature)):
+            # don't show the location of these global features
+            pass
+        elif isinstance(layout, rd.DynamicLayout) and rule.meta.scopes.dynamic == capa.rules.Scope.CALL:
+            # if we're in call scope, then the call will have been rendered at the top
+            # of the output, so don't re-render it again for each feature.
+            pass
+        elif isinstance(feature, (frzf.OSFeature, frzf.ArchFeature, frzf.FormatFeature)):
+            pass
+        else:
+            render_locations(ostream, layout, match.locations, indent)
         ostream.write("\n")
     else:
         # like:
@@ -193,15 +240,19 @@ def render_feature(ostream, match: rd.Match, feature: frzf.Feature, indent=0):
             ostream.write("  " * (indent + 1))
             ostream.write("- ")
             ostream.write(rutils.bold2(render_string_value(capture)))
-            render_locations(ostream, locations)
+            if isinstance(layout, rd.DynamicLayout) and rule.meta.scopes.dynamic == capa.rules.Scope.CALL:
+                # like above, don't re-render calls when in call scope.
+                pass
+            else:
+                render_locations(ostream, layout, locations, indent=indent)
             ostream.write("\n")
 
 
-def render_node(ostream, match: rd.Match, node: rd.Node, indent=0):
+def render_node(ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, node: rd.Node, indent: int):
     if isinstance(node, rd.StatementNode):
-        render_statement(ostream, match, node.statement, indent=indent)
+        render_statement(ostream, layout, match, node.statement, indent=indent)
     elif isinstance(node, rd.FeatureNode):
-        render_feature(ostream, match, node.feature, indent=indent)
+        render_feature(ostream, layout, rule, match, node.feature, indent=indent)
     else:
         raise RuntimeError("unexpected node type: " + str(node))
 
@@ -214,7 +265,7 @@ MODE_SUCCESS = "success"
 MODE_FAILURE = "failure"
 
 
-def render_match(ostream, match: rd.Match, indent=0, mode=MODE_SUCCESS):
+def render_match(ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, indent=0, mode=MODE_SUCCESS):
     child_mode = mode
     if mode == MODE_SUCCESS:
         # display only nodes that evaluated successfully.
@@ -246,10 +297,10 @@ def render_match(ostream, match: rd.Match, indent=0, mode=MODE_SUCCESS):
     else:
         raise RuntimeError("unexpected mode: " + mode)
 
-    render_node(ostream, match, match.node, indent=indent)
+    render_node(ostream, layout, rule, match, match.node, indent=indent)
 
     for child in match.children:
-        render_match(ostream, child, indent=indent + 1, mode=child_mode)
+        render_match(ostream, layout, rule, child, indent=indent + 1, mode=child_mode)
 
 
 def render_rules(ostream, doc: rd.ResultDocument):
@@ -260,7 +311,8 @@ def render_rules(ostream, doc: rd.ResultDocument):
         check for OutputDebugString error
         namespace  anti-analysis/anti-debugging/debugger-detection
         author     michael.hunhoff@mandiant.com
-        scope      function
+        static scope:   function
+        dynamic scope:  process
         mbc        Anti-Behavioral Analysis::Detect Debugger::OutputDebugString
         function @ 0x10004706
           and:
@@ -268,13 +320,20 @@ def render_rules(ostream, doc: rd.ResultDocument):
             api: kernel32.GetLastError @ 0x10004A87
             api: kernel32.OutputDebugString @ 0x10004767, 0x10004787, 0x10004816, 0x10004895
     """
-    functions_by_bb: Dict[capa.features.address.Address, capa.features.address.Address] = {}
-    for finfo in doc.meta.analysis.layout.functions:
-        faddress = finfo.address.to_capa()
+    import capa.render.verbose as v
 
-        for bb in finfo.matched_basic_blocks:
-            bbaddress = bb.address.to_capa()
-            functions_by_bb[bbaddress] = faddress
+    functions_by_bb: Dict[capa.features.address.Address, capa.features.address.Address] = {}
+    if isinstance(doc.meta.analysis, rd.StaticAnalysis):
+        for finfo in doc.meta.analysis.layout.functions:
+            faddress = finfo.address.to_capa()
+
+            for bb in finfo.matched_basic_blocks:
+                bbaddress = bb.address.to_capa()
+                functions_by_bb[bbaddress] = faddress
+    elif isinstance(doc.meta.analysis, rd.DynamicAnalysis):
+        pass
+    else:
+        raise ValueError("invalid analysis field in the document's meta")
 
     had_match = False
 
@@ -323,7 +382,13 @@ def render_rules(ostream, doc: rd.ResultDocument):
 
         rows.append(("author", ", ".join(rule.meta.authors)))
 
-        rows.append(("scope", rule.meta.scope.value))
+        if doc.meta.flavor == rd.Flavor.STATIC:
+            assert rule.meta.scopes.static is not None
+            rows.append(("scope", rule.meta.scopes.static.value))
+
+        if doc.meta.flavor == rd.Flavor.DYNAMIC:
+            assert rule.meta.scopes.dynamic is not None
+            rows.append(("scope", rule.meta.scopes.dynamic.value))
 
         if rule.meta.attack:
             rows.append(("att&ck", ", ".join([rutils.format_parts_id(v) for v in rule.meta.attack])))
@@ -339,7 +404,7 @@ def render_rules(ostream, doc: rd.ResultDocument):
 
         ostream.writeln(tabulate.tabulate(rows, tablefmt="plain"))
 
-        if rule.meta.scope == capa.rules.FILE_SCOPE:
+        if capa.rules.Scope.FILE in rule.meta.scopes:
             matches = doc.rules[rule.meta.name].matches
             if len(matches) != 1:
                 # i think there should only ever be one match per file-scope rule,
@@ -347,22 +412,42 @@ def render_rules(ostream, doc: rd.ResultDocument):
                 # but i'm not 100% sure if this is/will always be true.
                 # so, lets be explicit about our assumptions and raise an exception if they fail.
                 raise RuntimeError(f"unexpected file scope match count: {len(matches)}")
-            first_address, first_match = matches[0]
-            render_match(ostream, first_match, indent=0)
+            _, first_match = matches[0]
+            render_match(ostream, doc.meta.analysis.layout, rule, first_match, indent=0)
         else:
             for location, match in sorted(doc.rules[rule.meta.name].matches):
-                ostream.write(rule.meta.scope)
-                ostream.write(" @ ")
-                ostream.write(capa.render.verbose.format_address(location))
+                if doc.meta.flavor == rd.Flavor.STATIC:
+                    assert rule.meta.scopes.static is not None
+                    ostream.write(rule.meta.scopes.static.value)
+                    ostream.write(" @ ")
+                    ostream.write(capa.render.verbose.format_address(location))
 
-                if rule.meta.scope == capa.rules.BASIC_BLOCK_SCOPE:
-                    ostream.write(
-                        " in function "
-                        + capa.render.verbose.format_address(frz.Address.from_capa(functions_by_bb[location.to_capa()]))
-                    )
+                    if rule.meta.scopes.static == capa.rules.Scope.BASIC_BLOCK:
+                        func = frz.Address.from_capa(functions_by_bb[location.to_capa()])
+                        ostream.write(f" in function {capa.render.verbose.format_address(func)}")
+
+                elif doc.meta.flavor == rd.Flavor.DYNAMIC:
+                    assert rule.meta.scopes.dynamic is not None
+                    assert isinstance(doc.meta.analysis.layout, rd.DynamicLayout)
+
+                    ostream.write(rule.meta.scopes.dynamic.value)
+
+                    ostream.write(" @ ")
+
+                    if rule.meta.scopes.dynamic == capa.rules.Scope.PROCESS:
+                        ostream.write(v.render_process(doc.meta.analysis.layout, location))
+                    elif rule.meta.scopes.dynamic == capa.rules.Scope.THREAD:
+                        ostream.write(v.render_thread(doc.meta.analysis.layout, location))
+                    elif rule.meta.scopes.dynamic == capa.rules.Scope.CALL:
+                        ostream.write(hanging_indent(v.render_call(doc.meta.analysis.layout, location), indent=1))
+                    else:
+                        capa.helpers.assert_never(rule.meta.scopes.dynamic)
+
+                else:
+                    capa.helpers.assert_never(doc.meta.flavor)
 
                 ostream.write("\n")
-                render_match(ostream, match, indent=1)
+                render_match(ostream, doc.meta.analysis.layout, rule, match, indent=1)
                 if rule.meta.lib:
                     # only show first match
                     break

capa/rules/__init__.py

@@ -25,7 +25,8 @@ except ImportError:
     # https://github.com/python/mypy/issues/1153
     from backports.functools_lru_cache import lru_cache  # type: ignore
 
-from typing import Any, Set, Dict, List, Tuple, Union, Iterator
+from typing import Any, Set, Dict, List, Tuple, Union, Iterator, Optional
+from dataclasses import asdict, dataclass
 
 import yaml
 import pydantic
@@ -36,11 +37,13 @@ import capa.perf
 import capa.engine as ceng
 import capa.features
 import capa.optimizer
+import capa.features.com
 import capa.features.file
 import capa.features.insn
 import capa.features.common
 import capa.features.basicblock
 from capa.engine import Statement, FeatureSet
+from capa.features.com import ComType
 from capa.features.common import MAX_BYTES_FEATURE_SIZE, Feature
 from capa.features.address import Address
 
@@ -59,7 +62,7 @@ META_KEYS = (
     "authors",
     "description",
     "lib",
-    "scope",
+    "scopes",
     "att&ck",
     "mbc",
     "references",
@@ -74,28 +77,113 @@ HIDDEN_META_KEYS = ("capa/nursery", "capa/path")
 
 class Scope(str, Enum):
     FILE = "file"
+    PROCESS = "process"
+    THREAD = "thread"
+    CALL = "call"
     FUNCTION = "function"
     BASIC_BLOCK = "basic block"
     INSTRUCTION = "instruction"
 
+    # used only to specify supported features per scope.
+    # not used to validate rules.
+    GLOBAL = "global"
 
-FILE_SCOPE = Scope.FILE.value
-FUNCTION_SCOPE = Scope.FUNCTION.value
-BASIC_BLOCK_SCOPE = Scope.BASIC_BLOCK.value
-INSTRUCTION_SCOPE = Scope.INSTRUCTION.value
-# used only to specify supported features per scope.
-# not used to validate rules.
-GLOBAL_SCOPE = "global"
+    @classmethod
+    def to_yaml(cls, representer, node):
+        return representer.represent_str(f"{node.value}")
+
+
+# these literals are used to check if the flavor
+# of a rule is correct.
+STATIC_SCOPES = {
+    Scope.FILE,
+    Scope.GLOBAL,
+    Scope.FUNCTION,
+    Scope.BASIC_BLOCK,
+    Scope.INSTRUCTION,
+}
+DYNAMIC_SCOPES = {
+    Scope.FILE,
+    Scope.GLOBAL,
+    Scope.PROCESS,
+    Scope.THREAD,
+    Scope.CALL,
+}
+
+
+@dataclass
+class Scopes:
+    # when None, the scope is not supported by a rule
+    static: Optional[Scope] = None
+    # when None, the scope is not supported by a rule
+    dynamic: Optional[Scope] = None
+
+    def __contains__(self, scope: Scope) -> bool:
+        return (scope == self.static) or (scope == self.dynamic)
+
+    def __repr__(self) -> str:
+        if self.static and self.dynamic:
+            return f"static-scope: {self.static}, dynamic-scope: {self.dynamic}"
+        elif self.static:
+            return f"static-scope: {self.static}"
+        elif self.dynamic:
+            return f"dynamic-scope: {self.dynamic}"
+        else:
+            raise ValueError("invalid rules class. at least one scope must be specified")
+
+    @classmethod
+    def from_dict(self, scopes: Dict[str, str]) -> "Scopes":
+        # make local copy so we don't make changes outside of this routine.
+        # we'll use the value None to indicate the scope is not supported.
+        scopes_: Dict[str, Optional[str]] = dict(scopes)
+
+        # mark non-specified scopes as invalid
+        if "static" not in scopes_:
+            raise InvalidRule("static scope must be provided")
+        if "dynamic" not in scopes_:
+            raise InvalidRule("dynamic scope must be provided")
+
+        # check the syntax of the meta `scopes` field
+        if sorted(scopes_) != ["dynamic", "static"]:
+            raise InvalidRule("scope flavors can be either static or dynamic")
+
+        if scopes_["static"] == "unsupported":
+            scopes_["static"] = None
+        if scopes_["dynamic"] == "unsupported":
+            scopes_["dynamic"] = None
+
+        # unspecified is used to indicate a rule is yet to be migrated.
+        # TODO(williballenthin): this scope term should be removed once all rules have been migrated.
+        # https://github.com/mandiant/capa/issues/1747
+        if scopes_["static"] == "unspecified":
+            scopes_["static"] = None
+        if scopes_["dynamic"] == "unspecified":
+            scopes_["dynamic"] = None
+
+        if (not scopes_["static"]) and (not scopes_["dynamic"]):
+            raise InvalidRule("invalid scopes value. At least one scope must be specified")
+
+        # check that all the specified scopes are valid
+        if scopes_["static"] and scopes_["static"] not in STATIC_SCOPES:
+            raise InvalidRule(f"{scopes_['static']} is not a valid static scope")
+
+        if scopes_["dynamic"] and scopes_["dynamic"] not in DYNAMIC_SCOPES:
+            raise InvalidRule(f"{scopes_['dynamic']} is not a valid dynamic scope")
+
+        return Scopes(
+            static=Scope(scopes_["static"]) if scopes_["static"] else None,
+            dynamic=Scope(scopes_["dynamic"]) if scopes_["dynamic"] else None,
+        )
 
 
 SUPPORTED_FEATURES: Dict[str, Set] = {
-    GLOBAL_SCOPE: {
+    Scope.GLOBAL: {
         # these will be added to other scopes, see below.
         capa.features.common.OS,
         capa.features.common.Arch,
         capa.features.common.Format,
     },
-    FILE_SCOPE: {
+    Scope.FILE: {
         capa.features.common.MatchedRule,
         capa.features.file.Export,
         capa.features.file.Import,
@@ -108,7 +196,19 @@ SUPPORTED_FEATURES: Dict[str, Set] = {
         capa.features.common.Characteristic("mixed mode"),
         capa.features.common.Characteristic("forwarded export"),
     },
-    FUNCTION_SCOPE: {
+    Scope.PROCESS: {
+        capa.features.common.MatchedRule,
+    },
+    Scope.THREAD: set(),
+    Scope.CALL: {
+        capa.features.common.MatchedRule,
+        capa.features.common.Regex,
+        capa.features.common.String,
+        capa.features.common.Substring,
+        capa.features.insn.API,
+        capa.features.insn.Number,
+    },
+    Scope.FUNCTION: {
         capa.features.common.MatchedRule,
         capa.features.basicblock.BasicBlock,
         capa.features.common.Characteristic("calls from"),
@@ -117,13 +217,13 @@ SUPPORTED_FEATURES: Dict[str, Set] = {
         capa.features.common.Characteristic("recursive call"),
         # plus basic block scope features, see below
     },
-    BASIC_BLOCK_SCOPE: {
+    Scope.BASIC_BLOCK: {
         capa.features.common.MatchedRule,
         capa.features.common.Characteristic("tight loop"),
         capa.features.common.Characteristic("stack string"),
         # plus instruction scope features, see below
     },
-    INSTRUCTION_SCOPE: {
+    Scope.INSTRUCTION: {
         capa.features.common.MatchedRule,
         capa.features.insn.API,
         capa.features.insn.Property,
@@ -148,15 +248,24 @@ SUPPORTED_FEATURES: Dict[str, Set] = {
 }
 
 # global scope features are available in all other scopes
-SUPPORTED_FEATURES[INSTRUCTION_SCOPE].update(SUPPORTED_FEATURES[GLOBAL_SCOPE])
-SUPPORTED_FEATURES[BASIC_BLOCK_SCOPE].update(SUPPORTED_FEATURES[GLOBAL_SCOPE])
-SUPPORTED_FEATURES[FUNCTION_SCOPE].update(SUPPORTED_FEATURES[GLOBAL_SCOPE])
-SUPPORTED_FEATURES[FILE_SCOPE].update(SUPPORTED_FEATURES[GLOBAL_SCOPE])
+SUPPORTED_FEATURES[Scope.INSTRUCTION].update(SUPPORTED_FEATURES[Scope.GLOBAL])
+SUPPORTED_FEATURES[Scope.BASIC_BLOCK].update(SUPPORTED_FEATURES[Scope.GLOBAL])
+SUPPORTED_FEATURES[Scope.FUNCTION].update(SUPPORTED_FEATURES[Scope.GLOBAL])
+SUPPORTED_FEATURES[Scope.FILE].update(SUPPORTED_FEATURES[Scope.GLOBAL])
+SUPPORTED_FEATURES[Scope.PROCESS].update(SUPPORTED_FEATURES[Scope.GLOBAL])
+SUPPORTED_FEATURES[Scope.THREAD].update(SUPPORTED_FEATURES[Scope.GLOBAL])
+SUPPORTED_FEATURES[Scope.CALL].update(SUPPORTED_FEATURES[Scope.GLOBAL])
+
+
+# all call scope features are also thread features
+SUPPORTED_FEATURES[Scope.THREAD].update(SUPPORTED_FEATURES[Scope.CALL])
+# all thread scope features are also process features
+SUPPORTED_FEATURES[Scope.PROCESS].update(SUPPORTED_FEATURES[Scope.THREAD])
 
 # all instruction scope features are also basic block features
-SUPPORTED_FEATURES[BASIC_BLOCK_SCOPE].update(SUPPORTED_FEATURES[INSTRUCTION_SCOPE])
+SUPPORTED_FEATURES[Scope.BASIC_BLOCK].update(SUPPORTED_FEATURES[Scope.INSTRUCTION])
 # all basic block scope features are also function scope features
-SUPPORTED_FEATURES[FUNCTION_SCOPE].update(SUPPORTED_FEATURES[BASIC_BLOCK_SCOPE])
+SUPPORTED_FEATURES[Scope.FUNCTION].update(SUPPORTED_FEATURES[Scope.BASIC_BLOCK])
 
 
 class InvalidRule(ValueError):
@@ -194,22 +303,66 @@ class InvalidRuleSet(ValueError):
         return str(self)
 
 
-def ensure_feature_valid_for_scope(scope: str, feature: Union[Feature, Statement]):
+def ensure_feature_valid_for_scopes(scopes: Scopes, feature: Union[Feature, Statement]):
+    # construct a dict of all supported features
+    supported_features: Set = set()
+    if scopes.static:
+        supported_features.update(SUPPORTED_FEATURES[scopes.static])
+    if scopes.dynamic:
+        supported_features.update(SUPPORTED_FEATURES[scopes.dynamic])
+
     # if the given feature is a characteristic,
     # check that is a valid characteristic for the given scope.
     if (
         isinstance(feature, capa.features.common.Characteristic)
         and isinstance(feature.value, str)
-        and capa.features.common.Characteristic(feature.value) not in SUPPORTED_FEATURES[scope]
+        and capa.features.common.Characteristic(feature.value) not in supported_features
     ):
-        raise InvalidRule(f"feature {feature} not supported for scope {scope}")
+        raise InvalidRule(f"feature {feature} not supported for scopes {scopes}")
 
     if not isinstance(feature, capa.features.common.Characteristic):
         # features of this scope that are not Characteristics will be Type instances.
         # check that the given feature is one of these types.
-        types_for_scope = filter(lambda t: isinstance(t, type), SUPPORTED_FEATURES[scope])
-        if not isinstance(feature, tuple(types_for_scope)):  # type: ignore
-            raise InvalidRule(f"feature {feature} not supported for scope {scope}")
+        types_for_scope = filter(lambda t: isinstance(t, type), supported_features)
+        if not isinstance(feature, tuple(types_for_scope)):
+            raise InvalidRule(f"feature {feature} not supported for scopes {scopes}")
+
+
+def translate_com_feature(com_name: str, com_type: ComType) -> ceng.Statement:
+    com_db = capa.features.com.load_com_database(com_type)
+    guids: Optional[List[str]] = com_db.get(com_name)
+    if not guids:
+        logger.error(" %s doesn't exist in COM %s database", com_name, com_type)
+        raise InvalidRule(f"'{com_name}' doesn't exist in COM {com_type} database")
+
+    com_features: List[Feature] = []
+    for guid in guids:
+        hex_chars = guid.replace("-", "")
+        h = [hex_chars[i : i + 2] for i in range(0, len(hex_chars), 2)]
+        reordered_hex_pairs = [
+            h[3],
+            h[2],
+            h[1],
+            h[0],
+            h[5],
+            h[4],
+            h[7],
+            h[6],
+            h[8],
+            h[9],
+            h[10],
+            h[11],
+            h[12],
+            h[13],
+            h[14],
+            h[15],
+        ]
+        guid_bytes = bytes.fromhex("".join(reordered_hex_pairs))
+        prefix = capa.features.com.COM_PREFIXES[com_type]
+        symbol = prefix + com_name
+        com_features.append(capa.features.common.String(guid, f"{symbol} as GUID string"))
+        com_features.append(capa.features.common.Bytes(guid_bytes, f"{symbol} as bytes"))
+    return ceng.Or(com_features)
 
 
 def parse_int(s: str) -> int:
@@ -417,53 +570,103 @@ def pop_statement_description_entry(d):
     return description["description"]
 
 
-def build_statements(d, scope: str):
+def trim_dll_part(api: str) -> str:
+    # ordinal imports, like ws2_32.#1, keep dll
+    if ".#" in api:
+        return api
+
+    # kernel32.CreateFileA
+    if api.count(".") == 1:
+        if "::" not in api:
+            # skip System.Convert::FromBase64String
+            api = api.split(".")[1]
+    return api
+
+
+def build_statements(d, scopes: Scopes):
     if len(d.keys()) > 2:
         raise InvalidRule("too many statements")
 
     key = list(d.keys())[0]
     description = pop_statement_description_entry(d[key])
     if key == "and":
-        return ceng.And([build_statements(dd, scope) for dd in d[key]], description=description)
+        return ceng.And([build_statements(dd, scopes) for dd in d[key]], description=description)
     elif key == "or":
-        return ceng.Or([build_statements(dd, scope) for dd in d[key]], description=description)
+        return ceng.Or([build_statements(dd, scopes) for dd in d[key]], description=description)
     elif key == "not":
         if len(d[key]) != 1:
             raise InvalidRule("not statement must have exactly one child statement")
-        return ceng.Not(build_statements(d[key][0], scope), description=description)
+        return ceng.Not(build_statements(d[key][0], scopes), description=description)
     elif key.endswith(" or more"):
         count = int(key[: -len("or more")])
-        return ceng.Some(count, [build_statements(dd, scope) for dd in d[key]], description=description)
+        return ceng.Some(count, [build_statements(dd, scopes) for dd in d[key]], description=description)
     elif key == "optional":
         # `optional` is an alias for `0 or more`
         # which is useful for documenting behaviors,
         # like with `write file`, we might say that `WriteFile` is optionally found alongside `CreateFileA`.
-        return ceng.Some(0, [build_statements(dd, scope) for dd in d[key]], description=description)
+        return ceng.Some(0, [build_statements(dd, scopes) for dd in d[key]], description=description)
+
+    elif key == "process":
+        if Scope.FILE not in scopes:
+            raise InvalidRule("process subscope supported only for file scope")
+
+        if len(d[key]) != 1:
+            raise InvalidRule("subscope must have exactly one child statement")
+
+        return ceng.Subscope(
+            Scope.PROCESS, build_statements(d[key][0], Scopes(dynamic=Scope.PROCESS)), description=description
+        )
+
+    elif key == "thread":
+        if all(s not in scopes for s in (Scope.FILE, Scope.PROCESS)):
+            raise InvalidRule("thread subscope supported only for the process scope")
+
+        if len(d[key]) != 1:
+            raise InvalidRule("subscope must have exactly one child statement")
+
+        return ceng.Subscope(
+            Scope.THREAD, build_statements(d[key][0], Scopes(dynamic=Scope.THREAD)), description=description
+        )
+
+    elif key == "call":
+        if all(s not in scopes for s in (Scope.FILE, Scope.PROCESS, Scope.THREAD)):
+            raise InvalidRule("call subscope supported only for the process and thread scopes")
+
+        if len(d[key]) != 1:
+            raise InvalidRule("subscope must have exactly one child statement")
+
+        return ceng.Subscope(
+            Scope.CALL, build_statements(d[key][0], Scopes(dynamic=Scope.CALL)), description=description
+        )
 
     elif key == "function":
-        if scope != FILE_SCOPE:
+        if Scope.FILE not in scopes:
             raise InvalidRule("function subscope supported only for file scope")
 
         if len(d[key]) != 1:
             raise InvalidRule("subscope must have exactly one child statement")
 
-        return ceng.Subscope(FUNCTION_SCOPE, build_statements(d[key][0], FUNCTION_SCOPE), description=description)
+        return ceng.Subscope(
+            Scope.FUNCTION, build_statements(d[key][0], Scopes(static=Scope.FUNCTION)), description=description
+        )
 
     elif key == "basic block":
-        if scope != FUNCTION_SCOPE:
+        if Scope.FUNCTION not in scopes:
             raise InvalidRule("basic block subscope supported only for function scope")
 
         if len(d[key]) != 1:
             raise InvalidRule("subscope must have exactly one child statement")
 
-        return ceng.Subscope(BASIC_BLOCK_SCOPE, build_statements(d[key][0], BASIC_BLOCK_SCOPE), description=description)
+        return ceng.Subscope(
+            Scope.BASIC_BLOCK, build_statements(d[key][0], Scopes(static=Scope.BASIC_BLOCK)), description=description
+        )
 
     elif key == "instruction":
-        if scope not in (FUNCTION_SCOPE, BASIC_BLOCK_SCOPE):
+        if all(s not in scopes for s in (Scope.FUNCTION, Scope.BASIC_BLOCK)):
             raise InvalidRule("instruction subscope supported only for function and basic block scope")
 
         if len(d[key]) == 1:
-            statements = build_statements(d[key][0], INSTRUCTION_SCOPE)
+            statements = build_statements(d[key][0], Scopes(static=Scope.INSTRUCTION))
         else:
             # for instruction subscopes, we support a shorthand in which the top level AND is implied.
             # the following are equivalent:
@@ -477,9 +680,9 @@ def build_statements(d, scope: str):
             #       - arch: i386
             #       - mnemonic: cmp
             #
-            statements = ceng.And([build_statements(dd, INSTRUCTION_SCOPE) for dd in d[key]])
+            statements = ceng.And([build_statements(dd, Scopes(static=Scope.INSTRUCTION)) for dd in d[key]])
 
-        return ceng.Subscope(INSTRUCTION_SCOPE, statements, description=description)
+        return ceng.Subscope(Scope.INSTRUCTION, statements, description=description)
 
     elif key.startswith("count(") and key.endswith(")"):
         # e.g.:
@@ -507,6 +710,10 @@ def build_statements(d, scope: str):
             #     count(number(0x100 = description))
             if term != "string":
                 value, description = parse_description(arg, term)
+
+                if term == "api":
+                    value = trim_dll_part(value)
+
                 feature = Feature(value, description=description)
             else:
                 # arg is string (which doesn't support inline descriptions), like:
@@ -518,7 +725,7 @@ def build_statements(d, scope: str):
                 feature = Feature(arg)
         else:
             feature = Feature()
-        ensure_feature_valid_for_scope(scope, feature)
+        ensure_feature_valid_for_scopes(scopes, feature)
 
         count = d[key]
         if isinstance(count, int):
@@ -552,7 +759,7 @@ def build_statements(d, scope: str):
             feature = capa.features.insn.OperandNumber(index, value, description=description)
         except ValueError as e:
             raise InvalidRule(str(e)) from e
-        ensure_feature_valid_for_scope(scope, feature)
+        ensure_feature_valid_for_scopes(scopes, feature)
         return feature
 
     elif key.startswith("operand[") and key.endswith("].offset"):
@@ -568,7 +775,7 @@ def build_statements(d, scope: str):
             feature = capa.features.insn.OperandOffset(index, value, description=description)
         except ValueError as e:
             raise InvalidRule(str(e)) from e
-        ensure_feature_valid_for_scope(scope, feature)
+        ensure_feature_valid_for_scopes(scopes, feature)
         return feature
 
     elif (
@@ -588,17 +795,30 @@ def build_statements(d, scope: str):
             feature = capa.features.insn.Property(value, access=access, description=description)
         except ValueError as e:
             raise InvalidRule(str(e)) from e
-        ensure_feature_valid_for_scope(scope, feature)
+        ensure_feature_valid_for_scopes(scopes, feature)
         return feature
 
+    elif key.startswith("com/"):
+        com_type_name = str(key[len("com/") :])
+        try:
+            com_type = ComType(com_type_name)
+        except ValueError:
+            raise InvalidRule(f"unexpected COM type: {com_type_name}")
+        value, description = parse_description(d[key], key, d.get("description"))
+        return translate_com_feature(value, com_type)
+
     else:
         Feature = parse_feature(key)
         value, description = parse_description(d[key], key, d.get("description"))
+
+        if key == "api":
+            value = trim_dll_part(value)
+
         try:
             feature = Feature(value, description=description)
         except ValueError as e:
             raise InvalidRule(str(e)) from e
-        ensure_feature_valid_for_scope(scope, feature)
+        ensure_feature_valid_for_scopes(scopes, feature)
         return feature
 
 
@@ -611,10 +831,10 @@ def second(s: List[Any]) -> Any:
 
 
 class Rule:
-    def __init__(self, name: str, scope: str, statement: Statement, meta, definition=""):
+    def __init__(self, name: str, scopes: Scopes, statement: Statement, meta, definition=""):
         super().__init__()
         self.name = name
-        self.scope = scope
+        self.scopes = scopes
         self.statement = statement
         self.meta = meta
         self.definition = definition
@@ -623,7 +843,7 @@ class Rule:
         return f"Rule(name={self.name})"
 
     def __repr__(self):
-        return f"Rule(scope={self.scope}, name={self.name})"
+        return f"Rule(scope={self.scopes}, name={self.name})"
 
     def get_dependencies(self, namespaces):
         """
@@ -681,13 +901,19 @@ class Rule:
                 # the name is a randomly generated, hopefully unique value.
                 # ideally, this won't every be rendered to a user.
                 name = self.name + "/" + uuid.uuid4().hex
+                if subscope.scope in STATIC_SCOPES:
+                    scopes = Scopes(static=subscope.scope)
+                elif subscope.scope in DYNAMIC_SCOPES:
+                    scopes = Scopes(dynamic=subscope.scope)
+                else:
+                    raise InvalidRule(f"scope {subscope.scope} is not a valid subscope")
                 new_rule = Rule(
                     name,
-                    subscope.scope,
+                    scopes,
                     subscope.child,
                     {
                         "name": name,
-                        "scope": subscope.scope,
+                        "scopes": asdict(scopes),
                         # these derived rules are never meant to be inspected separately,
                         # they are dependencies for the parent rule,
                         # so mark it as such.
@@ -712,6 +938,9 @@ class Rule:
             for child in statement.get_children():
                 yield from self._extract_subscope_rules_rec(child)
 
+    def is_file_limitation_rule(self) -> bool:
+        return self.meta.get("namespace", "") == "internal/limitation/file"
+
     def is_subscope_rule(self):
         return bool(self.meta.get("capa/subscope-rule", False))
 
@@ -774,9 +1003,21 @@ class Rule:
     def from_dict(cls, d: Dict[str, Any], definition: str) -> "Rule":
         meta = d["rule"]["meta"]
         name = meta["name"]
+
         # if scope is not specified, default to function scope.
         # this is probably the mode that rule authors will start with.
-        scope = meta.get("scope", FUNCTION_SCOPE)
+        # each rule has two scopes, a static-flavor scope, and a
+        # dynamic-flavor one. which one is used depends on the analysis type.
+        if "scope" in meta:
+            raise InvalidRule(f"legacy rule detected (rule.meta.scope), please update to the new syntax: {name}")
+        elif "scopes" in meta:
+            scopes_ = meta.get("scopes")
+        else:
+            raise InvalidRule("please specify at least one of this rule's (static/dynamic) scopes")
+        if not isinstance(scopes_, dict):
+            raise InvalidRule("the scopes field must contain a dictionary specifying the scopes")
+
+        scopes: Scopes = Scopes.from_dict(scopes_)
         statements = d["rule"]["features"]
 
         # the rule must start with a single logic node.
@@ -787,16 +1028,13 @@ class Rule:
         if isinstance(statements[0], ceng.Subscope):
             raise InvalidRule("top level statement may not be a subscope")
 
-        if scope not in SUPPORTED_FEATURES.keys():
-            raise InvalidRule("{:s} is not a supported scope".format(scope))
-
         meta = d["rule"]["meta"]
         if not isinstance(meta.get("att&ck", []), list):
             raise InvalidRule("ATT&CK mapping must be a list")
         if not isinstance(meta.get("mbc", []), list):
             raise InvalidRule("MBC mapping must be a list")
 
-        return cls(name, scope, build_statements(statements[0], scope), meta, definition)
+        return cls(name, scopes, build_statements(statements[0], scopes), meta, definition)
 
     @staticmethod
     @lru_cache()
@@ -824,7 +1062,7 @@ class Rule:
 
         # leave quotes unchanged.
         # manually verified this property exists, even if mypy complains.
-        y.preserve_quotes = True  # type: ignore
+        y.preserve_quotes = True
 
         # indent lists by two spaces below their parent
         #
@@ -836,7 +1074,7 @@ class Rule:
 
         # avoid word wrapping
         # manually verified this property exists, even if mypy complains.
-        y.width = 4096  # type: ignore
+        y.width = 4096
 
         return y
 
@@ -895,10 +1133,8 @@ class Rule:
                 del meta[k]
         for k, v in self.meta.items():
             meta[k] = v
-
         # the name and scope of the rule instance overrides anything in meta.
         meta["name"] = self.name
-        meta["scope"] = self.scope
 
         def move_to_end(m, k):
             # ruamel.yaml uses an ordereddict-like structure to track maps (CommentedMap).
@@ -919,7 +1155,6 @@ class Rule:
             if key in META_KEYS:
                 continue
             move_to_end(meta, key)
-
         # save off the existing hidden meta values,
         # emit the document,
         # and re-add the hidden meta.
@@ -974,12 +1209,11 @@ class Rule:
         return doc
 
 
-def get_rules_with_scope(rules, scope) -> List[Rule]:
+def get_rules_with_scope(rules, scope: Scope) -> List[Rule]:
     """
     from the given collection of rules, select those with the given scope.
-    `scope` is one of the capa.rules.*_SCOPE constants.
     """
-    return [rule for rule in rules if rule.scope == scope]
+    return [rule for rule in rules if scope in rule.scopes]
 
 
 def get_rules_and_dependencies(rules: List[Rule], rule_name: str) -> Iterator[Rule]:
@@ -1104,7 +1338,10 @@ class RuleSet:
         capa.engine.match(ruleset.file_rules, ...)
     """
 
-    def __init__(self, rules: List[Rule]):
+    def __init__(
+        self,
+        rules: List[Rule],
+    ):
         super().__init__()
 
         ensure_rules_are_unique(rules)
@@ -1126,15 +1363,23 @@ class RuleSet:
 
         rules = capa.optimizer.optimize_rules(rules)
 
-        self.file_rules = self._get_rules_for_scope(rules, FILE_SCOPE)
-        self.function_rules = self._get_rules_for_scope(rules, FUNCTION_SCOPE)
-        self.basic_block_rules = self._get_rules_for_scope(rules, BASIC_BLOCK_SCOPE)
-        self.instruction_rules = self._get_rules_for_scope(rules, INSTRUCTION_SCOPE)
+        self.file_rules = self._get_rules_for_scope(rules, Scope.FILE)
+        self.process_rules = self._get_rules_for_scope(rules, Scope.PROCESS)
+        self.thread_rules = self._get_rules_for_scope(rules, Scope.THREAD)
+        self.call_rules = self._get_rules_for_scope(rules, Scope.CALL)
+        self.function_rules = self._get_rules_for_scope(rules, Scope.FUNCTION)
+        self.basic_block_rules = self._get_rules_for_scope(rules, Scope.BASIC_BLOCK)
+        self.instruction_rules = self._get_rules_for_scope(rules, Scope.INSTRUCTION)
         self.rules = {rule.name: rule for rule in rules}
         self.rules_by_namespace = index_rules_by_namespace(rules)
 
         # unstable
         (self._easy_file_rules_by_feature, self._hard_file_rules) = self._index_rules_by_feature(self.file_rules)
+        (self._easy_process_rules_by_feature, self._hard_process_rules) = self._index_rules_by_feature(
+            self.process_rules
+        )
+        (self._easy_thread_rules_by_feature, self._hard_thread_rules) = self._index_rules_by_feature(self.thread_rules)
+        (self._easy_call_rules_by_feature, self._hard_call_rules) = self._index_rules_by_feature(self.call_rules)
         (self._easy_function_rules_by_feature, self._hard_function_rules) = self._index_rules_by_feature(
             self.function_rules
         )
@@ -1380,16 +1625,25 @@ class RuleSet:
         except that it may be more performant.
         """
         easy_rules_by_feature = {}
-        if scope is Scope.FILE:
+        if scope == Scope.FILE:
             easy_rules_by_feature = self._easy_file_rules_by_feature
             hard_rule_names = self._hard_file_rules
-        elif scope is Scope.FUNCTION:
+        elif scope == Scope.PROCESS:
+            easy_rules_by_feature = self._easy_process_rules_by_feature
+            hard_rule_names = self._hard_process_rules
+        elif scope == Scope.THREAD:
+            easy_rules_by_feature = self._easy_thread_rules_by_feature
+            hard_rule_names = self._hard_thread_rules
+        elif scope == Scope.CALL:
+            easy_rules_by_feature = self._easy_call_rules_by_feature
+            hard_rule_names = self._hard_call_rules
+        elif scope == Scope.FUNCTION:
             easy_rules_by_feature = self._easy_function_rules_by_feature
             hard_rule_names = self._hard_function_rules
-        elif scope is Scope.BASIC_BLOCK:
+        elif scope == Scope.BASIC_BLOCK:
             easy_rules_by_feature = self._easy_basic_block_rules_by_feature
             hard_rule_names = self._hard_basic_block_rules
-        elif scope is Scope.INSTRUCTION:
+        elif scope == Scope.INSTRUCTION:
             easy_rules_by_feature = self._easy_instruction_rules_by_feature
             hard_rule_names = self._hard_instruction_rules
         else:

capa/sigs/README.md

@@ -1,4 +1,4 @@
-# capa/sigs
+# capa FLIRT signatures
 
 This directory contains FLIRT signatures that capa uses to identify library functions.
 Typically, capa will ignore library functions, which reduces false positives and improves runtime.

doc/installation.md

@@ -35,12 +35,6 @@ $ unzip v4.0.0.zip
 $ capa -r /path/to/capa-rules suspicious.exe
 ```
 
-This technique also doesn't set up the default library identification [signatures](https://github.com/mandiant/capa/tree/master/sigs). You can pass the signature directory using the `-s` argument.
-For example, to run capa with both a rule path and a signature path:
-```console
-$ capa -s /path/to/capa-sigs suspicious.exe
-```
-
 Alternatively, see Method 3 below.
 
 ### 2. Use capa
@@ -105,27 +99,28 @@ To install these development dependencies, run:
 
 We use [pre-commit](https://pre-commit.com/) so that its trivial to run the same linters & configuration locally as in CI.
 
-Run all linters liks:
+Run all linters like:
 
-    ❯ pre-commit run --all-files
+    ❯ pre-commit run --hook-stage=manual --all-files
     isort....................................................................Passed
     black....................................................................Passed
     ruff.....................................................................Passed
     flake8...................................................................Passed
     mypy.....................................................................Passed
+    pytest (fast)............................................................Passed
 
 Or run a single linter like:
 
-    ❯ pre-commit run --all-files isort
+    ❯ pre-commit run --all-files --hook-stage=manual isort
     isort....................................................................Passed
 
 
 Importantly, you can configure pre-commit to run automatically before every commit by running:
 
-    ❯ pre-commit install --hook-type pre-commit
+    ❯ pre-commit install --hook-type=pre-commit
     pre-commit installed at .git/hooks/pre-commit
 
-    ❯ pre-commit install --hook-type pre-push
+    ❯ pre-commit install --hook-type=pre-push
     pre-commit installed at .git/hooks/pre-push
 
 This way you can ensure that you don't commit code style or formatting offenses.

pyproject.toml

@@ -36,19 +36,19 @@ dependencies = [
     "pyyaml==6.0.1",
     "tabulate==0.9.0",
     "colorama==0.4.6",
-    "termcolor==2.3.0",
-    "wcwidth==0.2.6",
+    "termcolor==2.4.0",
+    "wcwidth==0.2.13",
     "ida-settings==2.1.0",
     "viv-utils[flirt]==0.7.9",
     "halo==0.0.31",
     "networkx==3.1",
-    "ruamel.yaml==0.17.32",
+    "ruamel.yaml==0.18.5",
     "vivisect==1.1.1",
     "pefile==2023.2.7",
-    "pyelftools==0.29",
-    "dnfile==0.13.0",
+    "pyelftools==0.30",
+    "dnfile==0.14.1",
     "dncil==1.0.2",
-    "pydantic==2.1.1",
+    "pydantic==2.4.0",
     "protobuf==4.23.4",
 ]
 dynamic = ["version"]
@@ -61,26 +61,26 @@ packages = ["capa"]
 
 [project.optional-dependencies]
 dev = [
-    "pre-commit==3.3.3",
-    "pytest==7.4.0",
+    "pre-commit==3.5.0",
+    "pytest==7.4.4",
     "pytest-sugar==0.9.7",
     "pytest-instafail==0.5.0",
     "pytest-cov==4.1.0",
-    "flake8==6.1.0",
-    "flake8-bugbear==23.7.10",
-    "flake8-encodings==0.5.0.post1",
+    "flake8==7.0.0",
+    "flake8-bugbear==23.12.2",
+    "flake8-encodings==0.5.1",
     "flake8-comprehensions==3.14.0",
     "flake8-logging-format==0.9.0",
-    "flake8-no-implicit-concat==0.3.4",
+    "flake8-no-implicit-concat==0.3.5",
     "flake8-print==5.0.0",
     "flake8-todos==0.3.0",
-    "flake8-simplify==0.20.0",
+    "flake8-simplify==0.21.0",
     "flake8-use-pathlib==0.3.0",
     "flake8-copyright==0.2.4",
-    "ruff==0.0.285",
-    "black==23.7.0",
-    "isort==5.11.4",
-    "mypy==1.5.1",
+    "ruff==0.1.13",
+    "black==23.12.1",
+    "isort==5.13.2",
+    "mypy==1.8.0",
     "psutil==5.9.2",
     "stix2==3.0.1",
     "requests==2.31.0",
@@ -89,16 +89,16 @@ dev = [
     "types-backports==0.1.3",
     "types-colorama==0.4.15.11",
     "types-PyYAML==6.0.8",
-    "types-tabulate==0.9.0.3",
+    "types-tabulate==0.9.0.20240106",
     "types-termcolor==1.1.4",
     "types-psutil==5.8.23",
-    "types_requests==2.31.0.2",
+    "types_requests==2.31.0.20240106",
     "types-protobuf==4.23.0.3",
 ]
 build = [
-    "pyinstaller==5.10.1",
-    "setuptools==68.0.0",
-    "build==0.10.0"
+    "pyinstaller==6.3.0",
+    "setuptools==69.0.3",
+    "build==1.0.3"
 ]
 
 [project.urls]

scripts/bulk-process.py

@@ -75,6 +75,7 @@ import capa
 import capa.main
 import capa.rules
 import capa.render.json
+import capa.capabilities.common
 import capa.render.result_document as rd
 from capa.features.common import OS_AUTO
 
@@ -112,7 +113,7 @@ def get_capa_results(args):
         extractor = capa.main.get_extractor(
             path, format, os_, capa.main.BACKEND_VIV, sigpaths, should_save_workspace, disable_progress=True
         )
-    except capa.main.UnsupportedFormatError:
+    except capa.exceptions.UnsupportedFormatError:
         # i'm 100% sure if multiprocessing will reliably raise exceptions across process boundaries.
         # so instead, return an object with explicit success/failure status.
         #
@@ -123,7 +124,7 @@ def get_capa_results(args):
             "status": "error",
             "error": f"input file does not appear to be a PE file: {path}",
         }
-    except capa.main.UnsupportedRuntimeError:
+    except capa.exceptions.UnsupportedRuntimeError:
         return {
             "path": path,
             "status": "error",
@@ -136,11 +137,9 @@ def get_capa_results(args):
             "error": f"unexpected error: {e}",
         }
 
-    meta = capa.main.collect_metadata([], path, format, os_, [], extractor)
-    capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True)
+    capabilities, counts = capa.capabilities.common.find_capabilities(rules, extractor, disable_progress=True)
 
-    meta.analysis.feature_counts = counts["feature_counts"]
-    meta.analysis.library_functions = counts["library_functions"]
+    meta = capa.main.collect_metadata([], path, format, os_, [], extractor, counts)
     meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
 
     doc = rd.ResultDocument.from_capa(meta, rules, capabilities)

scripts/capa2yara.py

@@ -61,7 +61,22 @@ var_names = ["".join(letters) for letters in itertools.product(string.ascii_lowe
 
 
 # this have to be the internal names used by capa.py which are sometimes different to the ones written out in the rules, e.g. "2 or more" is "Some", count is Range
-unsupported = ["characteristic", "mnemonic", "offset", "subscope", "Range"]
+unsupported = [
+    "characteristic",
+    "mnemonic",
+    "offset",
+    "subscope",
+    "Range",
+    "os",
+    "property",
+    "format",
+    "class",
+    "operand[0].number",
+    "operand[1].number",
+    "substring",
+    "arch",
+    "namespace",
+]
 # further idea: shorten this list, possible stuff:
 # - 2 or more strings: e.g.
 # -- https://github.com/mandiant/capa-rules/blob/master/collection/file-managers/gather-direct-ftp-information.yml
@@ -90,8 +105,7 @@ condition_header = """
 condition_rule = """
 private rule capa_pe_file : CAPA {
     meta:
-        description = "match in PE files. used by all further CAPA rules"
-        author = "Arnim Rupp"
+        description = "Match in PE files. Used by other CAPA rules"
     condition:
         uint16be(0) == 0x4d5a
         or uint16be(0) == 0x558b
@@ -566,7 +580,7 @@ def convert_rules(rules, namespaces, cround, make_priv):
             logger.info("skipping already converted rule capa: %s - yara rule: %s", rule.name, rule_name)
             continue
 
-        logger.info("-------------------------- DOING RULE CAPA: %s - yara rule: ", rule.name, rule_name)
+        logger.info("-------------------------- DOING RULE CAPA: %s - yara rule: %s", rule.name, rule_name)
         if "capa/path" in rule.meta:
             url = get_rule_url(rule.meta["capa/path"])
         else:
@@ -603,7 +617,12 @@ def convert_rules(rules, namespaces, cround, make_priv):
                 meta_name = meta
                 # e.g. 'examples:' can be a list
                 seen_hashes = []
-                if isinstance(metas[meta], list):
+                if isinstance(metas[meta], dict):
+                    if meta_name == "scopes":
+                        yara_meta += "\t" + "static scope" + ' = "' + metas[meta]["static"] + '"\n'
+                        yara_meta += "\t" + "dynamic scope" + ' = "' + metas[meta]["dynamic"] + '"\n'
+
+                elif isinstance(metas[meta], list):
                     if meta_name == "examples":
                         meta_name = "hash"
                     if meta_name == "att&ck":

scripts/capa_as_library.py

@@ -19,6 +19,7 @@ import capa.features
 import capa.render.json
 import capa.render.utils as rutils
 import capa.render.default
+import capa.capabilities.common
 import capa.render.result_document as rd
 import capa.features.freeze.features as frzf
 from capa.features.common import OS_AUTO, FORMAT_AUTO
@@ -175,13 +176,10 @@ def capa_details(rules_path: Path, file_path: Path, output_format="dictionary"):
     extractor = capa.main.get_extractor(
         file_path, FORMAT_AUTO, OS_AUTO, capa.main.BACKEND_VIV, [], False, disable_progress=True
     )
-    capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True)
+    capabilities, counts = capa.capabilities.common.find_capabilities(rules, extractor, disable_progress=True)
 
     # collect metadata (used only to make rendering more complete)
-    meta = capa.main.collect_metadata([], file_path, FORMAT_AUTO, OS_AUTO, [rules_path], extractor)
-
-    meta.analysis.feature_counts = counts["feature_counts"]
-    meta.analysis.library_functions = counts["library_functions"]
+    meta = capa.main.collect_metadata([], file_path, FORMAT_AUTO, OS_AUTO, [rules_path], extractor, counts)
     meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
 
     capa_output: Any = False

scripts/import-to-ida.py

@@ -90,7 +90,7 @@ def main():
             continue
         if rule.meta.is_subscope_rule:
             continue
-        if rule.meta.scope != capa.rules.Scope.FUNCTION:
+        if rule.meta.scopes.static == capa.rules.Scope.FUNCTION:
             continue
 
         ns = rule.meta.namespace

scripts/lint.py

@@ -41,6 +41,7 @@ import capa.rules
 import capa.engine
 import capa.helpers
 import capa.features.insn
+import capa.capabilities.common
 from capa.rules import Rule, RuleSet
 from capa.features.common import OS_AUTO, String, Feature, Substring
 from capa.render.result_document import RuleMetadata
@@ -151,20 +152,74 @@ class NamespaceDoesntMatchRulePath(Lint):
         return rule.meta["namespace"] not in get_normpath(rule.meta["capa/path"])
 
 
-class MissingScope(Lint):
-    name = "missing scope"
-    recommendation = "Add meta.scope so that the scope is explicit (defaults to `function`)"
+class MissingScopes(Lint):
+    name = "missing scopes"
+    recommendation = (
+        "Add meta.scopes with both the static (meta.scopes.static) and dynamic (meta.scopes.dynamic) scopes"
+    )
 
     def check_rule(self, ctx: Context, rule: Rule):
-        return "scope" not in rule.meta
+        return "scopes" not in rule.meta
 
 
-class InvalidScope(Lint):
-    name = "invalid scope"
-    recommendation = "Use only file, function, basic block, or instruction rule scopes"
+class MissingStaticScope(Lint):
+    name = "missing static scope"
+    recommendation = (
+        "Add a static scope for the rule (file, function, basic block, instruction, or unspecified/unsupported)"
+    )
 
     def check_rule(self, ctx: Context, rule: Rule):
-        return rule.meta.get("scope") not in ("file", "function", "basic block", "instruction")
+        return "static" not in rule.meta.get("scopes")
+
+
+class MissingDynamicScope(Lint):
+    name = "missing dynamic scope"
+    recommendation = "Add a dynamic scope for the rule (file, process, thread, call, or unspecified/unsupported)"
+
+    def check_rule(self, ctx: Context, rule: Rule):
+        return "dynamic" not in rule.meta.get("scopes")
+
+
+class InvalidStaticScope(Lint):
+    name = "invalid static scope"
+    recommendation = (
+        "For the static scope, use either: file, function, basic block, instruction, or unspecified/unsupported"
+    )
+
+    def check_rule(self, ctx: Context, rule: Rule):
+        return rule.meta.get("scopes").get("static") not in (
+            "file",
+            "function",
+            "basic block",
+            "instruction",
+            "unspecified",
+            "unsupported",
+        )
+
+
+class InvalidDynamicScope(Lint):
+    name = "invalid static scope"
+    recommendation = "For the dynamic scope, use either: file, process, thread, call, or unspecified/unsupported"
+
+    def check_rule(self, ctx: Context, rule: Rule):
+        return rule.meta.get("scopes").get("dynamic") not in (
+            "file",
+            "process",
+            "thread",
+            "call",
+            "unspecified",
+            "unsupported",
+        )
+
+
+class InvalidScopes(Lint):
+    name = "invalid scopes"
+    recommendation = "At least one scope (static or dynamic) must be specified"
+
+    def check_rule(self, ctx: Context, rule: Rule):
+        return (rule.meta.get("scopes").get("static") in ("unspecified", "unsupported")) and (
+            rule.meta.get("scopes").get("dynamic") in ("unspecified", "unsupported")
+        )
 
 
 class MissingAuthors(Lint):
@@ -305,14 +360,14 @@ def get_sample_capabilities(ctx: Context, path: Path) -> Set[str]:
     elif nice_path.name.endswith(capa.helpers.EXTENSIONS_SHELLCODE_64):
         format_ = "sc64"
     else:
-        format_ = capa.main.get_auto_format(nice_path)
+        format_ = capa.helpers.get_auto_format(nice_path)
 
     logger.debug("analyzing sample: %s", nice_path)
     extractor = capa.main.get_extractor(
         nice_path, format_, OS_AUTO, capa.main.BACKEND_VIV, DEFAULT_SIGNATURES, False, disable_progress=True
     )
 
-    capabilities, _ = capa.main.find_capabilities(ctx.rules, extractor, disable_progress=True)
+    capabilities, _ = capa.capabilities.common.find_capabilities(ctx.rules, extractor, disable_progress=True)
     # mypy doesn't seem to be happy with the MatchResults type alias & set(...keys())?
     # so we ignore a few types here.
     capabilities = set(capabilities.keys())  # type: ignore
@@ -700,14 +755,18 @@ def lint_name(ctx: Context, rule: Rule):
     return run_lints(NAME_LINTS, ctx, rule)
 
 
-SCOPE_LINTS = (
-    MissingScope(),
-    InvalidScope(),
+SCOPES_LINTS = (
+    MissingScopes(),
+    MissingStaticScope(),
+    MissingDynamicScope(),
+    InvalidStaticScope(),
+    InvalidDynamicScope(),
+    InvalidScopes(),
 )
 
 
 def lint_scope(ctx: Context, rule: Rule):
-    return run_lints(SCOPE_LINTS, ctx, rule)
+    return run_lints(SCOPES_LINTS, ctx, rule)
 
 
 META_LINTS = (

scripts/linter-data.json

@@ -43,7 +43,8 @@
       "T1598": "Phishing for Information",
       "T1598.001": "Phishing for Information::Spearphishing Service",
       "T1598.002": "Phishing for Information::Spearphishing Attachment",
-      "T1598.003": "Phishing for Information::Spearphishing Link"
+      "T1598.003": "Phishing for Information::Spearphishing Link",
+      "T1598.004": "Phishing for Information::Spearphishing Voice"
     },
     "Resource Development": {
       "T1583": "Acquire Infrastructure",
@@ -111,7 +112,9 @@
       "T1566": "Phishing",
       "T1566.001": "Phishing::Spearphishing Attachment",
       "T1566.002": "Phishing::Spearphishing Link",
-      "T1566.003": "Phishing::Spearphishing via Service"
+      "T1566.003": "Phishing::Spearphishing via Service",
+      "T1566.004": "Phishing::Spearphishing Voice",
+      "T1659": "Content Injection"
     },
     "Execution": {
       "T1047": "Windows Management Instrumentation",
@@ -175,6 +178,7 @@
       "T1098.003": "Account Manipulation::Additional Cloud Roles",
       "T1098.004": "Account Manipulation::SSH Authorized Keys",
       "T1098.005": "Account Manipulation::Device Registration",
+      "T1098.006": "Account Manipulation::Additional Container Cluster Roles",
       "T1133": "External Remote Services",
       "T1136": "Create Account",
       "T1136.001": "Create Account::Local Account",
@@ -264,7 +268,8 @@
       "T1574.010": "Hijack Execution Flow::Services File Permissions Weakness",
       "T1574.011": "Hijack Execution Flow::Services Registry Permissions Weakness",
       "T1574.012": "Hijack Execution Flow::COR_PROFILER",
-      "T1574.013": "Hijack Execution Flow::KernelCallbackTable"
+      "T1574.013": "Hijack Execution Flow::KernelCallbackTable",
+      "T1653": "Power Settings"
     },
     "Privilege Escalation": {
       "T1037": "Boot or Logon Initialization Scripts",
@@ -298,6 +303,13 @@
       "T1078.002": "Valid Accounts::Domain Accounts",
       "T1078.003": "Valid Accounts::Local Accounts",
       "T1078.004": "Valid Accounts::Cloud Accounts",
+      "T1098": "Account Manipulation",
+      "T1098.001": "Account Manipulation::Additional Cloud Credentials",
+      "T1098.002": "Account Manipulation::Additional Email Delegate Permissions",
+      "T1098.003": "Account Manipulation::Additional Cloud Roles",
+      "T1098.004": "Account Manipulation::SSH Authorized Keys",
+      "T1098.005": "Account Manipulation::Device Registration",
+      "T1098.006": "Account Manipulation::Additional Container Cluster Roles",
       "T1134": "Access Token Manipulation",
       "T1134.001": "Access Token Manipulation::Token Impersonation/Theft",
       "T1134.002": "Access Token Manipulation::Create Process with Token",
@@ -349,6 +361,7 @@
       "T1548.002": "Abuse Elevation Control Mechanism::Bypass User Account Control",
       "T1548.003": "Abuse Elevation Control Mechanism::Sudo and Sudo Caching",
       "T1548.004": "Abuse Elevation Control Mechanism::Elevated Execution with Prompt",
+      "T1548.005": "Abuse Elevation Control Mechanism::Temporary Elevated Cloud Access",
       "T1574": "Hijack Execution Flow",
       "T1574.001": "Hijack Execution Flow::DLL Search Order Hijacking",
       "T1574.002": "Hijack Execution Flow::DLL Side-Loading",
@@ -379,6 +392,7 @@
       "T1027.009": "Obfuscated Files or Information::Embedded Payloads",
       "T1027.010": "Obfuscated Files or Information::Command Obfuscation",
       "T1027.011": "Obfuscated Files or Information::Fileless Storage",
+      "T1027.012": "Obfuscated Files or Information::LNK Icon Smuggling",
       "T1036": "Masquerading",
       "T1036.001": "Masquerading::Invalid Code Signature",
       "T1036.002": "Masquerading::Right-to-Left Override",
@@ -388,6 +402,7 @@
       "T1036.006": "Masquerading::Space after Filename",
       "T1036.007": "Masquerading::Double File Extension",
       "T1036.008": "Masquerading::Masquerade File Type",
+      "T1036.009": "Masquerading::Break Process Trees",
       "T1055": "Process Injection",
       "T1055.001": "Process Injection::Dynamic-link Library Injection",
       "T1055.002": "Process Injection::Portable Executable Injection",
@@ -475,6 +490,7 @@
       "T1548.002": "Abuse Elevation Control Mechanism::Bypass User Account Control",
       "T1548.003": "Abuse Elevation Control Mechanism::Sudo and Sudo Caching",
       "T1548.004": "Abuse Elevation Control Mechanism::Elevated Execution with Prompt",
+      "T1548.005": "Abuse Elevation Control Mechanism::Temporary Elevated Cloud Access",
       "T1550": "Use Alternate Authentication Material",
       "T1550.001": "Use Alternate Authentication Material::Application Access Token",
       "T1550.002": "Use Alternate Authentication Material::Pass the Hash",
@@ -503,10 +519,11 @@
       "T1562.004": "Impair Defenses::Disable or Modify System Firewall",
       "T1562.006": "Impair Defenses::Indicator Blocking",
       "T1562.007": "Impair Defenses::Disable or Modify Cloud Firewall",
-      "T1562.008": "Impair Defenses::Disable Cloud Logs",
+      "T1562.008": "Impair Defenses::Disable or Modify Cloud Logs",
       "T1562.009": "Impair Defenses::Safe Mode Boot",
       "T1562.010": "Impair Defenses::Downgrade Attack",
       "T1562.011": "Impair Defenses::Spoof Security Alerting",
+      "T1562.012": "Impair Defenses::Disable or Modify Linux Audit System",
       "T1564": "Hide Artifacts",
       "T1564.001": "Hide Artifacts::Hidden Files and Directories",
       "T1564.002": "Hide Artifacts::Hidden Users",
@@ -518,6 +535,7 @@
       "T1564.008": "Hide Artifacts::Email Hiding Rules",
       "T1564.009": "Hide Artifacts::Resource Forking",
       "T1564.010": "Hide Artifacts::Process Argument Spoofing",
+      "T1564.011": "Hide Artifacts::Ignore Process Interrupts",
       "T1574": "Hijack Execution Flow",
       "T1574.001": "Hijack Execution Flow::DLL Search Order Hijacking",
       "T1574.002": "Hijack Execution Flow::DLL Side-Loading",
@@ -536,6 +554,7 @@
       "T1578.002": "Modify Cloud Compute Infrastructure::Create Cloud Instance",
       "T1578.003": "Modify Cloud Compute Infrastructure::Delete Cloud Instance",
       "T1578.004": "Modify Cloud Compute Infrastructure::Revert Cloud Instance",
+      "T1578.005": "Modify Cloud Compute Infrastructure::Modify Cloud Compute Configurations",
       "T1599": "Network Boundary Bridging",
       "T1599.001": "Network Boundary Bridging::Network Address Translation Traversal",
       "T1600": "Weaken Encryption",
@@ -548,7 +567,8 @@
       "T1612": "Build Image on Host",
       "T1620": "Reflective Code Loading",
       "T1622": "Debugger Evasion",
-      "T1647": "Plist File Modification"
+      "T1647": "Plist File Modification",
+      "T1656": "Impersonation"
     },
     "Credential Access": {
       "T1003": "OS Credential Dumping",
@@ -591,6 +611,7 @@
       "T1555.003": "Credentials from Password Stores::Credentials from Web Browsers",
       "T1555.004": "Credentials from Password Stores::Windows Credential Manager",
       "T1555.005": "Credentials from Password Stores::Password Managers",
+      "T1555.006": "Credentials from Password Stores::Cloud Secrets Management Stores",
       "T1556": "Modify Authentication Process",
       "T1556.001": "Modify Authentication Process::Domain Controller Authentication",
       "T1556.002": "Modify Authentication Process::Password Filter DLL",
@@ -621,6 +642,7 @@
       "T1012": "Query Registry",
       "T1016": "System Network Configuration Discovery",
       "T1016.001": "System Network Configuration Discovery::Internet Connection Discovery",
+      "T1016.002": "System Network Configuration Discovery::Wi-Fi Discovery",
       "T1018": "Remote System Discovery",
       "T1033": "System Owner/User Discovery",
       "T1040": "Network Sniffing",
@@ -659,7 +681,8 @@
       "T1615": "Group Policy Discovery",
       "T1619": "Cloud Storage Object Discovery",
       "T1622": "Debugger Evasion",
-      "T1652": "Device Driver Discovery"
+      "T1652": "Device Driver Discovery",
+      "T1654": "Log Enumeration"
     },
     "Lateral Movement": {
       "T1021": "Remote Services",
@@ -670,6 +693,7 @@
       "T1021.005": "Remote Services::VNC",
       "T1021.006": "Remote Services::Windows Remote Management",
       "T1021.007": "Remote Services::Cloud Services",
+      "T1021.008": "Remote Services::Direct Cloud VM Connections",
       "T1072": "Software Deployment Tools",
       "T1080": "Taint Shared Content",
       "T1091": "Replication Through Removable Media",
@@ -763,7 +787,8 @@
       "T1572": "Protocol Tunneling",
       "T1573": "Encrypted Channel",
       "T1573.001": "Encrypted Channel::Symmetric Cryptography",
-      "T1573.002": "Encrypted Channel::Asymmetric Cryptography"
+      "T1573.002": "Encrypted Channel::Asymmetric Cryptography",
+      "T1659": "Content Injection"
     },
     "Exfiltration": {
       "T1011": "Exfiltration Over Other Network Medium",
@@ -783,7 +808,8 @@
       "T1567": "Exfiltration Over Web Service",
       "T1567.001": "Exfiltration Over Web Service::Exfiltration to Code Repository",
       "T1567.002": "Exfiltration Over Web Service::Exfiltration to Cloud Storage",
-      "T1567.003": "Exfiltration Over Web Service::Exfiltration to Text Storage Sites"
+      "T1567.003": "Exfiltration Over Web Service::Exfiltration to Text Storage Sites",
+      "T1567.004": "Exfiltration Over Web Service::Exfiltration Over Webhook"
     },
     "Impact": {
       "T1485": "Data Destruction",
@@ -811,7 +837,8 @@
       "T1565": "Data Manipulation",
       "T1565.001": "Data Manipulation::Stored Data Manipulation",
       "T1565.002": "Data Manipulation::Transmitted Data Manipulation",
-      "T1565.003": "Data Manipulation::Runtime Data Manipulation"
+      "T1565.003": "Data Manipulation::Runtime Data Manipulation",
+      "T1657": "Financial Theft"
     }
   },
   "mbc": {

scripts/profile-time.py

@@ -54,6 +54,7 @@ import capa.helpers
 import capa.features
 import capa.features.common
 import capa.features.freeze
+import capa.capabilities.common
 
 logger = logging.getLogger("capa.profile")
 
@@ -114,7 +115,7 @@ def main(argv=None):
 
         def do_iteration():
             capa.perf.reset()
-            capa.main.find_capabilities(rules, extractor, disable_progress=True)
+            capa.capabilities.common.find_capabilities(rules, extractor, disable_progress=True)
             pbar.update(1)
 
         samples = timeit.repeat(do_iteration, number=args.number, repeat=args.repeat)

scripts/setup-linter-dependencies.py

@@ -47,7 +47,7 @@ from typing import Dict, List
 from pathlib import Path
 
 import requests
-from stix2 import Filter, MemoryStore, AttackPattern  # type: ignore
+from stix2 import Filter, MemoryStore, AttackPattern
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
 

scripts/show-capabilities-by-function.py

@@ -74,10 +74,12 @@ import capa.exceptions
 import capa.render.utils as rutils
 import capa.render.verbose
 import capa.features.freeze
+import capa.capabilities.common
 import capa.render.result_document as rd
 from capa.helpers import get_file_taste
 from capa.features.common import FORMAT_AUTO
 from capa.features.freeze import Address
+from capa.features.extractors.base_extractor import FeatureExtractor, StaticFeatureExtractor
 
 logger = logging.getLogger("capa.show-capabilities-by-function")
 
@@ -101,6 +103,7 @@ def render_matches_by_function(doc: rd.ResultDocument):
           - send HTTP request
           - connect to HTTP server
     """
+    assert isinstance(doc.meta.analysis, rd.StaticAnalysis)
     functions_by_bb: Dict[Address, Address] = {}
     for finfo in doc.meta.analysis.layout.functions:
         faddress = finfo.address
@@ -113,10 +116,10 @@ def render_matches_by_function(doc: rd.ResultDocument):
 
     matches_by_function = collections.defaultdict(set)
     for rule in rutils.capability_rules(doc):
-        if rule.meta.scope == capa.rules.FUNCTION_SCOPE:
+        if capa.rules.Scope.FUNCTION in rule.meta.scopes:
             for addr, _ in rule.matches:
                 matches_by_function[addr].add(rule.meta.name)
-        elif rule.meta.scope == capa.rules.BASIC_BLOCK_SCOPE:
+        elif capa.rules.Scope.BASIC_BLOCK in rule.meta.scopes:
             for addr, _ in rule.matches:
                 function = functions_by_bb[addr]
                 matches_by_function[function].add(rule.meta.name)
@@ -167,7 +170,7 @@ def main(argv=None):
 
     if (args.format == "freeze") or (args.format == FORMAT_AUTO and capa.features.freeze.is_freeze(taste)):
         format_ = "freeze"
-        extractor = capa.features.freeze.load(Path(args.sample).read_bytes())
+        extractor: FeatureExtractor = capa.features.freeze.load(Path(args.sample).read_bytes())
     else:
         format_ = args.format
         should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None)
@@ -176,6 +179,7 @@ def main(argv=None):
             extractor = capa.main.get_extractor(
                 args.sample, args.format, args.os, args.backend, sig_paths, should_save_workspace
             )
+            assert isinstance(extractor, StaticFeatureExtractor)
         except capa.exceptions.UnsupportedFormatError:
             capa.helpers.log_unsupported_format_error()
             return -1
@@ -183,14 +187,12 @@ def main(argv=None):
             capa.helpers.log_unsupported_runtime_error()
             return -1
 
-    meta = capa.main.collect_metadata(argv, args.sample, format_, args.os, args.rules, extractor)
-    capabilities, counts = capa.main.find_capabilities(rules, extractor)
+    capabilities, counts = capa.capabilities.common.find_capabilities(rules, extractor)
 
-    meta.analysis.feature_counts = counts["feature_counts"]
-    meta.analysis.library_functions = counts["library_functions"]
+    meta = capa.main.collect_metadata(argv, args.sample, format_, args.os, args.rules, extractor, counts)
     meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
 
-    if capa.main.has_file_limitation(rules, capabilities):
+    if capa.capabilities.common.has_file_limitation(rules, capabilities):
         # bail if capa encountered file limitation e.g. a packed binary
         # do show the output in verbose mode, though.
         if not (args.verbose or args.vverbose or args.json):