Merge pull request #1648 from mandiant/fix/issue-1622

prep v6.0.0a1

.github/CONTRIBUTING.md

@@ -159,12 +159,25 @@ The process described here has several goals:
 
 Please follow these steps to have your contribution considered by the maintainers:
 
+0. Sign the [Contributor License Agreement](#contributor-license-agreement)
 1. Follow the [styleguides](#styleguides)
 2. Update the CHANGELOG and add tests and documentation. In case they are not needed, indicate it in [the PR template](pull_request_template.md).
 3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing? </summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
 
 While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
 
+### Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution,
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to <https://cla.developers.google.com/> to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
 ## Styleguides
 
 ### Git Commit Messages

.github/flake8.ini

@@ -0,0 +1,41 @@
+[flake8]
+max-line-length = 120
+
+extend-ignore =
+    # E203: whitespace before ':'  (black does this)
+    E203,
+    # F401: `foo` imported but unused  (prefer ruff)
+    F401,
+    # F811 Redefinition of unused `foo`  (prefer ruff)
+    F811,
+    # E501 line too long  (prefer black)
+    E501,
+    # B010 Do not call setattr with a constant attribute value
+    B010,
+    # G200 Logging statement uses exception in arguments
+    G200,
+    # SIM102 Use a single if-statement instead of nested if-statements
+    # doesn't provide a space for commenting or logical separation of conditions
+    SIM102,
+    # SIM114 Use logical or and a single body
+    # makes logic trees too complex
+    SIM114,
+    # SIM117 Use 'with Foo, Bar:' instead of multiple with statements
+    # makes lines too long
+    SIM117
+
+per-file-ignores =
+    # T201 print found.
+    #
+    # scripts are meant to print output
+    scripts/*: T201
+    # capa.exe is meant to print output
+    capa/main.py: T201
+    # IDA tests emit results to output window so need to print
+    tests/test_ida_features.py: T201
+    # utility used to find the Binary Ninja API via invoking python.exe
+    capa/features/extractors/binja/find_binja_api.py: T201
+
+copyright-check = True
+copyright-min-file-size = 1 
+copyright-regexp = Copyright \(C\) 2023 Mandiant, Inc. All Rights Reserved.

.github/mypy/mypy.ini

@@ -42,6 +42,9 @@ ignore_missing_imports = True
 [mypy-idautils.*]
 ignore_missing_imports = True
 
+[mypy-ida_auto.*]
+ignore_missing_imports = True
+
 [mypy-ida_bytes.*]
 ignore_missing_imports = True
 

.github/pyinstaller/hooks/hook-vivisect.py

@@ -38,39 +38,36 @@ hiddenimports = [
     "vivisect",
     "vivisect.analysis",
     "vivisect.analysis.amd64",
-    "vivisect.analysis.amd64",
     "vivisect.analysis.amd64.emulation",
     "vivisect.analysis.amd64.golang",
     "vivisect.analysis.crypto",
-    "vivisect.analysis.crypto",
     "vivisect.analysis.crypto.constants",
     "vivisect.analysis.elf",
     "vivisect.analysis.elf.elfplt",
     "vivisect.analysis.elf.elfplt_late",
     "vivisect.analysis.elf.libc_start_main",
     "vivisect.analysis.generic",
-    "vivisect.analysis.generic",
     "vivisect.analysis.generic.codeblocks",
     "vivisect.analysis.generic.emucode",
     "vivisect.analysis.generic.entrypoints",
     "vivisect.analysis.generic.funcentries",
     "vivisect.analysis.generic.impapi",
+    "vivisect.analysis.generic.linker",
     "vivisect.analysis.generic.mkpointers",
+    "vivisect.analysis.generic.noret",
     "vivisect.analysis.generic.pointers",
     "vivisect.analysis.generic.pointertables",
     "vivisect.analysis.generic.relocations",
     "vivisect.analysis.generic.strconst",
     "vivisect.analysis.generic.switchcase",
+    "vivisect.analysis.generic.symswitchcase",
     "vivisect.analysis.generic.thunks",
-    "vivisect.analysis.generic.noret",
-    "vivisect.analysis.i386",
     "vivisect.analysis.i386",
     "vivisect.analysis.i386.calling",
     "vivisect.analysis.i386.golang",
     "vivisect.analysis.i386.importcalls",
     "vivisect.analysis.i386.instrhook",
-    "vivisect.analysis.i386.thunk_bx",
-    "vivisect.analysis.ms",
+    "vivisect.analysis.i386.thunk_reg",
     "vivisect.analysis.ms",
     "vivisect.analysis.ms.hotpatch",
     "vivisect.analysis.ms.localhints",
@@ -81,8 +78,40 @@ hiddenimports = [
     "vivisect.impapi.posix.amd64",
     "vivisect.impapi.posix.i386",
     "vivisect.impapi.windows",
+    "vivisect.impapi.windows.advapi_32",
+    "vivisect.impapi.windows.advapi_64",
     "vivisect.impapi.windows.amd64",
+    "vivisect.impapi.windows.gdi_32",
+    "vivisect.impapi.windows.gdi_64",
     "vivisect.impapi.windows.i386",
+    "vivisect.impapi.windows.kernel_32",
+    "vivisect.impapi.windows.kernel_64",
+    "vivisect.impapi.windows.msvcr100_32",
+    "vivisect.impapi.windows.msvcr100_64",
+    "vivisect.impapi.windows.msvcr110_32",
+    "vivisect.impapi.windows.msvcr110_64",
+    "vivisect.impapi.windows.msvcr120_32",
+    "vivisect.impapi.windows.msvcr120_64",
+    "vivisect.impapi.windows.msvcr71_32",
+    "vivisect.impapi.windows.msvcr80_32",
+    "vivisect.impapi.windows.msvcr80_64",
+    "vivisect.impapi.windows.msvcr90_32",
+    "vivisect.impapi.windows.msvcr90_64",
+    "vivisect.impapi.windows.msvcrt_32",
+    "vivisect.impapi.windows.msvcrt_64",
+    "vivisect.impapi.windows.ntdll_32",
+    "vivisect.impapi.windows.ntdll_64",
+    "vivisect.impapi.windows.ole_32",
+    "vivisect.impapi.windows.ole_64",
+    "vivisect.impapi.windows.rpcrt4_32",
+    "vivisect.impapi.windows.rpcrt4_64",
+    "vivisect.impapi.windows.shell_32",
+    "vivisect.impapi.windows.shell_64",
+    "vivisect.impapi.windows.user_32",
+    "vivisect.impapi.windows.user_64",
+    "vivisect.impapi.windows.ws2plus_32",
+    "vivisect.impapi.windows.ws2plus_64",
+    "vivisect.impapi.winkern",
     "vivisect.impapi.winkern.i386",
     "vivisect.impapi.winkern.amd64",
     "vivisect.parsers.blob",

.github/pyinstaller/pyinstaller.spec

@@ -61,6 +61,7 @@ a = Analysis(
         "qt5",
         "pyqtwebengine",
         "pyasn1",
+        "binaryninja",
     ],
 )
 

.github/ruff.toml

@@ -0,0 +1,43 @@
+# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E", "F"]
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# E402 module level import not at top of file
+# E722 do not use bare 'except'
+# E501 line too long
+ignore = ["E402", "E722", "E501"]
+
+line-length = 120
+
+exclude = [
+    # Exclude a variety of commonly ignored directories.
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+    # protobuf generated files
+    "*_pb2.py", 
+    "*_pb2.pyi"
+]

.github/tox.ini

@@ -1,10 +0,0 @@
-[pycodestyle]
-; E402: module level import not at top of file
-; W503: line break before binary operator
-; E231 missing whitespace after ',' (emitted by black)
-; E203 whitespace before ':' (emitted by black)
-ignore = E402,W503,E203,E231
-max-line-length = 160
-statistics = True
-count = True
-exclude = .*

.github/workflows/build.yml

@@ -6,6 +6,9 @@ on:
   release:
     types: [edited, published]
 
+permissions:
+  contents: write
+
 jobs:
   build:
     name: PyInstaller for ${{ matrix.os }}
@@ -15,7 +18,7 @@ jobs:
       fail-fast: true
       matrix:
         include:
-          - os: ubuntu-18.04
+          - os: ubuntu-20.04
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
@@ -36,7 +39,7 @@ jobs:
         uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
           python-version: 3.8
-      - if: matrix.os == 'ubuntu-18.04'
+      - if: matrix.os == 'ubuntu-20.04'
         run: sudo apt-get install -y libyaml-dev
       - name: Upgrade pip, setuptools
         run: python -m pip install --upgrade pip setuptools
@@ -65,10 +68,7 @@ jobs:
       matrix:
         include:
           # OSs not already tested above
-          - os: ubuntu-18.04
-            artifact_name: capa
-            asset_name: linux
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             artifact_name: capa
             asset_name: linux
           - os: windows-2022

.github/workflows/changelog.yml

@@ -7,6 +7,8 @@ on:
   pull_request_target:
     types: [opened, edited, synchronize]
 
+permissions: read-all
+
 jobs:
   check_changelog:
     # no need to check for dependency updates via dependabot

.github/workflows/publish.yml

@@ -1,29 +1,48 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
+# use PyPI trusted publishing, as described here: 
+# https://blog.trailofbits.com/2023/05/23/trusted-publishing-a-new-benchmark-for-packaging-security/
 name: publish to pypi
 
 on:
   release:
     types: [published]
 
+permissions:
+  contents: write
+
 jobs:
-  deploy:
-    runs-on: ubuntu-20.04
+  pypi-publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+    permissions:
+      id-token: write
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       - name: Set up Python
         uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
-          python-version: '3.7'
+          python-version: '3.8'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+          pip install -e .[build]
+      - name: build package
         run: |
-          python setup.py sdist bdist_wheel
-          twine upload --skip-existing dist/*
+          python -m build
+      - name: upload package artifacts
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ matrix.asset_name }}
+          path: dist/*
+      - name: upload package to GH Release
+        uses: svenstaro/upload-release-action@2728235f7dc9ff598bd86ce3c274b74f802d2208 # v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN}}
+          file: dist/*
+          tag: ${{ github.ref }}
+      - name: publish package
+        uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0  # release/v1
+        with:
+          skip-existing: true
+          verbose: true
+          print-hash: true

.github/workflows/tag.yml

@@ -4,6 +4,8 @@ on:
   release:
     types: [published]
 
+permissions: read-all
+
 jobs:
   tag:
     name: Tag capa rules

.github/workflows/tests.yml

@@ -6,6 +6,8 @@ on:
   pull_request:
     branches: [ master ]
 
+permissions: read-all
+
 # save workspaces to speed up testing
 env:
   CAPA_SAVE_WORKSPACE: "True"
@@ -27,20 +29,23 @@ jobs:
     steps:
     - name: Checkout capa
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-    - name: Set up Python 3.8
+    # use latest available python to take advantage of best performance
+    - name: Set up Python 3.11
       uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
-        python-version: "3.8"
+        python-version: "3.11"
     - name: Install dependencies
       run: pip install -e .[dev]
+    - name: Lint with ruff
+      run: pre-commit run ruff
     - name: Lint with isort
-      run: isort --profile black --length-sort --line-width 120 -c .
+      run: pre-commit run isort
     - name: Lint with black
-      run: black -l 120 --check .
-    - name: Lint with pycodestyle
-      run: pycodestyle --show-source capa/ scripts/ tests/
+      run: pre-commit run black
+    - name: Lint with flake8
+      run: pre-commit run flake8
     - name: Check types with mypy
-      run: mypy --config-file .github/mypy/mypy.ini --check-untyped-defs capa/ scripts/ tests/
+      run:  pre-commit run mypy
 
   rule_linter:
     runs-on: ubuntu-20.04
@@ -49,12 +54,12 @@ jobs:
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       with:
         submodules: recursive
-    - name: Set up Python 3.8
+    - name: Set up Python 3.11
       uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
-        python-version: "3.8"
+        python-version: "3.11"
     - name: Install capa
-      run: pip install -e .
+      run: pip install -e .[dev]
     - name: Run rule linter
       run: python scripts/lint.py rules/
 
@@ -67,13 +72,15 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-2019, macos-11]
         # across all operating systems
-        python-version: ["3.7", "3.11"]
+        python-version: ["3.8", "3.11"]
         include:
           # on Ubuntu run these as well
           - os: ubuntu-20.04
             python-version: "3.8"
           - os: ubuntu-20.04
             python-version: "3.9"
+          - os: ubuntu-20.04
+            python-version: "3.10"
     steps:
     - name: Checkout capa with submodules
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -90,3 +97,45 @@ jobs:
       run: pip install -e .[dev]
     - name: Run tests
       run: pytest -v tests/
+
+  binja-tests:
+    name: Binary Ninja tests for ${{ matrix.python-version }}
+    env:
+      BN_SERIAL: ${{ secrets.BN_SERIAL }}
+    runs-on: ubuntu-20.04
+    needs: [code_style, rule_linter]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.11"]
+    steps:
+    - name: Checkout capa with submodules
+      # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
+      if: ${{ env.BN_SERIAL != 0 }}
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      with:
+        submodules: recursive
+    - name: Set up Python ${{ matrix.python-version }}
+      if: ${{ env.BN_SERIAL != 0 }}
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install pyyaml
+      if: ${{ env.BN_SERIAL != 0 }}
+      run: sudo apt-get install -y libyaml-dev
+    - name: Install capa
+      if: ${{ env.BN_SERIAL != 0 }}
+      run: pip install -e .[dev]
+    - name: install Binary Ninja
+      if: ${{ env.BN_SERIAL != 0 }}
+      run: |
+        mkdir ./.github/binja
+        curl "https://raw.githubusercontent.com/Vector35/binaryninja-api/6812c97/scripts/download_headless.py" -o ./.github/binja/download_headless.py
+        python ./.github/binja/download_headless.py --serial ${{ env.BN_SERIAL }} --output .github/binja/BinaryNinja-headless.zip
+        unzip .github/binja/BinaryNinja-headless.zip -d .github/binja/
+        python .github/binja/binaryninja/scripts/install_api.py --install-on-root --silent
+    - name: Run tests
+      if: ${{ env.BN_SERIAL != 0 }}
+      env:
+        BN_LICENSE: ${{ secrets.BN_LICENSE }}
+      run: pytest -v tests/test_binja_features.py  # explicitly refer to the binja tests for performance. other tests run above.

.gitignore

@@ -108,17 +108,21 @@ venv.bak/
 *.viv
 *.idb
 *.i64
+.vscode
+
 !rules/lib
 
-# hooks/ci.sh output
-isort-output.log
-black-output.log
-rule-linter-output.log
-.vscode
 scripts/perf/*.txt
 scripts/perf/*.svg
 scripts/perf/*.zip
+
 .direnv
 .envrc
 .DS_Store
 */.DS_Store
+Pipfile
+Pipfile.lock
+/cache/
+.github/binja/binaryninja
+.github/binja/download_headless.py
+.github/binja/BinaryNinja-headless.zip

.pre-commit-config.yaml

@@ -0,0 +1,111 @@
+# install the pre-commit hooks:
+#
+#    ❯ pre-commit install --hook-type pre-commit
+#    pre-commit installed at .git/hooks/pre-commit
+#
+#    ❯ pre-commit install --hook-type pre-push
+#    pre-commit installed at .git/hooks/pre-push
+#
+# run all linters liks:
+#
+#    ❯ pre-commit run --all-files
+#    isort....................................................................Passed
+#    black....................................................................Passed
+#    ruff.....................................................................Passed
+#    flake8...................................................................Passed
+#    mypy.....................................................................Passed
+#
+# run a single linter like:
+#
+#    ❯ pre-commit run --all-files isort
+#    isort....................................................................Passed
+
+repos:
+-   repo: local
+    hooks:
+    -   id: isort
+        name: isort
+        stages: [commit, push]
+        language: system
+        entry: isort
+        args: 
+        -   "--length-sort"
+        -   "--profile"
+        -   "black"
+        -   "--line-length=120"
+        -   "--skip-glob"
+        -   "*_pb2.py"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: black
+        name: black
+        stages: [commit, push]
+        language: system
+        entry: black
+        args: 
+        -   "--line-length=120"
+        -   "--extend-exclude"
+        -   ".*_pb2.py"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: ruff
+        name: ruff
+        stages: [commit, push]
+        language: system
+        entry: ruff
+        args: 
+        -   "check"
+        -   "--config"
+        -   ".github/ruff.toml"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: flake8
+        name: flake8
+        stages: [commit, push]
+        language: system
+        entry: flake8
+        args: 
+        -   "--config"
+        -   ".github/flake8.ini"
+        -   "--extend-exclude"
+        -   "capa/render/proto/capa_pb2.py"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false
+
+-   repo: local
+    hooks:
+    -   id: mypy
+        name: mypy
+        stages: [commit, push]
+        language: system
+        entry: mypy
+        args: 
+        -   "--check-untyped-defs"
+        -   "--ignore-missing-imports"
+        -   "--config-file=.github/mypy/mypy.ini"
+        -   "capa/"
+        -   "scripts/"
+        -   "tests/"
+        always_run: true
+        pass_filenames: false

CHANGELOG.md

@@ -4,11 +4,116 @@
 
 ### New Features
 
-- new cli flag `--os` to override auto-detected operating system for a sample @captainGeech42
-
 ### Breaking Changes
 
-### New Rules (20)
+### New Rules (0)
+
+-
+
+### Bug Fixes
+
+### capa explorer IDA Pro plugin
+
+### Development
+
+### Raw diffs
+- [capa v6.0.0...master](https://github.com/mandiant/capa/compare/v6.0.0...master)
+- [capa-rules v6.0.0...master](https://github.com/mandiant/capa-rules/compare/v6.0.0...master)
+
+## v6.0.0
+
+capa v6.0 brings many bug fixes and quality improvements, including 64 rule updates and 26 new rules. We're now publishing to PyPI via [Trusted Publishing](https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/) and have migrated to using a `pyproject.toml` file. @Aayush-Goel-04 contributed a lot of new code across many files, so please welcome them to the project, along with @anders-v @crowface28 @dkelly2e @RonnieSalomonsen and @ejfocampo as first-time rule contributors!
+
+For those that use capa as a library, we've introduced some limited breaking changes that better represent data types (versus less-structured data like dictionaries and strings). With the recent deprecation, we've also dropped support for Python 3.7.
+
+### New Features
+- add script to detect feature overlap between new and existing capa rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
+- extract forwarded exports from PE files #1624 @williballenthin
+- extract function and API names from ELF symtab entries @yelhamer https://github.com/mandiant/capa-rules/issues/736
+- use fancy box drawing characters for default output #1586 @williballenthin
+
+### Breaking Changes
+- use a class to represent Metadata (not dict) #1411 @Aayush-Goel-04 @manasghandat
+- use pathlib.Path to represent file paths #1534 @Aayush-Goel-04
+- Python 3.8 is now the minimum supported Python version #1578 @williballenthin
+- Require a Contributor License Agreement (CLA) for PRs going forward #1642 @williballenthin
+
+### New Rules (26)
+
+- load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
+- nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
+- data-manipulation/encryption/aes/encrypt-data-using-aes-mixcolumns-step @mr-tz
+- linking/static/aplib/linked-against-aplib still@teamt5.org
+- communication/mailslot/read-from-mailslot nick.simonian@mandiant.com
+- nursery/hash-data-using-sha512managed-in-dotnet jonathanlepore@google.com
+- nursery/compiled-with-exescript jonathanlepore@google.com
+- nursery/check-for-sandbox-via-mac-address-ouis-in-dotnet jonathanlepore@google.com
+- host-interaction/hardware/enumerate-devices-by-category @mr-tz
+- host-interaction/service/continue-service @mr-tz
+- host-interaction/service/pause-service @mr-tz
+- persistence/exchange/act-as-exchange-transport-agent jakub.jozwiak@mandiant.com
+- host-interaction/file-system/create-virtual-file-system-in-dotnet jakub.jozwiak@mandiant.com
+- compiler/cx_freeze/compiled-with-cx_freeze @mr-tz jakub.jozwiak@mandiant.com
+- communication/socket/create-vmci-socket jakub.jozwiak@mandiant.com
+- persistence/office/act-as-excel-xll-add-in jakub.jozwiak@mandiant.com
+- persistence/office/act-as-office-com-add-in jakub.jozwiak@mandiant.com
+- persistence/office/act-as-word-wll-add-in jakub.jozwiak@mandiant.com
+- anti-analysis/anti-debugging/debugger-evasion/hide-thread-from-debugger michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com
+- host-interaction/memory/create-new-application-domain-in-dotnet jakub.jozwiak@mandiant.com
+- host-interaction/gui/switch-active-desktop jakub.jozwiak@mandiant.com
+- host-interaction/service/query-service-configuration @mr-tz
+- anti-analysis/anti-av/patch-event-tracing-for-windows-function jakub.jozwiak@mandiant.com
+- data-manipulation/encoding/xor/covertly-decode-and-write-data-to-windows-directory-using-indirect-calls dan.kelly@mandiant.com
+- linking/runtime-linking/resolve-function-by-brute-ratel-badger-hash jakub.jozwiak@mandiant.com
+
+### Bug Fixes
+- extractor: add a Binary Ninja test that asserts its version #1487 @xusheng6
+- extractor: update Binary Ninja stack string detection after the new constant outlining feature #1473 @xusheng6
+- extractor: update vivisect Arch extraction #1334 @mr-tz
+- extractor: avoid Binary Ninja exception when analyzing certain files #1441 @xusheng6 
+- symtab: fix struct.unpack() format for 64-bit ELF files @yelhamer
+- symtab: safeguard against ZeroDivisionError for files containing a symtab with a null entry size @yelhamer
+- improve ELF strtab and needed parsing @mr-tz
+- better handle exceptional cases when parsing ELF files #1458 @Aayush-Goel-04
+- improved testing coverage for Binary Ninja backend #1446 @Aayush-Goel-04
+- add logging and print redirect to tqdm for capa main #749 @Aayush-Goel-04
+- extractor: fix binja installation path detection does not work with Python 3.11
+- tests: refine the IDA test runner script #1513 @williballenthin
+- output: don't leave behind traces of progress bar @williballenthin
+- import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin
+- main: don't show spinner when emitting debug messages #1636 @williballenthin
+
+### capa explorer IDA Pro plugin
+
+### Development
+- update ATT&CK/MBC data for linting #1568 @mr-tz
+- log time taken to analyze each function #1290 @williballenthin
+- tests: make fixture available via conftest.py #1592 @williballenthin
+- publish via PyPI trusted publishing #1491 @williballenthin
+- migrate to pyproject.toml #1301 @williballenthin
+- use [pre-commit](https://pre-commit.com/) to invoke linters #1579 @williballenthin
+
+
+### Raw diffs
+- [capa v5.1.0...v6.0.0](https://github.com/mandiant/capa/compare/v5.1.0...v6.0.0a1)
+- [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0a1)
+
+## v5.1.0
+capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and a output color format that's easier to read.
+
+Over 25 capa rules have been added and improved.
+
+Thanks for all the support, especially to @xusheng6, @captainGeech42, @ggold7046, @manasghandat, @ooprathamm, @linpeiyu164, @yelhamer, @HongThatCong, @naikordian, @stevemk14ebr, @emtuls, @raymondlleong, @bkojusner, @joren485, and everyone else who submitted bugs and provided feedback!
+
+### New Features
+- add protobuf format for result documents #1219 @williballenthin @mr-tz 
+- extractor: add Binary Ninja feature extractor @xusheng6
+- new cli flag `--os` to override auto-detected operating system for a sample @captainGeech42
+- change colour/highlight to "cyan" instead of "blue" for better readability #1384 @ggold7046
+- add new format to parse output json back to capa #1396 @ooprathamm
+- parse ELF symbols' names to guess OS #1403 @yelhamer
+
+### New Rules (26)
 
 - persistence/scheduled-tasks/schedule-task-via-at joren485
 - data-manipulation/prng/generate-random-numbers-via-rtlgenrandom william.ballenthin@mandiant.com
@@ -30,22 +135,28 @@
 - nursery/hash-data-using-ripemd256 raymond.leong@mandiant.com
 - nursery/hash-data-using-ripemd320 raymond.leong@mandiant.com
 - nursery/set-web-proxy-in-dotnet michael.hunhoff@mandiant.com
--
+- nursery/check-for-windows-sandbox-via-subdirectory echernofsky@google.com
+- nursery/enumerate-pe-sections-in-dotnet @mr-tz
+- nursery/destroy-software-breakpoint-capability echernofsky@google.com
+- nursery/send-data-to-internet michael.hunhoff@mandiant.com
+- nursery/compiled-with-cx_freeze @mr-tz
+- nursery/contain-a-thread-local-storage-tls-section-in-dotnet michael.hunhoff@mandiant.com
 
 ### Bug Fixes
+- extractor: interface of cache modified to prevent extracting file and global features multiple times @stevemk14ebr
+- extractor: removed '.dynsym' as the library name for ELF imports #1318 @stevemk14ebr 
 - extractor: fix vivisect loop detection corner case #1310 @mr-tz
 - match: extend OS characteristic to match OS_ANY to all supported OSes #1324 @mike-hunhoff
-- extractor: fix IDA and vivisect string and bytes features overlap and tests #1327 #1336 @xusheng6 
+- extractor: fix IDA and vivisect string and bytes features overlap and tests #1327 #1336 @xusheng6
 
 ### capa explorer IDA Pro plugin
+- rule generator plugin now loads faster when jumping between functions @stevemk14ebr
 - fix exception when plugin loaded in IDA hosted under idat #1341 @mike-hunhoff
 - improve embedded PE detection performance and reduce FP potential #1344 @mike-hunhoff
 
-### Development
-
 ### Raw diffs
-- [capa v5.0.0...master](https://github.com/mandiant/capa/compare/v5.0.0...master)
-- [capa-rules v5.0.0...master](https://github.com/mandiant/capa-rules/compare/v5.0.0...master)
+- [capa v5.0.0...v5.1.0](https://github.com/mandiant/capa/compare/v5.0.0...v5.1.0)
+- [capa-rules v5.0.0...v5.1.0](https://github.com/mandiant/capa-rules/compare/v5.0.0...v5.1.0)
 
 
 ## v5.0.0 (2023-02-08)

LICENSE.txt

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (C) 2020 Mandiant, Inc.
+   Copyright (C) 2023 Mandiant, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-787-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-823-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)

capa/engine.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,7 +8,7 @@
 
 import copy
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator, cast
+from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator
 
 import capa.perf
 import capa.features.common
@@ -71,7 +71,7 @@ class Statement:
             yield child
 
         if hasattr(self, "children"):
-            for child in getattr(self, "children"):
+            for child in self.children:
                 assert isinstance(child, (Statement, Feature))
                 yield child
 
@@ -83,7 +83,7 @@ class Statement:
                 self.child = new
 
         if hasattr(self, "children"):
-            children = getattr(self, "children")
+            children = self.children
             for i, child in enumerate(children):
                 if child is existing:
                     children[i] = new

capa/exceptions.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 class UnsupportedRuntimeError(RuntimeError):
     pass
 

capa/features/address.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import abc
 
 

capa/features/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/common.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -100,7 +100,10 @@ class Result:
         return self.success
 
 
-class Feature(abc.ABC):
+class Feature(abc.ABC):  # noqa: B024
+    # this is an abstract class, since we don't want anyone to instantiate it directly,
+    # but it doesn't have any abstract methods.
+
     def __init__(
         self,
         value: Union[str, int, float, bytes],
@@ -124,7 +127,12 @@ class Feature(abc.ABC):
         return self.name == other.name and self.value == other.value
 
     def __lt__(self, other):
-        # TODO: this is a huge hack!
+        # implementing sorting by serializing to JSON is a huge hack.
+        # its slow, inelegant, and probably doesn't work intuitively;
+        # however, we only use it for deterministic output, so it's good enough for now.
+
+        # circular import
+        # we should fix if this wasn't already a huge hack.
         import capa.features.freeze.features
 
         return (
@@ -267,7 +275,7 @@ class _MatchedSubstring(Substring):
         self.matches = matches
 
     def __str__(self):
-        matches = ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys()))
+        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
         assert isinstance(self.value, str)
         return f'substring("{self.value}", matches = {matches})'
 
@@ -359,7 +367,7 @@ class _MatchedRegex(Regex):
         self.matches = matches
 
     def __str__(self):
-        matches = ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys()))
+        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
         assert isinstance(self.value, str)
         return f"regex(string =~ {self.value}, matches = {matches})"
 
@@ -450,6 +458,7 @@ FORMAT_AUTO = "auto"
 FORMAT_SC32 = "sc32"
 FORMAT_SC64 = "sc64"
 FORMAT_FREEZE = "freeze"
+FORMAT_RESULT = "result"
 FORMAT_UNKNOWN = "unknown"
 
 

capa/features/extractors/base_extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/binja/basicblock.py

@@ -0,0 +1,183 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import string
+import struct
+from typing import Tuple, Iterator
+
+from binaryninja import Function, Settings
+from binaryninja import BasicBlock as BinjaBasicBlock
+from binaryninja import (
+    BinaryView,
+    SymbolType,
+    RegisterValueType,
+    VariableSourceType,
+    MediumLevelILSetVar,
+    MediumLevelILOperation,
+    MediumLevelILBasicBlock,
+    MediumLevelILInstruction,
+)
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
+
+use_const_outline: bool = False
+settings: Settings = Settings()
+if settings.contains("analysis.outlining.builtins") and settings.get_bool("analysis.outlining.builtins"):
+    use_const_outline = True
+
+
+def get_printable_len_ascii(s: bytes) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    count = 0
+    for c in s:
+        if c == 0:
+            return count
+        if c < 127 and chr(c) in string.printable:
+            count += 1
+    return count
+
+
+def get_printable_len_wide(s: bytes) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    if all(c == 0x00 for c in s[1::2]):
+        return get_printable_len_ascii(s[::2])
+    return 0
+
+
+def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
+    bv: BinaryView = f.view
+
+    if il.operation != MediumLevelILOperation.MLIL_CALL:
+        return 0
+
+    target = il.dest
+    if target.operation not in [MediumLevelILOperation.MLIL_CONST, MediumLevelILOperation.MLIL_CONST_PTR]:
+        return 0
+
+    addr = target.value.value
+    sym = bv.get_symbol_at(addr)
+    if not sym or sym.type != SymbolType.LibraryFunctionSymbol:
+        return 0
+
+    if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
+        return 0
+
+    if len(il.params) < 2:
+        return 0
+
+    dest = il.params[0]
+    if dest.operation != MediumLevelILOperation.MLIL_ADDRESS_OF:
+        return 0
+
+    var = dest.src
+    if var.source_type != VariableSourceType.StackVariableSourceType:
+        return 0
+
+    src = il.params[1]
+    if src.value.type != RegisterValueType.ConstantDataAggregateValue:
+        return 0
+
+    s = f.get_constant_data(RegisterValueType.ConstantDataAggregateValue, src.value.value)
+    return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
+
+
+def get_printable_len(il: MediumLevelILSetVar) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    width = il.dest.type.width
+    value = il.src.value.value
+
+    if width == 1:
+        chars = struct.pack("<B", value & 0xFF)
+    elif width == 2:
+        chars = struct.pack("<H", value & 0xFFFF)
+    elif width == 4:
+        chars = struct.pack("<I", value & 0xFFFFFFFF)
+    elif width == 8:
+        chars = struct.pack("<Q", value & 0xFFFFFFFFFFFFFFFF)
+    else:
+        return 0
+
+    def is_printable_ascii(chars_: bytes):
+        return all(c < 127 and chr(c) in string.printable for c in chars_)
+
+    def is_printable_utf16le(chars_: bytes):
+        if all(c == 0x00 for c in chars_[1::2]):
+            return is_printable_ascii(chars_[::2])
+
+    if is_printable_ascii(chars):
+        return width
+
+    if is_printable_utf16le(chars):
+        return width // 2
+
+    return 0
+
+
+def is_mov_imm_to_stack(il: MediumLevelILInstruction) -> bool:
+    """verify instruction moves immediate onto stack"""
+    if il.operation != MediumLevelILOperation.MLIL_SET_VAR:
+        return False
+
+    if il.src.operation != MediumLevelILOperation.MLIL_CONST:
+        return False
+
+    if il.dest.source_type != VariableSourceType.StackVariableSourceType:
+        return False
+
+    return True
+
+
+def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
+    """check basic block for stackstring indicators
+
+    true if basic block contains enough moves of constant bytes to the stack
+    """
+    count = 0
+    for il in bb:
+        if use_const_outline:
+            count += get_stack_string_len(f, il)
+        else:
+            if is_mov_imm_to_stack(il):
+                count += get_printable_len(il)
+
+    if count > MIN_STACKSTRING_LEN:
+        return True
+    return False
+
+
+def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract stackstring indicators from basic block"""
+    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
+    if bb[1] is not None and bb_contains_stackstring(fh.inner, bb[1]):
+        yield Characteristic("stack string"), bbh.address
+
+
+def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract tight loop indicators from a basic block"""
+    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
+    for edge in bb[0].outgoing_edges:
+        if edge.target.start == bb[0].start:
+            yield Characteristic("tight loop"), bbh.address
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract basic block features"""
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, addr in bb_handler(fh, bbh):
+            yield feature, addr
+    yield BasicBlock(), bbh.address
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    extract_bb_stackstring,
+)

capa/features/extractors/binja/extractor.py

@@ -0,0 +1,75 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Tuple, Iterator
+
+import binaryninja as binja
+
+import capa.features.extractors.elf
+import capa.features.extractors.binja.file
+import capa.features.extractors.binja.insn
+import capa.features.extractors.binja.global_
+import capa.features.extractors.binja.function
+import capa.features.extractors.binja.basicblock
+from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+
+
+class BinjaFeatureExtractor(FeatureExtractor):
+    def __init__(self, bv: binja.BinaryView):
+        super().__init__()
+        self.bv = bv
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv))
+        self.global_features.extend(capa.features.extractors.binja.global_.extract_os(self.bv))
+        self.global_features.extend(capa.features.extractors.binja.global_.extract_arch(self.bv))
+
+    def get_base_address(self):
+        return AbsoluteVirtualAddress(self.bv.start)
+
+    def extract_global_features(self):
+        yield from self.global_features
+
+    def extract_file_features(self):
+        yield from capa.features.extractors.binja.file.extract_features(self.bv)
+
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        for f in self.bv.functions:
+            yield FunctionHandle(address=AbsoluteVirtualAddress(f.start), inner=f)
+
+    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binja.function.extract_features(fh)
+
+    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
+        f: binja.Function = fh.inner
+        # Set up a MLIL basic block dict look up to associate the disassembly basic block with its MLIL basic block
+        mlil_lookup = {}
+        for mlil_bb in f.mlil.basic_blocks:
+            mlil_lookup[mlil_bb.source_block.start] = mlil_bb
+
+        for bb in f.basic_blocks:
+            mlil_bb = mlil_lookup.get(bb.start)
+
+            yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=(bb, mlil_bb))
+
+    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.binja.basicblock.extract_features(fh, bbh)
+
+    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
+        import capa.features.extractors.binja.helpers as binja_helpers
+
+        bb: Tuple[binja.BasicBlock, binja.MediumLevelILBasicBlock] = bbh.inner
+        addr = bb[0].start
+
+        for text, length in bb[0]:
+            insn = binja_helpers.DisassemblyInstruction(addr, length, text)
+            yield InsnHandle(address=AbsoluteVirtualAddress(addr), inner=insn)
+            addr += length
+
+    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
+        yield from capa.features.extractors.binja.insn.extract_features(fh, bbh, ih)

capa/features/extractors/binja/file.py

@@ -0,0 +1,167 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import struct
+from typing import Tuple, Iterator
+
+from binaryninja import Segment, BinaryView, SymbolType, SymbolBinding
+
+import capa.features.extractors.common
+import capa.features.extractors.helpers
+import capa.features.extractors.strings
+from capa.features.file import Export, Import, Section, FunctionName
+from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
+from capa.features.extractors.binja.helpers import unmangle_c_name
+
+
+def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[int, int]]:
+    """check segment for embedded PE
+
+    adapted for binja from:
+    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
+    """
+    mz_xor = [
+        (
+            capa.features.extractors.helpers.xor_static(b"MZ", i),
+            capa.features.extractors.helpers.xor_static(b"PE", i),
+            i,
+        )
+        for i in range(256)
+    ]
+
+    todo = []
+    # If this is the first segment of the binary, skip the first bytes. Otherwise, there will always be a matched
+    # PE at the start of the binaryview.
+    start = seg.start
+    if bv.view_type == "PE" and start == bv.start:
+        start += 1
+
+    for mzx, pex, i in mz_xor:
+        for off, _ in bv.find_all_data(start, seg.end, mzx):
+            todo.append((off, mzx, pex, i))
+
+    while len(todo):
+        off, mzx, pex, i = todo.pop()
+
+        # The MZ header has one field we will check e_lfanew is at 0x3c
+        e_lfanew = off + 0x3C
+
+        if seg.end < (e_lfanew + 4):
+            continue
+
+        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(bv.read(e_lfanew, 4), i))[0]
+
+        peoff = off + newoff
+        if seg.end < (peoff + 2):
+            continue
+
+        if bv.read(peoff, 2) == pex:
+            yield off, i
+
+
+def extract_file_embedded_pe(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract embedded PE features"""
+    for seg in bv.segments:
+        for ea, _ in check_segment_for_pe(bv, seg):
+            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
+
+
+def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract function exports"""
+    for sym in bv.get_symbols_of_type(SymbolType.FunctionSymbol):
+        if sym.binding in [SymbolBinding.GlobalBinding, SymbolBinding.WeakBinding]:
+            name = sym.short_name
+            yield Export(name), AbsoluteVirtualAddress(sym.address)
+            unmangled_name = unmangle_c_name(name)
+            if name != unmangled_name:
+                yield Export(unmangled_name), AbsoluteVirtualAddress(sym.address)
+
+
+def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract function imports
+
+    1. imports by ordinal:
+     - modulename.#ordinal
+
+    2. imports by name, results in two features to support importname-only
+       matching:
+     - modulename.importname
+     - importname
+    """
+    for sym in bv.get_symbols_of_type(SymbolType.ImportAddressSymbol):
+        lib_name = str(sym.namespace)
+        addr = AbsoluteVirtualAddress(sym.address)
+        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym.short_name):
+            yield Import(name), addr
+
+        ordinal = sym.ordinal
+        if ordinal != 0 and (lib_name != ""):
+            ordinal_name = f"#{ordinal}"
+            for name in capa.features.extractors.helpers.generate_symbols(lib_name, ordinal_name):
+                yield Import(name), addr
+
+
+def extract_file_section_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract section names"""
+    for name, section in bv.sections.items():
+        yield Section(name), AbsoluteVirtualAddress(section.start)
+
+
+def extract_file_strings(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract ASCII and UTF-16 LE strings"""
+    for s in bv.strings:
+        yield String(s.value), FileOffsetAddress(s.start)
+
+
+def extract_file_function_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract the names of statically-linked library functions.
+    """
+    for sym_name in bv.symbols:
+        for sym in bv.symbols[sym_name]:
+            if sym.type == SymbolType.LibraryFunctionSymbol:
+                name = sym.short_name
+                yield FunctionName(name), sym.address
+                if name.startswith("_"):
+                    # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                    # extract features for both the mangled and un-mangled representations.
+                    # e.g. `_fwrite` -> `fwrite`
+                    # see: https://stackoverflow.com/a/2628384/87207
+                    yield FunctionName(name[1:]), sym.address
+
+
+def extract_file_format(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    view_type = bv.view_type
+    if view_type in ["PE", "COFF"]:
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif view_type == "ELF":
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    elif view_type == "Raw":
+        # no file type to return when processing a binary file, but we want to continue processing
+        return
+    else:
+        raise NotImplementedError(f"unexpected file format: {view_type}")
+
+
+def extract_features(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    """extract file features"""
+    for file_handler in FILE_HANDLERS:
+        for feature, addr in file_handler(bv):
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_strings,
+    extract_file_section_names,
+    extract_file_embedded_pe,
+    extract_file_function_names,
+    extract_file_format,
+)

capa/features/extractors/binja/find_binja_api.py

@@ -0,0 +1,35 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import subprocess
+from pathlib import Path
+
+# When the script gets executed as a standalone executable (via PyInstaller), `import binaryninja` does not work because
+# we have excluded the binaryninja module in `pyinstaller.spec`. The trick here is to call the system Python and try
+# to find out the path of the binaryninja module that has been installed.
+# Note, including the binaryninja module in the `pyintaller.spec` would not work, since the binaryninja module tries to
+# find the binaryninja core e.g., `libbinaryninjacore.dylib`, using a relative path. And this does not work when the
+# binaryninja module is extracted by the PyInstaller.
+code = r"""
+from pathlib import Path
+from importlib import util
+spec = util.find_spec('binaryninja')
+if spec is not None:
+    if len(spec.submodule_search_locations) > 0:
+            path = Path(spec.submodule_search_locations[0])
+            # encode the path with utf8 then convert to hex, make sure it can be read and restored properly
+            print(str(path.parent).encode('utf8').hex())
+"""
+
+
+def find_binja_path() -> Path:
+    raw_output = subprocess.check_output(["python", "-c", code]).decode("ascii").strip()
+    return Path(bytes.fromhex(raw_output).decode("utf8"))
+
+
+if __name__ == "__main__":
+    print(find_binja_path())

capa/features/extractors/binja/function.py

@@ -0,0 +1,68 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Tuple, Iterator
+
+from binaryninja import Function, BinaryView, LowLevelILOperation
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors import loops
+from capa.features.extractors.base_extractor import FunctionHandle
+
+
+def extract_function_calls_to(fh: FunctionHandle):
+    """extract callers to a function"""
+    func: Function = fh.inner
+
+    for caller in func.caller_sites:
+        # Everything that is a code reference to the current function is considered a caller, which actually includes
+        # many other references that are NOT a caller. For example, an instruction `push function_start` will also be
+        # considered a caller to the function
+        if caller.llil is not None and caller.llil.operation in [
+            LowLevelILOperation.LLIL_CALL,
+            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+            LowLevelILOperation.LLIL_JUMP,
+            LowLevelILOperation.LLIL_TAILCALL,
+        ]:
+            yield Characteristic("calls to"), AbsoluteVirtualAddress(caller.address)
+
+
+def extract_function_loop(fh: FunctionHandle):
+    """extract loop indicators from a function"""
+    func: Function = fh.inner
+
+    edges = []
+
+    # construct control flow graph
+    for bb in func.basic_blocks:
+        for edge in bb.outgoing_edges:
+            edges.append((bb.start, edge.target.start))
+
+    if loops.has_loop(edges):
+        yield Characteristic("loop"), fh.address
+
+
+def extract_recursive_call(fh: FunctionHandle):
+    """extract recursive function call"""
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+    if bv is None:
+        return
+
+    for ref in bv.get_code_refs(func.start):
+        if ref.function == func:
+            yield Characteristic("recursive call"), fh.address
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)

capa/features/extractors/binja/global_.py

@@ -0,0 +1,60 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+from typing import Tuple, Iterator
+
+from binaryninja import BinaryView
+
+from capa.features.common import OS, OS_MACOS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
+from capa.features.address import NO_ADDRESS, Address
+
+logger = logging.getLogger(__name__)
+
+
+def extract_os(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    name = bv.platform.name
+    if "-" in name:
+        name = name.split("-")[0]
+
+    if name == "windows":
+        yield OS(OS_WINDOWS), NO_ADDRESS
+
+    elif name == "macos":
+        yield OS(OS_MACOS), NO_ADDRESS
+
+    elif name in ["linux", "freebsd", "decree"]:
+        yield OS(name), NO_ADDRESS
+
+    else:
+        # we likely end up here:
+        #  1. handling shellcode, or
+        #  2. handling a new file format (e.g. macho)
+        #
+        # for (1) we can't do much - its shellcode and all bets are off.
+        # we could maybe accept a further CLI argument to specify the OS,
+        # but i think this would be rarely used.
+        # rules that rely on OS conditions will fail to match on shellcode.
+        #
+        # for (2), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported file format: %s, will not guess OS", name)
+        return
+
+
+def extract_arch(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
+    arch = bv.arch.name
+    if arch == "x86_64":
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+    elif arch == "x86":
+        yield Arch(ARCH_I386), NO_ADDRESS
+    else:
+        # we likely end up here:
+        #  1. handling a new architecture (e.g. aarch64)
+        #
+        # for (1), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported architecture: %s", arch)
+        return

capa/features/extractors/binja/helpers.py

@@ -0,0 +1,53 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import re
+from typing import List, Callable
+from dataclasses import dataclass
+
+from binaryninja import LowLevelILInstruction
+from binaryninja.architecture import InstructionTextToken
+
+
+@dataclass
+class DisassemblyInstruction:
+    address: int
+    length: int
+    text: List[InstructionTextToken]
+
+
+LLIL_VISITOR = Callable[[LowLevelILInstruction, LowLevelILInstruction, int], bool]
+
+
+def visit_llil_exprs(il: LowLevelILInstruction, func: LLIL_VISITOR):
+    # BN does not really support operand index at the disassembly level, so use the LLIL operand index as a substitute.
+    # Note, this is NOT always guaranteed to be the same as disassembly operand.
+    for i, op in enumerate(il.operands):
+        if isinstance(op, LowLevelILInstruction) and func(op, il, i):
+            visit_llil_exprs(op, func)
+
+
+def unmangle_c_name(name: str) -> str:
+    # https://learn.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170#FormatC
+    # Possible variations for BaseThreadInitThunk:
+    # @BaseThreadInitThunk@12
+    # _BaseThreadInitThunk
+    # _BaseThreadInitThunk@12
+    # It is also possible for a function to have a `Stub` appended to its name:
+    # _lstrlenWStub@4
+
+    # A small optimization to avoid running the regex too many times
+    # this still increases the unit test execution time from 170s to 200s, should be able to accelerate it
+    #
+    # TODO(xusheng): performance optimizations to improve test execution time
+    # https://github.com/mandiant/capa/issues/1610
+    if name[0] in ["@", "_"]:
+        match = re.match(r"^[@|_](.*?)(Stub)?(@\d+)?$", name)
+        if match:
+            return match.group(1)
+
+    return name

capa/features/extractors/binja/insn.py

@@ -0,0 +1,582 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Any, List, Tuple, Iterator, Optional
+
+from binaryninja import Function
+from binaryninja import BasicBlock as BinjaBasicBlock
+from binaryninja import (
+    BinaryView,
+    ILRegister,
+    SymbolType,
+    BinaryReader,
+    RegisterValueType,
+    LowLevelILOperation,
+    LowLevelILInstruction,
+)
+
+import capa.features.extractors.helpers
+from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
+from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.binja.helpers import DisassemblyInstruction, visit_llil_exprs
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+# security cookie checks may perform non-zeroing XORs, these are expected within a certain
+# byte range within the first and returning basic blocks, this helps to reduce FP features
+SECURITY_COOKIE_BYTES_DELTA = 0x40
+
+
+# check if a function is a stub function to another function/symbol. The criteria is:
+# 1. The function must only have one basic block
+# 2. The function must only make one call/jump to another address
+# If the function being checked is a stub function, returns the target address. Otherwise, return None.
+def is_stub_function(bv: BinaryView, addr: int) -> Optional[int]:
+    funcs = bv.get_functions_at(addr)
+    for func in funcs:
+        if len(func.basic_blocks) != 1:
+            continue
+
+        call_count = 0
+        call_target = None
+        for il in func.llil.instructions:
+            if il.operation in [
+                LowLevelILOperation.LLIL_CALL,
+                LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+                LowLevelILOperation.LLIL_JUMP,
+                LowLevelILOperation.LLIL_TAILCALL,
+            ]:
+                call_count += 1
+                if il.dest.value.type in [
+                    RegisterValueType.ImportedAddressValue,
+                    RegisterValueType.ConstantValue,
+                    RegisterValueType.ConstantPointerValue,
+                ]:
+                    call_target = il.dest.value.value
+
+        if call_count == 1 and call_target is not None:
+            return call_target
+
+    return None
+
+
+def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction API features
+
+    example:
+       call dword [0x00473038]
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    for llil in func.get_llils_at(ih.address):
+        if llil.operation in [
+            LowLevelILOperation.LLIL_CALL,
+            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+            LowLevelILOperation.LLIL_JUMP,
+            LowLevelILOperation.LLIL_TAILCALL,
+        ]:
+            if llil.dest.value.type not in [
+                RegisterValueType.ImportedAddressValue,
+                RegisterValueType.ConstantValue,
+                RegisterValueType.ConstantPointerValue,
+            ]:
+                continue
+            address = llil.dest.value.value
+            candidate_addrs = [address]
+            stub_addr = is_stub_function(bv, address)
+            if stub_addr is not None:
+                candidate_addrs.append(stub_addr)
+
+            for address in candidate_addrs:
+                sym = func.view.get_symbol_at(address)
+                if sym is None or sym.type not in [SymbolType.ImportAddressSymbol, SymbolType.ImportedFunctionSymbol]:
+                    continue
+
+                sym_name = sym.short_name
+
+                lib_name = ""
+                import_lib = bv.lookup_imported_object_library(sym.address)
+                if import_lib is not None:
+                    lib_name = import_lib[0].name
+                    if lib_name.endswith(".dll"):
+                        lib_name = lib_name[:-4]
+                    elif lib_name.endswith(".so"):
+                        lib_name = lib_name[:-3]
+
+                for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name):
+                    yield API(name), ih.address
+
+                if sym_name.startswith("_"):
+                    for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name[1:]):
+                        yield API(name), ih.address
+
+
+def extract_insn_number_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction number features
+    example:
+        push    3136B0h         ; dwControlCode
+    """
+    func: Function = fh.inner
+
+    results: List[Tuple[Any[Number, OperandNumber], Address]] = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation == LowLevelILOperation.LLIL_LOAD:
+            return False
+
+        if il.operation not in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            return True
+
+        for op in parent.operands:
+            if isinstance(op, ILRegister) and op.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
+                return False
+            elif isinstance(op, LowLevelILInstruction) and op.operation == LowLevelILOperation.LLIL_REG:
+                if op.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
+                    return False
+
+        raw_value = il.value.value
+        if parent.operation == LowLevelILOperation.LLIL_SUB:
+            raw_value = -raw_value
+
+        results.append((Number(raw_value), ih.address))
+        results.append((OperandNumber(index, raw_value), ih.address))
+
+        return False
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse referenced byte sequences
+    example:
+        push    offset iid_004118d4_IShellLinkA ; riid
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    candidate_addrs = set()
+
+    llil = func.get_llil_at(ih.address)
+    if llil is None or llil.operation in [LowLevelILOperation.LLIL_CALL, LowLevelILOperation.LLIL_CALL_STACK_ADJUST]:
+        return
+
+    for ref in bv.get_code_refs_from(ih.address):
+        if ref == ih.address:
+            continue
+
+        if len(bv.get_functions_containing(ref)) > 0:
+            continue
+
+        candidate_addrs.add(ref)
+
+    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            value = il.value.value
+            if value > 0:
+                candidate_addrs.add(value)
+            return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    for addr in candidate_addrs:
+        extracted_bytes = bv.read(addr, MAX_BYTES_FEATURE_SIZE)
+        if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
+            if bv.get_string_at(addr) is None:
+                # don't extract byte features for obvious strings
+                yield Bytes(extracted_bytes), ih.address
+
+
+def extract_insn_string_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction string features
+
+    example:
+        push offset aAcr     ; "ACR  > "
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    candidate_addrs = set()
+
+    # collect candidate address from code refs directly
+    for ref in bv.get_code_refs_from(ih.address):
+        if ref == ih.address:
+            continue
+
+        if len(bv.get_functions_containing(ref)) > 0:
+            continue
+
+        candidate_addrs.add(ref)
+
+    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            value = il.value.value
+            if value > 0:
+                candidate_addrs.add(value)
+            return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    # Now we have all the candidate address, check them for string or pointer to string
+    br = BinaryReader(bv)
+    for addr in candidate_addrs:
+        found = bv.get_string_at(addr)
+        if found:
+            yield String(found.value), ih.address
+
+        br.seek(addr)
+        pointer = None
+        if bv.arch.address_size == 4:
+            pointer = br.read32()
+        elif bv.arch.address_size == 8:
+            pointer = br.read64()
+
+        if pointer is not None:
+            found = bv.get_string_at(pointer)
+            if found:
+                yield String(found.value), ih.address
+
+
+def extract_insn_offset_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction structure offset features
+
+    example:
+        .text:0040112F cmp [esi+4], ebx
+    """
+    func: Function = fh.inner
+
+    results: List[Tuple[Any[Offset, OperandOffset], Address]] = []
+    address_size = func.view.arch.address_size * 8
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        #  The most common case, read/write dereference to something like `dword [eax+0x28]`
+        if il.operation in [LowLevelILOperation.LLIL_ADD, LowLevelILOperation.LLIL_SUB]:
+            left = il.left
+            right = il.right
+            # Exclude offsets based on stack/franme pointers
+            if left.operation == LowLevelILOperation.LLIL_REG and left.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
+                return True
+
+            if right.operation != LowLevelILOperation.LLIL_CONST:
+                return True
+
+            raw_value = right.value.value
+            # If this is not a dereference, then this must be an add and the offset must be in the range \
+            # [0, MAX_STRUCTURE_SIZE]. For example,
+            # add eax, 0x10,
+            # lea ebx, [eax + 1]
+            if parent.operation not in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
+                if il.operation != LowLevelILOperation.LLIL_ADD or (not 0 < raw_value < MAX_STRUCTURE_SIZE):
+                    return False
+
+            if address_size > 0:
+                # BN also encodes the constant value as two's complement, we need to restore its original value
+                value = capa.features.extractors.helpers.twos_complement(raw_value, address_size)
+            else:
+                value = raw_value
+
+            results.append((Offset(value), ih.address))
+            results.append((OperandOffset(index, value), ih.address))
+            return False
+
+        # An edge case: for code like `push dword [esi]`, we need to generate a feature for offset 0x0
+        elif il.operation in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
+            if il.operands[0].operation == LowLevelILOperation.LLIL_REG:
+                results.append((Offset(0), ih.address))
+                results.append((OperandOffset(index, 0), ih.address))
+                return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def is_nzxor_stack_cookie(f: Function, bb: BinjaBasicBlock, llil: LowLevelILInstruction) -> bool:
+    """check if nzxor exists within stack cookie delta"""
+    # TODO(xusheng): use LLIL SSA to do more accurate analysis
+    # https://github.com/mandiant/capa/issues/1609
+
+    reg_names = []
+    if llil.left.operation == LowLevelILOperation.LLIL_REG:
+        reg_names.append(llil.left.src.name)
+
+    if llil.right.operation == LowLevelILOperation.LLIL_REG:
+        reg_names.append(llil.right.src.name)
+
+    # stack cookie reg should be stack/frame pointer
+    if not any(reg in ["ebp", "esp", "rbp", "rsp", "sp"] for reg in reg_names):
+        return False
+
+    # expect security cookie init in first basic block within first bytes (instructions)
+    if len(bb.incoming_edges) == 0 and llil.address < (bb.start + SECURITY_COOKIE_BYTES_DELTA):
+        return True
+
+    # ... or within last bytes (instructions) before a return
+    if len(bb.outgoing_edges) == 0 and llil.address > (bb.end - SECURITY_COOKIE_BYTES_DELTA):
+        return True
+
+    return False
+
+
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction non-zeroing XOR instruction
+    ignore expected non-zeroing XORs, e.g. security cookies
+    """
+    func: Function = fh.inner
+
+    results = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        # If the two operands of the xor instruction are the same, the LLIL will be translated to other instructions,
+        # e.g., <llil: eax = 0>, (LLIL_SET_REG). So we do not need to check whether the two operands are the same.
+        if il.operation == LowLevelILOperation.LLIL_XOR:
+            # Exclude cases related to the stack cookie
+            if is_nzxor_stack_cookie(fh.inner, bbh.inner[0], il):
+                return False
+            results.append((Characteristic("nzxor"), ih.address))
+            return False
+        else:
+            return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_mnemonic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction mnemonic features"""
+    insn: DisassemblyInstruction = ih.inner
+    yield Mnemonic(insn.text[0].text), ih.address
+
+
+def extract_insn_obfs_call_plus_5_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse call $+5 instruction from the given instruction.
+    """
+    insn: DisassemblyInstruction = ih.inner
+    if insn.text[0].text == "call" and insn.text[2].text == "$+5" and insn.length == 5:
+        yield Characteristic("call $+5"), ih.address
+
+
+def extract_insn_peb_access_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction peb access
+
+    fs:[0x30] on x86, gs:[0x60] on x64
+    """
+    func: Function = fh.inner
+
+    results = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILOperation, index: int) -> bool:
+        if il.operation != LowLevelILOperation.LLIL_LOAD:
+            return True
+
+        src = il.src
+        if src.operation != LowLevelILOperation.LLIL_ADD:
+            return True
+
+        left = src.left
+        right = src.right
+
+        if left.operation != LowLevelILOperation.LLIL_REG:
+            return True
+
+        reg = left.src.name
+
+        if right.operation != LowLevelILOperation.LLIL_CONST:
+            return True
+
+        value = right.value.value
+        if (reg, value) not in (("fsbase", 0x30), ("gsbase", 0x60)):
+            return True
+
+        results.append((Characteristic("peb access"), ih.address))
+        return False
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_segment_access_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction fs or gs access"""
+    func: Function = fh.inner
+
+    results = []
+
+    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
+        if il.operation == LowLevelILOperation.LLIL_REG:
+            reg = il.src.name
+            if reg == "fsbase":
+                results.append((Characteristic("fs access"), ih.address))
+                return False
+            elif reg == "gsbase":
+                results.append((Characteristic("gs access"), ih.address))
+                return False
+            return False
+
+        return True
+
+    for llil in func.get_llils_at(ih.address):
+        visit_llil_exprs(llil, llil_checker)
+
+    yield from results
+
+
+def extract_insn_cross_section_cflow(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    if bv is None:
+        return
+
+    seg1 = bv.get_segment_at(ih.address)
+    sections1 = bv.get_sections_at(ih.address)
+    for ref in bv.get_code_refs_from(ih.address):
+        if len(bv.get_functions_at(ref)) == 0:
+            continue
+
+        seg2 = bv.get_segment_at(ref)
+        sections2 = bv.get_sections_at(ref)
+        if seg1 != seg2 or sections1 != sections2:
+            yield Characteristic("cross section flow"), ih.address
+
+
+def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract functions calls from features
+
+    most relevant at the function scope, however, its most efficient to extract at the instruction scope
+    """
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+
+    if bv is None:
+        return
+
+    for il in func.get_llils_at(ih.address):
+        if il.operation not in [
+            LowLevelILOperation.LLIL_CALL,
+            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+            LowLevelILOperation.LLIL_TAILCALL,
+        ]:
+            continue
+
+        dest = il.dest
+        if dest.operation == LowLevelILOperation.LLIL_CONST_PTR:
+            value = dest.value.value
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
+        elif dest.operation == LowLevelILOperation.LLIL_CONST:
+            yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value)
+        elif dest.operation == LowLevelILOperation.LLIL_LOAD:
+            indirect_src = dest.src
+            if indirect_src.operation == LowLevelILOperation.LLIL_CONST_PTR:
+                value = indirect_src.value.value
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
+            elif indirect_src.operation == LowLevelILOperation.LLIL_CONST:
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(indirect_src.value)
+        elif dest.operation == LowLevelILOperation.LLIL_REG:
+            if dest.value.type in [
+                RegisterValueType.ImportedAddressValue,
+                RegisterValueType.ConstantValue,
+                RegisterValueType.ConstantPointerValue,
+            ]:
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value.value)
+
+
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
+    does not include calls like => call ds:dword_ABD4974
+
+    most relevant at the function or basic block scope;
+    however, its most efficient to extract at the instruction scope
+    """
+    func: Function = fh.inner
+
+    llil = func.get_llil_at(ih.address)
+    if llil is None or llil.operation not in [
+        LowLevelILOperation.LLIL_CALL,
+        LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
+        LowLevelILOperation.LLIL_TAILCALL,
+    ]:
+        return
+
+    if llil.dest.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+        return
+
+    if llil.dest.operation == LowLevelILOperation.LLIL_LOAD:
+        src = llil.dest.src
+        if src.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
+            return
+
+    yield Characteristic("indirect call"), ih.address
+
+
+def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract instruction features"""
+    for inst_handler in INSTRUCTION_HANDLERS:
+        for feature, ea in inst_handler(f, bbh, insn):
+            yield feature, ea
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_bytes_features,
+    extract_insn_string_features,
+    extract_insn_offset_features,
+    extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    extract_insn_obfs_call_plus_5_characteristic_features,
+    extract_insn_peb_access_characteristic_features,
+    extract_insn_cross_section_cflow,
+    extract_insn_segment_access_features,
+    extract_function_calls_from,
+    extract_function_indirect_call_characteristic_features,
+)

capa/features/extractors/common.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import io
 import logging
 import binascii
@@ -12,11 +19,14 @@ import capa.features.extractors.pefile
 import capa.features.extractors.strings
 from capa.features.common import (
     OS,
+    OS_ANY,
     OS_AUTO,
+    ARCH_ANY,
     FORMAT_PE,
     FORMAT_ELF,
     OS_WINDOWS,
     FORMAT_FREEZE,
+    FORMAT_RESULT,
     Arch,
     Format,
     String,
@@ -27,6 +37,11 @@ from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress
 
 logger = logging.getLogger(__name__)
 
+# match strings for formats
+MATCH_PE = b"MZ"
+MATCH_ELF = b"\x7fELF"
+MATCH_RESULT = b'{"meta":'
+
 
 def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[String, Address]]:
     """
@@ -40,12 +55,14 @@ def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[String, Address]]:
 
 
 def extract_format(buf) -> Iterator[Tuple[Feature, Address]]:
-    if buf.startswith(b"MZ"):
+    if buf.startswith(MATCH_PE):
         yield Format(FORMAT_PE), NO_ADDRESS
-    elif buf.startswith(b"\x7fELF"):
+    elif buf.startswith(MATCH_ELF):
         yield Format(FORMAT_ELF), NO_ADDRESS
     elif is_freeze(buf):
         yield Format(FORMAT_FREEZE), NO_ADDRESS
+    elif buf.startswith(MATCH_RESULT):
+        yield Format(FORMAT_RESULT), NO_ADDRESS
     else:
         # we likely end up here:
         #  1. handling a file format (e.g. macho)
@@ -56,10 +73,13 @@ def extract_format(buf) -> Iterator[Tuple[Feature, Address]]:
 
 
 def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
-    if buf.startswith(b"MZ"):
+    if buf.startswith(MATCH_PE):
         yield from capa.features.extractors.pefile.extract_file_arch(pe=pefile.PE(data=buf))
 
-    elif buf.startswith(b"\x7fELF"):
+    elif buf.startswith(MATCH_RESULT):
+        yield Arch(ARCH_ANY), NO_ADDRESS
+
+    elif buf.startswith(MATCH_ELF):
         with contextlib.closing(io.BytesIO(buf)) as f:
             arch = capa.features.extractors.elf.detect_elf_arch(f)
 
@@ -88,9 +108,11 @@ def extract_os(buf, os=OS_AUTO) -> Iterator[Tuple[Feature, Address]]:
     if os != OS_AUTO:
         yield OS(os), NO_ADDRESS
 
-    if buf.startswith(b"MZ"):
+    if buf.startswith(MATCH_PE):
         yield OS(OS_WINDOWS), NO_ADDRESS
-    elif buf.startswith(b"\x7fELF"):
+    elif buf.startswith(MATCH_RESULT):
+        yield OS(OS_ANY), NO_ADDRESS
+    elif buf.startswith(MATCH_ELF):
         with contextlib.closing(io.BytesIO(buf)) as f:
             os = capa.features.extractors.elf.detect_elf_os(f)
 

capa/features/extractors/dnfile/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -9,6 +9,7 @@
 from __future__ import annotations
 
 from typing import Dict, List, Tuple, Union, Iterator, Optional
+from pathlib import Path
 
 import dnfile
 from dncil.cil.opcode import OpCodes
@@ -52,25 +53,25 @@ class DnFileFeatureExtractorCache:
             self.types[type_.token] = type_
 
     def get_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.imports.get(token, None)
+        return self.imports.get(token)
 
     def get_native_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.native_imports.get(token, None)
+        return self.native_imports.get(token)
 
     def get_method(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.methods.get(token, None)
+        return self.methods.get(token)
 
     def get_field(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.fields.get(token, None)
+        return self.fields.get(token)
 
     def get_type(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.types.get(token, None)
+        return self.types.get(token)
 
 
 class DnfileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
+    def __init__(self, path: Path):
         super().__init__()
-        self.pe: dnfile.dnPE = dnfile.dnPE(path)
+        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
 
         # pre-compute .NET token lookup tables; each .NET method has access to this cache for feature extraction
         # most relevant at instruction scope
@@ -119,7 +120,7 @@ class DnfileFeatureExtractor(FeatureExtractor):
                 address: DNTokenAddress = DNTokenAddress(insn.operand.value)
 
                 # record call to destination method; note: we only consider MethodDef methods for destinations
-                dest: Optional[FunctionHandle] = methods.get(address, None)
+                dest: Optional[FunctionHandle] = methods.get(address)
                 if dest is not None:
                     dest.ctx["calls_to"].add(fh.address)
 

capa/features/extractors/dnfile/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/dnfile/function.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/dnfile/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -52,7 +52,7 @@ def resolve_dotnet_token(pe: dnfile.dnPE, token: Token) -> Union[dnfile.base.MDT
             return InvalidToken(token.value)
         return user_string
 
-    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(token.table, None)
+    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(token.table)
     if table is None:
         # table index is not valid
         return InvalidToken(token.value)
@@ -204,7 +204,7 @@ def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
                 continue
 
             token: int = calculate_dotnet_token_value(method.table.number, method.row_index)
-            access: Optional[str] = accessor_map.get(token, None)
+            access: Optional[str] = accessor_map.get(token)
 
             method_name: str = method.row.Name
             if method_name.startswith(("get_", "set_")):

capa/features/extractors/dnfile/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -9,7 +9,7 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Dict, Tuple, Union, Iterator, Optional
+from typing import TYPE_CHECKING, Tuple, Union, Iterator, Optional
 
 if TYPE_CHECKING:
     from capa.features.extractors.dnfile.extractor import DnFileFeatureExtractorCache

capa/features/extractors/dnfile/types.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,11 +6,10 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from enum import Enum
-from typing import Union, Optional
+from typing import Optional
 
 
-class DnType(object):
+class DnType:
     def __init__(self, token: int, class_: str, namespace: str = "", member: str = "", access: Optional[str] = None):
         self.token: int = token
         self.access: Optional[str] = access

capa/features/extractors/dnfile_.py

@@ -1,5 +1,13 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import logging
 from typing import Tuple, Iterator
+from pathlib import Path
 
 import dnfile
 import pefile
@@ -74,10 +82,10 @@ GLOBAL_HANDLERS = (
 
 
 class DnfileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
+    def __init__(self, path: Path):
         super().__init__()
-        self.path: str = path
-        self.pe: dnfile.dnPE = dnfile.dnPE(path)
+        self.path: Path = path
+        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
 
     def get_base_address(self) -> AbsoluteVirtualAddress:
         return AbsoluteVirtualAddress(0x0)

capa/features/extractors/dotnetfile.py

@@ -1,5 +1,13 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import logging
-from typing import Tuple, Iterator, cast
+from typing import Tuple, Iterator
+from pathlib import Path
 
 import dnfile
 import pefile
@@ -158,10 +166,10 @@ GLOBAL_HANDLERS = (
 
 
 class DotnetFileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
+    def __init__(self, path: Path):
         super().__init__()
-        self.path: str = path
-        self.pe: dnfile.dnPE = dnfile.dnPE(path)
+        self.path: Path = path
+        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
 
     def get_base_address(self):
         return NO_ADDRESS

capa/features/extractors/elf.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -24,7 +24,7 @@ def align(v, alignment):
         return v + (alignment - remainder)
 
 
-def read_cstr(buf, offset):
+def read_cstr(buf, offset) -> str:
     s = buf[offset:]
     s, _, _ = s.partition(b"\x00")
     return s.decode("utf-8")
@@ -88,8 +88,23 @@ class Shdr:
     offset: int
     size: int
     link: int
+    entsize: int
     buf: bytes
 
+    @classmethod
+    def from_viv(cls, section, buf: bytes) -> "Shdr":
+        return cls(
+            section.sh_name,
+            section.sh_type,
+            section.sh_flags,
+            section.sh_addr,
+            section.sh_offset,
+            section.sh_size,
+            section.sh_link,
+            section.sh_entsize,
+            buf,
+        )
+
 
 class ELF:
     def __init__(self, f: BinaryIO):
@@ -320,12 +335,12 @@ class ELF:
         shent = self.shbuf[shent_offset : shent_offset + self.e_shentsize]
 
         if self.bitness == 32:
-            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link = struct.unpack_from(
-                self.endian + "IIIIIII", shent, 0x0
+            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link, _, _, sh_entsize = struct.unpack_from(
+                self.endian + "IIIIIIIIII", shent, 0x0
             )
         elif self.bitness == 64:
-            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link = struct.unpack_from(
-                self.endian + "IIQQQQI", shent, 0x0
+            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link, _, _, sh_entsize = struct.unpack_from(
+                self.endian + "IIQQQQIIQQ", shent, 0x0
             )
         else:
             raise NotImplementedError()
@@ -337,7 +352,7 @@ class ELF:
         if len(buf) != sh_size:
             raise ValueError("failed to read section header content")
 
-        return Shdr(sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link, buf)
+        return Shdr(sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link, sh_entsize, buf)
 
     @property
     def section_headers(self):
@@ -396,7 +411,7 @@ class ELF:
                 # there should be vn_cnt of these.
                 # each entry describes an ABI name required by the shared object.
                 vna_offset = vn_offset + vn_aux
-                for i in range(vn_cnt):
+                for _ in range(vn_cnt):
                     # ElfXX_Vernaux layout is the same on 32 and 64 bit
                     _, _, _, vna_name, vna_next = struct.unpack_from(self.endian + "IHHII", shdr.buf, vna_offset)
 
@@ -457,10 +472,12 @@ class ELF:
         for d_tag, d_val in self.dynamic_entries:
             if d_tag == DT_STRTAB:
                 strtab_addr = d_val
+                break
 
         for d_tag, d_val in self.dynamic_entries:
             if d_tag == DT_STRSZ:
                 strtab_size = d_val
+                break
 
         if strtab_addr is None:
             return None
@@ -470,8 +487,10 @@ class ELF:
 
         strtab_offset = None
         for shdr in self.section_headers:
-            if shdr.addr <= strtab_addr < shdr.addr + shdr.size:
+            # the section header address should be defined
+            if shdr.addr and shdr.addr <= strtab_addr < shdr.addr + shdr.size:
                 strtab_offset = shdr.offset + (strtab_addr - shdr.addr)
+                break
 
         if strtab_offset is None:
             return None
@@ -500,7 +519,27 @@ class ELF:
             if d_tag != DT_NEEDED:
                 continue
 
-            yield read_cstr(strtab, d_val)
+            try:
+                yield read_cstr(strtab, d_val)
+            except UnicodeDecodeError as e:
+                logger.warning("failed to read DT_NEEDED entry: %s", str(e))
+
+    @property
+    def symtab(self) -> Optional[Tuple[Shdr, Shdr]]:
+        """
+        fetch the Shdr for the symtab and the associated strtab.
+        """
+        SHT_SYMTAB = 0x2
+        for shdr in self.section_headers:
+            if shdr.type != SHT_SYMTAB:
+                continue
+
+            # the linked section contains strings referenced by the symtab structures.
+            strtab_shdr = self.parse_section_header(shdr.link)
+
+            return shdr, strtab_shdr
+
+        return None
 
 
 @dataclass
@@ -603,11 +642,101 @@ class SHNote:
         return ABITag(os, kmajor, kminor, kpatch)
 
 
-def guess_os_from_osabi(elf) -> Optional[OS]:
+@dataclass
+class Symbol:
+    name_offset: int
+    value: int
+    size: int
+    info: int
+    other: int
+    shndx: int
+
+
+class SymTab:
+    def __init__(
+        self,
+        endian: str,
+        bitness: int,
+        symtab: Shdr,
+        strtab: Shdr,
+    ) -> None:
+        self.symbols: List[Symbol] = []
+
+        self.symtab = symtab
+        self.strtab = strtab
+
+        self._parse(endian, bitness, symtab.buf)
+
+    def _parse(self, endian: str, bitness: int, symtab_buf: bytes) -> None:
+        """
+        return the symbol's information in
+        the order specified by sys/elf32.h
+        """
+        if self.symtab.entsize == 0:
+            return
+
+        for i in range(int(len(self.symtab.buf) / self.symtab.entsize)):
+            if bitness == 32:
+                name_offset, value, size, info, other, shndx = struct.unpack_from(
+                    endian + "IIIBBH", symtab_buf, i * self.symtab.entsize
+                )
+            elif bitness == 64:
+                name_offset, info, other, shndx, value, size = struct.unpack_from(
+                    endian + "IBBHQQ", symtab_buf, i * self.symtab.entsize
+                )
+
+            self.symbols.append(Symbol(name_offset, value, size, info, other, shndx))
+
+    def get_name(self, symbol: Symbol) -> str:
+        """
+        fetch a symbol's name from symtab's
+        associated strings' section (SHT_STRTAB)
+        """
+        if not self.strtab:
+            raise ValueError("no strings found")
+
+        for i in range(symbol.name_offset, self.strtab.size):
+            if self.strtab.buf[i] == 0:
+                return self.strtab.buf[symbol.name_offset : i].decode("utf-8")
+
+        raise ValueError("symbol name not found")
+
+    def get_symbols(self) -> Iterator[Symbol]:
+        """
+        return a tuple: (name, value, size, info, other, shndx)
+        for each symbol contained in the symbol table
+        """
+        yield from self.symbols
+
+    @classmethod
+    def from_Elf(cls, ElfBinary) -> Optional["SymTab"]:
+        endian = "<" if ElfBinary.getEndian() == 0 else ">"
+        bitness = ElfBinary.bits
+
+        SHT_SYMTAB = 0x2
+        for section in ElfBinary.sections:
+            if section.sh_info & SHT_SYMTAB:
+                strtab_section = ElfBinary.sections[section.sh_link]
+                sh_symtab = Shdr.from_viv(section, ElfBinary.readAtOffset(section.sh_offset, section.sh_size))
+                sh_strtab = Shdr.from_viv(
+                    strtab_section, ElfBinary.readAtOffset(strtab_section.sh_offset, strtab_section.sh_size)
+                )
+
+        try:
+            return cls(endian, bitness, sh_symtab, sh_strtab)
+        except NameError:
+            return None
+        except Exception:
+            # all exceptions that could be encountered by
+            # cls._parse() imply a faulty symbol's table.
+            raise CorruptElfFile("malformed symbol's table")
+
+
+def guess_os_from_osabi(elf: ELF) -> Optional[OS]:
     return elf.ei_osabi
 
 
-def guess_os_from_ph_notes(elf) -> Optional[OS]:
+def guess_os_from_ph_notes(elf: ELF) -> Optional[OS]:
     # search for PT_NOTE sections that specify an OS
     # for example, on Linux there is a GNU section with minimum kernel version
     PT_NOTE = 0x4
@@ -646,7 +775,7 @@ def guess_os_from_ph_notes(elf) -> Optional[OS]:
     return None
 
 
-def guess_os_from_sh_notes(elf) -> Optional[OS]:
+def guess_os_from_sh_notes(elf: ELF) -> Optional[OS]:
     # search for notes stored in sections that aren't visible in program headers.
     # e.g. .note.Linux in Linux kernel modules.
     SHT_NOTE = 0x7
@@ -679,7 +808,7 @@ def guess_os_from_sh_notes(elf) -> Optional[OS]:
     return None
 
 
-def guess_os_from_linker(elf) -> Optional[OS]:
+def guess_os_from_linker(elf: ELF) -> Optional[OS]:
     # search for recognizable dynamic linkers (interpreters)
     # for example, on linux, we see file paths like: /lib64/ld-linux-x86-64.so.2
     linker = elf.linker
@@ -689,12 +818,12 @@ def guess_os_from_linker(elf) -> Optional[OS]:
     return None
 
 
-def guess_os_from_abi_versions_needed(elf) -> Optional[OS]:
+def guess_os_from_abi_versions_needed(elf: ELF) -> Optional[OS]:
     # then lets look for GLIBC symbol versioning requirements.
     # this will let us guess about linux/hurd in some cases.
 
     versions_needed = elf.versions_needed
-    if any(map(lambda abi: abi.startswith("GLIBC"), itertools.chain(*versions_needed.values()))):
+    if any(abi.startswith("GLIBC") for abi in itertools.chain(*versions_needed.values())):
         # there are any GLIBC versions needed
 
         if elf.e_machine != "i386":
@@ -720,7 +849,7 @@ def guess_os_from_abi_versions_needed(elf) -> Optional[OS]:
     return None
 
 
-def guess_os_from_needed_dependencies(elf) -> Optional[OS]:
+def guess_os_from_needed_dependencies(elf: ELF) -> Optional[OS]:
     for needed in elf.needed:
         if needed.startswith("libmachuser.so"):
             return OS.HURD
@@ -730,29 +859,91 @@ def guess_os_from_needed_dependencies(elf) -> Optional[OS]:
     return None
 
 
+def guess_os_from_symtab(elf: ELF) -> Optional[OS]:
+    shdrs = elf.symtab
+    if not shdrs:
+        # executable does not contain a symbol table
+        # or the symbol's names are stripped
+        return None
+
+    symtab_shdr, strtab_shdr = shdrs
+    symtab = SymTab(elf.endian, elf.bitness, symtab_shdr, strtab_shdr)
+
+    keywords = {
+        OS.LINUX: [
+            "linux",
+            "/linux/",
+        ],
+    }
+
+    for symbol in symtab.get_symbols():
+        sym_name = symtab.get_name(symbol)
+
+        for os, hints in keywords.items():
+            if any(hint in sym_name for hint in hints):
+                return os
+
+    return None
+
+
 def detect_elf_os(f) -> str:
     """
     f: type Union[BinaryIO, IDAIO]
     """
-    elf = ELF(f)
+    try:
+        elf = ELF(f)
+    except Exception as e:
+        logger.warning("Error parsing ELF file: %s", e)
+        return "unknown"
 
-    osabi_guess = guess_os_from_osabi(elf)
-    logger.debug("guess: osabi: %s", osabi_guess)
+    try:
+        osabi_guess = guess_os_from_osabi(elf)
+        logger.debug("guess: osabi: %s", osabi_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from OSABI: %s", e)
+        osabi_guess = None
 
-    ph_notes_guess = guess_os_from_ph_notes(elf)
-    logger.debug("guess: ph notes: %s", ph_notes_guess)
+    try:
+        ph_notes_guess = guess_os_from_ph_notes(elf)
+        logger.debug("guess: ph notes: %s", ph_notes_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from program header notes: %s", e)
+        ph_notes_guess = None
 
-    sh_notes_guess = guess_os_from_sh_notes(elf)
-    logger.debug("guess: sh notes: %s", sh_notes_guess)
+    try:
+        sh_notes_guess = guess_os_from_sh_notes(elf)
+        logger.debug("guess: sh notes: %s", sh_notes_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from section header notes: %s", e)
+        sh_notes_guess = None
 
-    linker_guess = guess_os_from_linker(elf)
-    logger.debug("guess: linker: %s", linker_guess)
+    try:
+        linker_guess = guess_os_from_linker(elf)
+        logger.debug("guess: linker: %s", linker_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from linker: %s", e)
+        linker_guess = None
 
-    abi_versions_needed_guess = guess_os_from_abi_versions_needed(elf)
-    logger.debug("guess: ABI versions needed: %s", abi_versions_needed_guess)
+    try:
+        abi_versions_needed_guess = guess_os_from_abi_versions_needed(elf)
+        logger.debug("guess: ABI versions needed: %s", abi_versions_needed_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from ABI versions needed: %s", e)
+        abi_versions_needed_guess = None
 
-    needed_dependencies_guess = guess_os_from_needed_dependencies(elf)
-    logger.debug("guess: needed dependencies: %s", needed_dependencies_guess)
+    try:
+        needed_dependencies_guess = guess_os_from_needed_dependencies(elf)
+        logger.debug("guess: needed dependencies: %s", needed_dependencies_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from needed dependencies: %s", e)
+        needed_dependencies_guess = None
+
+    try:
+        symtab_guess = guess_os_from_symtab(elf)
+        logger.debug("guess: pertinent symbol name: %s", symtab_guess)
+    except Exception as e:
+        logger.warning("Error guessing OS from symbol table: %s", e)
+        symtab_guess = None
 
     ret = None
 
@@ -774,6 +965,9 @@ def detect_elf_os(f) -> str:
     elif needed_dependencies_guess:
         ret = needed_dependencies_guess
 
+    elif symtab_guess:
+        ret = symtab_guess
+
     return ret.value if ret is not None else "unknown"
 
 

capa/features/extractors/elffile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,6 +8,7 @@
 import io
 import logging
 from typing import Tuple, Iterator
+from pathlib import Path
 
 from elftools.elf.elffile import ELFFile, SymbolTableSection
 
@@ -36,8 +37,8 @@ def extract_file_import_names(elf, **kwargs):
 
         for _, symbol in enumerate(section.iter_symbols()):
             if symbol.name and symbol.entry.st_info.type == "STT_FUNC":
-                # TODO symbol address
-                # TODO symbol version info?
+                # TODO(williballenthin): extract symbol address
+                # https://github.com/mandiant/capa/issues/1608
                 yield Import(symbol.name), FileOffsetAddress(0x0)
 
 
@@ -68,7 +69,6 @@ def extract_file_format(**kwargs):
 
 
 def extract_file_arch(elf, **kwargs):
-    # TODO merge with capa.features.extractors.elf.detect_elf_arch()
     arch = elf.get_machine_arch()
     if arch == "x86":
         yield Arch("i386"), NO_ADDRESS
@@ -85,7 +85,8 @@ def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, i
 
 
 FILE_HANDLERS = (
-    # TODO extract_file_export_names,
+    # TODO(williballenthin): implement extract_file_export_names
+    # https://github.com/mandiant/capa/issues/1607
     extract_file_import_names,
     extract_file_section_names,
     extract_file_strings,
@@ -107,11 +108,10 @@ GLOBAL_HANDLERS = (
 
 
 class ElfFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
+    def __init__(self, path: Path):
         super().__init__()
-        self.path = path
-        with open(self.path, "rb") as f:
-            self.elf = ELFFile(io.BytesIO(f.read()))
+        self.path: Path = path
+        self.elf = ELFFile(io.BytesIO(path.read_bytes()))
 
     def get_base_address(self):
         # virtual address of the first segment with type LOAD
@@ -120,15 +120,13 @@ class ElfFeatureExtractor(FeatureExtractor):
                 return AbsoluteVirtualAddress(segment.header.p_vaddr)
 
     def extract_global_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = self.path.read_bytes()
 
         for feature, addr in extract_global_features(self.elf, buf):
             yield feature, addr
 
     def extract_file_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = self.path.read_bytes()
 
         for feature, addr in extract_file_features(self.elf, buf):
             yield feature, addr

capa/features/extractors/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -70,6 +70,23 @@ def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
             yield symbol[:-1]
 
 
+def reformat_forwarded_export_name(forwarded_name: str) -> str:
+    """
+    a forwarded export has a DLL name/path an symbol name.
+    we want the former to be lowercase, and the latter to be verbatim.
+    """
+
+    # use rpartition so we can split on separator between dll and name.
+    # the dll name can be a full path, like in the case of
+    # ef64d6d7c34250af8e21a10feb931c9b
+    # which i assume means the path can have embedded periods.
+    # so we don't want the first period, we want the last.
+    forwarded_dll, _, forwarded_symbol = forwarded_name.rpartition(".")
+    forwarded_dll = forwarded_dll.lower()
+
+    return f"{forwarded_dll}.{forwarded_symbol}"
+
+
 def all_zeros(bytez: bytes) -> bool:
     return all(b == 0 for b in builtins.bytes(bytez))
 

capa/features/extractors/ida/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -104,19 +104,3 @@ BASIC_BLOCK_HANDLERS = (
     extract_bb_tight_loop,
     extract_bb_stackstring,
 )
-
-
-def main():
-    features = []
-    for fhandle in helpers.get_functions(skip_thunks=True, skip_libs=True):
-        f: idaapi.func_t = fhandle.inner
-        for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
-            features.extend(list(extract_features(fhandle, bb)))
-
-    import pprint
-
-    pprint.pprint(features)
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ida/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/ida/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -12,6 +12,7 @@ from typing import Tuple, Iterator
 import idc
 import idaapi
 import idautils
+import ida_entry
 
 import capa.features.extractors.common
 import capa.features.extractors.helpers
@@ -83,8 +84,14 @@ def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
 
 def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
     """extract function exports"""
-    for _, _, ea, name in idautils.Entries():
-        yield Export(name), AbsoluteVirtualAddress(ea)
+    for _, ordinal, ea, name in idautils.Entries():
+        forwarded_name = ida_entry.get_entry_forwarder(ordinal)
+        if forwarded_name is None:
+            yield Export(name), AbsoluteVirtualAddress(ea)
+        else:
+            forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
+            yield Export(forwarded_name), AbsoluteVirtualAddress(ea)
+            yield Characteristic("forwarded export"), AbsoluteVirtualAddress(ea)
 
 
 def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
@@ -199,14 +206,3 @@ FILE_HANDLERS = (
     extract_file_function_names,
     extract_file_format,
 )
-
-
-def main():
-    """ """
-    import pprint
-
-    pprint.pprint(list(extract_features()))
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ida/function.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -50,18 +50,3 @@ def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
 
 
 FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
-
-
-def main():
-    """ """
-    features = []
-    for fhandle in capa.features.extractors.ida.helpers.get_functions(skip_thunks=True, skip_libs=True):
-        features.extend(list(extract_features(fhandle)))
-
-    import pprint
-
-    pprint.pprint(features)
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ida/global_.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import logging
 import contextlib
 from typing import Tuple, Iterator

capa/features/extractors/ida/helpers.py

@@ -1,10 +1,11 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import functools
 from typing import Any, Dict, Tuple, Iterator, Optional
 
 import idc
@@ -27,7 +28,8 @@ def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
     """
     seqstr = " ".join([f"{b:02x}" for b in seq])
     while True:
-        # TODO find_binary: Deprecated. Please use ida_bytes.bin_search() instead.
+        # TODO(mike-hunhoff): find_binary is deprecated. Please use ida_bytes.bin_search() instead.
+        # https://github.com/mandiant/capa/issues/1606
         ea = idaapi.find_binary(start, end, seqstr, 0, idaapi.SEARCH_DOWN)
         if ea == idaapi.BADADDR:
             break
@@ -80,9 +82,22 @@ def get_segment_buffer(seg: idaapi.segment_t) -> bytes:
     return buff if buff else b""
 
 
+def inspect_import(imports, library, ea, function, ordinal):
+    if function and function.startswith("__imp_"):
+        # handle mangled PE imports
+        function = function[len("__imp_") :]
+
+    if function and "@@" in function:
+        # handle mangled ELF imports, like "fopen@@glibc_2.2.5"
+        function, _, _ = function.partition("@@")
+
+    imports[ea] = (library.lower(), function, ordinal)
+    return True
+
+
 def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
     """get file imports"""
-    imports = {}
+    imports: Dict[int, Tuple[str, str, int]] = {}
 
     for idx in range(idaapi.get_import_module_qty()):
         library = idaapi.get_import_module_name(idx)
@@ -90,22 +105,15 @@ def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
         if not library:
             continue
 
-        # IDA uses section names for the library of ELF imports, like ".dynsym"
-        library = library.lstrip(".")
+        # IDA uses section names for the library of ELF imports, like ".dynsym".
+        # These are not useful to us, we may need to expand this list over time
+        # TODO(williballenthin): find all section names used by IDA
+        # https://github.com/mandiant/capa/issues/1419
+        if library == ".dynsym":
+            library = ""
 
-        def inspect_import(ea, function, ordinal):
-            if function and function.startswith("__imp_"):
-                # handle mangled PE imports
-                function = function[len("__imp_") :]
-
-            if function and "@@" in function:
-                # handle mangled ELF imports, like "fopen@@glibc_2.2.5"
-                function, _, _ = function.partition("@@")
-
-            imports[ea] = (library.lower(), function, ordinal)
-            return True
-
-        idaapi.enum_import_names(idx, inspect_import)
+        cb = functools.partial(inspect_import, imports, library)
+        idaapi.enum_import_names(idx, cb)
 
     return imports
 
@@ -114,7 +122,7 @@ def get_file_externs() -> Dict[int, Tuple[str, str, int]]:
     externs = {}
 
     for seg in get_segments(skip_header_segments=True):
-        if not (seg.type == ida_segment.SEG_XTRN):
+        if seg.type != ida_segment.SEG_XTRN:
             continue
 
         for ea in idautils.Functions(seg.start_ea, seg.end_ea):
@@ -267,20 +275,18 @@ def is_op_offset(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
 
 def is_sp_modified(insn: idaapi.insn_t) -> bool:
     """determine if instruction modifies SP, ESP, RSP"""
-    for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
-        if op.reg == idautils.procregs.sp.reg and is_op_write(insn, op):
-            # register is stack and written
-            return True
-    return False
+    return any(
+        op.reg == idautils.procregs.sp.reg and is_op_write(insn, op)
+        for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,))
+    )
 
 
 def is_bp_modified(insn: idaapi.insn_t) -> bool:
     """check if instruction modifies BP, EBP, RBP"""
-    for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
-        if op.reg == idautils.procregs.bp.reg and is_op_write(insn, op):
-            # register is base and written
-            return True
-    return False
+    return any(
+        op.reg == idautils.procregs.bp.reg and is_op_write(insn, op)
+        for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,))
+    )
 
 
 def is_frame_register(reg: int) -> bool:
@@ -326,10 +332,7 @@ def mask_op_val(op: idaapi.op_t) -> int:
 
 def is_function_recursive(f: idaapi.func_t) -> bool:
     """check if function is recursive"""
-    for ref in idautils.CodeRefsTo(f.start_ea, True):
-        if f.contains(ref):
-            return True
-    return False
+    return any(f.contains(ref) for ref in idautils.CodeRefsTo(f.start_ea, True))
 
 
 def is_basic_block_tight_loop(bb: idaapi.BasicBlock) -> bool:
@@ -378,8 +381,7 @@ def find_data_reference_from_insn(insn: idaapi.insn_t, max_depth: int = 10) -> i
 def get_function_blocks(f: idaapi.func_t) -> Iterator[idaapi.BasicBlock]:
     """yield basic blocks contained in specified function"""
     # leverage idaapi.FC_NOEXT flag to ignore useless external blocks referenced by the function
-    for block in idaapi.FlowChart(f, flags=(idaapi.FC_PREDS | idaapi.FC_NOEXT)):
-        yield block
+    yield from idaapi.FlowChart(f, flags=(idaapi.FC_PREDS | idaapi.FC_NOEXT))
 
 
 def is_basic_block_return(bb: idaapi.BasicBlock) -> bool:

capa/features/extractors/ida/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -73,7 +73,7 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
     """
     insn: idaapi.insn_t = ih.inner
 
-    if not insn.get_canon_mnem() in ("call", "jmp"):
+    if insn.get_canon_mnem() not in ("call", "jmp"):
         return
 
     # check calls to imported functions
@@ -216,7 +216,7 @@ def extract_insn_offset_features(
 
         p_info = capa.features.extractors.ida.helpers.get_op_phrase_info(op)
 
-        op_off = p_info.get("offset", None)
+        op_off = p_info.get("offset")
         if op_off is None:
             continue
 
@@ -398,14 +398,16 @@ def extract_insn_peb_access_characteristic_features(
     if insn.itype not in (idaapi.NN_push, idaapi.NN_mov):
         return
 
-    if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
+    if all(op.type != idaapi.o_mem for op in insn.ops):
         # try to optimize for only memory references
         return
 
     disasm = idc.GetDisasm(insn.ea)
 
     if " fs:30h" in disasm or " gs:60h" in disasm:
-        # TODO: replace above with proper IDA
+        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
+        # scanning the disassembly text is a hack.
+        # https://github.com/mandiant/capa/issues/1605
         yield Characteristic("peb access"), ih.address
 
 
@@ -419,18 +421,22 @@ def extract_insn_segment_access_features(
     """
     insn: idaapi.insn_t = ih.inner
 
-    if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
+    if all(op.type != idaapi.o_mem for op in insn.ops):
         # try to optimize for only memory references
         return
 
     disasm = idc.GetDisasm(insn.ea)
 
     if " fs:" in disasm:
-        # TODO: replace above with proper IDA
+        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
+        # scanning the disassembly text is a hack.
+        # https://github.com/mandiant/capa/issues/1605
         yield Characteristic("fs access"), ih.address
 
     if " gs:" in disasm:
-        # TODO: replace above with proper IDA
+        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
+        # scanning the disassembly text is a hack.
+        # https://github.com/mandiant/capa/issues/1605
         yield Characteristic("gs access"), ih.address
 
 
@@ -441,7 +447,7 @@ def extract_insn_cross_section_cflow(
     insn: idaapi.insn_t = ih.inner
 
     for ref in idautils.CodeRefsFrom(insn.ea, False):
-        if ref in get_imports(fh.ctx).keys():
+        if ref in get_imports(fh.ctx):
             # ignore API calls
             continue
         if not idaapi.getseg(ref):
@@ -501,20 +507,3 @@ INSTRUCTION_HANDLERS = (
     extract_function_calls_from,
     extract_function_indirect_call_characteristic_features,
 )
-
-
-def main():
-    """ """
-    features = []
-    for f in capa.features.extractors.ida.helpers.get_functions(skip_thunks=True, skip_libs=True):
-        for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
-            for insn in capa.features.extractors.ida.helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
-                features.extend(list(extract_features(f, bb, insn)))
-
-    import pprint
-
-    pprint.pprint(features)
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/loops.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/null.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 from typing import Dict, List, Tuple
 from dataclasses import dataclass
 

capa/features/extractors/pefile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,6 +7,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import logging
+from pathlib import Path
 
 import pefile
 
@@ -39,8 +40,20 @@ def extract_file_export_names(pe, **kwargs):
                 name = export.name.partition(b"\x00")[0].decode("ascii")
             except UnicodeDecodeError:
                 continue
-            va = base_address + export.address
-            yield Export(name), AbsoluteVirtualAddress(va)
+
+            if export.forwarder is None:
+                va = base_address + export.address
+                yield Export(name), AbsoluteVirtualAddress(va)
+
+            else:
+                try:
+                    forwarded_name = export.forwarder.partition(b"\x00")[0].decode("ascii")
+                except UnicodeDecodeError:
+                    continue
+                forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
+                va = base_address + export.address
+                yield Export(forwarded_name), AbsoluteVirtualAddress(va)
+                yield Characteristic("forwarded export"), AbsoluteVirtualAddress(va)
 
 
 def extract_file_import_names(pe, **kwargs):
@@ -173,23 +186,21 @@ GLOBAL_HANDLERS = (
 
 
 class PefileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
+    def __init__(self, path: Path):
         super().__init__()
-        self.path = path
-        self.pe = pefile.PE(path)
+        self.path: Path = path
+        self.pe = pefile.PE(str(path))
 
     def get_base_address(self):
         return AbsoluteVirtualAddress(self.pe.OPTIONAL_HEADER.ImageBase)
 
     def extract_global_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = Path(self.path).read_bytes()
 
         yield from extract_global_features(self.pe, buf)
 
     def extract_file_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
+        buf = Path(self.path).read_bytes()
 
         yield from extract_file_features(self.pe, buf)
 

capa/features/extractors/strings.py

@@ -1,6 +1,6 @@
 # strings code from FLOSS, https://github.com/mandiant/flare-floss
 #
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -9,6 +9,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import re
+import contextlib
 from collections import namedtuple
 
 ASCII_BYTE = r" !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t".encode(
@@ -81,24 +82,5 @@ def extract_unicode_strings(buf, n=4):
         reg = b"((?:[%s]\x00){%d,})" % (ASCII_BYTE, n)
         r = re.compile(reg)
     for match in r.finditer(buf):
-        try:
+        with contextlib.suppress(UnicodeDecodeError):
             yield String(match.group().decode("utf-16"), match.start())
-        except UnicodeDecodeError:
-            pass
-
-
-def main():
-    import sys
-
-    with open(sys.argv[1], "rb") as f:
-        b = f.read()
-
-    for s in extract_ascii_strings(b):
-        print("0x{:x}: {:s}".format(s.offset, s.s))
-
-    for s in extract_unicode_strings(b):
-        print("0x{:x}: {:s}".format(s.offset, s.s))
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/viv/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -92,7 +92,6 @@ def is_mov_imm_to_stack(instr: envi.archs.i386.disasm.i386Opcode) -> bool:
     if not src.isImmed():
         return False
 
-    # TODO what about 64-bit operands?
     if not isinstance(dst, envi.archs.i386.disasm.i386SibOper) and not isinstance(
         dst, envi.archs.i386.disasm.i386RegMemOper
     ):

capa/features/extractors/viv/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,7 +6,8 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import logging
-from typing import List, Tuple, Iterator
+from typing import Any, Dict, List, Tuple, Iterator
+from pathlib import Path
 
 import viv_utils
 import viv_utils.flirt
@@ -25,12 +26,11 @@ logger = logging.getLogger(__name__)
 
 
 class VivisectFeatureExtractor(FeatureExtractor):
-    def __init__(self, vw, path, os):
+    def __init__(self, vw, path: Path, os):
         super().__init__()
         self.vw = vw
         self.path = path
-        with open(self.path, "rb") as f:
-            self.buf = f.read()
+        self.buf = path.read_bytes()
 
         # pre-compute these because we'll yield them at *every* scope.
         self.global_features: List[Tuple[Feature, Address]] = []
@@ -49,8 +49,11 @@ class VivisectFeatureExtractor(FeatureExtractor):
         yield from capa.features.extractors.viv.file.extract_features(self.vw, self.buf)
 
     def get_functions(self) -> Iterator[FunctionHandle]:
+        cache: Dict[str, Any] = {}
         for va in sorted(self.vw.getFunctions()):
-            yield FunctionHandle(address=AbsoluteVirtualAddress(va), inner=viv_utils.Function(self.vw, va))
+            yield FunctionHandle(
+                address=AbsoluteVirtualAddress(va), inner=viv_utils.Function(self.vw, va), ctx={"cache": cache}
+            )
 
     def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
         yield from capa.features.extractors.viv.function.extract_features(fh)

capa/features/extractors/viv/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,6 +8,7 @@
 from typing import Tuple, Iterator
 
 import PE.carve as pe_carve  # vivisect PE
+import vivisect
 import viv_utils
 import viv_utils.flirt
 
@@ -16,7 +17,7 @@ import capa.features.extractors.common
 import capa.features.extractors.helpers
 import capa.features.extractors.strings
 from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import String, Feature, Characteristic
+from capa.features.common import Feature, Characteristic
 from capa.features.address import Address, FileOffsetAddress, AbsoluteVirtualAddress
 
 
@@ -25,10 +26,35 @@ def extract_file_embedded_pe(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]
         yield Characteristic("embedded pe"), FileOffsetAddress(offset)
 
 
-def extract_file_export_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
+def get_first_vw_filename(vw: vivisect.VivWorkspace):
+    # vivisect associates metadata with each file that its loaded into the workspace.
+    # capa only loads a single file into each workspace.
+    # so to access the metadata for the file in question, we can just take the first one.
+    # otherwise, we'd have to pass around the module name of the file we're analyzing,
+    # which is a pain.
+    #
+    # so this is a simplifying assumption.
+    return next(iter(vw.filemeta.keys()))
+
+
+def extract_file_export_names(vw: vivisect.VivWorkspace, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     for va, _, name, _ in vw.getExports():
         yield Export(name), AbsoluteVirtualAddress(va)
 
+    if vw.getMeta("Format") == "pe":
+        pe = vw.parsedbin
+        baseaddr = pe.IMAGE_NT_HEADERS.OptionalHeader.ImageBase
+        for rva, _, forwarded_name in vw.getFileMeta(get_first_vw_filename(vw), "forwarders"):
+            try:
+                forwarded_name = forwarded_name.partition(b"\x00")[0].decode("ascii")
+            except UnicodeDecodeError:
+                continue
+
+            forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
+            va = baseaddr + rva
+            yield Export(forwarded_name), AbsoluteVirtualAddress(va)
+            yield Characteristic("forwarded export"), AbsoluteVirtualAddress(va)
+
 
 def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     """

capa/features/extractors/viv/function.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -11,9 +11,11 @@ import envi
 import viv_utils
 import vivisect.const
 
+from capa.features.file import FunctionName
 from capa.features.common import Feature, Characteristic
 from capa.features.address import Address, AbsoluteVirtualAddress
 from capa.features.extractors import loops
+from capa.features.extractors.elf import SymTab
 from capa.features.extractors.base_extractor import FunctionHandle
 
 
@@ -30,6 +32,28 @@ def interface_extract_function_XXX(fh: FunctionHandle) -> Iterator[Tuple[Feature
     raise NotImplementedError
 
 
+def extract_function_symtab_names(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    if fh.inner.vw.metadata["Format"] == "elf":
+        # the file's symbol table gets added to the metadata of the vivisect workspace.
+        # this is in order to eliminate the computational overhead of refetching symtab each time.
+        if "symtab" not in fh.ctx["cache"]:
+            try:
+                fh.ctx["cache"]["symtab"] = SymTab.from_Elf(fh.inner.vw.parsedbin)
+            except Exception:
+                fh.ctx["cache"]["symtab"] = None
+
+        symtab = fh.ctx["cache"]["symtab"]
+        if symtab:
+            for symbol in symtab.get_symbols():
+                sym_name = symtab.get_name(symbol)
+                sym_value = symbol.value
+                sym_info = symbol.info
+
+                STT_FUNC = 0x2
+                if sym_value == fh.address and sym_info & STT_FUNC != 0:
+                    yield FunctionName(sym_name), fh.address
+
+
 def extract_function_calls_to(fhandle: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
     f: viv_utils.Function = fhandle.inner
     for src, _, _, _ in f.vw.getXrefsTo(f.va, rtype=vivisect.const.REF_CODE):
@@ -79,4 +103,8 @@ def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
             yield feature, addr
 
 
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)
+FUNCTION_HANDLERS = (
+    extract_function_symtab_names,
+    extract_function_calls_to,
+    extract_function_loop,
+)

capa/features/extractors/viv/global_.py

@@ -1,9 +1,13 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import logging
 from typing import Tuple, Iterator
 
-import envi.archs.i386
-import envi.archs.amd64
-
 from capa.features.common import ARCH_I386, ARCH_AMD64, Arch, Feature
 from capa.features.address import NO_ADDRESS, Address
 
@@ -11,10 +15,11 @@ logger = logging.getLogger(__name__)
 
 
 def extract_arch(vw) -> Iterator[Tuple[Feature, Address]]:
-    if isinstance(vw.arch, envi.archs.amd64.Amd64Module):
+    arch = vw.getMeta("Architecture")
+    if arch == "amd64":
         yield Arch(ARCH_AMD64), NO_ADDRESS
 
-    elif isinstance(vw.arch, envi.archs.i386.i386Module):
+    elif arch == "i386":
         yield Arch(ARCH_I386), NO_ADDRESS
 
     else:

capa/features/extractors/viv/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/viv/indirect_calls.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,7 +7,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import collections
-from typing import Set, List, Deque, Tuple, Union, Optional
+from typing import Set, List, Deque, Tuple, Optional
 
 import envi
 import vivisect.const
@@ -71,7 +71,7 @@ class NotFoundError(Exception):
     pass
 
 
-def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int, None]]:
+def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Optional[int]]:
     """
     scan backwards from the given address looking for assignments to the given register.
     if a constant, return that value.
@@ -87,8 +87,8 @@ def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int
     raises:
       NotFoundError: when the definition cannot be found.
     """
-    q = collections.deque()  # type: Deque[int]
-    seen = set([])  # type: Set[int]
+    q: Deque[int] = collections.deque()
+    seen: Set[int] = set()
 
     q.extend(get_previous_instructions(vw, va))
     while q:

capa/features/extractors/viv/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -22,6 +22,7 @@ import capa.features.extractors.viv.helpers
 from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
 from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Feature, Characteristic
 from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.elf import SymTab
 from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
 from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_indirect_call
 
@@ -109,6 +110,26 @@ def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterato
         if not target:
             return
 
+        if f.vw.metadata["Format"] == "elf":
+            if "symtab" not in fh.ctx["cache"]:
+                # the symbol table gets stored as a function's attribute in order to avoid running
+                # this code everytime the call is made, thus preventing the computational overhead.
+                try:
+                    fh.ctx["cache"]["symtab"] = SymTab.from_Elf(f.vw.parsedbin)
+                except Exception:
+                    fh.ctx["cache"]["symtab"] = None
+
+            symtab = fh.ctx["cache"]["symtab"]
+            if symtab:
+                for symbol in symtab.get_symbols():
+                    sym_name = symtab.get_name(symbol)
+                    sym_value = symbol.value
+                    sym_info = symbol.info
+
+                    STT_FUNC = 0x2
+                    if sym_value == target and sym_info & STT_FUNC != 0:
+                        yield API(sym_name), ih.address
+
         if viv_utils.flirt.is_library_function(f.vw, target):
             name = viv_utils.get_function_name(f.vw, target)
             yield API(name), ih.address
@@ -267,16 +288,16 @@ def extract_insn_bytes_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera
         else:
             continue
 
-        for v in derefs(f.vw, v):
+        for vv in derefs(f.vw, v):
             try:
-                buf = read_bytes(f.vw, v)
+                buf = read_bytes(f.vw, vv)
             except envi.exc.SegmentationViolation:
                 continue
 
             if capa.features.extractors.helpers.all_zeros(buf):
                 continue
 
-            if f.vw.isProbablyString(v) or f.vw.isProbablyUnicode(v):
+            if f.vw.isProbablyString(vv) or f.vw.isProbablyUnicode(vv):
                 # don't extract byte features for obvious strings
                 continue
 
@@ -330,7 +351,6 @@ def is_security_cookie(f, bb, insn) -> bool:
     if oper.isReg() and oper.reg not in [
         envi.archs.i386.regs.REG_ESP,
         envi.archs.i386.regs.REG_EBP,
-        # TODO: do x64 support for real.
         envi.archs.amd64.regs.REG_RBP,
         envi.archs.amd64.regs.REG_RSP,
     ]:
@@ -390,9 +410,7 @@ def extract_insn_obfs_call_plus_5_characteristic_features(f, bb, ih: InsnHandle)
         if insn.va + 5 == insn.opers[0].getOperValue(insn):
             yield Characteristic("call $+5"), ih.address
 
-    if isinstance(insn.opers[0], envi.archs.i386.disasm.i386ImmMemOper) or isinstance(
-        insn.opers[0], envi.archs.amd64.disasm.Amd64RipRelOper
-    ):
+    if isinstance(insn.opers[0], (envi.archs.i386.disasm.i386ImmMemOper, envi.archs.amd64.disasm.Amd64RipRelOper)):
         if insn.va + 5 == insn.opers[0].getOperAddr(insn):
             yield Characteristic("call $+5"), ih.address
 
@@ -401,7 +419,6 @@ def extract_insn_peb_access_characteristic_features(f, bb, ih: InsnHandle) -> It
     """
     parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
     """
-    # TODO handle where fs/gs are loaded into a register or onto the stack and used later
     insn: envi.Opcode = ih.inner
 
     if insn.mnem not in ["push", "mov"]:
@@ -625,7 +642,6 @@ def extract_op_offset_features(
         if oper.reg == envi.archs.i386.regs.REG_EBP:
             return
 
-        # TODO: do x64 support for real.
         if oper.reg == envi.archs.amd64.regs.REG_RBP:
             return
 
@@ -679,9 +695,9 @@ def extract_op_string_features(
     else:
         return
 
-    for v in derefs(f.vw, v):
+    for vv in derefs(f.vw, v):
         try:
-            s = read_string(f.vw, v).rstrip("\x00")
+            s = read_string(f.vw, vv).rstrip("\x00")
         except ValueError:
             continue
         else:

capa/features/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/freeze/__init__.py

@@ -1,7 +1,7 @@
 """
 capa freeze file format: `| capa0000 | + zlib(utf-8(json(...)))`
 
-Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
 You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations
 import zlib
 import logging
 from enum import Enum
-from typing import Any, List, Tuple, Union
+from typing import List, Tuple, Union
 
 from pydantic import Field, BaseModel
 
@@ -268,7 +268,8 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
                     basic_block=bbaddr,
                     address=Address.from_capa(addr),
                     feature=feature_from_capa(feature),
-                )
+                )  # type: ignore
+                # Mypy is unable to recognise `basic_block` as a argument due to alias
                 for feature, addr in extractor.extract_basic_block_features(f, bb)
             ]
 
@@ -287,38 +288,41 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
                 instructions.append(
                     InstructionFeatures(
                         address=iaddr,
-                        features=ifeatures,
+                        features=tuple(ifeatures),
                     )
                 )
 
             basic_blocks.append(
                 BasicBlockFeatures(
                     address=bbaddr,
-                    features=bbfeatures,
-                    instructions=instructions,
+                    features=tuple(bbfeatures),
+                    instructions=tuple(instructions),
                 )
             )
 
         function_features.append(
             FunctionFeatures(
                 address=faddr,
-                features=ffeatures,
+                features=tuple(ffeatures),
                 basic_blocks=basic_blocks,
-            )
+            )  # type: ignore
+            # Mypy is unable to recognise `basic_blocks` as a argument due to alias
         )
 
     features = Features(
         global_=global_features,
-        file=file_features,
-        functions=function_features,
-    )
+        file=tuple(file_features),
+        functions=tuple(function_features),
+    )  # type: ignore
+    # Mypy is unable to recognise `global_` as a argument due to alias
 
     freeze = Freeze(
         version=2,
         base_address=Address.from_capa(extractor.get_base_address()),
         extractor=Extractor(name=extractor.__class__.__name__),
         features=features,
-    )
+    )  # type: ignore
+    # Mypy is unable to recognise `base_address` as a argument due to alias
 
     return freeze.json()
 
@@ -378,6 +382,7 @@ def load(buf: bytes) -> capa.features.extractors.base_extractor.FeatureExtractor
 def main(argv=None):
     import sys
     import argparse
+    from pathlib import Path
 
     import capa.main
 
@@ -394,8 +399,7 @@ def main(argv=None):
 
     extractor = capa.main.get_extractor(args.sample, args.format, args.os, args.backend, sigpaths, False)
 
-    with open(args.output, "wb") as f:
-        f.write(dump(extractor))
+    Path(args.output).write_bytes(dump(extractor))
 
     return 0
 

capa/features/freeze/features.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import binascii
 from typing import Union, Optional
 
@@ -101,59 +108,79 @@ class FeatureModel(BaseModel):
 
 def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
     if isinstance(f, capa.features.common.OS):
+        assert isinstance(f.value, str)
         return OSFeature(os=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Arch):
+        assert isinstance(f.value, str)
         return ArchFeature(arch=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Format):
+        assert isinstance(f.value, str)
         return FormatFeature(format=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.MatchedRule):
+        assert isinstance(f.value, str)
         return MatchFeature(match=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Characteristic):
+        assert isinstance(f.value, str)
         return CharacteristicFeature(characteristic=f.value, description=f.description)
 
     elif isinstance(f, capa.features.file.Export):
+        assert isinstance(f.value, str)
         return ExportFeature(export=f.value, description=f.description)
 
     elif isinstance(f, capa.features.file.Import):
-        return ImportFeature(import_=f.value, description=f.description)
+        assert isinstance(f.value, str)
+        return ImportFeature(import_=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `import_` as a argument due to alias
 
     elif isinstance(f, capa.features.file.Section):
+        assert isinstance(f.value, str)
         return SectionFeature(section=f.value, description=f.description)
 
     elif isinstance(f, capa.features.file.FunctionName):
-        return FunctionNameFeature(function_name=f.value, description=f.description)
+        assert isinstance(f.value, str)
+        return FunctionNameFeature(function_name=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `function_name` as a argument due to alias
 
     # must come before check for String due to inheritance
     elif isinstance(f, capa.features.common.Substring):
+        assert isinstance(f.value, str)
         return SubstringFeature(substring=f.value, description=f.description)
 
     # must come before check for String due to inheritance
     elif isinstance(f, capa.features.common.Regex):
+        assert isinstance(f.value, str)
         return RegexFeature(regex=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.String):
+        assert isinstance(f.value, str)
         return StringFeature(string=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Class):
-        return ClassFeature(class_=f.value, description=f.description)
+        assert isinstance(f.value, str)
+        return ClassFeature(class_=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `class_` as a argument due to alias
 
     elif isinstance(f, capa.features.common.Namespace):
+        assert isinstance(f.value, str)
         return NamespaceFeature(namespace=f.value, description=f.description)
 
     elif isinstance(f, capa.features.basicblock.BasicBlock):
         return BasicBlockFeature(description=f.description)
 
     elif isinstance(f, capa.features.insn.API):
+        assert isinstance(f.value, str)
         return APIFeature(api=f.value, description=f.description)
 
     elif isinstance(f, capa.features.insn.Property):
+        assert isinstance(f.value, str)
         return PropertyFeature(property=f.value, access=f.access, description=f.description)
 
     elif isinstance(f, capa.features.insn.Number):
+        assert isinstance(f.value, (int, float))
         return NumberFeature(number=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Bytes):
@@ -162,16 +189,22 @@ def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
         return BytesFeature(bytes=binascii.hexlify(buf).decode("ascii"), description=f.description)
 
     elif isinstance(f, capa.features.insn.Offset):
+        assert isinstance(f.value, int)
         return OffsetFeature(offset=f.value, description=f.description)
 
     elif isinstance(f, capa.features.insn.Mnemonic):
+        assert isinstance(f.value, str)
         return MnemonicFeature(mnemonic=f.value, description=f.description)
 
     elif isinstance(f, capa.features.insn.OperandNumber):
-        return OperandNumberFeature(index=f.index, operand_number=f.value, description=f.description)
+        assert isinstance(f.value, int)
+        return OperandNumberFeature(index=f.index, operand_number=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `operand_number` as a argument due to alias
 
     elif isinstance(f, capa.features.insn.OperandOffset):
-        return OperandOffsetFeature(index=f.index, operand_offset=f.value, description=f.description)
+        assert isinstance(f.value, int)
+        return OperandOffsetFeature(index=f.index, operand_offset=f.value, description=f.description)  # type: ignore
+        # Mypy is unable to recognise `operand_offset` as a argument due to alias
 
     else:
         raise NotImplementedError(f"feature_from_capa({type(f)}) not implemented")

capa/features/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -53,6 +53,15 @@ class Property(_AccessFeature):
 
 class Number(Feature):
     def __init__(self, value: Union[int, float], description=None):
+        """
+        args:
+          value (int or float): positive or negative integer, or floating point number.
+
+        the range of the value is:
+          - if positive, the range of u64
+          - if negative, the range of i64
+          - if floating, the range and precision of double
+        """
         super().__init__(value, description=description)
 
     def get_value_str(self):
@@ -61,7 +70,7 @@ class Number(Feature):
         elif isinstance(self.value, float):
             return str(self.value)
         else:
-            raise ValueError("invalid value type")
+            raise ValueError(f"invalid value type {type(self.value)}")
 
 
 # max recognized structure size (and therefore, offset size)
@@ -70,6 +79,14 @@ MAX_STRUCTURE_SIZE = 0x10000
 
 class Offset(Feature):
     def __init__(self, value: int, description=None):
+        """
+        args:
+          value (int): the offset, which can be positive or negative.
+
+        the range of the value is:
+          - if positive, the range of u64
+          - if negative, the range of i64
+        """
         super().__init__(value, description=description)
 
     def get_value_str(self):
@@ -92,7 +109,7 @@ MAX_OPERAND_INDEX = MAX_OPERAND_COUNT - 1
 class _Operand(Feature, abc.ABC):
     # superclass: don't use directly
     # subclasses should set self.name and provide the value string formatter
-    def __init__(self, index: int, value: int, description=None):
+    def __init__(self, index: int, value: Union[int, float], description=None):
         super().__init__(value, description=description)
         self.index = index
 
@@ -108,13 +125,26 @@ class OperandNumber(_Operand):
     NAMES = [f"operand[{i}].number" for i in range(MAX_OPERAND_COUNT)]
 
     # operand[i].number: 0x12
-    def __init__(self, index: int, value: int, description=None):
+    def __init__(self, index: int, value: Union[int, float], description=None):
+        """
+        args:
+          value (int or float): positive or negative integer, or floating point number.
+
+        the range of the value is:
+          - if positive, the range of u64
+          - if negative, the range of i64
+          - if floating, the range and precision of double
+        """
         super().__init__(index, value, description=description)
         self.name = self.NAMES[index]
 
     def get_value_str(self) -> str:
-        assert isinstance(self.value, int)
-        return hex(self.value)
+        if isinstance(self.value, int):
+            return capa.helpers.hex(self.value)
+        elif isinstance(self.value, float):
+            return str(self.value)
+        else:
+            raise ValueError("invalid value type")
 
 
 class OperandOffset(_Operand):
@@ -123,6 +153,14 @@ class OperandOffset(_Operand):
 
     # operand[i].offset: 0x12
     def __init__(self, index: int, value: int, description=None):
+        """
+        args:
+          value (int): the offset, which can be positive or negative.
+
+        the range of the value is:
+          - if positive, the range of u64
+          - if negative, the range of i64
+        """
         super().__init__(index, value, description=description)
         self.name = self.NAMES[index]
 

capa/helpers.py

@@ -1,13 +1,18 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import os
+import inspect
 import logging
+import contextlib
+import importlib.util
 from typing import NoReturn
+from pathlib import Path
+
+import tqdm
 
 from capa.exceptions import UnsupportedFormatError
 from capa.features.common import FORMAT_PE, FORMAT_SC32, FORMAT_SC64, FORMAT_DOTNET, FORMAT_UNKNOWN, Format
@@ -27,36 +32,32 @@ def hex(n: int) -> str:
         return f"0x{(n):X}"
 
 
-def get_file_taste(sample_path: str) -> bytes:
-    if not os.path.exists(sample_path):
+def get_file_taste(sample_path: Path) -> bytes:
+    if not sample_path.exists():
         raise IOError(f"sample path {sample_path} does not exist or cannot be accessed")
-    with open(sample_path, "rb") as f:
-        taste = f.read(8)
+    taste = sample_path.open("rb").read(8)
     return taste
 
 
 def is_runtime_ida():
-    try:
-        import idc
-    except ImportError:
-        return False
-    else:
-        return True
+    return importlib.util.find_spec("idc") is not None
 
 
-def assert_never(value: NoReturn) -> NoReturn:
-    assert False, f"Unhandled value: {value} ({type(value).__name__})"
+def assert_never(value) -> NoReturn:
+    # careful: python -O will remove this assertion.
+    # but this is only used for type checking, so it's ok.
+    assert False, f"Unhandled value: {value} ({type(value).__name__})"  # noqa: B011
 
 
-def get_format_from_extension(sample: str) -> str:
-    if sample.endswith(EXTENSIONS_SHELLCODE_32):
+def get_format_from_extension(sample: Path) -> str:
+    if sample.name.endswith(EXTENSIONS_SHELLCODE_32):
         return FORMAT_SC32
-    elif sample.endswith(EXTENSIONS_SHELLCODE_64):
+    elif sample.name.endswith(EXTENSIONS_SHELLCODE_64):
         return FORMAT_SC64
     return FORMAT_UNKNOWN
 
 
-def get_auto_format(path: str) -> str:
+def get_auto_format(path: Path) -> str:
     format_ = get_format(path)
     if format_ == FORMAT_UNKNOWN:
         format_ = get_format_from_extension(path)
@@ -65,13 +66,12 @@ def get_auto_format(path: str) -> str:
     return format_
 
 
-def get_format(sample: str) -> str:
+def get_format(sample: Path) -> str:
     # imported locally to avoid import cycle
     from capa.features.extractors.common import extract_format
     from capa.features.extractors.dnfile_ import DnfileFeatureExtractor
 
-    with open(sample, "rb") as f:
-        buf = f.read()
+    buf = sample.read_bytes()
 
     for feature, _ in extract_format(buf):
         if feature == Format(FORMAT_PE):
@@ -85,6 +85,39 @@ def get_format(sample: str) -> str:
     return FORMAT_UNKNOWN
 
 
+@contextlib.contextmanager
+def redirecting_print_to_tqdm(disable_progress):
+    """
+    tqdm (progress bar) expects to have fairly tight control over console output.
+    so calls to `print()` will break the progress bar and make things look bad.
+    so, this context manager temporarily replaces the `print` implementation
+    with one that is compatible with tqdm.
+    via: https://stackoverflow.com/a/42424890/87207
+    """
+    old_print = print  # noqa: T202 [reserved word print used]
+
+    def new_print(*args, **kwargs):
+        # If tqdm.tqdm.write raises error, use builtin print
+        if disable_progress:
+            old_print(*args, **kwargs)
+        else:
+            try:
+                tqdm.tqdm.write(*args, **kwargs)
+            except Exception:
+                old_print(*args, **kwargs)
+
+    try:
+        # Globally replace print with new_print.
+        # Verified this works manually on Python 3.11:
+        #     >>> import inspect
+        #     >>> inspect.builtins
+        #     <module 'builtins' (built-in)>
+        inspect.builtins.print = new_print  # type: ignore
+        yield
+    finally:
+        inspect.builtins.print = old_print  # type: ignore
+
+
 def log_unsupported_format_error():
     logger.error("-" * 80)
     logger.error(" Input file does not appear to be a PE or ELF file.")
@@ -118,7 +151,7 @@ def log_unsupported_runtime_error():
     logger.error("-" * 80)
     logger.error(" Unsupported runtime or Python interpreter.")
     logger.error(" ")
-    logger.error(" capa supports running under Python 3.7 and higher.")
+    logger.error(" capa supports running under Python 3.8 and higher.")
     logger.error(" ")
     logger.error(
         " If you're seeing this message on the command line, please ensure you're running a supported Python version."

capa/ida/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -9,7 +9,8 @@ import json
 import logging
 import datetime
 import contextlib
-from typing import Optional
+from typing import List, Optional
+from pathlib import Path
 
 import idc
 import idaapi
@@ -22,7 +23,8 @@ import capa
 import capa.version
 import capa.render.utils as rutils
 import capa.features.common
-import capa.render.result_document
+import capa.features.freeze
+import capa.render.result_document as rdoc
 from capa.features.address import AbsoluteVirtualAddress
 
 logger = logging.getLogger("capa")
@@ -45,7 +47,8 @@ NETNODE_RULES_CACHE_ID = "rules-cache-id"
 
 
 def inform_user_ida_ui(message):
-    idaapi.info(f"{message}. Please refer to IDA Output window for more information.")
+    # this isn't a logger, this is IDA's logging facility
+    idaapi.info(f"{message}. Please refer to IDA Output window for more information.")  # noqa: G004
 
 
 def is_supported_ida_version():
@@ -53,7 +56,7 @@ def is_supported_ida_version():
     if version < 7.4 or version >= 9:
         warning_msg = "This plugin does not support your IDA Pro version"
         logger.warning(warning_msg)
-        logger.warning("Your IDA Pro version is: %s. Supported versions are: IDA >= 7.4 and IDA < 9.0." % version)
+        logger.warning("Your IDA Pro version is: %s. Supported versions are: IDA >= 7.4 and IDA < 9.0.", version)
         return False
     return True
 
@@ -118,7 +121,7 @@ def get_file_sha256():
     return sha256
 
 
-def collect_metadata(rules):
+def collect_metadata(rules: List[Path]):
     """ """
     md5 = get_file_md5()
     sha256 = get_file_sha256()
@@ -140,37 +143,35 @@ def collect_metadata(rules):
     else:
         os = "unknown os"
 
-    return {
-        "timestamp": datetime.datetime.now().isoformat(),
-        "argv": [],
-        "sample": {
-            "md5": md5,
-            "sha1": "",  # not easily accessible
-            "sha256": sha256,
-            "path": idaapi.get_input_file_path(),
-        },
-        "analysis": {
-            "format": idaapi.get_file_type_name(),
-            "arch": arch,
-            "os": os,
-            "extractor": "ida",
-            "rules": rules,
-            "base_address": idaapi.get_imagebase(),
-            "layout": {
+    return rdoc.Metadata(
+        timestamp=datetime.datetime.now(),
+        version=capa.version.__version__,
+        argv=(),
+        sample=rdoc.Sample(
+            md5=md5,
+            sha1="",  # not easily accessible
+            sha256=sha256,
+            path=idaapi.get_input_file_path(),
+        ),
+        analysis=rdoc.Analysis(
+            format=idaapi.get_file_type_name(),
+            arch=arch,
+            os=os,
+            extractor="ida",
+            rules=tuple(r.resolve().absolute().as_posix() for r in rules),
+            base_address=capa.features.freeze.Address.from_capa(idaapi.get_imagebase()),
+            layout=rdoc.Layout(
+                functions=(),
                 # this is updated after capabilities have been collected.
                 # will look like:
                 #
                 # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
-            },
+            ),
             # ignore these for now - not used by IDA plugin.
-            "feature_counts": {
-                "file": {},
-                "functions": {},
-            },
-            "library_functions": {},
-        },
-        "version": capa.version.__version__,
-    }
+            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
+            library_functions=(),
+        ),
+    )
 
 
 class IDAIO:
@@ -213,16 +214,16 @@ def idb_contains_cached_results() -> bool:
         n = netnode.Netnode(CAPA_NETNODE)
         return bool(n.get(NETNODE_RESULTS))
     except netnode.NetnodeCorruptError as e:
-        logger.error("%s", e, exc_info=True)
+        logger.exception(str(e))
         return False
 
 
-def load_and_verify_cached_results() -> Optional[capa.render.result_document.ResultDocument]:
+def load_and_verify_cached_results() -> Optional[rdoc.ResultDocument]:
     """verifies that cached results have valid (mapped) addresses for the current database"""
     logger.debug("loading cached capa results from netnode '%s'", CAPA_NETNODE)
 
     n = netnode.Netnode(CAPA_NETNODE)
-    doc = capa.render.result_document.ResultDocument.parse_obj(json.loads(n[NETNODE_RESULTS]))
+    doc = rdoc.ResultDocument.parse_obj(json.loads(n[NETNODE_RESULTS]))
 
     for rule in rutils.capability_rules(doc):
         for location_, _ in rule.matches:

capa/ida/plugin/README.md

@@ -95,7 +95,7 @@ can update using the `Settings` button.
 
 ### Requirements
 
-capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:
+capa explorer supports Python versions >= 3.8.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:
 
 * IDA 7.4
 * IDA 7.5
@@ -105,7 +105,7 @@ capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The
 * IDA 8.1
 * IDA 8.2
 
-capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x).
+capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.8.x).
 
 If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues).
 

capa/ida/plugin/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -67,7 +67,16 @@ class CapaExplorerPlugin(idaapi.plugin_t):
           arg (int): bitflag. Setting LSB enables automatic analysis upon
           loading. The other bits are currently undefined. See `form.Options`.
         """
-        self.form = CapaExplorerForm(self.PLUGIN_NAME, arg)
+        if not self.form:
+            self.form = CapaExplorerForm(self.PLUGIN_NAME, arg)
+        else:
+            widget = idaapi.find_widget(self.form.form_title)
+            if widget:
+                idaapi.activate_widget(widget, True)
+            else:
+                self.form.Show()
+                self.form.load_capa_results(False, True)
+
         return True
 
 

capa/ida/plugin/cache.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -10,7 +10,7 @@ from __future__ import annotations
 
 import itertools
 import collections
-from typing import Set, Dict, List, Tuple, Union, Optional
+from typing import Set, Dict, Tuple, Union, Optional
 
 import capa.engine
 from capa.rules import Scope, RuleSet
@@ -37,18 +37,21 @@ class CapaRuleGenFeatureCacheNode:
         self.children: Set[CapaRuleGenFeatureCacheNode] = set()
 
     def __hash__(self):
-        # TODO: unique enough?
+        # TODO(mike-hunhoff): confirm this is unique enough
+        # https://github.com/mandiant/capa/issues/1604
         return hash((self.address,))
 
     def __eq__(self, other):
         if not isinstance(other, type(self)):
             return NotImplemented
-        # TODO: unique enough?
+        # TODO(mike-hunhoff): confirm this is unique enough
+        # https://github.com/mandiant/capa/issues/1604
         return self.address == other.address
 
 
 class CapaRuleGenFeatureCache:
-    def __init__(self, fh_list: List[FunctionHandle], extractor: CapaExplorerFeatureExtractor):
+    def __init__(self, extractor: CapaExplorerFeatureExtractor):
+        self.extractor = extractor
         self.global_features: FeatureSet = collections.defaultdict(set)
 
         self.file_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(None, None)
@@ -56,12 +59,11 @@ class CapaRuleGenFeatureCache:
         self.bb_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
         self.insn_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
 
-        self._find_global_features(extractor)
-        self._find_file_features(extractor)
-        self._find_function_and_below_features(fh_list, extractor)
+        self._find_global_features()
+        self._find_file_features()
 
-    def _find_global_features(self, extractor: CapaExplorerFeatureExtractor):
-        for feature, addr in extractor.extract_global_features():
+    def _find_global_features(self):
+        for feature, addr in self.extractor.extract_global_features():
             # not all global features may have virtual addresses.
             # if not, then at least ensure the feature shows up in the index.
             # the set of addresses will still be empty.
@@ -71,46 +73,45 @@ class CapaRuleGenFeatureCache:
                 if feature not in self.global_features:
                     self.global_features[feature] = set()
 
-    def _find_file_features(self, extractor: CapaExplorerFeatureExtractor):
+    def _find_file_features(self):
         # not all file features may have virtual addresses.
         # if not, then at least ensure the feature shows up in the index.
         # the set of addresses will still be empty.
-        for feature, addr in extractor.extract_file_features():
+        for feature, addr in self.extractor.extract_file_features():
             if addr is not None:
                 self.file_node.features[feature].add(addr)
             else:
                 if feature not in self.file_node.features:
                     self.file_node.features[feature] = set()
 
-    def _find_function_and_below_features(self, fh_list: List[FunctionHandle], extractor: CapaExplorerFeatureExtractor):
-        for fh in fh_list:
-            f_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(fh, self.file_node)
+    def _find_function_and_below_features(self, fh: FunctionHandle):
+        f_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(fh, self.file_node)
 
-            # extract basic block and below features
-            for bbh in extractor.get_basic_blocks(fh):
-                bb_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(bbh, f_node)
+        # extract basic block and below features
+        for bbh in self.extractor.get_basic_blocks(fh):
+            bb_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(bbh, f_node)
 
-                # extract instruction features
-                for ih in extractor.get_instructions(fh, bbh):
-                    inode: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(ih, bb_node)
+            # extract instruction features
+            for ih in self.extractor.get_instructions(fh, bbh):
+                inode: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(ih, bb_node)
 
-                    for feature, addr in extractor.extract_insn_features(fh, bbh, ih):
-                        inode.features[feature].add(addr)
+                for feature, addr in self.extractor.extract_insn_features(fh, bbh, ih):
+                    inode.features[feature].add(addr)
 
-                    self.insn_nodes[inode.address] = inode
+                self.insn_nodes[inode.address] = inode
 
-                # extract basic block features
-                for feature, addr in extractor.extract_basic_block_features(fh, bbh):
-                    bb_node.features[feature].add(addr)
+            # extract basic block features
+            for feature, addr in self.extractor.extract_basic_block_features(fh, bbh):
+                bb_node.features[feature].add(addr)
 
-                # store basic block features in cache and function parent
-                self.bb_nodes[bb_node.address] = bb_node
+            # store basic block features in cache and function parent
+            self.bb_nodes[bb_node.address] = bb_node
 
-            # extract function features
-            for feature, addr in extractor.extract_function_features(fh):
-                f_node.features[feature].add(addr)
+        # extract function features
+        for feature, addr in self.extractor.extract_function_features(fh):
+            f_node.features[feature].add(addr)
 
-            self.func_nodes[f_node.address] = f_node
+        self.func_nodes[f_node.address] = f_node
 
     def _find_instruction_capabilities(
         self, ruleset: RuleSet, insn: CapaRuleGenFeatureCacheNode
@@ -155,7 +156,7 @@ class CapaRuleGenFeatureCache:
     def find_code_capabilities(
         self, ruleset: RuleSet, fh: FunctionHandle
     ) -> Tuple[FeatureSet, MatchResults, MatchResults, MatchResults]:
-        f_node: Optional[CapaRuleGenFeatureCacheNode] = self.func_nodes.get(fh.address, None)
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self._get_cached_func_node(fh)
         if f_node is None:
             return {}, {}, {}, {}
 
@@ -195,8 +196,16 @@ class CapaRuleGenFeatureCache:
         _, matches = ruleset.match(Scope.FILE, features, NO_ADDRESS)
         return features, matches
 
+    def _get_cached_func_node(self, fh: FunctionHandle) -> Optional[CapaRuleGenFeatureCacheNode]:
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self.func_nodes.get(fh.address)
+        if f_node is None:
+            # function is not in our cache, do extraction now
+            self._find_function_and_below_features(fh)
+            f_node = self.func_nodes.get(fh.address)
+        return f_node
+
     def get_all_function_features(self, fh: FunctionHandle) -> FeatureSet:
-        f_node: Optional[CapaRuleGenFeatureCacheNode] = self.func_nodes.get(fh.address, None)
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self._get_cached_func_node(fh)
         if f_node is None:
             return {}
 

capa/ida/plugin/capa_explorer.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/error.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/form.py

@@ -1,16 +1,17 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import os
 import copy
 import logging
 import itertools
 import collections
+from enum import IntFlag
 from typing import Any, List, Optional
+from pathlib import Path
 
 import idaapi
 import ida_kernwin
@@ -57,9 +58,6 @@ CAPA_OFFICIAL_RULESET_URL = f"https://github.com/mandiant/capa-rules/releases/ta
 CAPA_RULESET_DOC_URL = "https://github.com/mandiant/capa/blob/master/doc/rules.md"
 
 
-from enum import IntFlag
-
-
 class Options(IntFlag):
     NO_ANALYSIS = 0  # No auto analysis
     ANALYZE_AUTO = 1  # Runs the analysis when starting the explorer, see details below
@@ -73,10 +71,9 @@ AnalyzeOptionsText = {
 }
 
 
-def write_file(path, data):
+def write_file(path: Path, data):
     """ """
-    with open(path, "wb") as save_file:
-        save_file.write(data)
+    path.write_bytes(data)
 
 
 def trim_function_name(f, max_length=25):
@@ -192,8 +189,10 @@ class CapaExplorerForm(idaapi.PluginForm):
         # caches used to speed up capa explorer analysis - these must be init to None
         self.resdoc_cache: Optional[capa.render.result_document.ResultDocument] = None
         self.program_analysis_ruleset_cache: Optional[capa.rules.RuleSet] = None
-        self.rulegen_ruleset_cache: Optional[capa.rules.RuleSet] = None
+        self.feature_extractor: Optional[CapaExplorerFeatureExtractor] = None
+        self.rulegen_feature_extractor: Optional[CapaExplorerFeatureExtractor] = None
         self.rulegen_feature_cache: Optional[CapaRuleGenFeatureCache] = None
+        self.rulegen_ruleset_cache: Optional[capa.rules.RuleSet] = None
         self.rulegen_current_function: Optional[FunctionHandle] = None
 
         # models
@@ -536,7 +535,7 @@ class CapaExplorerForm(idaapi.PluginForm):
         @param new_ea: destination ea
         @param old_ea: source ea
         """
-        if not self.view_tabs.currentIndex() in (0, 1):
+        if self.view_tabs.currentIndex() not in (0, 1):
             return
 
         if idaapi.get_widget_type(widget) != idaapi.BWN_DISASM:
@@ -574,10 +573,10 @@ class CapaExplorerForm(idaapi.PluginForm):
 
     def ensure_capa_settings_rule_path(self):
         try:
-            path: str = settings.user.get(CAPA_SETTINGS_RULE_PATH, "")
+            path: Path = Path(settings.user.get(CAPA_SETTINGS_RULE_PATH, ""))
 
             # resolve rules directory - check self and settings first, then ask user
-            if not os.path.exists(path):
+            if not path.exists():
                 # configure rules selection messagebox
                 rules_message = QtWidgets.QMessageBox()
                 rules_message.setIcon(QtWidgets.QMessageBox.Information)
@@ -585,7 +584,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                 rules_message.setText("You must specify a directory containing capa rules before running analysis.")
                 rules_message.setInformativeText(
                     "Click 'Ok' to specify a local directory of rules or you can download and extract the official "
-                    f"rules from the URL listed in the details."
+                    + "rules from the URL listed in the details."
                 )
                 rules_message.setDetailedText(f"{CAPA_OFFICIAL_RULESET_URL}")
                 rules_message.setStandardButtons(QtWidgets.QMessageBox.Ok | QtWidgets.QMessageBox.Cancel)
@@ -595,24 +594,25 @@ class CapaExplorerForm(idaapi.PluginForm):
                 if pressed == QtWidgets.QMessageBox.Cancel:
                     raise UserCancelledError()
 
-                path = self.ask_user_directory()
+                path = Path(self.ask_user_directory())
                 if not path:
                     raise UserCancelledError()
 
-                if not os.path.exists(path):
-                    logger.error("rule path %s does not exist or cannot be accessed" % path)
+                if not path.exists():
+                    logger.error("rule path %s does not exist or cannot be accessed", path)
                     return False
 
-                settings.user[CAPA_SETTINGS_RULE_PATH] = path
-        except UserCancelledError as e:
+                settings.user[CAPA_SETTINGS_RULE_PATH] = str(path)
+        except UserCancelledError:
             capa.ida.helpers.inform_user_ida_ui("Analysis requires capa rules")
             logger.warning(
-                f"You must specify a directory containing capa rules before running analysis. Download and extract the official rules from {CAPA_OFFICIAL_RULESET_URL} (recommended)."
+                "You must specify a directory containing capa rules before running analysis.%s",
+                f"Download and extract the official rules from {CAPA_OFFICIAL_RULESET_URL} (recommended).",
             )
             return False
         except Exception as e:
             capa.ida.helpers.inform_user_ida_ui("Failed to load capa rules")
-            logger.error("Failed to load capa rules (error: %s).", e, exc_info=True)
+            logger.exception("Failed to load capa rules (error: %s).", e)
             return False
 
         if ida_kernwin.user_cancelled():
@@ -626,7 +626,7 @@ class CapaExplorerForm(idaapi.PluginForm):
         if not self.ensure_capa_settings_rule_path():
             return False
 
-        rule_path: str = settings.user.get(CAPA_SETTINGS_RULE_PATH, "")
+        rule_path: Path = Path(settings.user.get(CAPA_SETTINGS_RULE_PATH, ""))
         try:
 
             def on_load_rule(_, i, total):
@@ -645,9 +645,9 @@ class CapaExplorerForm(idaapi.PluginForm):
 
             logger.error("Failed to load capa rules from %s (error: %s).", settings.user[CAPA_SETTINGS_RULE_PATH], e)
             logger.error(
-                "Make sure your file directory contains properly "
-                "formatted capa rules. You can download and extract the official rules from %s. "
-                "Or, for more details, see the rules documentation here: %s",
+                "Make sure your file directory contains properly "  # noqa: G003 [logging statement uses +]
+                + "formatted capa rules. You can download and extract the official rules from %s. "
+                + "Or, for more details, see the rules documentation here: %s",
                 CAPA_OFFICIAL_RULESET_URL,
                 CAPA_RULESET_DOC_URL,
             )
@@ -705,14 +705,15 @@ class CapaExplorerForm(idaapi.PluginForm):
 
                         capa.ida.helpers.inform_user_ida_ui("Cached results were generated using different capas rules")
                         logger.warning(
-                            "capa is showing you cached results from a previous analysis run. Your rules have changed since and you should reanalyze the program to see new results."
+                            "capa is showing you cached results from a previous analysis run.%s ",
+                            "Your rules have changed since and you should reanalyze the program to see new results.",
                         )
                         view_status_rules = "no rules matched for cache"
 
                     cached_results_time = self.resdoc_cache.meta.timestamp.strftime("%Y-%m-%d %H:%M:%S")
                     new_view_status = f"capa rules: {view_status_rules}, cached results (created {cached_results_time})"
                 except Exception as e:
-                    logger.error("Failed to load cached capa results (error: %s).", e, exc_info=True)
+                    logger.exception("Failed to load cached capa results (error: %s).", e)
                     return False
             else:
                 # load results from fresh anlaysis
@@ -725,13 +726,11 @@ class CapaExplorerForm(idaapi.PluginForm):
                     update_wait_box(f"{text} ({self.process_count} of {self.process_total})")
                     self.process_count += 1
 
-                update_wait_box("initializing feature extractor")
-
                 try:
-                    extractor = CapaExplorerFeatureExtractor()
-                    extractor.indicator.progress.connect(slot_progress_feature_extraction)
+                    self.feature_extractor = CapaExplorerFeatureExtractor()
+                    self.feature_extractor.indicator.progress.connect(slot_progress_feature_extraction)
                 except Exception as e:
-                    logger.error("Failed to initialize feature extractor (error: %s).", e, exc_info=True)
+                    logger.exception("Failed to initialize feature extractor (error: %s)", e)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -741,9 +740,9 @@ class CapaExplorerForm(idaapi.PluginForm):
                 update_wait_box("calculating analysis")
 
                 try:
-                    self.process_total += len(tuple(extractor.get_functions()))
+                    self.process_total += len(tuple(self.feature_extractor.get_functions()))
                 except Exception as e:
-                    logger.error("Failed to calculate analysis (error: %s).", e, exc_info=True)
+                    logger.exception("Failed to calculate analysis (error: %s).", e)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -767,15 +766,19 @@ class CapaExplorerForm(idaapi.PluginForm):
                 update_wait_box("extracting features")
 
                 try:
-                    meta = capa.ida.helpers.collect_metadata([settings.user[CAPA_SETTINGS_RULE_PATH]])
-                    capabilities, counts = capa.main.find_capabilities(ruleset, extractor, disable_progress=True)
-                    meta["analysis"].update(counts)
-                    meta["analysis"]["layout"] = capa.main.compute_layout(ruleset, extractor, capabilities)
+                    meta = capa.ida.helpers.collect_metadata([Path(settings.user[CAPA_SETTINGS_RULE_PATH])])
+                    capabilities, counts = capa.main.find_capabilities(
+                        ruleset, self.feature_extractor, disable_progress=True
+                    )
+
+                    meta.analysis.feature_counts = counts["feature_counts"]
+                    meta.analysis.library_functions = counts["library_functions"]
+                    meta.analysis.layout = capa.main.compute_layout(ruleset, self.feature_extractor, capabilities)
                 except UserCancelledError:
                     logger.info("User cancelled analysis.")
                     return False
                 except Exception as e:
-                    logger.error("Failed to extract capabilities from database (error: %s)", e, exc_info=True)
+                    logger.exception("Failed to extract capabilities from database (error: %s)", e)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -787,7 +790,8 @@ class CapaExplorerForm(idaapi.PluginForm):
                 try:
                     # support binary files specifically for x86/AMD64 shellcode
                     # warn user binary file is loaded but still allow capa to process it
-                    # TODO: check specific architecture of binary files based on how user configured IDA processors
+                    # TODO(mike-hunhoff): check specific architecture of binary files based on how user configured IDA processors
+                    # https://github.com/mandiant/capa/issues/1603
                     if idaapi.get_file_type_name() == "Binary file":
                         logger.warning("-" * 80)
                         logger.warning(" Input file appears to be a binary file.")
@@ -808,7 +812,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                     if capa.main.has_file_limitation(ruleset, capabilities, is_standalone=False):
                         capa.ida.helpers.inform_user_ida_ui("capa encountered file limitation warnings during analysis")
                 except Exception as e:
-                    logger.error("Failed to check for file limitations (error: %s)", e, exc_info=True)
+                    logger.exception("Failed to check for file limitations (error: %s)", e)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -822,7 +826,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                         meta, ruleset, capabilities
                     )
                 except Exception as e:
-                    logger.error("Failed to collect results (error: %s)", e, exc_info=True)
+                    logger.exception("Failed to collect results (error: %s)", e)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -838,7 +842,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                     capa.ida.helpers.save_rules_cache_id(ruleset_id)
                     logger.info("Saved cached results to database")
                 except Exception as e:
-                    logger.error("Failed to save results to database (error: %s)", e, exc_info=True)
+                    logger.exception("Failed to save results to database (error: %s)", e)
                     return False
                 user_settings = settings.user[CAPA_SETTINGS_RULE_PATH]
                 count_source_rules = self.program_analysis_ruleset_cache.source_rule_count
@@ -859,7 +863,7 @@ class CapaExplorerForm(idaapi.PluginForm):
 
             self.model_data.render_capa_doc(self.resdoc_cache, self.view_show_results_by_function.isChecked())
         except Exception as e:
-            logger.error("Failed to render results (error: %s)", e, exc_info=True)
+            logger.exception("Failed to render results (error: %s)", e)
             return False
 
         self.set_view_status_label(new_view_status)
@@ -911,7 +915,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                 has_cache: bool = capa.ida.helpers.idb_contains_cached_results()
             except Exception as e:
                 capa.ida.helpers.inform_user_ida_ui("Failed to check for cached results, reanalyzing program")
-                logger.error("Failed to check for cached results (error: %s)", e, exc_info=True)
+                logger.exception("Failed to check for cached results (error: %s)", e)
                 return False
 
             if ida_kernwin.user_cancelled():
@@ -931,7 +935,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                         ] = capa.ida.helpers.load_and_verify_cached_results()
                     except Exception as e:
                         capa.ida.helpers.inform_user_ida_ui("Failed to verify cached results, reanalyzing program")
-                        logger.error("Failed to verify cached results (error: %s)", e, exc_info=True)
+                        logger.exception("Failed to verify cached results (error: %s)", e)
                         return False
 
                     if results is None:
@@ -944,9 +948,9 @@ class CapaExplorerForm(idaapi.PluginForm):
                         "Reanalyze program",
                         "",
                         ida_kernwin.ASKBTN_YES,
-                        f"This database contains capa results generated on "
-                        f"{results.meta.timestamp.strftime('%Y-%m-%d at %H:%M:%S')}.\n"
-                        f"Load existing data or analyze program again?",
+                        "This database contains capa results generated on "
+                        + results.meta.timestamp.strftime("%Y-%m-%d at %H:%M:%S")
+                        + ".\nLoad existing data or analyze program again?",
                     )
 
                     if btn_id == ida_kernwin.ASKBTN_CANCEL:
@@ -973,26 +977,21 @@ class CapaExplorerForm(idaapi.PluginForm):
         # so we'll work with a local copy of the ruleset.
         ruleset = copy.deepcopy(self.rulegen_ruleset_cache)
 
-        # clear feature cache
-        if self.rulegen_feature_cache is not None:
-            self.rulegen_feature_cache = None
-
         # clear cached function
         if self.rulegen_current_function is not None:
             self.rulegen_current_function = None
 
-        if ida_kernwin.user_cancelled():
-            logger.info("User cancelled analysis.")
-            return False
-
-        update_wait_box("Initializing feature extractor")
-
-        try:
-            # must use extractor to get function, as capa analysis requires casted object
-            extractor = CapaExplorerFeatureExtractor()
-        except Exception as e:
-            logger.error("Failed to initialize feature extractor (error: %s)", e, exc_info=True)
-            return False
+        # these are init once objects, create on tab change
+        if self.rulegen_feature_cache is None or self.rulegen_feature_extractor is None:
+            try:
+                update_wait_box("performing one-time file analysis")
+                self.rulegen_feature_extractor = CapaExplorerFeatureExtractor()
+                self.rulegen_feature_cache = CapaRuleGenFeatureCache(self.rulegen_feature_extractor)
+            except Exception as e:
+                logger.exception("Failed to initialize feature extractor (error: %s)", e)
+                return False
+        else:
+            logger.info("Reusing prior rulegen cache")
 
         if ida_kernwin.user_cancelled():
             logger.info("User cancelled analysis.")
@@ -1004,24 +1003,9 @@ class CapaExplorerForm(idaapi.PluginForm):
         try:
             f = idaapi.get_func(idaapi.get_screen_ea())
             if f is not None:
-                self.rulegen_current_function = extractor.get_function(f.start_ea)
+                self.rulegen_current_function = self.rulegen_feature_extractor.get_function(f.start_ea)
         except Exception as e:
-            logger.error("Failed to resolve function at address 0x%X (error: %s)", f.start_ea, e, exc_info=True)
-            return False
-
-        if ida_kernwin.user_cancelled():
-            logger.info("User cancelled analysis.")
-            return False
-
-        # extract features
-        try:
-            fh_list: List[FunctionHandle] = []
-            if self.rulegen_current_function is not None:
-                fh_list.append(self.rulegen_current_function)
-
-            self.rulegen_feature_cache = CapaRuleGenFeatureCache(fh_list, extractor)
-        except Exception as e:
-            logger.error("Failed to extract features (error: %s)", e, exc_info=True)
+            logger.exception("Failed to resolve function at address 0x%X (error: %s)", f.start_ea, e)
             return False
 
         if ida_kernwin.user_cancelled():
@@ -1047,7 +1031,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                     for addr, _ in result:
                         all_function_features[capa.features.common.MatchedRule(name)].add(addr)
         except Exception as e:
-            logger.error("Failed to generate rule matches (error: %s)", e, exc_info=True)
+            logger.exception("Failed to generate rule matches (error: %s)", e)
             return False
 
         if ida_kernwin.user_cancelled():
@@ -1068,7 +1052,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                 for addr, _ in result:
                     all_file_features[capa.features.common.MatchedRule(name)].add(addr)
         except Exception as e:
-            logger.error("Failed to generate file rule matches (error: %s)", e, exc_info=True)
+            logger.exception("Failed to generate file rule matches (error: %s)", e)
             return False
 
         if ida_kernwin.user_cancelled():
@@ -1091,7 +1075,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                 f"capa rules: {settings.user[CAPA_SETTINGS_RULE_PATH]} ({settings.user[CAPA_SETTINGS_RULE_PATH]} rules)"
             )
         except Exception as e:
-            logger.error("Failed to render views (error: %s)", e, exc_info=True)
+            logger.exception("Failed to render views (error: %s)", e)
             return False
 
         return True
@@ -1176,7 +1160,7 @@ class CapaExplorerForm(idaapi.PluginForm):
             assert self.rulegen_ruleset_cache is not None
             assert self.rulegen_feature_cache is not None
         except Exception as e:
-            logger.error("Failed to access cache (error: %s)", e, exc_info=True)
+            logger.exception("Failed to access cache (error: %s)", e)
             self.set_rulegen_status("Error: see console output for more details")
             return
 
@@ -1220,11 +1204,11 @@ class CapaExplorerForm(idaapi.PluginForm):
                 self.set_rulegen_status(f"Failed to create function rule matches from rule set ({e})")
                 return
 
-            if rule.scope == capa.rules.Scope.FUNCTION and rule.name in func_matches.keys():
+            if rule.scope == capa.rules.Scope.FUNCTION and rule.name in func_matches:
                 is_match = True
-            elif rule.scope == capa.rules.Scope.BASIC_BLOCK and rule.name in bb_matches.keys():
+            elif rule.scope == capa.rules.Scope.BASIC_BLOCK and rule.name in bb_matches:
                 is_match = True
-            elif rule.scope == capa.rules.Scope.INSTRUCTION and rule.name in insn_matches.keys():
+            elif rule.scope == capa.rules.Scope.INSTRUCTION and rule.name in insn_matches:
                 is_match = True
         elif rule.scope == capa.rules.Scope.FILE:
             try:
@@ -1232,7 +1216,7 @@ class CapaExplorerForm(idaapi.PluginForm):
             except Exception as e:
                 self.set_rulegen_status(f"Failed to create file rule matches from rule set ({e})")
                 return
-            if rule.name in file_matches.keys():
+            if rule.name in file_matches:
                 is_match = True
         else:
             is_match = False
@@ -1259,7 +1243,6 @@ class CapaExplorerForm(idaapi.PluginForm):
         elif index == 1:
             self.set_view_status_label(self.view_status_label_rulegen_cache)
             self.view_status_label_analysis_cache = status_prev
-
             self.view_reset_button.setText("Clear")
 
     def slot_rulegen_editor_update(self):
@@ -1323,8 +1306,8 @@ class CapaExplorerForm(idaapi.PluginForm):
 
         s = self.resdoc_cache.json().encode("utf-8")
 
-        path = self.ask_user_capa_json_file()
-        if not path:
+        path = Path(self.ask_user_capa_json_file())
+        if not path.exists():
             return
 
         write_file(path, s)
@@ -1336,8 +1319,8 @@ class CapaExplorerForm(idaapi.PluginForm):
             idaapi.info("No rule to save.")
             return
 
-        path = self.ask_user_capa_rule_file()
-        if not path:
+        path = Path(self.ask_user_capa_rule_file())
+        if not path.exists():
             return
 
         write_file(path, s)

capa/ida/plugin/hooks.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -30,7 +30,7 @@ class CapaExplorerIdaHooks(idaapi.UI_Hooks):
 
         @retval must be 0
         """
-        self.process_action_handle = self.process_action_hooks.get(name, None)
+        self.process_action_handle = self.process_action_hooks.get(name)
 
         if self.process_action_handle:
             self.process_action_handle(self.process_action_meta)

capa/ida/plugin/icon.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import base64
 
 # this is just `capa/.github/icon.png`.

capa/ida/plugin/item.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -130,8 +130,7 @@ class CapaExplorerDataItem:
 
     def children(self) -> Iterator["CapaExplorerDataItem"]:
         """yield children"""
-        for child in self._children:
-            yield child
+        yield from self._children
 
     def removeChildren(self):
         """remove children"""

capa/ida/plugin/model.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -372,7 +372,8 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                     display += f" ({statement.description})"
                 return CapaExplorerDefaultItem(parent, display)
         elif isinstance(statement, rd.CompoundStatement) and statement.type == rd.CompoundStatementType.NOT:
-            # TODO: do we display 'not'
+            # TODO(mike-hunhoff): verify that we can display NOT statements
+            # https://github.com/mandiant/capa/issues/1602
             pass
         elif isinstance(statement, rd.SomeStatement):
             display = f"{statement.count} or more"
@@ -421,12 +422,13 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         @param doc: result doc
         """
         if not match.success:
-            # TODO: display failed branches at some point? Help with debugging rules?
+            # TODO(mike-hunhoff): display failed branches at some point? Help with debugging rules?
+            # https://github.com/mandiant/capa/issues/1601
             return
 
         # optional statement with no successful children is empty
         if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
-            if not any(map(lambda m: m.success, match.children)):
+            if not any(m.success for m in match.children):
                 return
 
         if isinstance(match.node, rd.StatementNode):
@@ -626,7 +628,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             matched_rule_source = ""
 
             # check if match is a matched rule
-            matched_rule = doc.rules.get(feature.match, None)
+            matched_rule = doc.rules.get(feature.match)
             if matched_rule is not None:
                 matched_rule_source = matched_rule.source
 

capa/ida/plugin/proxy.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/view.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,6 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import re
+from typing import Dict, Optional
 from collections import Counter
 
 import idc
@@ -63,7 +64,7 @@ def parse_yaml_line(feature):
         feature, _, comment = feature.partition("#")
         feature, _, description = feature.partition("=")
 
-    return map(lambda o: o.strip(), (feature, description, comment))
+    return (o.strip() for o in (feature, description, comment))
 
 
 def parse_node_for_feature(feature, description, comment, depth):
@@ -93,7 +94,7 @@ def parse_node_for_feature(feature, description, comment, depth):
                 if name in ("string",):
                     display += f"{' '*depth}{feature}"
                     if comment:
-                        display += " # %s" % comment
+                        display += f" # {comment}"
                     display += f"\n{' '*(depth+2)}description: {description}\n"
                 else:
                     display += f"{' '*depth}- count({name}({value} = {description})): {count}"
@@ -498,12 +499,13 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
             rule_text += "\n  features:\n"
 
         for o in iterate_tree(self):
-            feature, description, comment = map(lambda o: o.strip(), tuple(o.text(i) for i in range(3)))
+            feature, description, comment = (o.strip() for o in tuple(o.text(i) for i in range(3)))
             rule_text += parse_node_for_feature(feature, description, comment, calc_item_depth(o))
 
-        # FIXME we avoid circular update by disabling signals when updating
+        # TODO(mike-hunhoff): we avoid circular update by disabling signals when updating
         # the preview. Preferably we would refactor the code to avoid this
-        # in the first place
+        # in the first place.
+        # https://github.com/mandiant/capa/issues/1600
         self.preview.blockSignals(True)
         self.preview.setPlainText(rule_text)
         self.preview.blockSignals(False)
@@ -646,7 +648,7 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
         counted = list(zip(Counter(features).keys(), Counter(features).values()))
 
         # single features
-        for k, v in filter(lambda t: t[1] == 1, counted):
+        for k, _ in filter(lambda t: t[1] == 1, counted):
             if isinstance(k, (capa.features.common.String,)):
                 value = f'"{capa.features.common.escape_string(k.get_value_str())}"'
             else:
@@ -682,10 +684,12 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
 
             # we don't add a new node for description; either set description column of parent's last child
             # or the parent itself
-            if parent.childCount():
-                parent.child(parent.childCount() - 1).setText(1, feature.lstrip("description:").lstrip())
-            else:
-                parent.setText(1, feature.lstrip("description:").lstrip())
+            if feature.startswith("description:"):
+                description = feature[len("description:") :].lstrip()
+                if parent.childCount():
+                    parent.child(parent.childCount() - 1).setText(1, description)
+                else:
+                    parent.setText(1, description)
             return None
         elif feature.startswith("- description:"):
             if not parent:
@@ -693,7 +697,8 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
                 return None
 
             # we don't add a new node for description; set the description column of the parent instead
-            parent.setText(1, feature.lstrip("- description:").lstrip())
+            description = feature[len("- description:") :].lstrip()
+            parent.setText(1, description)
             return None
 
         node = QtWidgets.QTreeWidgetItem(parent)
@@ -1010,7 +1015,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
 
         return o
 
-    def load_features(self, file_features, func_features={}):
+    def load_features(self, file_features, func_features: Optional[Dict] = None):
         """ """
         self.parse_features_for_tree(self.new_parent_node(self, ("File Scope",)), file_features)
         if func_features:
@@ -1219,8 +1224,7 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
             yield self.new_action(*action)
 
         # add default actions
-        for action in self.load_default_context_menu_actions(data):
-            yield action
+        yield from self.load_default_context_menu_actions(data)
 
     def load_default_context_menu(self, pos, item, model_index):
         """create default custom context menu

capa/main.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
 You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,38 +8,43 @@ Unless required by applicable law or agreed to in writing, software distributed
  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
+import io
 import os
 import sys
 import time
 import hashlib
 import logging
-import os.path
 import argparse
 import datetime
 import textwrap
 import itertools
 import contextlib
 import collections
-from typing import Any, Dict, List, Tuple, Callable
+from typing import Any, Dict, List, Tuple, Callable, Optional
+from pathlib import Path
 
 import halo
 import tqdm
 import colorama
+import tqdm.contrib.logging
 from pefile import PEFormatError
 from elftools.common.exceptions import ELFError
 
 import capa.perf
 import capa.rules
 import capa.engine
+import capa.helpers
 import capa.version
 import capa.render.json
 import capa.rules.cache
 import capa.render.default
 import capa.render.verbose
 import capa.features.common
-import capa.features.freeze
+import capa.features.freeze as frz
 import capa.render.vverbose
 import capa.features.extractors
+import capa.render.result_document
+import capa.render.result_document as rdoc
 import capa.features.extractors.common
 import capa.features.extractors.pefile
 import capa.features.extractors.dnfile_
@@ -53,6 +58,7 @@ from capa.helpers import (
     get_file_taste,
     get_auto_format,
     log_unsupported_os_error,
+    redirecting_print_to_tqdm,
     log_unsupported_arch_error,
     log_unsupported_format_error,
 )
@@ -69,6 +75,7 @@ from capa.features.common import (
     FORMAT_SC64,
     FORMAT_DOTNET,
     FORMAT_FREEZE,
+    FORMAT_RESULT,
 )
 from capa.features.address import NO_ADDRESS, Address
 from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
@@ -77,6 +84,8 @@ RULES_PATH_DEFAULT_STRING = "(embedded rules)"
 SIGNATURES_PATH_DEFAULT_STRING = "(embedded signatures)"
 BACKEND_VIV = "vivisect"
 BACKEND_DOTNET = "dotnet"
+BACKEND_BINJA = "binja"
+BACKEND_PEFILE = "pefile"
 
 E_MISSING_RULES = 10
 E_MISSING_FILE = 11
@@ -241,45 +250,61 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
     all_bb_matches = collections.defaultdict(list)  # type: MatchResults
     all_insn_matches = collections.defaultdict(list)  # type: MatchResults
 
-    meta = {
-        "feature_counts": {
-            "file": 0,
-            "functions": {},
-        },
-        "library_functions": {},
-    }  # type: Dict[str, Any]
+    feature_counts = rdoc.FeatureCounts(file=0, functions=())
+    library_functions: Tuple[rdoc.LibraryFunction, ...] = ()
 
-    pbar = tqdm.tqdm
-    if disable_progress:
-        # do not use tqdm to avoid unnecessary side effects when caller intends
-        # to disable progress completely
-        pbar = lambda s, *args, **kwargs: s
+    with redirecting_print_to_tqdm(disable_progress):
+        with tqdm.contrib.logging.logging_redirect_tqdm():
+            pbar = tqdm.tqdm
+            if disable_progress:
+                # do not use tqdm to avoid unnecessary side effects when caller intends
+                # to disable progress completely
+                def pbar(s, *args, **kwargs):
+                    return s
 
-    functions = list(extractor.get_functions())
-    n_funcs = len(functions)
+            functions = list(extractor.get_functions())
+            n_funcs = len(functions)
 
-    pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions")
-    for f in pb:
-        if extractor.is_library_function(f.address):
-            function_name = extractor.get_function_name(f.address)
-            logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
-            meta["library_functions"][f.address] = function_name
-            n_libs = len(meta["library_functions"])
-            percentage = round(100 * (n_libs / n_funcs))
-            if isinstance(pb, tqdm.tqdm):
-                pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
-            continue
+            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False)
+            for f in pb:
+                t0 = time.time()
+                if extractor.is_library_function(f.address):
+                    function_name = extractor.get_function_name(f.address)
+                    logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
+                    library_functions += (
+                        rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name),
+                    )
+                    n_libs = len(library_functions)
+                    percentage = round(100 * (n_libs / n_funcs))
+                    if isinstance(pb, tqdm.tqdm):
+                        pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
+                    continue
 
-        function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(ruleset, extractor, f)
-        meta["feature_counts"]["functions"][f.address] = feature_count
-        logger.debug("analyzed function 0x%x and extracted %d features", f.address, feature_count)
+                function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(
+                    ruleset, extractor, f
+                )
+                feature_counts.functions += (
+                    rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count),
+                )
+                t1 = time.time()
 
-        for rule_name, res in function_matches.items():
-            all_function_matches[rule_name].extend(res)
-        for rule_name, res in bb_matches.items():
-            all_bb_matches[rule_name].extend(res)
-        for rule_name, res in insn_matches.items():
-            all_insn_matches[rule_name].extend(res)
+                match_count = sum(len(res) for res in function_matches.values())
+                match_count += sum(len(res) for res in bb_matches.values())
+                match_count += sum(len(res) for res in insn_matches.values())
+                logger.debug(
+                    "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
+                    f.address,
+                    feature_count,
+                    match_count,
+                    t1 - t0,
+                )
+
+                for rule_name, res in function_matches.items():
+                    all_function_matches[rule_name].extend(res)
+                for rule_name, res in bb_matches.items():
+                    all_bb_matches[rule_name].extend(res)
+                for rule_name, res in insn_matches.items():
+                    all_insn_matches[rule_name].extend(res)
 
     # collection of features that captures the rule matches within function, BB, and instruction scopes.
     # mapping from feature (matched rule) to set of addresses at which it matched.
@@ -287,16 +312,15 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
     for rule_name, results in itertools.chain(
         all_function_matches.items(), all_bb_matches.items(), all_insn_matches.items()
     ):
-        locations = set(map(lambda p: p[0], results))
+        locations = {p[0] for p in results}
         rule = ruleset[rule_name]
         capa.engine.index_rule_matches(function_and_lower_features, rule, locations)
 
     all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, function_and_lower_features)
-    meta["feature_counts"]["file"] = feature_count
+    feature_counts.file = feature_count
 
-    matches = {
-        rule_name: results
-        for rule_name, results in itertools.chain(
+    matches = dict(
+        itertools.chain(
             # each rule exists in exactly one scope,
             # so there won't be any overlap among these following MatchResults,
             # and we can merge the dictionaries naively.
@@ -305,17 +329,20 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
             all_function_matches.items(),
             all_file_matches.items(),
         )
+    )
+
+    meta = {
+        "feature_counts": feature_counts,
+        "library_functions": library_functions,
     }
 
     return matches, meta
 
 
-# TODO move all to helpers?
-def has_rule_with_namespace(rules, capabilities, rule_cat):
-    for rule_name in capabilities.keys():
-        if rules.rules[rule_name].meta.get("namespace", "").startswith(rule_cat):
-            return True
-    return False
+def has_rule_with_namespace(rules: RuleSet, capabilities: MatchResults, namespace: str) -> bool:
+    return any(
+        rules.rules[rule_name].meta.get("namespace", "").startswith(namespace) for rule_name in capabilities.keys()
+    )
 
 
 def is_internal_rule(rule: Rule) -> bool:
@@ -348,26 +375,23 @@ def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalon
     return False
 
 
-def is_supported_format(sample: str) -> bool:
+def is_supported_format(sample: Path) -> bool:
     """
     Return if this is a supported file based on magic header values
     """
-    with open(sample, "rb") as f:
-        taste = f.read(0x100)
+    taste = sample.open("rb").read(0x100)
 
     return len(list(capa.features.extractors.common.extract_format(taste))) == 1
 
 
-def is_supported_arch(sample: str) -> bool:
-    with open(sample, "rb") as f:
-        buf = f.read()
+def is_supported_arch(sample: Path) -> bool:
+    buf = sample.read_bytes()
 
     return len(list(capa.features.extractors.common.extract_arch(buf))) == 1
 
 
-def get_arch(sample: str) -> str:
-    with open(sample, "rb") as f:
-        buf = f.read()
+def get_arch(sample: Path) -> str:
+    buf = sample.read_bytes()
 
     for feature, _ in capa.features.extractors.common.extract_arch(buf):
         assert isinstance(feature.value, str)
@@ -376,16 +400,14 @@ def get_arch(sample: str) -> str:
     return "unknown"
 
 
-def is_supported_os(sample: str) -> bool:
-    with open(sample, "rb") as f:
-        buf = f.read()
+def is_supported_os(sample: Path) -> bool:
+    buf = sample.read_bytes()
 
     return len(list(capa.features.extractors.common.extract_os(buf))) == 1
 
 
-def get_os(sample: str) -> str:
-    with open(sample, "rb") as f:
-        buf = f.read()
+def get_os(sample: Path) -> str:
+    buf = sample.read_bytes()
 
     for feature, _ in capa.features.extractors.common.extract_os(buf):
         assert isinstance(feature.value, str)
@@ -413,7 +435,7 @@ def is_running_standalone() -> bool:
     return hasattr(sys, "frozen") and hasattr(sys, "_MEIPASS")
 
 
-def get_default_root() -> str:
+def get_default_root() -> Path:
     """
     get the file system path to the default resources directory.
     under PyInstaller, this comes from _MEIPASS.
@@ -423,30 +445,28 @@ def get_default_root() -> str:
         # pylance/mypy don't like `sys._MEIPASS` because this isn't standard.
         # its injected by pyinstaller.
         # so we'll fetch this attribute dynamically.
-        return getattr(sys, "_MEIPASS")
+        assert hasattr(sys, "_MEIPASS")
+        return Path(sys._MEIPASS)
     else:
-        return os.path.join(os.path.dirname(__file__), "..")
+        return Path(__file__).resolve().parent.parent
 
 
-def get_default_signatures() -> List[str]:
+def get_default_signatures() -> List[Path]:
     """
     compute a list of file system paths to the default FLIRT signatures.
     """
-    sigs_path = os.path.join(get_default_root(), "sigs")
+    sigs_path = get_default_root() / "sigs"
     logger.debug("signatures path: %s", sigs_path)
 
     ret = []
-    for root, _, files in os.walk(sigs_path):
-        for file in files:
-            if not (file.endswith(".pat") or file.endswith(".pat.gz") or file.endswith(".sig")):
-                continue
-
-            ret.append(os.path.join(root, file))
+    for file in sigs_path.rglob("*"):
+        if file.is_file() and file.suffix.lower() in (".pat", ".pat.gz", ".sig"):
+            ret.append(file)
 
     return ret
 
 
-def get_workspace(path, format_, sigpaths):
+def get_workspace(path: Path, format_: str, sigpaths: List[Path]):
     """
     load the program at the given path into a vivisect workspace using the given format.
     also apply the given FLIRT signatures.
@@ -467,24 +487,23 @@ def get_workspace(path, format_, sigpaths):
     import viv_utils.flirt
 
     logger.debug("generating vivisect workspace for: %s", path)
-    # TODO should not be auto at this point, anymore
     if format_ == FORMAT_AUTO:
         if not is_supported_format(path):
             raise UnsupportedFormatError()
 
         # don't analyze, so that we can add our Flirt function analyzer first.
-        vw = viv_utils.getWorkspace(path, analyze=False, should_save=False)
+        vw = viv_utils.getWorkspace(str(path), analyze=False, should_save=False)
     elif format_ in {FORMAT_PE, FORMAT_ELF}:
-        vw = viv_utils.getWorkspace(path, analyze=False, should_save=False)
+        vw = viv_utils.getWorkspace(str(path), analyze=False, should_save=False)
     elif format_ == FORMAT_SC32:
         # these are not analyzed nor saved.
-        vw = viv_utils.getShellcodeWorkspaceFromFile(path, arch="i386", analyze=False)
+        vw = viv_utils.getShellcodeWorkspaceFromFile(str(path), arch="i386", analyze=False)
     elif format_ == FORMAT_SC64:
-        vw = viv_utils.getShellcodeWorkspaceFromFile(path, arch="amd64", analyze=False)
+        vw = viv_utils.getShellcodeWorkspaceFromFile(str(path), arch="amd64", analyze=False)
     else:
         raise ValueError("unexpected format: " + format_)
 
-    viv_utils.flirt.register_flirt_signature_analyzers(vw, sigpaths)
+    viv_utils.flirt.register_flirt_signature_analyzers(vw, [str(s) for s in sigpaths])
 
     vw.analyze()
 
@@ -492,13 +511,12 @@ def get_workspace(path, format_, sigpaths):
     return vw
 
 
-# TODO get_extractors -> List[FeatureExtractor]?
 def get_extractor(
-    path: str,
+    path: Path,
     format_: str,
-    os: str,
+    os_: str,
     backend: str,
-    sigpaths: List[str],
+    sigpaths: List[Path],
     should_save_workspace=False,
     disable_progress=False,
 ) -> FeatureExtractor:
@@ -515,7 +533,7 @@ def get_extractor(
         if not is_supported_arch(path):
             raise UnsupportedArchError()
 
-        if os == OS_AUTO and not is_supported_os(path):
+        if os_ == OS_AUTO and not is_supported_os(path):
             raise UnsupportedOSError()
 
     if format_ == FORMAT_DOTNET:
@@ -523,8 +541,39 @@ def get_extractor(
 
         return capa.features.extractors.dnfile.extractor.DnfileFeatureExtractor(path)
 
-    # default to use vivisect backend
-    else:
+    elif backend == BACKEND_BINJA:
+        from capa.features.extractors.binja.find_binja_api import find_binja_path
+
+        # When we are running as a standalone executable, we cannot directly import binaryninja
+        # We need to fist find the binja API installation path and add it into sys.path
+        if is_running_standalone():
+            bn_api = find_binja_path()
+            if bn_api.exists():
+                sys.path.append(str(bn_api))
+
+        try:
+            from binaryninja import BinaryView, BinaryViewType
+        except ImportError:
+            raise RuntimeError(
+                "Cannot import binaryninja module. Please install the Binary Ninja Python API first: "
+                + "https://docs.binary.ninja/dev/batch.html#install-the-api)."
+            )
+
+        import capa.features.extractors.binja.extractor
+
+        with halo.Halo(text="analyzing program", spinner="simpleDots", stream=sys.stderr, enabled=not disable_progress):
+            bv: BinaryView = BinaryViewType.get_view_of_file(str(path))
+            if bv is None:
+                raise RuntimeError(f"Binary Ninja cannot open file {path}")
+
+        return capa.features.extractors.binja.extractor.BinjaFeatureExtractor(bv)
+
+    elif backend == BACKEND_PEFILE:
+        import capa.features.extractors.pefile
+
+        return capa.features.extractors.pefile.PefileFeatureExtractor(path)
+
+    elif backend == BACKEND_VIV:
         import capa.features.extractors.viv.extractor
 
         with halo.Halo(text="analyzing program", spinner="simpleDots", stream=sys.stderr, enabled=not disable_progress):
@@ -540,11 +589,14 @@ def get_extractor(
             else:
                 logger.debug("CAPA_SAVE_WORKSPACE unset, not saving workspace")
 
-        return capa.features.extractors.viv.extractor.VivisectFeatureExtractor(vw, path, os)
+        return capa.features.extractors.viv.extractor.VivisectFeatureExtractor(vw, path, os_)
+
+    else:
+        raise ValueError("unexpected backend: " + backend)
 
 
-def get_file_extractors(sample: str, format_: str) -> List[FeatureExtractor]:
-    file_extractors: List[FeatureExtractor] = list()
+def get_file_extractors(sample: Path, format_: str) -> List[FeatureExtractor]:
+    file_extractors: List[FeatureExtractor] = []
 
     if format_ == FORMAT_PE:
         file_extractors.append(capa.features.extractors.pefile.PefileFeatureExtractor(sample))
@@ -559,7 +611,7 @@ def get_file_extractors(sample: str, format_: str) -> List[FeatureExtractor]:
     return file_extractors
 
 
-def is_nursery_rule_path(path: str) -> bool:
+def is_nursery_rule_path(path: Path) -> bool:
     """
     The nursery is a spot for rules that have not yet been fully polished.
     For example, they may not have references to public example of a technique.
@@ -569,21 +621,21 @@ def is_nursery_rule_path(path: str) -> bool:
     When nursery rules are loaded, their metadata section should be updated with:
       `nursery=True`.
     """
-    return "nursery" in path
+    return "nursery" in path.parts
 
 
-def collect_rule_file_paths(rule_paths: List[str]) -> List[str]:
+def collect_rule_file_paths(rule_paths: List[Path]) -> List[Path]:
     """
     collect all rule file paths, including those in subdirectories.
     """
     rule_file_paths = []
     for rule_path in rule_paths:
-        if not os.path.exists(rule_path):
+        if not rule_path.exists():
             raise IOError(f"rule path {rule_path} does not exist or cannot be accessed")
 
-        if os.path.isfile(rule_path):
+        if rule_path.is_file():
             rule_file_paths.append(rule_path)
-        elif os.path.isdir(rule_path):
+        elif rule_path.is_dir():
             logger.debug("reading rules from directory %s", rule_path)
             for root, _, files in os.walk(rule_path):
                 if ".git" in root:
@@ -600,14 +652,12 @@ def collect_rule_file_paths(rule_paths: List[str]) -> List[str]:
                             # other things maybe are rules, but are mis-named.
                             logger.warning("skipping non-.yml file: %s", file)
                         continue
-                    rule_path = os.path.join(root, file)
-                    rule_file_paths.append(rule_path)
-
+                    rule_file_paths.append(Path(root) / file)
     return rule_file_paths
 
 
 # TypeAlias. note: using `foo: TypeAlias = bar` is Python 3.10+
-RulePath = str
+RulePath = Path
 
 
 def on_load_rule_default(_path: RulePath, i: int, _total: int) -> None:
@@ -627,17 +677,13 @@ def get_rules(
     """
     if cache_dir is None:
         cache_dir = capa.rules.cache.get_default_cache_directory()
-
     # rule_paths may contain directory paths,
     # so search for file paths recursively.
     rule_file_paths = collect_rule_file_paths(rule_paths)
 
     # this list is parallel to `rule_file_paths`:
     # rule_file_paths[i] corresponds to rule_contents[i].
-    rule_contents = []
-    for file_path in rule_file_paths:
-        with open(file_path, "rb") as f:
-            rule_contents.append(f.read())
+    rule_contents = [file_path.read_bytes() for file_path in rule_file_paths]
 
     ruleset = capa.rules.cache.load_cached_ruleset(cache_dir, rule_contents)
     if ruleset is not None:
@@ -654,9 +700,8 @@ def get_rules(
         except capa.rules.InvalidRule:
             raise
         else:
-            rule.meta["capa/path"] = path
-            if is_nursery_rule_path(path):
-                rule.meta["capa/nursery"] = True
+            rule.meta["capa/path"] = path.as_posix()
+            rule.meta["capa/nursery"] = is_nursery_rule_path(path)
 
             rules.append(rule)
             logger.debug("loaded rule: '%s' with scope: %s", rule.name, rule.scope)
@@ -668,27 +713,25 @@ def get_rules(
     return ruleset
 
 
-def get_signatures(sigs_path):
-    if not os.path.exists(sigs_path):
+def get_signatures(sigs_path: Path) -> List[Path]:
+    if not sigs_path.exists():
         raise IOError(f"signatures path {sigs_path} does not exist or cannot be accessed")
 
-    paths = []
-    if os.path.isfile(sigs_path):
+    paths: List[Path] = []
+    if sigs_path.is_file():
         paths.append(sigs_path)
-    elif os.path.isdir(sigs_path):
-        logger.debug("reading signatures from directory %s", os.path.abspath(os.path.normpath(sigs_path)))
-        for root, _, files in os.walk(sigs_path):
-            for file in files:
-                if file.endswith((".pat", ".pat.gz", ".sig")):
-                    sig_path = os.path.join(root, file)
-                    paths.append(sig_path)
+    elif sigs_path.is_dir():
+        logger.debug("reading signatures from directory %s", sigs_path.resolve())
+        for file in sigs_path.rglob("*"):
+            if file.is_file() and file.suffix.lower() in (".pat", ".pat.gz", ".sig"):
+                paths.append(file)
 
-    # nicely normalize and format path so that debugging messages are clearer
-    paths = [os.path.abspath(os.path.normpath(path)) for path in paths]
+    # Convert paths to their absolute and normalized forms
+    paths = [path.resolve().absolute() for path in paths]
 
     # load signatures in deterministic order: the alphabetic sorting of filename.
     # this means that `0_sigs.pat` loads before `1_sigs.pat`.
-    paths = sorted(paths, key=os.path.basename)
+    paths = sorted(paths, key=lambda path: path.name)
 
     for path in paths:
         logger.debug("found signature file: %s", path)
@@ -698,58 +741,58 @@ def get_signatures(sigs_path):
 
 def collect_metadata(
     argv: List[str],
-    sample_path: str,
+    sample_path: Path,
     format_: str,
     os_: str,
-    rules_path: List[str],
+    rules_path: List[Path],
     extractor: capa.features.extractors.base_extractor.FeatureExtractor,
-):
+) -> rdoc.Metadata:
     md5 = hashlib.md5()
     sha1 = hashlib.sha1()
     sha256 = hashlib.sha256()
 
-    with open(sample_path, "rb") as f:
-        buf = f.read()
+    buf = sample_path.read_bytes()
 
     md5.update(buf)
     sha1.update(buf)
     sha256.update(buf)
 
-    if rules_path != [RULES_PATH_DEFAULT_STRING]:
-        rules_path = [os.path.abspath(os.path.normpath(r)) for r in rules_path]
-
-    format_ = get_format(sample_path) if format_ == FORMAT_AUTO else f"{format_} (manual)"
+    rules = tuple(r.resolve().absolute().as_posix() for r in rules_path)
+    format_ = get_format(sample_path) if format_ == FORMAT_AUTO else format_
     arch = get_arch(sample_path)
-    os_ = get_os(sample_path) if os_ == OS_AUTO else f"{os_} (manual)"
+    os_ = get_os(sample_path) if os_ == OS_AUTO else os_
 
-    return {
-        "timestamp": datetime.datetime.now().isoformat(),
-        "version": capa.version.__version__,
-        "argv": argv,
-        "sample": {
-            "md5": md5.hexdigest(),
-            "sha1": sha1.hexdigest(),
-            "sha256": sha256.hexdigest(),
-            "path": os.path.normpath(sample_path),
-        },
-        "analysis": {
-            "format": format_,
-            "arch": arch,
-            "os": os_,
-            "extractor": extractor.__class__.__name__,
-            "rules": rules_path,
-            "base_address": extractor.get_base_address(),
-            "layout": {
+    return rdoc.Metadata(
+        timestamp=datetime.datetime.now(),
+        version=capa.version.__version__,
+        argv=tuple(argv) if argv else None,
+        sample=rdoc.Sample(
+            md5=md5.hexdigest(),
+            sha1=sha1.hexdigest(),
+            sha256=sha256.hexdigest(),
+            path=sample_path.resolve().absolute().as_posix(),
+        ),
+        analysis=rdoc.Analysis(
+            format=format_,
+            arch=arch,
+            os=os_,
+            extractor=extractor.__class__.__name__,
+            rules=rules,
+            base_address=frz.Address.from_capa(extractor.get_base_address()),
+            layout=rdoc.Layout(
+                functions=(),
                 # this is updated after capabilities have been collected.
                 # will look like:
                 #
                 # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
-            },
-        },
-    }
+            ),
+            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
+            library_functions=(),
+        ),
+    )
 
 
-def compute_layout(rules, extractor, capabilities):
+def compute_layout(rules, extractor, capabilities) -> rdoc.Layout:
     """
     compute a metadata structure that links basic blocks
     to the functions in which they're found.
@@ -774,16 +817,19 @@ def compute_layout(rules, extractor, capabilities):
                 assert addr in functions_by_bb
                 matched_bbs.add(addr)
 
-    layout = {
-        "functions": {
-            f: {
-                "matched_basic_blocks": [bb for bb in bbs if bb in matched_bbs]
-                # this object is open to extension in the future,
+    layout = rdoc.Layout(
+        functions=tuple(
+            rdoc.FunctionLayout(
+                address=frz.Address.from_capa(f),
+                matched_basic_blocks=tuple(
+                    rdoc.BasicBlockLayout(address=frz.Address.from_capa(bb)) for bb in bbs if bb in matched_bbs
+                )  # this object is open to extension in the future,
                 # such as with the function name, etc.
-            }
+            )
             for f, bbs in bbs_by_function.items()
-        }
-    }
+            if len([bb for bb in bbs if bb in matched_bbs]) > 0
+        )
+    )
 
     return layout
 
@@ -873,7 +919,7 @@ def install_common_args(parser, wanted=None):
             "--backend",
             type=str,
             help="select the backend to use",
-            choices=(BACKEND_VIV,),
+            choices=(BACKEND_VIV, BACKEND_BINJA, BACKEND_PEFILE),
             default=BACKEND_VIV,
         )
 
@@ -884,12 +930,12 @@ def install_common_args(parser, wanted=None):
             (OS_MACOS,),
             (OS_WINDOWS,),
         ]
-        os_help = ", ".join(["%s (%s)" % (o[0], o[1]) if len(o) == 2 else o[0] for o in oses])
+        os_help = ", ".join([f"{o[0]} ({o[1]})" if len(o) == 2 else o[0] for o in oses])
         parser.add_argument(
             "--os",
             choices=[o[0] for o in oses],
             default=OS_AUTO,
-            help="select sample OS: %s" % os_help,
+            help=f"select sample OS: {os_help}",
         )
 
     if "rules" in wanted:
@@ -942,12 +988,20 @@ def handle_common_args(args):
     # disable vivisect-related logging, it's verbose and not relevant for capa users
     set_vivisect_log_level(logging.CRITICAL)
 
-    # Since Python 3.8 cp65001 is an alias to utf_8, but not for Python < 3.8
-    # TODO: remove this code when only supporting Python 3.8+
-    # https://stackoverflow.com/a/3259271/87207
-    import codecs
-
-    codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None)
+    if isinstance(sys.stdout, io.TextIOWrapper) or hasattr(sys.stdout, "reconfigure"):
+        # from sys.stdout type hint:
+        #
+        # TextIO is used instead of more specific types for the standard streams,
+        # since they are often monkeypatched at runtime. At startup, the objects
+        # are initialized to instances of TextIOWrapper.
+        #
+        # To use methods from TextIOWrapper, use an isinstance check to ensure that
+        # the streams have not been overridden:
+        #
+        # if isinstance(sys.stdout, io.TextIOWrapper):
+        #    sys.stdout.reconfigure(...)
+        sys.stdout.reconfigure(encoding="utf-8")
+    colorama.just_fix_windows_console()
 
     if args.color == "always":
         colorama.init(strip=False)
@@ -962,8 +1016,11 @@ def handle_common_args(args):
     else:
         raise RuntimeError("unexpected --color value: " + args.color)
 
+    if hasattr(args, "sample"):
+        args.sample = Path(args.sample)
+
     if hasattr(args, "rules"):
-        rules_paths: List[str] = []
+        rules_paths: List[Path] = []
 
         if args.rules == [RULES_PATH_DEFAULT_STRING]:
             logger.debug("-" * 80)
@@ -973,9 +1030,9 @@ def handle_common_args(args):
             logger.debug("     https://github.com/mandiant/capa-rules")
             logger.debug("-" * 80)
 
-            default_rule_path = os.path.join(get_default_root(), "rules")
+            default_rule_path = get_default_root() / "rules"
 
-            if not os.path.exists(default_rule_path):
+            if not default_rule_path.exists():
                 # when a users installs capa via pip,
                 # this pulls down just the source code - not the default rules.
                 # i'm not sure the default rules should even be written to the library directory,
@@ -987,10 +1044,9 @@ def handle_common_args(args):
             rules_paths.append(default_rule_path)
             args.is_default_rules = True
         else:
-            rules_paths = args.rules
-
-            if RULES_PATH_DEFAULT_STRING in rules_paths:
-                rules_paths.remove(RULES_PATH_DEFAULT_STRING)
+            for rule in args.rules:
+                if RULES_PATH_DEFAULT_STRING != rule:
+                    rules_paths.append(Path(rule))
 
             for rule_path in rules_paths:
                 logger.debug("using rules path: %s", rule_path)
@@ -1008,17 +1064,25 @@ def handle_common_args(args):
             )
             logger.debug("-" * 80)
 
-            sigs_path = os.path.join(get_default_root(), "sigs")
+            sigs_path = get_default_root() / "sigs"
+
+            if not sigs_path.exists():
+                logger.error(
+                    "Using default signature path, but it doesn't exist. "  # noqa: G003 [logging statement uses +]
+                    + "Please install the signatures first: "
+                    + "https://github.com/mandiant/capa/blob/master/doc/installation.md#method-2-using-capa-as-a-python-library."
+                )
+                raise IOError(f"signatures path {sigs_path} does not exist or cannot be accessed")
         else:
-            sigs_path = args.signatures
+            sigs_path = Path(args.signatures)
             logger.debug("using signatures path: %s", sigs_path)
 
         args.signatures = sigs_path
 
 
-def main(argv=None):
-    if sys.version_info < (3, 7):
-        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.7+")
+def main(argv: Optional[List[str]] = None):
+    if sys.version_info < (3, 8):
+        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+")
 
     if argv is None:
         argv = sys.argv[1:]
@@ -1083,7 +1147,7 @@ def main(argv=None):
 
     try:
         if is_running_standalone() and args.is_default_rules:
-            cache_dir = os.path.join(get_default_root(), "cache")
+            cache_dir = get_default_root() / "cache"
         else:
             cache_dir = capa.rules.cache.get_default_cache_directory()
 
@@ -1100,13 +1164,13 @@ def main(argv=None):
             rules = rules.filter_rules_by_meta(args.tag)
             logger.debug("selected %d rules", len(rules))
             for i, r in enumerate(rules.rules, 1):
-                # TODO don't display subscope rules?
                 logger.debug(" %d. %s", i, r)
+
     except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e:
         logger.error("%s", str(e))
         logger.error(
-            "Make sure your file directory contains properly formatted capa rules. You can download the standard "
-            "collection of capa rules from https://github.com/mandiant/capa-rules/releases."
+            "Make sure your file directory contains properly formatted capa rules. You can download the standard "  # noqa: G003 [logging statement uses +]
+            + "collection of capa rules from https://github.com/mandiant/capa-rules/releases."
         )
         logger.error(
             "Please ensure you're using the rules that correspond to your major version of capa (%s)",
@@ -1153,54 +1217,72 @@ def main(argv=None):
                 logger.debug("file limitation short circuit, won't analyze fully.")
                 return E_FILE_LIMITATION
 
-    if format_ == FORMAT_FREEZE:
-        with open(args.sample, "rb") as f:
-            extractor = capa.features.freeze.load(f.read())
+    meta: rdoc.Metadata
+    capabilities: MatchResults
+    counts: Dict[str, Any]
+
+    if format_ == FORMAT_RESULT:
+        # result document directly parses into meta, capabilities
+        result_doc = capa.render.result_document.ResultDocument.parse_file(args.sample)
+        meta, capabilities = result_doc.to_capa()
+
     else:
-        try:
-            if format_ == FORMAT_PE:
-                sig_paths = get_signatures(args.signatures)
-            else:
-                sig_paths = []
-                logger.debug("skipping library code matching: only have native PE signatures")
-        except IOError as e:
-            logger.error("%s", str(e))
-            return E_INVALID_SIG
+        # all other formats we must create an extractor
+        # and use that to extract meta and capabilities
 
-        should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None)
+        if format_ == FORMAT_FREEZE:
+            # freeze format deserializes directly into an extractor
+            extractor = frz.load(Path(args.sample).read_bytes())
+        else:
+            # all other formats we must create an extractor,
+            # such as viv, binary ninja, etc. workspaces
+            # and use those for extracting.
 
-        try:
-            extractor = get_extractor(
-                args.sample,
-                format_,
-                args.os,
-                args.backend,
-                sig_paths,
-                should_save_workspace,
-                disable_progress=args.quiet,
-            )
-        except UnsupportedFormatError:
-            log_unsupported_format_error()
-            return E_INVALID_FILE_TYPE
-        except UnsupportedArchError:
-            log_unsupported_arch_error()
-            return E_INVALID_FILE_ARCH
-        except UnsupportedOSError:
-            log_unsupported_os_error()
-            return E_INVALID_FILE_OS
+            try:
+                if format_ == FORMAT_PE:
+                    sig_paths = get_signatures(args.signatures)
+                else:
+                    sig_paths = []
+                    logger.debug("skipping library code matching: only have native PE signatures")
+            except IOError as e:
+                logger.error("%s", str(e))
+                return E_INVALID_SIG
 
-    meta = collect_metadata(argv, args.sample, args.format, args.os, args.rules, extractor)
+            should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None)
 
-    capabilities, counts = find_capabilities(rules, extractor, disable_progress=args.quiet)
-    meta["analysis"].update(counts)
-    meta["analysis"]["layout"] = compute_layout(rules, extractor, capabilities)
+            try:
+                extractor = get_extractor(
+                    args.sample,
+                    format_,
+                    args.os,
+                    args.backend,
+                    sig_paths,
+                    should_save_workspace,
+                    disable_progress=args.quiet or args.debug,
+                )
+            except UnsupportedFormatError:
+                log_unsupported_format_error()
+                return E_INVALID_FILE_TYPE
+            except UnsupportedArchError:
+                log_unsupported_arch_error()
+                return E_INVALID_FILE_ARCH
+            except UnsupportedOSError:
+                log_unsupported_os_error()
+                return E_INVALID_FILE_OS
 
-    if has_file_limitation(rules, capabilities):
-        # bail if capa encountered file limitation e.g. a packed binary
-        # do show the output in verbose mode, though.
-        if not (args.verbose or args.vverbose or args.json):
-            return E_FILE_LIMITATION
+        meta = collect_metadata(argv, args.sample, args.format, args.os, args.rules, extractor)
 
+        capabilities, counts = find_capabilities(rules, extractor, disable_progress=args.quiet)
+
+        meta.analysis.feature_counts = counts["feature_counts"]
+        meta.analysis.library_functions = counts["library_functions"]
+        meta.analysis.layout = compute_layout(rules, extractor, capabilities)
+
+        if has_file_limitation(rules, capabilities):
+            # bail if capa encountered file limitation e.g. a packed binary
+            # do show the output in verbose mode, though.
+            if not (args.verbose or args.vverbose or args.json):
+                return E_FILE_LIMITATION
     if args.json:
         print(capa.render.json.render(meta, rules, capabilities))
     elif args.vverbose:
@@ -1238,14 +1320,16 @@ def ida_main():
     logger.debug("     https://github.com/mandiant/capa-rules")
     logger.debug("-" * 80)
 
-    rules_path = os.path.join(get_default_root(), "rules")
+    rules_path = get_default_root() / "rules"
     logger.debug("rule path: %s", rules_path)
     rules = get_rules([rules_path])
 
     meta = capa.ida.helpers.collect_metadata([rules_path])
 
     capabilities, counts = find_capabilities(rules, capa.features.extractors.ida.extractor.IdaFeatureExtractor())
-    meta["analysis"].update(counts)
+
+    meta.analysis.feature_counts = counts["feature_counts"]
+    meta.analysis.library_functions = counts["library_functions"]
 
     if has_file_limitation(rules, capabilities, is_standalone=False):
         capa.ida.helpers.inform_user_ida_ui("capa encountered warnings during analysis")
@@ -1254,17 +1338,8 @@ def ida_main():
     print(capa.render.default.render(meta, rules, capabilities))
 
 
-def is_runtime_ida():
-    try:
-        import idc
-    except ImportError:
-        return False
-    else:
-        return True
-
-
 if __name__ == "__main__":
-    if is_runtime_ida():
+    if capa.helpers.is_runtime_ida():
         ida_main()
     else:
         sys.exit(main())

capa/optimizer.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import logging
 
 import capa.engine as ceng
@@ -22,7 +29,7 @@ def get_node_cost(node):
         # substring and regex features require a full scan of each string
         # which we anticipate is more expensive then a hash lookup feature (e.g. mnemonic or count).
         #
-        # TODO: compute the average cost of these feature relative to hash feature
+        # fun research: compute the average cost of these feature relative to hash feature
         # and adjust the factor accordingly.
         return 2
 

capa/perf.py

@@ -1,3 +1,10 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
 import typing
 import collections
 

capa/render/default.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -11,7 +11,6 @@ import collections
 import tabulate
 
 import capa.render.utils as rutils
-import capa.features.freeze as frz
 import capa.render.result_document as rd
 import capa.features.freeze.features as frzf
 from capa.rules import RuleSet
@@ -40,7 +39,7 @@ def render_meta(doc: rd.ResultDocument, ostream: StringIO):
         ("path", doc.meta.sample.path),
     ]
 
-    ostream.write(tabulate.tabulate(rows, tablefmt="psql"))
+    ostream.write(tabulate.tabulate(rows, tablefmt="mixed_outline"))
     ostream.write("\n")
 
 
@@ -49,7 +48,7 @@ def find_subrule_matches(doc: rd.ResultDocument):
     collect the rule names that have been matched as a subrule match.
     this way we can avoid displaying entries for things that are too specific.
     """
-    matches = set([])
+    matches = set()
 
     def rec(match: rd.Match):
         if not match.success:
@@ -65,7 +64,7 @@ def find_subrule_matches(doc: rd.ResultDocument):
             matches.add(match.node.feature.match)
 
     for rule in rutils.capability_rules(doc):
-        for address, match in rule.matches:
+        for _, match in rule.matches:
             rec(match)
 
     return matches
@@ -102,7 +101,7 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
 
     if rows:
         ostream.write(
-            tabulate.tabulate(rows, headers=[width("CAPABILITY", 50), width("NAMESPACE", 50)], tablefmt="psql")
+            tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline")
         )
         ostream.write("\n")
     else:
@@ -148,7 +147,7 @@ def render_attack(doc: rd.ResultDocument, ostream: StringIO):
     if rows:
         ostream.write(
             tabulate.tabulate(
-                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="psql"
+                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="mixed_grid"
             )
         )
         ostream.write("\n")
@@ -190,7 +189,9 @@ def render_mbc(doc: rd.ResultDocument, ostream: StringIO):
 
     if rows:
         ostream.write(
-            tabulate.tabulate(rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="psql")
+            tabulate.tabulate(
+                rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="mixed_grid"
+            )
         )
         ostream.write("\n")
 

capa/render/json.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/render/proto/__init__.py

@@ -0,0 +1,737 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+"""
+Convert capa results to protobuf format.
+The functionality here is similar to the various *from_capa functions, e.g. ResultDocument.from_capa() or
+feature_from_capa.
+
+For few classes we can rely on the proto json parser (e.g. RuleMetadata).
+
+For most classes (e.g. RuleMatches) conversion is tricky, because we use natively unsupported types (e.g. tuples),
+several classes with unions, and more complex layouts. So, it's more straight forward to convert explicitly vs.
+massaging the data so the protobuf json parser works.
+
+Of note, the 3 in `syntax = "proto3"` has nothing to do with the 2 in capa_pb2.py;
+see details in https://github.com/grpc/grpc/issues/15444#issuecomment-396442980.
+
+First compile the protobuf to generate an API file and a mypy stub file
+$ protoc.exe --python_out=. --mypy_out=. <path_to_proto> (e.g. capa/render/proto/capa.proto)
+
+Alternatively, --pyi_out=. can be used to generate a Python Interface file that supports development
+"""
+import datetime
+from typing import Any, Dict, Union
+
+import google.protobuf.json_format
+
+import capa.rules
+import capa.features.freeze as frz
+import capa.render.proto.capa_pb2 as capa_pb2
+import capa.render.result_document as rd
+import capa.features.freeze.features as frzf
+from capa.helpers import assert_never
+from capa.features.freeze import AddressType
+
+
+def dict_tuple_to_list_values(d: Dict) -> Dict:
+    o = {}
+    for k, v in d.items():
+        if isinstance(v, tuple):
+            o[k] = list(v)
+        else:
+            o[k] = v
+    return o
+
+
+def int_to_pb2(v: int) -> capa_pb2.Integer:
+    if v < -2_147_483_648:
+        raise ValueError(f"value underflow: {v}")
+    if v > 0xFFFFFFFFFFFFFFFF:
+        raise ValueError(f"value overflow: {v}")
+
+    if v < 0:
+        return capa_pb2.Integer(i=v)
+    else:
+        return capa_pb2.Integer(u=v)
+
+
+def number_to_pb2(v: Union[int, float]) -> capa_pb2.Number:
+    if isinstance(v, float):
+        return capa_pb2.Number(f=v)
+    elif isinstance(v, int):
+        i = int_to_pb2(v)
+        if v < 0:
+            return capa_pb2.Number(i=i.i)
+        else:
+            return capa_pb2.Number(u=i.u)
+    else:
+        assert_never(v)
+
+
+def addr_to_pb2(addr: frz.Address) -> capa_pb2.Address:
+    if addr.type is AddressType.ABSOLUTE:
+        assert isinstance(addr.value, int)
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_ABSOLUTE, v=int_to_pb2(addr.value))
+
+    elif addr.type is AddressType.RELATIVE:
+        assert isinstance(addr.value, int)
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_RELATIVE, v=int_to_pb2(addr.value))
+
+    elif addr.type is AddressType.FILE:
+        assert isinstance(addr.value, int)
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_FILE, v=int_to_pb2(addr.value))
+
+    elif addr.type is AddressType.DN_TOKEN:
+        assert isinstance(addr.value, int)
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_DN_TOKEN, v=int_to_pb2(addr.value))
+
+    elif addr.type is AddressType.DN_TOKEN_OFFSET:
+        assert isinstance(addr.value, tuple)
+        token, offset = addr.value
+        assert isinstance(token, int)
+        assert isinstance(offset, int)
+        return capa_pb2.Address(
+            type=capa_pb2.AddressType.ADDRESSTYPE_DN_TOKEN_OFFSET,
+            token_offset=capa_pb2.Token_Offset(token=int_to_pb2(token), offset=offset),
+        )
+
+    elif addr.type is AddressType.NO_ADDRESS:
+        # value == None, so only set type
+        return capa_pb2.Address(type=capa_pb2.AddressType.ADDRESSTYPE_NO_ADDRESS)
+
+    else:
+        assert_never(addr)
+
+
+def scope_to_pb2(scope: capa.rules.Scope) -> capa_pb2.Scope.ValueType:
+    if scope == capa.rules.Scope.FILE:
+        return capa_pb2.Scope.SCOPE_FILE
+    elif scope == capa.rules.Scope.FUNCTION:
+        return capa_pb2.Scope.SCOPE_FUNCTION
+    elif scope == capa.rules.Scope.BASIC_BLOCK:
+        return capa_pb2.Scope.SCOPE_BASIC_BLOCK
+    elif scope == capa.rules.Scope.INSTRUCTION:
+        return capa_pb2.Scope.SCOPE_INSTRUCTION
+    else:
+        assert_never(scope)
+
+
+def metadata_to_pb2(meta: rd.Metadata) -> capa_pb2.Metadata:
+    return capa_pb2.Metadata(
+        timestamp=str(meta.timestamp),
+        version=meta.version,
+        argv=meta.argv,
+        sample=google.protobuf.json_format.ParseDict(meta.sample.dict(), capa_pb2.Sample()),
+        analysis=capa_pb2.Analysis(
+            format=meta.analysis.format,
+            arch=meta.analysis.arch,
+            os=meta.analysis.os,
+            extractor=meta.analysis.extractor,
+            rules=list(meta.analysis.rules),
+            base_address=addr_to_pb2(meta.analysis.base_address),
+            layout=capa_pb2.Layout(
+                functions=[
+                    capa_pb2.FunctionLayout(
+                        address=addr_to_pb2(f.address),
+                        matched_basic_blocks=[
+                            capa_pb2.BasicBlockLayout(address=addr_to_pb2(bb.address)) for bb in f.matched_basic_blocks
+                        ],
+                    )
+                    for f in meta.analysis.layout.functions
+                ]
+            ),
+            feature_counts=capa_pb2.FeatureCounts(
+                file=meta.analysis.feature_counts.file,
+                functions=[
+                    capa_pb2.FunctionFeatureCount(address=addr_to_pb2(f.address), count=f.count)
+                    for f in meta.analysis.feature_counts.functions
+                ],
+            ),
+            library_functions=[
+                capa_pb2.LibraryFunction(address=addr_to_pb2(lf.address), name=lf.name)
+                for lf in meta.analysis.library_functions
+            ],
+        ),
+    )
+
+
+def statement_to_pb2(statement: rd.Statement) -> capa_pb2.StatementNode:
+    if isinstance(statement, rd.RangeStatement):
+        return capa_pb2.StatementNode(
+            range=capa_pb2.RangeStatement(
+                type="range",
+                description=statement.description,
+                min=statement.min,
+                max=statement.max,
+                child=feature_to_pb2(statement.child),
+            ),
+            type="statement",
+        )
+
+    elif isinstance(statement, rd.SomeStatement):
+        return capa_pb2.StatementNode(
+            some=capa_pb2.SomeStatement(type=statement.type, description=statement.description, count=statement.count),
+            type="statement",
+        )
+
+    elif isinstance(statement, rd.SubscopeStatement):
+        return capa_pb2.StatementNode(
+            subscope=capa_pb2.SubscopeStatement(
+                type=statement.type,
+                description=statement.description,
+                scope=scope_to_pb2(statement.scope),
+            ),
+            type="statement",
+        )
+
+    elif isinstance(statement, rd.CompoundStatement):
+        return capa_pb2.StatementNode(
+            compound=capa_pb2.CompoundStatement(type=statement.type, description=statement.description),
+            type="statement",
+        )
+
+    else:
+        assert_never(statement)
+
+
+def feature_to_pb2(f: frzf.Feature) -> capa_pb2.FeatureNode:
+    if isinstance(f, frzf.OSFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", os=capa_pb2.OSFeature(type=f.type, os=f.os, description=f.description)
+        )
+
+    elif isinstance(f, frzf.ArchFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", arch=capa_pb2.ArchFeature(type=f.type, arch=f.arch, description=f.description)
+        )
+
+    elif isinstance(f, frzf.FormatFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", format=capa_pb2.FormatFeature(type=f.type, format=f.format, description=f.description)
+        )
+
+    elif isinstance(f, frzf.MatchFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            match=capa_pb2.MatchFeature(
+                type=f.type,
+                match=f.match,
+                description=f.description,
+            ),
+        )
+
+    elif isinstance(f, frzf.CharacteristicFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            characteristic=capa_pb2.CharacteristicFeature(
+                type=f.type, characteristic=f.characteristic, description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.ExportFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", export=capa_pb2.ExportFeature(type=f.type, export=f.export, description=f.description)
+        )
+
+    elif isinstance(f, frzf.ImportFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", import_=capa_pb2.ImportFeature(type=f.type, import_=f.import_, description=f.description)
+        )
+
+    elif isinstance(f, frzf.SectionFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", section=capa_pb2.SectionFeature(type=f.type, section=f.section, description=f.description)
+        )
+
+    elif isinstance(f, frzf.FunctionNameFeature):
+        return capa_pb2.FeatureNode(
+            type="function name",
+            function_name=capa_pb2.FunctionNameFeature(
+                type=f.type, function_name=f.function_name, description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.SubstringFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            substring=capa_pb2.SubstringFeature(type=f.type, substring=f.substring, description=f.description),
+        )
+
+    elif isinstance(f, frzf.RegexFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", regex=capa_pb2.RegexFeature(type=f.type, regex=f.regex, description=f.description)
+        )
+
+    elif isinstance(f, frzf.StringFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            string=capa_pb2.StringFeature(
+                type=f.type,
+                string=f.string,
+                description=f.description,
+            ),
+        )
+
+    elif isinstance(f, frzf.ClassFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", class_=capa_pb2.ClassFeature(type=f.type, class_=f.class_, description=f.description)
+        )
+
+    elif isinstance(f, frzf.NamespaceFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            namespace=capa_pb2.NamespaceFeature(type=f.type, namespace=f.namespace, description=f.description),
+        )
+
+    elif isinstance(f, frzf.APIFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", api=capa_pb2.APIFeature(type=f.type, api=f.api, description=f.description)
+        )
+
+    elif isinstance(f, frzf.PropertyFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            property_=capa_pb2.PropertyFeature(
+                type=f.type, access=f.access, property_=f.property, description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.NumberFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            number=capa_pb2.NumberFeature(type=f.type, number=number_to_pb2(f.number), description=f.description),
+        )
+
+    elif isinstance(f, frzf.BytesFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", bytes=capa_pb2.BytesFeature(type=f.type, bytes=f.bytes, description=f.description)
+        )
+
+    elif isinstance(f, frzf.OffsetFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            offset=capa_pb2.OffsetFeature(type=f.type, offset=int_to_pb2(f.offset), description=f.description),
+        )
+
+    elif isinstance(f, frzf.MnemonicFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            mnemonic=capa_pb2.MnemonicFeature(type=f.type, mnemonic=f.mnemonic, description=f.description),
+        )
+
+    elif isinstance(f, frzf.OperandNumberFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            operand_number=capa_pb2.OperandNumberFeature(
+                type=f.type, index=f.index, operand_number=int_to_pb2(f.operand_number), description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.OperandOffsetFeature):
+        return capa_pb2.FeatureNode(
+            type="feature",
+            operand_offset=capa_pb2.OperandOffsetFeature(
+                type=f.type, index=f.index, operand_offset=int_to_pb2(f.operand_offset), description=f.description
+            ),
+        )
+
+    elif isinstance(f, frzf.BasicBlockFeature):
+        return capa_pb2.FeatureNode(
+            type="feature", basic_block=capa_pb2.BasicBlockFeature(type=f.type, description=f.description)
+        )
+
+    else:
+        assert_never(f)
+
+
+def node_to_pb2(node: rd.Node) -> Union[capa_pb2.FeatureNode, capa_pb2.StatementNode]:
+    if isinstance(node, rd.StatementNode):
+        return statement_to_pb2(node.statement)
+
+    elif isinstance(node, rd.FeatureNode):
+        return feature_to_pb2(node.feature)
+
+    else:
+        assert_never(node)
+
+
+def match_to_pb2(match: rd.Match) -> capa_pb2.Match:
+    node = node_to_pb2(match.node)
+    children = list(map(match_to_pb2, match.children))
+    locations = list(map(addr_to_pb2, match.locations))
+
+    if isinstance(node, capa_pb2.StatementNode):
+        return capa_pb2.Match(
+            success=match.success,
+            statement=node,
+            children=children,
+            locations=locations,
+            captures={},
+        )
+
+    elif isinstance(node, capa_pb2.FeatureNode):
+        return capa_pb2.Match(
+            success=match.success,
+            feature=node,
+            children=children,
+            locations=locations,
+            captures={
+                capture: capa_pb2.Addresses(address=list(map(addr_to_pb2, locs)))
+                for capture, locs in match.captures.items()
+            },
+        )
+
+    else:
+        assert_never(match)
+
+
+def rule_metadata_to_pb2(rule_metadata: rd.RuleMetadata) -> capa_pb2.RuleMetadata:
+    # after manual type conversions to the RuleMetadata, we can rely on the protobuf json parser
+    # conversions include tuple -> list and rd.Enum -> proto.enum
+    meta = dict_tuple_to_list_values(rule_metadata.dict())
+    meta["scope"] = scope_to_pb2(meta["scope"])
+    meta["attack"] = list(map(dict_tuple_to_list_values, meta.get("attack", [])))
+    meta["mbc"] = list(map(dict_tuple_to_list_values, meta.get("mbc", [])))
+
+    return google.protobuf.json_format.ParseDict(meta, capa_pb2.RuleMetadata())
+
+
+def doc_to_pb2(doc: rd.ResultDocument) -> capa_pb2.ResultDocument:
+    rule_matches: Dict[str, capa_pb2.RuleMatches] = {}
+    for rule_name, matches in doc.rules.items():
+        m = capa_pb2.RuleMatches(
+            meta=rule_metadata_to_pb2(matches.meta),
+            source=matches.source,
+            matches=[
+                capa_pb2.Pair_Address_Match(address=addr_to_pb2(addr), match=match_to_pb2(match))
+                for addr, match in matches.matches
+            ],
+        )
+        rule_matches[rule_name] = m
+
+    r = capa_pb2.ResultDocument(meta=metadata_to_pb2(doc.meta), rules=rule_matches)
+
+    return r
+
+
+def int_from_pb2(v: capa_pb2.Integer) -> int:
+    type = v.WhichOneof("value")
+    if type == "u":
+        return v.u
+    elif type == "i":
+        return v.i
+    else:
+        assert_never(type)
+
+
+def number_from_pb2(v: capa_pb2.Number) -> Union[int, float]:
+    type = v.WhichOneof("value")
+    if type == "u":
+        return v.u
+    elif type == "i":
+        return v.i
+    elif type == "f":
+        return v.f
+    else:
+        assert_never(type)
+
+
+def addr_from_pb2(addr: capa_pb2.Address) -> frz.Address:
+    if addr.type == capa_pb2.AddressType.ADDRESSTYPE_ABSOLUTE:
+        return frz.Address(type=frz.AddressType.ABSOLUTE, value=int_from_pb2(addr.v))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_RELATIVE:
+        return frz.Address(type=frz.AddressType.RELATIVE, value=int_from_pb2(addr.v))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_FILE:
+        return frz.Address(type=frz.AddressType.FILE, value=int_from_pb2(addr.v))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_DN_TOKEN:
+        return frz.Address(type=frz.AddressType.DN_TOKEN, value=int_from_pb2(addr.v))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_DN_TOKEN_OFFSET:
+        token = int_from_pb2(addr.token_offset.token)
+        offset = addr.token_offset.offset
+        return frz.Address(type=frz.AddressType.DN_TOKEN_OFFSET, value=(token, offset))
+
+    elif addr.type == capa_pb2.AddressType.ADDRESSTYPE_NO_ADDRESS:
+        return frz.Address(type=frz.AddressType.NO_ADDRESS, value=None)
+
+    else:
+        assert_never(addr)
+
+
+def scope_from_pb2(scope: capa_pb2.Scope.ValueType) -> capa.rules.Scope:
+    if scope == capa_pb2.Scope.SCOPE_FILE:
+        return capa.rules.Scope.FILE
+    elif scope == capa_pb2.Scope.SCOPE_FUNCTION:
+        return capa.rules.Scope.FUNCTION
+    elif scope == capa_pb2.Scope.SCOPE_BASIC_BLOCK:
+        return capa.rules.Scope.BASIC_BLOCK
+    elif scope == capa_pb2.Scope.SCOPE_INSTRUCTION:
+        return capa.rules.Scope.INSTRUCTION
+    else:
+        assert_never(scope)
+
+
+def metadata_from_pb2(meta: capa_pb2.Metadata) -> rd.Metadata:
+    return rd.Metadata(
+        timestamp=datetime.datetime.fromisoformat(meta.timestamp),
+        version=meta.version,
+        argv=tuple(meta.argv) if meta.argv else None,
+        sample=rd.Sample(
+            md5=meta.sample.md5,
+            sha1=meta.sample.sha1,
+            sha256=meta.sample.sha256,
+            path=meta.sample.path,
+        ),
+        analysis=rd.Analysis(
+            format=meta.analysis.format,
+            arch=meta.analysis.arch,
+            os=meta.analysis.os,
+            extractor=meta.analysis.extractor,
+            rules=tuple(meta.analysis.rules),
+            base_address=addr_from_pb2(meta.analysis.base_address),
+            layout=rd.Layout(
+                functions=tuple(
+                    [
+                        rd.FunctionLayout(
+                            address=addr_from_pb2(f.address),
+                            matched_basic_blocks=tuple(
+                                [
+                                    rd.BasicBlockLayout(address=addr_from_pb2(bb.address))
+                                    for bb in f.matched_basic_blocks
+                                ]
+                            ),
+                        )
+                        for f in meta.analysis.layout.functions
+                    ]
+                )
+            ),
+            feature_counts=rd.FeatureCounts(
+                file=meta.analysis.feature_counts.file,
+                functions=tuple(
+                    [
+                        rd.FunctionFeatureCount(address=addr_from_pb2(f.address), count=f.count)
+                        for f in meta.analysis.feature_counts.functions
+                    ]
+                ),
+            ),
+            library_functions=tuple(
+                [
+                    rd.LibraryFunction(address=addr_from_pb2(lf.address), name=lf.name)
+                    for lf in meta.analysis.library_functions
+                ]
+            ),
+        ),
+    )
+
+
+def statement_from_pb2(statement: capa_pb2.StatementNode) -> rd.Statement:
+    type_ = statement.WhichOneof("statement")
+
+    if type_ == "range":
+        return rd.RangeStatement(
+            min=statement.range.min,
+            max=statement.range.max,
+            child=feature_from_pb2(statement.range.child),
+            description=statement.range.description or None,
+        )
+    elif type_ == "some":
+        return rd.SomeStatement(
+            count=statement.some.count,
+            description=statement.some.description or None,
+        )
+    elif type_ == "subscope":
+        return rd.SubscopeStatement(
+            scope=scope_from_pb2(statement.subscope.scope),
+            description=statement.subscope.description or None,
+        )
+    elif type_ == "compound":
+        return rd.CompoundStatement(
+            type=statement.compound.type,
+            description=statement.compound.description or None,
+        )
+    else:
+        assert_never(type_)
+
+
+def feature_from_pb2(f: capa_pb2.FeatureNode) -> frzf.Feature:
+    type_ = f.WhichOneof("feature")
+
+    # mypy gets angry below because ff may have a different type in each branch,
+    # even though we don't use ff outside each branch.
+    # so we just let mypy know that ff might be any type to silence that warning.
+    # upstream issue: https://github.com/python/mypy/issues/6233
+    ff: Any
+
+    if type_ == "os":
+        ff = f.os
+        return frzf.OSFeature(os=ff.os, description=ff.description or None)
+    elif type_ == "arch":
+        ff = f.arch
+        return frzf.ArchFeature(arch=ff.arch, description=ff.description or None)
+    elif type_ == "format":
+        ff = f.format
+        return frzf.FormatFeature(format=ff.format, description=ff.description or None)
+    elif type_ == "match":
+        ff = f.match
+        return frzf.MatchFeature(match=ff.match, description=ff.description or None)
+    elif type_ == "characteristic":
+        ff = f.characteristic
+        return frzf.CharacteristicFeature(characteristic=ff.characteristic, description=ff.description or None)
+    elif type_ == "export":
+        ff = f.export
+        return frzf.ExportFeature(export=ff.export, description=ff.description or None)
+    elif type_ == "import_":
+        ff = f.import_
+        return frzf.ImportFeature(import_=ff.import_, description=ff.description or None)  # type: ignore
+        # Mypy is unable to recognize `import_` as an argument
+    elif type_ == "section":
+        ff = f.section
+        return frzf.SectionFeature(section=ff.section, description=ff.description or None)
+    elif type_ == "function_name":
+        ff = f.function_name
+        return frzf.FunctionNameFeature(function_name=ff.function_name, description=ff.description or None)  # type: ignore
+    elif type_ == "substring":
+        ff = f.substring
+        return frzf.SubstringFeature(substring=ff.substring, description=ff.description or None)
+    elif type_ == "regex":
+        ff = f.regex
+        return frzf.RegexFeature(regex=ff.regex, description=ff.description or None)
+    elif type_ == "string":
+        ff = f.string
+        return frzf.StringFeature(string=ff.string, description=ff.description or None)
+    elif type_ == "class_":
+        ff = f.class_
+        return frzf.ClassFeature(class_=ff.class_, description=ff.description or None)  # type: ignore
+        # Mypy is unable to recognize `class_` as an argument due to aliasing
+    elif type_ == "namespace":
+        ff = f.namespace
+        return frzf.NamespaceFeature(namespace=ff.namespace, description=ff.description or None)
+    elif type_ == "api":
+        ff = f.api
+        return frzf.APIFeature(api=ff.api, description=ff.description or None)
+    elif type_ == "property_":
+        ff = f.property_
+        return frzf.PropertyFeature(property=ff.property_, access=ff.access or None, description=ff.description or None)
+    elif type_ == "number":
+        ff = f.number
+        return frzf.NumberFeature(number=number_from_pb2(ff.number), description=ff.description or None)
+    elif type_ == "bytes":
+        ff = f.bytes
+        return frzf.BytesFeature(bytes=ff.bytes, description=ff.description or None)
+    elif type_ == "offset":
+        ff = f.offset
+        return frzf.OffsetFeature(offset=int_from_pb2(ff.offset), description=ff.description or None)
+    elif type_ == "mnemonic":
+        ff = f.mnemonic
+        return frzf.MnemonicFeature(mnemonic=ff.mnemonic, description=ff.description or None)
+    elif type_ == "operand_number":
+        ff = f.operand_number
+        return frzf.OperandNumberFeature(
+            index=ff.index, operand_number=number_from_pb2(ff.operand_number), description=ff.description or None
+        )  # type: ignore
+    elif type_ == "operand_offset":
+        ff = f.operand_offset
+        return frzf.OperandOffsetFeature(
+            index=ff.index, operand_offset=int_from_pb2(ff.operand_offset), description=ff.description or None
+        )  # type: ignore
+        # Mypy is unable to recognize `operand_offset` as an argument due to aliasing
+    elif type_ == "basic_block":
+        ff = f.basic_block
+        return frzf.BasicBlockFeature(description=ff.description or None)
+    else:
+        assert_never(type_)
+
+
+def match_from_pb2(match: capa_pb2.Match) -> rd.Match:
+    children = list(map(match_from_pb2, match.children))
+    locations = list(map(addr_from_pb2, match.locations))
+
+    node_type = match.WhichOneof("node")
+    if node_type == "statement":
+        return rd.Match(
+            success=match.success,
+            node=rd.StatementNode(statement=statement_from_pb2(match.statement)),
+            children=tuple(children),
+            locations=tuple(locations),
+            captures={},
+        )
+    elif node_type == "feature":
+        return rd.Match(
+            success=match.success,
+            node=rd.FeatureNode(feature=feature_from_pb2(match.feature)),
+            children=tuple(children),
+            locations=tuple(locations),
+            captures={capture: tuple(map(addr_from_pb2, locs.address)) for capture, locs in match.captures.items()},
+        )
+    else:
+        assert_never(node_type)
+
+
+def attack_from_pb2(pb: capa_pb2.AttackSpec) -> rd.AttackSpec:
+    return rd.AttackSpec(
+        parts=tuple(pb.parts),
+        tactic=pb.tactic,
+        technique=pb.technique,
+        subtechnique=pb.subtechnique,
+        id=pb.id,
+    )
+
+
+def mbc_from_pb2(pb: capa_pb2.MBCSpec) -> rd.MBCSpec:
+    return rd.MBCSpec(
+        parts=tuple(pb.parts),
+        objective=pb.objective,
+        behavior=pb.behavior,
+        method=pb.method,
+        id=pb.id,
+    )
+
+
+def maec_from_pb2(pb: capa_pb2.MaecMetadata) -> rd.MaecMetadata:
+    return rd.MaecMetadata(
+        analysis_conclusion=pb.analysis_conclusion or None,
+        analysis_conclusion_ov=pb.analysis_conclusion_ov or None,
+        malware_family=pb.malware_family or None,
+        malware_category=pb.malware_category or None,
+        malware_category_ov=pb.malware_category_ov or None,
+    )  # type: ignore
+    # Mypy is unable to recognise arguments due to alias
+
+
+def rule_metadata_from_pb2(pb: capa_pb2.RuleMetadata) -> rd.RuleMetadata:
+    return rd.RuleMetadata(
+        name=pb.name,
+        namespace=pb.namespace or None,
+        authors=tuple(pb.authors),
+        scope=scope_from_pb2(pb.scope),
+        attack=tuple([attack_from_pb2(attack) for attack in pb.attack]),
+        mbc=tuple([mbc_from_pb2(mbc) for mbc in pb.mbc]),
+        references=tuple(pb.references),
+        examples=tuple(pb.examples),
+        description=pb.description,
+        lib=pb.lib,
+        is_subscope_rule=pb.is_subscope_rule,
+        maec=maec_from_pb2(pb.maec),
+    )  # type: ignore
+    # Mypy is unable to recognise `attack` and `is_subscope_rule` as arguments due to alias
+
+
+def doc_from_pb2(doc: capa_pb2.ResultDocument) -> rd.ResultDocument:
+    rule_matches: Dict[str, rd.RuleMatches] = {}
+    for rule_name, matches in doc.rules.items():
+        m = rd.RuleMatches(
+            meta=rule_metadata_from_pb2(matches.meta),
+            source=matches.source,
+            matches=tuple([(addr_from_pb2(pair.address), match_from_pb2(pair.match)) for pair in matches.matches]),
+        )
+        rule_matches[rule_name] = m
+
+    return rd.ResultDocument(meta=metadata_from_pb2(doc.meta), rules=rule_matches)

capa/render/proto/capa.proto

@@ -0,0 +1,364 @@
+syntax = "proto3";
+
+message APIFeature {
+  string type = 1;
+  string api = 2;
+  optional string description = 3;
+}
+
+message Address {
+  AddressType type = 1;
+  oneof value {
+    Integer v = 2;
+    Token_Offset token_offset = 3;
+  };
+}
+
+enum AddressType {
+  ADDRESSTYPE_UNSPECIFIED = 0;
+  ADDRESSTYPE_ABSOLUTE = 1;
+  ADDRESSTYPE_RELATIVE = 2;
+  ADDRESSTYPE_FILE = 3;
+  ADDRESSTYPE_DN_TOKEN = 4;
+  ADDRESSTYPE_DN_TOKEN_OFFSET = 5;
+  ADDRESSTYPE_NO_ADDRESS = 6;
+}
+
+message Analysis {
+  string format = 1;
+  string arch = 2;
+  string os = 3;
+  string extractor = 4;
+  repeated string rules = 5;
+  Address base_address = 6;
+  Layout layout = 7;
+  FeatureCounts feature_counts = 8;
+  repeated LibraryFunction library_functions = 9;
+}
+
+message ArchFeature {
+  string type = 1;
+  string arch = 2;
+  optional string description = 3;
+}
+
+message AttackSpec {
+  repeated string parts = 1;
+  string tactic = 2;
+  string technique = 3;
+  string subtechnique = 4;
+  string id = 5;
+}
+
+message BasicBlockFeature {
+  string type = 1;
+  optional string description = 2;
+}
+
+message BasicBlockLayout {
+  Address address = 1;
+}
+
+message BytesFeature {
+  string type = 1;
+  string bytes = 2;
+  optional string description = 3;
+}
+
+message CharacteristicFeature {
+  string type = 1;
+  string characteristic = 2;
+  optional string description = 3;
+}
+
+message ClassFeature {
+  string type = 1;
+  string class_ = 2;  // class is protected Python keyword
+  optional string description = 3;
+}
+
+message CompoundStatement {
+  string type = 1;
+  optional string description = 2;
+}
+
+message ExportFeature {
+  string type = 1;
+  string export = 2;
+  optional string description = 3;
+}
+
+message FeatureCounts {
+  uint64 file = 1;
+  repeated FunctionFeatureCount functions = 2;
+}
+
+message FeatureNode {
+  string type = 1;
+  oneof feature {
+    OSFeature os = 2;
+    ArchFeature arch = 3;
+    FormatFeature format = 4;
+    MatchFeature match = 5;
+    CharacteristicFeature characteristic = 6;
+    ExportFeature export = 7;
+    ImportFeature import_ = 8;  // import is Python keyword
+    SectionFeature section = 9;
+    FunctionNameFeature function_name = 10;
+    SubstringFeature substring = 11;
+    RegexFeature regex = 12;
+    StringFeature string = 13;
+    ClassFeature class_ = 14;
+    NamespaceFeature namespace = 15;
+    APIFeature api = 16;
+    PropertyFeature property_ = 17;  // property is a Python top-level decorator name
+    NumberFeature number = 18;
+    BytesFeature bytes = 19;
+    OffsetFeature offset = 20;
+    MnemonicFeature mnemonic = 21;
+    OperandNumberFeature operand_number = 22;
+    OperandOffsetFeature operand_offset = 23;
+    BasicBlockFeature basic_block = 24;
+  };
+}
+
+message FormatFeature {
+  string type = 1;
+  string format = 2;
+  optional string description = 3;
+}
+
+message FunctionFeatureCount {
+  Address address = 1;
+  uint64 count = 2;
+}
+
+message FunctionLayout {
+  Address address = 1;
+  repeated BasicBlockLayout matched_basic_blocks = 2;
+}
+
+message FunctionNameFeature {
+  string type = 1;
+  string function_name = 2;
+  optional string description = 3;
+}
+
+message ImportFeature {
+  string type = 1;
+  string import_ = 2;
+  optional string description = 3;
+}
+
+message Layout {
+  repeated FunctionLayout functions = 1;
+}
+
+message LibraryFunction {
+  Address address = 1;
+  string name = 2;
+}
+
+message MBCSpec {
+  repeated string parts = 1;
+  string objective = 2;
+  string behavior = 3;
+  string method = 4;
+  string id = 5;
+}
+
+message MaecMetadata {
+  string analysis_conclusion = 1;
+  string analysis_conclusion_ov = 2;
+  string malware_family = 3;
+  string malware_category = 4;
+  string malware_category_ov = 5;
+}
+
+message Match {
+  bool success = 1;
+  oneof node {
+    StatementNode statement = 2;
+    FeatureNode feature = 3;
+  };
+  repeated Match children = 5;
+  repeated Address locations = 6;
+  map <string, Addresses> captures = 7;
+}
+
+message MatchFeature {
+  string type = 1;
+  string match = 2;
+  optional string description = 3;
+}
+
+message Metadata {
+  string timestamp = 1;  // iso8601 format, like: 2019-01-01T00:00:00Z 
+  string version = 2;
+  repeated string argv = 3;
+  Sample sample = 4;
+  Analysis analysis = 5;
+}
+
+message MnemonicFeature {
+  string type = 1;
+  string mnemonic = 2;
+  optional string description = 3;
+}
+
+message NamespaceFeature {
+  string type = 1;
+  string namespace = 2;
+  optional string description = 3;
+}
+
+message NumberFeature {
+  string type = 1;
+  Number number = 2;  // this can be positive (range: u64), negative (range: i64), or a double.
+  optional string description = 5;
+}
+
+message OSFeature {
+  string type = 1;
+  string os = 2;
+  optional string description = 3;
+}
+
+message OffsetFeature {
+  string type = 1;
+  Integer offset = 2;  // offset can be negative
+  optional string description = 3;
+}
+
+message OperandNumberFeature {
+  string type = 1;
+  uint32 index = 2;
+  Integer operand_number = 3;  // this can be positive (range: u64), negative (range: i64), or a double.
+  optional string description = 4;
+}
+
+message OperandOffsetFeature {
+  string type = 1;
+  uint32 index = 2;
+  Integer operand_offset = 3;
+  optional string description = 4;
+}
+
+message PropertyFeature {
+  string type = 1;
+  string property_ = 2;  // property is a Python top-level decorator name
+  optional string access = 3;
+  optional string description = 4;
+}
+
+message RangeStatement {
+  string type = 1;
+  uint64 min = 2;
+  uint64 max = 3;
+  // reusing FeatureNode here to avoid duplication and list all features OSFeature, ArchFeature, ... again.
+  FeatureNode child = 4;
+  optional string description = 5;
+}
+
+message RegexFeature {
+  string type = 1;
+  string regex = 2;
+  optional string description = 3;
+}
+
+message ResultDocument {
+  Metadata meta = 1;
+  map <string, RuleMatches> rules = 2;
+}
+
+message RuleMatches {
+  RuleMetadata meta = 1;
+  string source = 2;
+  repeated Pair_Address_Match matches = 3;
+}
+
+message RuleMetadata {
+  string name = 1;
+  string namespace = 2;
+  repeated string authors = 3;
+  Scope scope = 4;
+  repeated AttackSpec attack = 5;
+  repeated MBCSpec mbc = 6;
+  repeated string references = 7;
+  repeated string examples = 8;
+  string description = 9;
+  bool lib = 10;
+  MaecMetadata maec = 11;
+  bool is_subscope_rule = 12;
+}
+
+message Sample {
+  string md5 = 1;
+  string sha1 = 2;
+  string sha256 = 3;
+  string path = 4;
+}
+
+enum Scope {
+  SCOPE_UNSPECIFIED = 0;
+  SCOPE_FILE = 1;
+  SCOPE_FUNCTION = 2;
+  SCOPE_BASIC_BLOCK = 3;
+  SCOPE_INSTRUCTION = 4;
+}
+
+message SectionFeature {
+  string type = 1;
+  string section = 2;
+  optional string description = 3;
+}
+
+message SomeStatement {
+  string type = 1;
+  uint32 count = 2;
+  optional string description = 3;
+}
+
+message StatementNode {
+  string type = 1;
+  oneof statement {
+    RangeStatement range = 2;
+    SomeStatement some = 3;
+    SubscopeStatement subscope = 4;
+    CompoundStatement compound = 5;
+  };
+}
+
+message StringFeature {
+  string type = 1;
+  string string = 2;
+  optional string description = 3;
+}
+
+message SubscopeStatement {
+  string type = 1;
+  Scope scope = 2;
+  optional string description = 3;
+}
+
+message SubstringFeature {
+  string type = 1;
+  string substring = 2;
+  optional string description = 3;
+}
+
+message Addresses { repeated Address address = 1; }
+
+message Pair_Address_Match {
+  Address address = 1;
+  Match match = 2;
+}
+
+message Token_Offset {
+  Integer token = 1;
+  uint64 offset = 2;  // offset is always >= 0
+}
+
+message Integer { oneof value { uint64 u = 1; sint64 i = 2; } }  // unsigned or signed int
+
+message Number { oneof value { uint64 u = 1; sint64 i = 2; double f = 3; } }

capa/render/result_document.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,7 +6,8 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import datetime
-from typing import Any, Dict, Tuple, Union, Optional
+import collections
+from typing import Dict, List, Tuple, Union, Optional
 
 from pydantic import Field, BaseModel
 
@@ -24,44 +25,50 @@ from capa.helpers import assert_never
 class FrozenModel(BaseModel):
     class Config:
         frozen = True
+        extra = "forbid"
 
 
-class Sample(FrozenModel):
+class Model(BaseModel):
+    class Config:
+        extra = "forbid"
+
+
+class Sample(Model):
     md5: str
     sha1: str
     sha256: str
     path: str
 
 
-class BasicBlockLayout(FrozenModel):
+class BasicBlockLayout(Model):
     address: frz.Address
 
 
-class FunctionLayout(FrozenModel):
+class FunctionLayout(Model):
     address: frz.Address
     matched_basic_blocks: Tuple[BasicBlockLayout, ...]
 
 
-class Layout(FrozenModel):
+class Layout(Model):
     functions: Tuple[FunctionLayout, ...]
 
 
-class LibraryFunction(FrozenModel):
+class LibraryFunction(Model):
     address: frz.Address
     name: str
 
 
-class FunctionFeatureCount(FrozenModel):
+class FunctionFeatureCount(Model):
     address: frz.Address
     count: int
 
 
-class FeatureCounts(FrozenModel):
+class FeatureCounts(Model):
     file: int
     functions: Tuple[FunctionFeatureCount, ...]
 
 
-class Analysis(FrozenModel):
+class Analysis(Model):
     format: str
     arch: str
     os: str
@@ -73,57 +80,13 @@ class Analysis(FrozenModel):
     library_functions: Tuple[LibraryFunction, ...]
 
 
-class Metadata(FrozenModel):
+class Metadata(Model):
     timestamp: datetime.datetime
     version: str
     argv: Optional[Tuple[str, ...]]
     sample: Sample
     analysis: Analysis
 
-    @classmethod
-    def from_capa(cls, meta: Any) -> "Metadata":
-        return cls(
-            timestamp=meta["timestamp"],
-            version=meta["version"],
-            argv=meta["argv"] if "argv" in meta else None,
-            sample=Sample(
-                md5=meta["sample"]["md5"],
-                sha1=meta["sample"]["sha1"],
-                sha256=meta["sample"]["sha256"],
-                path=meta["sample"]["path"],
-            ),
-            analysis=Analysis(
-                format=meta["analysis"]["format"],
-                arch=meta["analysis"]["arch"],
-                os=meta["analysis"]["os"],
-                extractor=meta["analysis"]["extractor"],
-                rules=meta["analysis"]["rules"],
-                base_address=frz.Address.from_capa(meta["analysis"]["base_address"]),
-                layout=Layout(
-                    functions=tuple(
-                        FunctionLayout(
-                            address=frz.Address.from_capa(address),
-                            matched_basic_blocks=tuple(
-                                BasicBlockLayout(address=frz.Address.from_capa(bb)) for bb in f["matched_basic_blocks"]
-                            ),
-                        )
-                        for address, f in meta["analysis"]["layout"]["functions"].items()
-                    )
-                ),
-                feature_counts=FeatureCounts(
-                    file=meta["analysis"]["feature_counts"]["file"],
-                    functions=tuple(
-                        FunctionFeatureCount(address=frz.Address.from_capa(address), count=count)
-                        for address, count in meta["analysis"]["feature_counts"]["functions"].items()
-                    ),
-                ),
-                library_functions=tuple(
-                    LibraryFunction(address=frz.Address.from_capa(address), name=name)
-                    for address, name in meta["analysis"]["library_functions"].items()
-                ),
-            ),
-        )
-
 
 class CompoundStatementType:
     AND = "and"
@@ -226,7 +189,55 @@ def node_from_capa(node: Union[capa.engine.Statement, capa.engine.Feature]) -> N
         assert_never(node)
 
 
-class Match(BaseModel):
+def node_to_capa(
+    node: Node, children: List[Union[capa.engine.Statement, capa.engine.Feature]]
+) -> Union[capa.engine.Statement, capa.engine.Feature]:
+    if isinstance(node, StatementNode):
+        if isinstance(node.statement, CompoundStatement):
+            if node.statement.type == CompoundStatementType.AND:
+                return capa.engine.And(description=node.statement.description, children=children)
+
+            elif node.statement.type == CompoundStatementType.OR:
+                return capa.engine.Or(description=node.statement.description, children=children)
+
+            elif node.statement.type == CompoundStatementType.NOT:
+                return capa.engine.Not(description=node.statement.description, child=children[0])
+
+            elif node.statement.type == CompoundStatementType.OPTIONAL:
+                return capa.engine.Some(description=node.statement.description, count=0, children=children)
+
+            else:
+                assert_never(node.statement.type)
+
+        elif isinstance(node.statement, SomeStatement):
+            return capa.engine.Some(
+                description=node.statement.description, count=node.statement.count, children=children
+            )
+
+        elif isinstance(node.statement, RangeStatement):
+            return capa.engine.Range(
+                description=node.statement.description,
+                min=node.statement.min,
+                max=node.statement.max,
+                child=node.statement.child.to_capa(),
+            )
+
+        elif isinstance(node.statement, SubscopeStatement):
+            return capa.engine.Subscope(
+                description=node.statement.description, scope=node.statement.scope, child=children[0]
+            )
+
+        else:
+            assert_never(node.statement)
+
+    elif isinstance(node, FeatureNode):
+        return node.feature.to_capa()
+
+    else:
+        assert_never(node)
+
+
+class Match(FrozenModel):
     """
     args:
       success: did the node match?
@@ -291,7 +302,7 @@ class Match(BaseModel):
                 # pull matches from the referenced rule into our tree here.
                 rule_name = name
                 rule = rules[rule_name]
-                rule_matches = {address: result for (address, result) in capabilities[rule_name]}
+                rule_matches = dict(capabilities[rule_name])
 
                 if rule.is_subscope_rule():
                     # for a subscope rule, fixup the node to be a scope node, rather than a match feature node.
@@ -336,7 +347,7 @@ class Match(BaseModel):
                         # we could introduce an intermediate node here.
                         # this would be a breaking change and require updates to the renderers.
                         # in the meantime, the above might be sufficient.
-                        rule_matches = {address: result for (address, result) in capabilities[rule.name]}
+                        rule_matches = dict(capabilities[rule.name])
                         for location in result.locations:
                             # doc[locations] contains all matches for the given namespace.
                             # for example, the feature might be `match: anti-analysis/packer`
@@ -353,9 +364,42 @@ class Match(BaseModel):
         return cls(
             success=success,
             node=node,
+            children=tuple(children),
+            locations=tuple(locations),
+            captures={capture: tuple(captures[capture]) for capture in captures},
+        )
+
+    def to_capa(self, rules_by_name: Dict[str, capa.rules.Rule]) -> capa.engine.Result:
+        children = [child.to_capa(rules_by_name) for child in self.children]
+        statement = node_to_capa(self.node, [child.statement for child in children])
+
+        if isinstance(self.node, FeatureNode):
+            feature = self.node.feature
+
+            if isinstance(feature, (frzf.SubstringFeature, frzf.RegexFeature)):
+                matches = {capture: {loc.to_capa() for loc in locs} for capture, locs in self.captures.items()}
+
+                if isinstance(feature, frzf.SubstringFeature):
+                    assert isinstance(statement, capa.features.common.Substring)
+                    statement = capa.features.common._MatchedSubstring(statement, matches)
+                elif isinstance(feature, frzf.RegexFeature):
+                    assert isinstance(statement, capa.features.common.Regex)
+                    statement = capa.features.common._MatchedRegex(statement, matches)
+                else:
+                    assert_never(feature)
+
+        # apparently we don't have to fixup match and subscope entries here.
+        # at least, default, verbose, and vverbose renderers seem to work well without any special handling here.
+        #
+        # children contains a single tree of results, corresponding to the logic of the matched rule.
+        # self.node.feature.match contains the name of the rule that was matched.
+        # so its all available to reconstruct, if necessary.
+
+        return capa.features.common.Result(
+            success=self.success,
+            statement=statement,
+            locations={loc.to_capa() for loc in self.locations},
             children=children,
-            locations=locations,
-            captures=captures,
         )
 
 
@@ -484,28 +528,30 @@ class RuleMetadata(FrozenModel):
             namespace=rule.meta.get("namespace"),
             authors=rule.meta.get("authors"),
             scope=capa.rules.Scope(rule.meta.get("scope")),
-            attack=list(map(AttackSpec.from_str, rule.meta.get("att&ck", []))),
-            mbc=list(map(MBCSpec.from_str, rule.meta.get("mbc", []))),
+            attack=tuple(map(AttackSpec.from_str, rule.meta.get("att&ck", []))),
+            mbc=tuple(map(MBCSpec.from_str, rule.meta.get("mbc", []))),
             references=rule.meta.get("references", []),
             examples=rule.meta.get("examples", []),
             description=rule.meta.get("description", ""),
             lib=rule.meta.get("lib", False),
-            capa_subscope=rule.meta.get("capa/subscope", False),
+            is_subscope_rule=rule.meta.get("capa/subscope", False),
             maec=MaecMetadata(
                 analysis_conclusion=rule.meta.get("maec/analysis-conclusion"),
                 analysis_conclusion_ov=rule.meta.get("maec/analysis-conclusion-ov"),
                 malware_family=rule.meta.get("maec/malware-family"),
                 malware_category=rule.meta.get("maec/malware-category"),
                 malware_category_ov=rule.meta.get("maec/malware-category-ov"),
-            ),
-        )
+            ),  # type: ignore
+            # Mypy is unable to recognise arguments due to alias
+        )  # type: ignore
+        # Mypy is unable to recognise arguments due to alias
 
     class Config:
         frozen = True
         allow_population_by_field_name = True
 
 
-class RuleMatches(BaseModel):
+class RuleMatches(FrozenModel):
     """
     args:
         meta: the metadata from the rule
@@ -517,12 +563,12 @@ class RuleMatches(BaseModel):
     matches: Tuple[Tuple[frz.Address, Match], ...]
 
 
-class ResultDocument(BaseModel):
+class ResultDocument(FrozenModel):
     meta: Metadata
     rules: Dict[str, RuleMatches]
 
     @classmethod
-    def from_capa(cls, meta, rules: RuleSet, capabilities: MatchResults) -> "ResultDocument":
+    def from_capa(cls, meta: Metadata, rules: RuleSet, capabilities: MatchResults) -> "ResultDocument":
         rule_matches: Dict[str, RuleMatches] = {}
         for rule_name, matches in capabilities.items():
             rule = rules[rule_name]
@@ -539,4 +585,22 @@ class ResultDocument(BaseModel):
                 ),
             )
 
-        return ResultDocument(meta=Metadata.from_capa(meta), rules=rule_matches)
+        return ResultDocument(meta=meta, rules=rule_matches)
+
+    def to_capa(self) -> Tuple[Metadata, Dict]:
+        capabilities: Dict[
+            str, List[Tuple[capa.features.address.Address, capa.features.common.Result]]
+        ] = collections.defaultdict(list)
+
+        # this doesn't quite work because we don't have the rule source for rules that aren't matched.
+        rules_by_name = {
+            rule_name: capa.rules.Rule.from_yaml(rule_match.source) for rule_name, rule_match in self.rules.items()
+        }
+
+        for rule_name, rule_match in self.rules.items():
+            for addr, match in rule_match.matches:
+                result: capa.engine.Result = match.to_capa(rules_by_name)
+
+                capabilities[rule_name].append((addr.to_capa(), result))
+
+        return self.meta, capabilities

capa/render/utils.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -16,7 +16,7 @@ import capa.render.result_document as rd
 
 def bold(s: str) -> str:
     """draw attention to the given string"""
-    return termcolor.colored(s, "blue")
+    return termcolor.colored(s, "cyan")
 
 
 def bold2(s: str) -> str:
@@ -37,7 +37,7 @@ def format_parts_id(data: Union[rd.AttackSpec, rd.MBCSpec]):
 
 def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]:
     """enumerate the rules in (namespace, name) order that are 'capability' rules (not lib/subscope/disposition/etc)."""
-    for _, _, rule in sorted(map(lambda rule: (rule.meta.namespace or "", rule.meta.name, rule), doc.rules.values())):
+    for _, _, rule in sorted((rule.meta.namespace or "", rule.meta.name, rule) for rule in doc.rules.values()):
         if rule.meta.lib:
             continue
         if rule.meta.is_subscope_rule:

capa/render/verbose.py

@@ -14,7 +14,7 @@ example::
                  0x10003415
                  0x10003797
 
-Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
 You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -96,8 +96,7 @@ def render_meta(ostream, doc: rd.ResultDocument):
         ("library function count", len(doc.meta.analysis.library_functions)),
         (
             "total feature count",
-            doc.meta.analysis.feature_counts.file
-            + sum(map(lambda f: f.count, doc.meta.analysis.feature_counts.functions)),
+            doc.meta.analysis.feature_counts.file + sum(f.count for f in doc.meta.analysis.feature_counts.functions),
         ),
     ]
 
@@ -141,7 +140,7 @@ def render_rules(ostream, doc: rd.ResultDocument):
             rows.append((key, v))
 
         if rule.meta.scope != capa.rules.FILE_SCOPE:
-            locations = list(map(lambda m: m[0], doc.rules[rule.meta.name].matches))
+            locations = [m[0] for m in doc.rules[rule.meta.name].matches]
             rows.append(("matches", "\n".join(map(format_address, locations))))
 
         ostream.writeln(tabulate.tabulate(rows, tablefmt="plain"))

capa/render/vverbose.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,7 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from typing import Dict, Iterable
+from typing import Dict, Iterable, Optional
 
 import tabulate
 
@@ -29,7 +29,7 @@ def render_locations(ostream, locations: Iterable[frz.Address]):
     # its possible to have an empty locations array here,
     # such as when we're in MODE_FAILURE and showing the logic
     # under a `not` statement (which will have no matched locations).
-    locations = list(sorted(locations))
+    locations = sorted(locations)
 
     if len(locations) == 0:
         return
@@ -129,6 +129,7 @@ def render_feature(ostream, match: rd.Match, feature: frzf.Feature, indent=0):
     ostream.write("  " * indent)
 
     key = feature.type
+    value: Optional[str]
     if isinstance(feature, frzf.BasicBlockFeature):
         # i don't think it makes sense to have standalone basic block features.
         # we don't parse them from rules, only things like: `count(basic block) > 1`
@@ -140,7 +141,7 @@ def render_feature(ostream, match: rd.Match, feature: frzf.Feature, indent=0):
         value = feature.class_
     else:
         # convert attributes to dictionary using aliased names, if applicable
-        value = feature.dict(by_alias=True).get(key, None)
+        value = feature.dict(by_alias=True).get(key)
 
     if value is None:
         raise ValueError(f"{key} contains None")
@@ -222,7 +223,7 @@ def render_match(ostream, match: rd.Match, indent=0, mode=MODE_SUCCESS):
 
         # optional statement with no successful children is empty
         if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
-            if not any(map(lambda m: m.success, match.children)):
+            if not any(m.success for m in match.children):
                 return
 
         # not statement, so invert the child mode to show failed evaluations
@@ -236,7 +237,7 @@ def render_match(ostream, match: rd.Match, indent=0, mode=MODE_SUCCESS):
 
         # optional statement with successful children is not relevant
         if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
-            if any(map(lambda m: m.success, match.children)):
+            if any(m.success for m in match.children):
                 return
 
         # not statement, so invert the child mode to show successful evaluations
@@ -277,7 +278,7 @@ def render_rules(ostream, doc: rd.ResultDocument):
 
     had_match = False
 
-    for _, _, rule in sorted(map(lambda rule: (rule.meta.namespace or "", rule.meta.name, rule), doc.rules.values())):
+    for _, _, rule in sorted((rule.meta.namespace or "", rule.meta.name, rule) for rule in doc.rules.values()):
         # default scope hides things like lib rules, malware-category rules, etc.
         # but in vverbose mode, we really want to show everything.
         #

capa/rules/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -14,6 +14,7 @@ import logging
 import binascii
 import collections
 from enum import Enum
+from pathlib import Path
 
 from capa.helpers import assert_never
 
@@ -510,7 +511,9 @@ def build_statements(d, scope: str):
                 # arg is string (which doesn't support inline descriptions), like:
                 #
                 #     count(string(error))
-                # TODO: what about embedded newlines?
+                #
+                # known problem that embedded newlines may not work here?
+                # this may become a problem (or not), so address it when encountered.
                 feature = Feature(arg)
         else:
             feature = Feature()
@@ -634,7 +637,7 @@ class Rule:
         Returns:
           List[str]: names of rules upon which this rule depends.
         """
-        deps: Set[str] = set([])
+        deps: Set[str] = set()
 
         def rec(statement):
             if isinstance(statement, capa.features.common.MatchedRule):
@@ -648,7 +651,7 @@ class Rule:
                 # but, namespaces tend to use `-` while rule names use ` `. so, unlikely, but possible.
                 if statement.value in namespaces:
                     # matches a namespace, so take precedence and don't even check rule names.
-                    deps.update(map(lambda r: r.name, namespaces[statement.value]))
+                    deps.update(r.name for r in namespaces[statement.value])
                 else:
                     # not a namespace, assume its a rule name.
                     assert isinstance(statement.value, str)
@@ -706,8 +709,7 @@ class Rule:
             # note: we cannot recurse into the subscope sub-tree,
             #  because its been replaced by a `match` statement.
             for child in statement.get_children():
-                for new_rule in self._extract_subscope_rules_rec(child):
-                    yield new_rule
+                yield from self._extract_subscope_rules_rec(child)
 
     def is_subscope_rule(self):
         return bool(self.meta.get("capa/subscope-rule", False))
@@ -733,8 +735,7 @@ class Rule:
         #   replace old node with reference to new rule
         #   yield new rule
 
-        for new_rule in self._extract_subscope_rules_rec(self.statement):
-            yield new_rule
+        yield from self._extract_subscope_rules_rec(self.statement)
 
     def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
@@ -778,7 +779,7 @@ class Rule:
             # on Windows, get WHLs from pyyaml.org/pypi
             logger.debug("using libyaml CLoader.")
             return yaml.CLoader
-        except:
+        except Exception:
             logger.debug("unable to import libyaml CLoader, falling back to Python yaml parser.")
             logger.debug("this will be slower to load rules.")
             return yaml.Loader
@@ -823,7 +824,7 @@ class Rule:
 
     @classmethod
     def from_yaml_file(cls, path, use_ruamel=False) -> "Rule":
-        with open(path, "rb") as f:
+        with Path(path).open("rb") as f:
             try:
                 rule = cls.from_yaml(f.read().decode("utf-8"), use_ruamel=use_ruamel)
                 # import here to avoid circular dependency
@@ -950,7 +951,7 @@ def get_rules_with_scope(rules, scope) -> List[Rule]:
     from the given collection of rules, select those with the given scope.
     `scope` is one of the capa.rules.*_SCOPE constants.
     """
-    return list(rule for rule in rules if rule.scope == scope)
+    return [rule for rule in rules if rule.scope == scope]
 
 
 def get_rules_and_dependencies(rules: List[Rule], rule_name: str) -> Iterator[Rule]:
@@ -961,7 +962,7 @@ def get_rules_and_dependencies(rules: List[Rule], rule_name: str) -> Iterator[Ru
     rules = list(rules)
     namespaces = index_rules_by_namespace(rules)
     rules_by_name = {rule.name: rule for rule in rules}
-    wanted = set([rule_name])
+    wanted = {rule_name}
 
     def rec(rule):
         wanted.add(rule.name)
@@ -976,7 +977,7 @@ def get_rules_and_dependencies(rules: List[Rule], rule_name: str) -> Iterator[Ru
 
 
 def ensure_rules_are_unique(rules: List[Rule]) -> None:
-    seen = set([])
+    seen = set()
     for rule in rules:
         if rule.name in seen:
             raise InvalidRule("duplicate rule name: " + rule.name)
@@ -1041,7 +1042,7 @@ def topologically_order_rules(rules: List[Rule]) -> List[Rule]:
     rules = list(rules)
     namespaces = index_rules_by_namespace(rules)
     rules_by_name = {rule.name: rule for rule in rules}
-    seen = set([])
+    seen = set()
     ret = []
 
     def rec(rule):
@@ -1190,7 +1191,6 @@ class RuleSet:
                     # so thats not helpful to decide how to downselect.
                     #
                     # and, a global rule will never be the sole selector in a rule.
-                    # TODO: probably want a lint for this.
                     pass
                 else:
                     # easy feature: hash lookup
@@ -1247,7 +1247,7 @@ class RuleSet:
                 # the set of subtypes of type A is unbounded,
                 # because any user might come along and create a new subtype B,
                 # so mypy can't reason about this set of types.
-                assert False, f"Unhandled value: {node} ({type(node).__name__})"
+                assert_never(node)
             else:
                 # programming error
                 assert_never(node)
@@ -1284,7 +1284,7 @@ class RuleSet:
         don't include auto-generated "subscope" rules.
         we want to include general "lib" rules here - even if they are not dependencies of other rules, see #398
         """
-        scope_rules: Set[Rule] = set([])
+        scope_rules: Set[Rule] = set()
 
         # we need to process all rules, not just rules with the given scope.
         # this is because rules with a higher scope, e.g. file scope, may have subscope rules
@@ -1329,7 +1329,7 @@ class RuleSet:
         TODO support -t=metafield <k>
         """
         rules = list(self.rules.values())
-        rules_filtered = set([])
+        rules_filtered = set()
         for rule in rules:
             for k, v in rule.meta.items():
                 if isinstance(v, str) and tag in v:

capa/rules/cache.py

@@ -1,10 +1,18 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import os
 import sys
 import zlib
 import pickle
 import hashlib
 import logging
-import os.path
 from typing import List, Optional
+from pathlib import Path
 from dataclasses import dataclass
 
 import capa.rules
@@ -28,7 +36,7 @@ def compute_cache_identifier(rule_content: List[bytes]) -> CacheIdentifier:
     hash.update(version.encode("utf-8"))
     hash.update(b"\x00")
 
-    rule_hashes = list(sorted([hashlib.sha256(buf).hexdigest() for buf in rule_content]))
+    rule_hashes = sorted([hashlib.sha256(buf).hexdigest() for buf in rule_content])
     for rule_hash in rule_hashes:
         hash.update(rule_hash.encode("ascii"))
         hash.update(b"\x00")
@@ -36,7 +44,7 @@ def compute_cache_identifier(rule_content: List[bytes]) -> CacheIdentifier:
     return hash.hexdigest()
 
 
-def get_default_cache_directory() -> str:
+def get_default_cache_directory() -> Path:
     # ref: https://github.com/mandiant/capa/issues/1212#issuecomment-1361259813
     #
     # Linux:   $XDG_CACHE_HOME/capa/
@@ -45,22 +53,22 @@ def get_default_cache_directory() -> str:
 
     # ref: https://stackoverflow.com/a/8220141/87207
     if sys.platform == "linux" or sys.platform == "linux2":
-        directory = os.environ.get("XDG_CACHE_HOME", os.path.join(os.environ["HOME"], ".cache", "capa"))
+        directory = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache" / "capa"))
     elif sys.platform == "darwin":
-        directory = os.path.join(os.environ["HOME"], "Library", "Caches", "capa")
+        directory = Path.home() / "Library" / "Caches" / "capa"
     elif sys.platform == "win32":
-        directory = os.path.join(os.environ["LOCALAPPDATA"], "flare", "capa", "cache")
+        directory = Path(os.environ["LOCALAPPDATA"]) / "flare" / "capa" / "cache"
     else:
         raise NotImplementedError(f"unsupported platform: {sys.platform}")
 
-    os.makedirs(directory, exist_ok=True)
+    directory.mkdir(parents=True, exist_ok=True)
 
     return directory
 
 
-def get_cache_path(cache_dir: str, id: CacheIdentifier) -> str:
+def get_cache_path(cache_dir: Path, id: CacheIdentifier) -> Path:
     filename = "capa-" + id[:8] + ".cache"
-    return os.path.join(cache_dir, filename)
+    return cache_dir / filename
 
 
 MAGIC = b"capa"
@@ -102,7 +110,7 @@ def compute_ruleset_cache_identifier(ruleset: capa.rules.RuleSet) -> CacheIdenti
     return compute_cache_identifier(rule_contents)
 
 
-def cache_ruleset(cache_dir: str, ruleset: capa.rules.RuleSet):
+def cache_ruleset(cache_dir: Path, ruleset: capa.rules.RuleSet):
     """
     cache the given ruleset to disk, using the given cache directory.
     this can subsequently be reloaded via `load_cached_ruleset`,
@@ -113,19 +121,18 @@ def cache_ruleset(cache_dir: str, ruleset: capa.rules.RuleSet):
     """
     id = compute_ruleset_cache_identifier(ruleset)
     path = get_cache_path(cache_dir, id)
-    if os.path.exists(path):
+    if path.exists():
         logger.debug("rule set already cached to %s", path)
         return
 
     cache = RuleCache(id, ruleset)
-    with open(path, "wb") as f:
-        f.write(cache.dump())
+    path.write_bytes(cache.dump())
 
     logger.debug("rule set cached to %s", path)
     return
 
 
-def load_cached_ruleset(cache_dir: str, rule_contents: List[bytes]) -> Optional[capa.rules.RuleSet]:
+def load_cached_ruleset(cache_dir: Path, rule_contents: List[bytes]) -> Optional[capa.rules.RuleSet]:
     """
     load a cached ruleset from disk, using the given cache directory.
     the raw rule contents are required here to prove that the rules haven't changed
@@ -136,20 +143,19 @@ def load_cached_ruleset(cache_dir: str, rule_contents: List[bytes]) -> Optional[
     """
     id = compute_cache_identifier(rule_contents)
     path = get_cache_path(cache_dir, id)
-    if not os.path.exists(path):
+    if not path.exists():
         logger.debug("rule set cache does not exist: %s", path)
         return None
 
     logger.debug("loading rule set from cache: %s", path)
-    with open(path, "rb") as f:
-        buf = f.read()
+    buf = path.read_bytes()
 
     try:
         cache = RuleCache.load(buf)
     except AssertionError:
         logger.debug("rule set cache is invalid: %s", path)
         # delete the cache that seems to be invalid.
-        os.remove(path)
+        path.unlink()
         return None
     else:
         return cache.ruleset

capa/version.py

@@ -1,4 +1,11 @@
-__version__ = "5.0.0"
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+__version__ = "6.0.0a1"
 
 
 def get_major_version():

doc/installation.md

@@ -93,28 +93,43 @@ For more details about creating and using virtual environments, check out the [v
 
 We use the following tools to ensure consistent code style and formatting:
   - [black](https://github.com/psf/black) code formatter
-  - [isort 5](https://pypi.org/project/isort/) code formatter
-  - [dos2unix](https://linux.die.net/man/1/dos2unix) for UNIX-style LF newlines
+  - [isort](https://pypi.org/project/isort/) code formatter
+  - [ruff](https://beta.ruff.rs/docs/) code linter
+  - [flake8](https://flake8.pycqa.org/en/latest/) code linter
+  - [mypy](https://mypy-lang.org/) type checking
   - [capafmt](https://github.com/mandiant/capa/blob/master/scripts/capafmt.py) rule formatter
 
 To install these development dependencies, run:
 
 `$ pip install -e /local/path/to/src[dev]`
 
-To check the code style, formatting and run the tests you can run the script `scripts/ci.sh`.
-You can run it with the argument `no_tests` to skip the tests and only run the code style and formatting: `scripts/ci.sh no_tests`
+We use [pre-commit](https://pre-commit.com/) so that its trivial to run the same linters & configuration locally as in CI.
 
-##### Setup hooks [optional]
+Run all linters liks:
 
-If you plan to contribute to capa, you may want to setup the provided hooks.
-Run `scripts/setup-hooks.sh` to set the following hooks up:
-- The `pre-commit` hook runs checks before every `git commit`.
-  It runs `scripts/ci.sh no_tests` aborting the commit if there are code style or rule linter offenses you need to fix.
-- The `pre-push` hook runs checks before every `git push`.
-  It runs `scripts/ci.sh` aborting the push if there are code style or rule linter offenses or if the tests fail.
-  This way you can ensure everything is alright before sending a pull request.
+    ❯ pre-commit run --all-files
+    isort....................................................................Passed
+    black....................................................................Passed
+    ruff.....................................................................Passed
+    flake8...................................................................Passed
+    mypy.....................................................................Passed
 
-You can skip the checks by using the `-n`/`--no-verify` git option.
+Or run a single linter like:
+
+    ❯ pre-commit run --all-files isort
+    isort....................................................................Passed
+
+
+Importantly, you can configure pre-commit to run automatically before every commit by running:
+
+    ❯ pre-commit install --hook-type pre-commit
+    pre-commit installed at .git/hooks/pre-commit
+
+    ❯ pre-commit install --hook-type pre-push
+    pre-commit installed at .git/hooks/pre-push
+
+This way you can ensure that you don't commit code style or formatting offenses.
+You can always temporarily skip the checks by using the `-n`/`--no-verify` git option.
 
 ### 3. Compile binary using PyInstaller
 We compile capa standalone binaries using PyInstaller. To reproduce the build process check out the source code as described above and follow the following steps.
@@ -126,6 +141,12 @@ Or install capa with build dependencies:
 
 `$ pip install -e /local/path/to/src[build]`
 
+#### Generate rule cache
+
+Generate cache for all rules in the `rules` folder and save the output in the `cache` folder.
+
+`$ python scripts/cache-ruleset.py rules/ cache/`
+
 #### Run Pyinstaller
 `$ pyinstaller .github/pyinstaller/pyinstaller.spec`
 

pyproject.toml

@@ -0,0 +1,112 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "flare-capa"
+authors = [
+    {name = "Willi Ballenthin", email = "william.ballenthin@mandiant.com"},
+    {name = "Moritz Raabe", email = "moritz.raabe@mandiant.com"},
+    {name = "Mike Hunhoff", email = "michael.hunhoff@mandiant.com"},
+]
+description = "The FLARE team's open-source tool to identify capabilities in executable files."
+license = {file = "LICENSE.txt"}
+requires-python = ">=3.8"
+keywords = ["malware analysis", "reverse engineering", "capability detection", "software behaviors", "capa", "FLARE"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "License :: OSI Approved :: Apache Software License",
+    "Natural Language :: English",
+    "Programming Language :: Python :: 3",
+    "Topic :: Security",
+]
+dependencies = [
+    "tqdm==4.65.0",
+    "pyyaml==6.0",
+    "tabulate==0.9.0",
+    "colorama==0.4.6",
+    "termcolor==2.3.0",
+    "wcwidth==0.2.6",
+    "ida-settings==2.1.0",
+    "viv-utils[flirt]==0.7.9",
+    "halo==0.0.31",
+    "networkx==3.1",
+    "ruamel.yaml==0.17.32",
+    "vivisect==1.1.1",
+    "pefile==2023.2.7",
+    "pyelftools==0.29",
+    "dnfile==0.13.0",
+    "dncil==1.0.2",
+    "pydantic==1.10.9",
+    "protobuf==4.23.4",
+]
+dynamic = ["version", "readme"]
+
+[tool.setuptools.dynamic]
+version = {attr = "capa.version.__version__"}
+readme = {file = "README.md"}
+
+[tool.setuptools]
+packages = ["capa"]
+
+[project.optional-dependencies]
+dev = [
+    "pre-commit==3.3.3",
+    "pytest==7.4.0",
+    "pytest-sugar==0.9.7",
+    "pytest-instafail==0.5.0",
+    "pytest-cov==4.1.0",
+    "flake8==6.0.0",
+    "flake8-bugbear==23.7.10",
+    "flake8-encodings==0.5.0.post1",
+    "flake8-comprehensions==3.14.0",
+    "flake8-logging-format==0.9.0",
+    "flake8-no-implicit-concat==0.3.4",
+    "flake8-print==5.0.0",
+    "flake8-todos==0.3.0",
+    "flake8-simplify==0.20.0",
+    "flake8-use-pathlib==0.3.0",
+    "flake8-copyright==0.2.4",
+    "ruff==0.0.278",
+    "black==23.7.0",
+    "isort==5.11.4",
+    "mypy==1.4.1",
+    "psutil==5.9.2",
+    "stix2==3.0.1",
+    "requests==2.31.0",
+    "mypy-protobuf==3.4.0",
+    # type stubs for mypy
+    "types-backports==0.1.3",
+    "types-colorama==0.4.15.11",
+    "types-PyYAML==6.0.8",
+    "types-tabulate==0.9.0.1",
+    "types-termcolor==1.1.4",
+    "types-psutil==5.8.23",
+    "types_requests==2.31.0.1",
+    "types-protobuf==4.23.0.1",
+]
+build = [
+    "pyinstaller==5.10.1",
+    "setuptools==68.0.0",
+    "build==0.10.0"
+]
+
+[project.urls]
+Homepage = "https://github.com/mandiant/capa"
+Repository = "https://github.com/mandiant/capa.git"
+Documentation = "https://github.com/mandiant/capa/tree/master/doc"
+Rules = "https://github.com/mandiant/capa-rules"
+"Rules Documentation" = "https://github.com/mandiant/capa-rules/tree/master/doc"
+
+[project.scripts]
+capa = "capa.main:main"