Update capa/features/extractors/dnfile/extractor.py

Co-authored-by: Moritz <mr-tz@users.noreply.github.com>

.devcontainer/devcontainer.json

@@ -41,7 +41,7 @@
 	// "forwardPorts": [],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "git submodule update --init && pip3 install --user -e .[dev] && pre-commit install",
+	"postCreateCommand": "git submodule update --init && pip3 install --user -e .[dev]",
 
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",

.github/CONTRIBUTING.md

@@ -159,25 +159,12 @@ The process described here has several goals:
 
 Please follow these steps to have your contribution considered by the maintainers:
 
-0. Sign the [Contributor License Agreement](#contributor-license-agreement)
 1. Follow the [styleguides](#styleguides)
 2. Update the CHANGELOG and add tests and documentation. In case they are not needed, indicate it in [the PR template](pull_request_template.md).
 3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing? </summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
 
 While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
 
-### Contributor License Agreement
-
-Contributions to this project must be accompanied by a Contributor License
-Agreement. You (or your employer) retain the copyright to your contribution,
-this simply gives us permission to use and redistribute your contributions as
-part of the project. Head over to <https://cla.developers.google.com/> to see
-your current agreements on file or to sign a new one.
-
-You generally only need to submit a CLA once, so if you've already submitted one
-(even if it was for a different project), you probably don't need to do it
-again.
-
 ## Styleguides
 
 ### Git Commit Messages

.github/flake8.ini

@@ -1,41 +0,0 @@
-[flake8]
-max-line-length = 120
-
-extend-ignore =
-    # E203: whitespace before ':'  (black does this)
-    E203,
-    # F401: `foo` imported but unused  (prefer ruff)
-    F401,
-    # F811 Redefinition of unused `foo`  (prefer ruff)
-    F811,
-    # E501 line too long  (prefer black)
-    E501,
-    # B010 Do not call setattr with a constant attribute value
-    B010,
-    # G200 Logging statement uses exception in arguments
-    G200,
-    # SIM102 Use a single if-statement instead of nested if-statements
-    # doesn't provide a space for commenting or logical separation of conditions
-    SIM102,
-    # SIM114 Use logical or and a single body
-    # makes logic trees too complex
-    SIM114,
-    # SIM117 Use 'with Foo, Bar:' instead of multiple with statements
-    # makes lines too long
-    SIM117
-
-per-file-ignores =
-    # T201 print found.
-    #
-    # scripts are meant to print output
-    scripts/*: T201
-    # capa.exe is meant to print output
-    capa/main.py: T201
-    # IDA tests emit results to output window so need to print
-    tests/test_ida_features.py: T201
-    # utility used to find the Binary Ninja API via invoking python.exe
-    capa/features/extractors/binja/find_binja_api.py: T201
-
-copyright-check = True
-copyright-min-file-size = 1 
-copyright-regexp = Copyright \(C\) 2023 Mandiant, Inc. All Rights Reserved.

.github/mypy/mypy.ini

@@ -42,9 +42,6 @@ ignore_missing_imports = True
 [mypy-idautils.*]
 ignore_missing_imports = True
 
-[mypy-ida_auto.*]
-ignore_missing_imports = True
-
 [mypy-ida_bytes.*]
 ignore_missing_imports = True
 
@@ -86,6 +83,3 @@ ignore_missing_imports = True
 
 [mypy-netnode.*]
 ignore_missing_imports = True
-
-[mypy-ghidra.*]
-ignore_missing_imports = True

.github/pyinstaller/hooks/hook-vivisect.py

@@ -38,36 +38,39 @@ hiddenimports = [
     "vivisect",
     "vivisect.analysis",
     "vivisect.analysis.amd64",
+    "vivisect.analysis.amd64",
     "vivisect.analysis.amd64.emulation",
     "vivisect.analysis.amd64.golang",
     "vivisect.analysis.crypto",
+    "vivisect.analysis.crypto",
     "vivisect.analysis.crypto.constants",
     "vivisect.analysis.elf",
     "vivisect.analysis.elf.elfplt",
     "vivisect.analysis.elf.elfplt_late",
     "vivisect.analysis.elf.libc_start_main",
     "vivisect.analysis.generic",
+    "vivisect.analysis.generic",
     "vivisect.analysis.generic.codeblocks",
     "vivisect.analysis.generic.emucode",
     "vivisect.analysis.generic.entrypoints",
     "vivisect.analysis.generic.funcentries",
     "vivisect.analysis.generic.impapi",
-    "vivisect.analysis.generic.linker",
     "vivisect.analysis.generic.mkpointers",
-    "vivisect.analysis.generic.noret",
     "vivisect.analysis.generic.pointers",
     "vivisect.analysis.generic.pointertables",
     "vivisect.analysis.generic.relocations",
     "vivisect.analysis.generic.strconst",
     "vivisect.analysis.generic.switchcase",
-    "vivisect.analysis.generic.symswitchcase",
     "vivisect.analysis.generic.thunks",
+    "vivisect.analysis.generic.noret",
+    "vivisect.analysis.i386",
     "vivisect.analysis.i386",
     "vivisect.analysis.i386.calling",
     "vivisect.analysis.i386.golang",
     "vivisect.analysis.i386.importcalls",
     "vivisect.analysis.i386.instrhook",
-    "vivisect.analysis.i386.thunk_reg",
+    "vivisect.analysis.i386.thunk_bx",
+    "vivisect.analysis.ms",
     "vivisect.analysis.ms",
     "vivisect.analysis.ms.hotpatch",
     "vivisect.analysis.ms.localhints",
@@ -78,40 +81,8 @@ hiddenimports = [
     "vivisect.impapi.posix.amd64",
     "vivisect.impapi.posix.i386",
     "vivisect.impapi.windows",
-    "vivisect.impapi.windows.advapi_32",
-    "vivisect.impapi.windows.advapi_64",
     "vivisect.impapi.windows.amd64",
-    "vivisect.impapi.windows.gdi_32",
-    "vivisect.impapi.windows.gdi_64",
     "vivisect.impapi.windows.i386",
-    "vivisect.impapi.windows.kernel_32",
-    "vivisect.impapi.windows.kernel_64",
-    "vivisect.impapi.windows.msvcr100_32",
-    "vivisect.impapi.windows.msvcr100_64",
-    "vivisect.impapi.windows.msvcr110_32",
-    "vivisect.impapi.windows.msvcr110_64",
-    "vivisect.impapi.windows.msvcr120_32",
-    "vivisect.impapi.windows.msvcr120_64",
-    "vivisect.impapi.windows.msvcr71_32",
-    "vivisect.impapi.windows.msvcr80_32",
-    "vivisect.impapi.windows.msvcr80_64",
-    "vivisect.impapi.windows.msvcr90_32",
-    "vivisect.impapi.windows.msvcr90_64",
-    "vivisect.impapi.windows.msvcrt_32",
-    "vivisect.impapi.windows.msvcrt_64",
-    "vivisect.impapi.windows.ntdll_32",
-    "vivisect.impapi.windows.ntdll_64",
-    "vivisect.impapi.windows.ole_32",
-    "vivisect.impapi.windows.ole_64",
-    "vivisect.impapi.windows.rpcrt4_32",
-    "vivisect.impapi.windows.rpcrt4_64",
-    "vivisect.impapi.windows.shell_32",
-    "vivisect.impapi.windows.shell_64",
-    "vivisect.impapi.windows.user_32",
-    "vivisect.impapi.windows.user_64",
-    "vivisect.impapi.windows.ws2plus_32",
-    "vivisect.impapi.windows.ws2plus_64",
-    "vivisect.impapi.winkern",
     "vivisect.impapi.winkern.i386",
     "vivisect.impapi.winkern.amd64",
     "vivisect.parsers.blob",

.github/pyinstaller/pyinstaller.spec

@@ -61,7 +61,6 @@ a = Analysis(
         "qt5",
         "pyqtwebengine",
         "pyasn1",
-        "binaryninja",
     ],
 )
 

.github/ruff.toml

@@ -1,43 +0,0 @@
-# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
-# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
-# McCabe complexity (`C901`) by default.
-select = ["E", "F"]
-
-# Allow autofix for all enabled rules (when `--fix`) is provided.
-fixable = ["ALL"]
-unfixable = []
-
-# E402 module level import not at top of file
-# E722 do not use bare 'except'
-# E501 line too long
-ignore = ["E402", "E722", "E501"]
-
-line-length = 120
-
-exclude = [
-    # Exclude a variety of commonly ignored directories.
-    ".bzr",
-    ".direnv",
-    ".eggs",
-    ".git",
-    ".git-rewrite",
-    ".hg",
-    ".mypy_cache",
-    ".nox",
-    ".pants.d",
-    ".pytype",
-    ".ruff_cache",
-    ".svn",
-    ".tox",
-    ".venv",
-    "__pypackages__",
-    "_build",
-    "buck-out",
-    "build",
-    "dist",
-    "node_modules",
-    "venv",
-    # protobuf generated files
-    "*_pb2.py", 
-    "*_pb2.pyi"
-]

.github/tox.ini

@@ -0,0 +1,10 @@
+[pycodestyle]
+; E402: module level import not at top of file
+; W503: line break before binary operator
+; E231 missing whitespace after ',' (emitted by black)
+; E203 whitespace before ':' (emitted by black)
+ignore = E402,W503,E203,E231
+max-line-length = 160
+statistics = True
+count = True
+exclude = .*

.github/workflows/build.yml

@@ -6,9 +6,6 @@ on:
   release:
     types: [edited, published]
 
-permissions:
-  contents: write
-
 jobs:
   build:
     name: PyInstaller for ${{ matrix.os }}
@@ -18,7 +15,7 @@ jobs:
       fail-fast: true
       matrix:
         include:
-          - os: ubuntu-20.04
+          - os: ubuntu-18.04
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
@@ -39,7 +36,7 @@ jobs:
         uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
           python-version: 3.8
-      - if: matrix.os == 'ubuntu-20.04'
+      - if: matrix.os == 'ubuntu-18.04'
         run: sudo apt-get install -y libyaml-dev
       - name: Upgrade pip, setuptools
         run: python -m pip install --upgrade pip setuptools
@@ -68,7 +65,10 @@ jobs:
       matrix:
         include:
           # OSs not already tested above
-          - os: ubuntu-22.04
+          - os: ubuntu-18.04
+            artifact_name: capa
+            asset_name: linux
+          - os: ubuntu-20.04
             artifact_name: capa
             asset_name: linux
           - os: windows-2022

.github/workflows/changelog.yml

@@ -7,8 +7,6 @@ on:
   pull_request_target:
     types: [opened, edited, synchronize]
 
-permissions: read-all
-
 jobs:
   check_changelog:
     # no need to check for dependency updates via dependabot

.github/workflows/publish.yml

@@ -1,41 +1,29 @@
-# use PyPI trusted publishing, as described here: 
-# https://blog.trailofbits.com/2023/05/23/trusted-publishing-a-new-benchmark-for-packaging-security/
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
 name: publish to pypi
 
 on:
   release:
     types: [published]
 
-permissions:
-  contents: write
-
 jobs:
-  pypi-publish:
-    runs-on: ubuntu-latest
-    environment:
-      name: release
-    permissions:
-      id-token: write
+  deploy:
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       - name: Set up Python
         uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
-          python-version: '3.8'
+          python-version: '3.7'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e .[build]
-      - name: build package
+          pip install setuptools wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          python -m build
-      - name: upload package artifacts
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        with:
-          path: dist/*
-      - name: publish package
-        uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0  # release/v1
-        with:
-          skip-existing: true
-          verbose: true
-          print-hash: true
+          python setup.py sdist bdist_wheel
+          twine upload --skip-existing dist/*

.github/workflows/tag.yml

@@ -4,8 +4,6 @@ on:
   release:
     types: [published]
 
-permissions: read-all
-
 jobs:
   tag:
     name: Tag capa rules

.github/workflows/tests.yml

@@ -6,8 +6,6 @@ on:
   pull_request:
     branches: [ master ]
 
-permissions: read-all
-
 # save workspaces to speed up testing
 env:
   CAPA_SAVE_WORKSPACE: "True"
@@ -29,23 +27,20 @@ jobs:
     steps:
     - name: Checkout capa
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-    # use latest available python to take advantage of best performance
-    - name: Set up Python 3.11
+    - name: Set up Python 3.8
       uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
-        python-version: "3.11"
+        python-version: "3.8"
     - name: Install dependencies
       run: pip install -e .[dev]
-    - name: Lint with ruff
-      run: pre-commit run ruff
     - name: Lint with isort
-      run: pre-commit run isort
+      run: isort --profile black --length-sort --line-width 120 -c .
     - name: Lint with black
-      run: pre-commit run black
-    - name: Lint with flake8
-      run: pre-commit run flake8
+      run: black -l 120 --check .
+    - name: Lint with pycodestyle
+      run: pycodestyle --show-source capa/ scripts/ tests/
     - name: Check types with mypy
-      run:  pre-commit run mypy
+      run: mypy --config-file .github/mypy/mypy.ini --check-untyped-defs capa/ scripts/ tests/
 
   rule_linter:
     runs-on: ubuntu-20.04
@@ -54,12 +49,12 @@ jobs:
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       with:
         submodules: recursive
-    - name: Set up Python 3.11
+    - name: Set up Python 3.8
       uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
       with:
-        python-version: "3.11"
+        python-version: "3.8"
     - name: Install capa
-      run: pip install -e .[dev]
+      run: pip install -e .
     - name: Run rule linter
       run: python scripts/lint.py rules/
 
@@ -72,15 +67,13 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-2019, macos-11]
         # across all operating systems
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.7", "3.11"]
         include:
           # on Ubuntu run these as well
           - os: ubuntu-20.04
             python-version: "3.8"
           - os: ubuntu-20.04
             python-version: "3.9"
-          - os: ubuntu-20.04
-            python-version: "3.10"
     steps:
     - name: Checkout capa with submodules
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -97,104 +90,3 @@ jobs:
       run: pip install -e .[dev]
     - name: Run tests
       run: pytest -v tests/
-
-  binja-tests:
-    name: Binary Ninja tests for ${{ matrix.python-version }}
-    env:
-      BN_SERIAL: ${{ secrets.BN_SERIAL }}
-    runs-on: ubuntu-20.04
-    needs: [code_style, rule_linter]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.11"]
-    steps:
-    - name: Checkout capa with submodules
-      # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
-      if: ${{ env.BN_SERIAL != 0 }}
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      with:
-        submodules: recursive
-    - name: Set up Python ${{ matrix.python-version }}
-      if: ${{ env.BN_SERIAL != 0 }}
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install pyyaml
-      if: ${{ env.BN_SERIAL != 0 }}
-      run: sudo apt-get install -y libyaml-dev
-    - name: Install capa
-      if: ${{ env.BN_SERIAL != 0 }}
-      run: pip install -e .[dev]
-    - name: install Binary Ninja
-      if: ${{ env.BN_SERIAL != 0 }}
-      run: |
-        mkdir ./.github/binja
-        curl "https://raw.githubusercontent.com/Vector35/binaryninja-api/6812c97/scripts/download_headless.py" -o ./.github/binja/download_headless.py
-        python ./.github/binja/download_headless.py --serial ${{ env.BN_SERIAL }} --output .github/binja/BinaryNinja-headless.zip
-        unzip .github/binja/BinaryNinja-headless.zip -d .github/binja/
-        python .github/binja/binaryninja/scripts/install_api.py --install-on-root --silent
-    - name: Run tests
-      if: ${{ env.BN_SERIAL != 0 }}
-      env:
-        BN_LICENSE: ${{ secrets.BN_LICENSE }}
-      run: pytest -v tests/test_binja_features.py  # explicitly refer to the binja tests for performance. other tests run above.
-
-  ghidra-tests:
-    name: Ghidra tests for ${{ matrix.python-version }}
-    runs-on: ubuntu-20.04
-    needs: [code_style, rule_linter]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.11"]
-        java-version: ["17"]
-        gradle-version: ["7.3"]
-        ghidra-version: ["10.3"]
-        public-version: ["PUBLIC_20230510"] # for ghidra releases
-        jep-version: ["4.1.1"]
-        ghidrathon-version: ["3.0.0"]
-    steps:
-    - name: Checkout capa with submodules
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      with:
-        submodules: true
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Set up Java ${{ matrix.java-version }}
-      uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3
-      with:
-        distribution: 'temurin'
-        java-version: ${{ matrix.java-version }}
-    - name: Set up Gradle ${{ matrix.gradle-version }} 
-      uses: gradle/gradle-build-action@40b6781dcdec2762ad36556682ac74e31030cfe2 # v2.5.1
-      with:
-        gradle-version: ${{ matrix.gradle-version }}
-    - name: Install Jep ${{ matrix.jep-version }} 
-      run : pip install jep==${{ matrix.jep-version }}
-    - name: Install Ghidra ${{ matrix.ghidra-version }} 
-      run: |
-        mkdir ./.github/ghidra
-        wget "https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_${{ matrix.ghidra-version }}_build/ghidra_${{ matrix.ghidra-version }}_${{ matrix.public-version }}.zip" -O ./.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip
-        unzip .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip -d .github/ghidra/
-    - name: Install Ghidrathon
-      run : |
-        mkdir ./.github/ghidrathon
-        curl -o ./.github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip "https://codeload.github.com/mandiant/Ghidrathon/zip/refs/tags/v${{ matrix.ghidrathon-version }}"
-        unzip .github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/
-        gradle -p ./.github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/ -PGHIDRA_INSTALL_DIR=$(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC
-        unzip .github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/dist/*.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions
-    - name: Install pyyaml
-      run: sudo apt-get install -y libyaml-dev
-    - name: Install capa
-      run: pip install -e .[dev] 
-    - name: Run tests
-      run: | 
-        mkdir ./.github/ghidra/project
-        .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/support/analyzeHeadless .github/ghidra/project ghidra_test -Import ./tests/data/mimikatz.exe_ -ScriptPath ./tests/ -PostScript test_ghidra_features.py > ../output.log
-        cat ../output.log
-        exit_code=$(cat ../output.log | grep exit | awk '{print $NF}')
-        exit $exit_code
- 

.gitignore

@@ -108,21 +108,17 @@ venv.bak/
 *.viv
 *.idb
 *.i64
-.vscode
-
 !rules/lib
 
+# hooks/ci.sh output
+isort-output.log
+black-output.log
+rule-linter-output.log
+.vscode
 scripts/perf/*.txt
 scripts/perf/*.svg
 scripts/perf/*.zip
-
 .direnv
 .envrc
 .DS_Store
 */.DS_Store
-Pipfile
-Pipfile.lock
-/cache/
-.github/binja/binaryninja
-.github/binja/download_headless.py
-.github/binja/BinaryNinja-headless.zip

.pre-commit-config.yaml

@@ -1,111 +0,0 @@
-# install the pre-commit hooks:
-#
-#    ❯ pre-commit install --hook-type pre-commit
-#    pre-commit installed at .git/hooks/pre-commit
-#
-#    ❯ pre-commit install --hook-type pre-push
-#    pre-commit installed at .git/hooks/pre-push
-#
-# run all linters liks:
-#
-#    ❯ pre-commit run --all-files
-#    isort....................................................................Passed
-#    black....................................................................Passed
-#    ruff.....................................................................Passed
-#    flake8...................................................................Passed
-#    mypy.....................................................................Passed
-#
-# run a single linter like:
-#
-#    ❯ pre-commit run --all-files isort
-#    isort....................................................................Passed
-
-repos:
--   repo: local
-    hooks:
-    -   id: isort
-        name: isort
-        stages: [commit, push]
-        language: system
-        entry: isort
-        args: 
-        -   "--length-sort"
-        -   "--profile"
-        -   "black"
-        -   "--line-length=120"
-        -   "--skip-glob"
-        -   "*_pb2.py"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false
-
--   repo: local
-    hooks:
-    -   id: black
-        name: black
-        stages: [commit, push]
-        language: system
-        entry: black
-        args: 
-        -   "--line-length=120"
-        -   "--extend-exclude"
-        -   ".*_pb2.py"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false
-
--   repo: local
-    hooks:
-    -   id: ruff
-        name: ruff
-        stages: [commit, push]
-        language: system
-        entry: ruff
-        args: 
-        -   "check"
-        -   "--config"
-        -   ".github/ruff.toml"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false
-
--   repo: local
-    hooks:
-    -   id: flake8
-        name: flake8
-        stages: [commit, push]
-        language: system
-        entry: flake8
-        args: 
-        -   "--config"
-        -   ".github/flake8.ini"
-        -   "--extend-exclude"
-        -   "capa/render/proto/capa_pb2.py"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false
-
--   repo: local
-    hooks:
-    -   id: mypy
-        name: mypy
-        stages: [commit, push]
-        language: system
-        entry: mypy
-        args: 
-        -   "--check-untyped-defs"
-        -   "--ignore-missing-imports"
-        -   "--config-file=.github/mypy/mypy.ini"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false

CHANGELOG.md

@@ -3,168 +3,11 @@
 ## master (unreleased)
 
 ### New Features
-- ghidra: add Ghidra feature extractor and supporting code #1770 @colton-gabertan
-- ghidra: add entry script helping users run capa against a loaded Ghidra database #1767 @mike-hunhoff
-- binja: add support for forwarded exports #1646 @xusheng6
-- binja: add support for symtab names #1504 @xusheng6
+- dotnet: add support for basic blocks #1326 @mike-hunhoff
 
 ### Breaking Changes
 
-### New Rules (1)
-
-- nursery/get-ntoskrnl-base-address @mr-tz
--
-
-### Bug Fixes
-- ghidra: fix ints_to_bytes performance #1761 @mike-hunhoff
-- binja: improve function call site detection @xusheng6
-- binja: use binaryninja.load to open files @xusheng6
-- binja: bump binja version to 3.5 #1789 @xusheng6
-
-### capa explorer IDA Pro plugin
-
-### Development
-
-### Raw diffs
-- [capa v6.1.0...master](https://github.com/mandiant/capa/compare/v6.1.0...master)
-- [capa-rules v6.1.0...master](https://github.com/mandiant/capa-rules/compare/v6.1.0...master)
-
-## v6.1.0
-
-capa v6.1.0 is a bug fix release, most notably fixing unhandled exceptions in the capa explorer IDA Pro plugin.
-@Aayush-Goel-04 put a lot of effort into improving code quality and adding a script for rule authors.
-The script shows which features are present in a sample but not referenced by any existing rule.
-You could use this script to find opportunities for new rules.
-
-Speaking of new rules, we have eight additions, coming from Ronnie, Jakub, Moritz, Ervin, and still@teamt5.org!
-
-### New Features
-- ELF: implement import and export name extractor #1607 #1608 @Aayush-Goel-04
-- bump pydantic from 1.10.9 to 2.1.1 #1582 @Aayush-Goel-04
-- develop script to highlight features not used during matching #331 @Aayush-Goel-04
-
-### New Rules (8)
-
-- executable/pe/export/forwarded-export ronnie.salomonsen@mandiant.com
-- host-interaction/bootloader/get-uefi-variable jakub.jozwiak@mandiant.com
-- host-interaction/bootloader/set-uefi-variable jakub.jozwiak@mandiant.com
-- nursery/enumerate-device-drivers-on-linux @mr-tz
-- anti-analysis/anti-vm/vm-detection/check-for-foreground-window-switch ervin.ocampo@mandiant.com
-- linking/static/sqlite3/linked-against-cppsqlite3 still@teamt5.org
-- linking/static/sqlite3/linked-against-sqlite3 still@teamt5.org
-
-### Bug Fixes
-
-- rules: fix forwarded export characteristic #1656 @RonnieSalomonsen
-- Binary Ninja: Fix stack string detection #1473 @xusheng6
-- linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin 
-- OS: detect Android ELF files #1705 @williballenthin
-- ELF: fix parsing of symtab #1704 @williballenthin
-- result document: don't use deprecated pydantic functions #1718 @williballenthin
-- pytest: don't mark IDA tests as pytest tests #1719 @williballenthin
-
-### capa explorer IDA Pro plugin
-- fix unhandled exception when resolving rule path #1693 @mike-hunhoff
-
-### Raw diffs
-- [capa v6.0.0...v6.1.0](https://github.com/mandiant/capa/compare/v6.0.0...v6.1.0)
-- [capa-rules v6.0.0...v6.1.0](https://github.com/mandiant/capa-rules/compare/v6.0.0...v6.1.0)
-
-## v6.0.0
-
-capa v6.0 brings many bug fixes and quality improvements, including 64 rule updates and 26 new rules. We're now publishing to PyPI via [Trusted Publishing](https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/) and have migrated to using a `pyproject.toml` file. @Aayush-Goel-04 contributed a lot of new code across many files, so please welcome them to the project, along with @anders-v @crowface28 @dkelly2e @RonnieSalomonsen and @ejfocampo as first-time rule contributors!
-
-For those that use capa as a library, we've introduced some limited breaking changes that better represent data types (versus less-structured data like dictionaries and strings). With the recent deprecation, we've also dropped support for Python 3.7.
-
-### New Features
-- add script to detect feature overlap between new and existing capa rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
-- extract forwarded exports from PE files #1624 @williballenthin
-- extract function and API names from ELF symtab entries @yelhamer https://github.com/mandiant/capa-rules/issues/736
-- use fancy box drawing characters for default output #1586 @williballenthin
-
-### Breaking Changes
-- use a class to represent Metadata (not dict) #1411 @Aayush-Goel-04 @manasghandat
-- use pathlib.Path to represent file paths #1534 @Aayush-Goel-04
-- Python 3.8 is now the minimum supported Python version #1578 @williballenthin
-- Require a Contributor License Agreement (CLA) for PRs going forward #1642 @williballenthin
-
-### New Rules (26)
-
-- load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
-- nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
-- data-manipulation/encryption/aes/encrypt-data-using-aes-mixcolumns-step @mr-tz
-- linking/static/aplib/linked-against-aplib still@teamt5.org
-- communication/mailslot/read-from-mailslot nick.simonian@mandiant.com
-- nursery/hash-data-using-sha512managed-in-dotnet jonathanlepore@google.com
-- nursery/compiled-with-exescript jonathanlepore@google.com
-- nursery/check-for-sandbox-via-mac-address-ouis-in-dotnet jonathanlepore@google.com
-- host-interaction/hardware/enumerate-devices-by-category @mr-tz
-- host-interaction/service/continue-service @mr-tz
-- host-interaction/service/pause-service @mr-tz
-- persistence/exchange/act-as-exchange-transport-agent jakub.jozwiak@mandiant.com
-- host-interaction/file-system/create-virtual-file-system-in-dotnet jakub.jozwiak@mandiant.com
-- compiler/cx_freeze/compiled-with-cx_freeze @mr-tz jakub.jozwiak@mandiant.com
-- communication/socket/create-vmci-socket jakub.jozwiak@mandiant.com
-- persistence/office/act-as-excel-xll-add-in jakub.jozwiak@mandiant.com
-- persistence/office/act-as-office-com-add-in jakub.jozwiak@mandiant.com
-- persistence/office/act-as-word-wll-add-in jakub.jozwiak@mandiant.com
-- anti-analysis/anti-debugging/debugger-evasion/hide-thread-from-debugger michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com
-- host-interaction/memory/create-new-application-domain-in-dotnet jakub.jozwiak@mandiant.com
-- host-interaction/gui/switch-active-desktop jakub.jozwiak@mandiant.com
-- host-interaction/service/query-service-configuration @mr-tz
-- anti-analysis/anti-av/patch-event-tracing-for-windows-function jakub.jozwiak@mandiant.com
-- data-manipulation/encoding/xor/covertly-decode-and-write-data-to-windows-directory-using-indirect-calls dan.kelly@mandiant.com
-- linking/runtime-linking/resolve-function-by-brute-ratel-badger-hash jakub.jozwiak@mandiant.com
-
-### Bug Fixes
-- extractor: add a Binary Ninja test that asserts its version #1487 @xusheng6
-- extractor: update Binary Ninja stack string detection after the new constant outlining feature #1473 @xusheng6
-- extractor: update vivisect Arch extraction #1334 @mr-tz
-- extractor: avoid Binary Ninja exception when analyzing certain files #1441 @xusheng6 
-- symtab: fix struct.unpack() format for 64-bit ELF files @yelhamer
-- symtab: safeguard against ZeroDivisionError for files containing a symtab with a null entry size @yelhamer
-- improve ELF strtab and needed parsing @mr-tz
-- better handle exceptional cases when parsing ELF files #1458 @Aayush-Goel-04
-- improved testing coverage for Binary Ninja backend #1446 @Aayush-Goel-04
-- add logging and print redirect to tqdm for capa main #749 @Aayush-Goel-04
-- extractor: fix binja installation path detection does not work with Python 3.11
-- tests: refine the IDA test runner script #1513 @williballenthin
-- output: don't leave behind traces of progress bar @williballenthin
-- import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin
-- main: don't show spinner when emitting debug messages #1636 @williballenthin
-- rules: add forwarded export characteristics to rule syntax file scope #1653 @RonnieSalomonsen
-
-### capa explorer IDA Pro plugin
-
-### Development
-- update ATT&CK/MBC data for linting #1568 @mr-tz
-- log time taken to analyze each function #1290 @williballenthin
-- tests: make fixture available via conftest.py #1592 @williballenthin
-- publish via PyPI trusted publishing #1491 @williballenthin
-- migrate to pyproject.toml #1301 @williballenthin
-- use [pre-commit](https://pre-commit.com/) to invoke linters #1579 @williballenthin
-
-
-### Raw diffs
-- [capa v5.1.0...v6.0.0](https://github.com/mandiant/capa/compare/v5.1.0...v6.0.0)
-- [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0)
-
-## v5.1.0
-capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and a output color format that's easier to read.
-
-Over 25 capa rules have been added and improved.
-
-Thanks for all the support, especially to @xusheng6, @captainGeech42, @ggold7046, @manasghandat, @ooprathamm, @linpeiyu164, @yelhamer, @HongThatCong, @naikordian, @stevemk14ebr, @emtuls, @raymondlleong, @bkojusner, @joren485, and everyone else who submitted bugs and provided feedback!
-
-### New Features
-- add protobuf format for result documents #1219 @williballenthin @mr-tz 
-- extractor: add Binary Ninja feature extractor @xusheng6
-- new cli flag `--os` to override auto-detected operating system for a sample @captainGeech42
-- change colour/highlight to "cyan" instead of "blue" for better readability #1384 @ggold7046
-- add new format to parse output json back to capa #1396 @ooprathamm
-- parse ELF symbols' names to guess OS #1403 @yelhamer
-
-### New Rules (26)
+### New Rules (9)
 
 - persistence/scheduled-tasks/schedule-task-via-at joren485
 - data-manipulation/prng/generate-random-numbers-via-rtlgenrandom william.ballenthin@mandiant.com
@@ -175,39 +18,18 @@ Thanks for all the support, especially to @xusheng6, @captainGeech42, @ggold7046
 - communication/http/reference-http-user-agent-string @mr-tz
 - communication/http/get-http-content-length william.ballenthin@mandiant.com
 - nursery/move-directory michael.hunhoff@mandiant.com
-- nursery/get-http-request-uri william.ballenthin@mandiant.com
-- nursery/create-zip-archive-in-dotnet michael.hunhoff@mandiant.com
-- nursery/extract-zip-archive-in-dotnet anushka.virgaonkar@mandiant.com michael.hunhoff@mandiant.com
-- data-manipulation/encryption/tea/decrypt-data-using-tea william.ballenthin@mandiant.com raymond.leong@mandiant.com
-- data-manipulation/encryption/tea/encrypt-data-using-tea william.ballenthin@mandiant.com raymond.leong@mandiant.com
-- data-manipulation/encryption/xtea/encrypt-data-using-xtea raymond.leong@mandiant.com
-- data-manipulation/encryption/xxtea/encrypt-data-using-xxtea raymond.leong@mandiant.com
-- nursery/hash-data-using-ripemd128 raymond.leong@mandiant.com
-- nursery/hash-data-using-ripemd256 raymond.leong@mandiant.com
-- nursery/hash-data-using-ripemd320 raymond.leong@mandiant.com
-- nursery/set-web-proxy-in-dotnet michael.hunhoff@mandiant.com
-- nursery/check-for-windows-sandbox-via-subdirectory echernofsky@google.com
-- nursery/enumerate-pe-sections-in-dotnet @mr-tz
-- nursery/destroy-software-breakpoint-capability echernofsky@google.com
-- nursery/send-data-to-internet michael.hunhoff@mandiant.com
-- nursery/compiled-with-cx_freeze @mr-tz
-- nursery/contain-a-thread-local-storage-tls-section-in-dotnet michael.hunhoff@mandiant.com
+-
 
 ### Bug Fixes
-- extractor: interface of cache modified to prevent extracting file and global features multiple times @stevemk14ebr
-- extractor: removed '.dynsym' as the library name for ELF imports #1318 @stevemk14ebr 
 - extractor: fix vivisect loop detection corner case #1310 @mr-tz
-- match: extend OS characteristic to match OS_ANY to all supported OSes #1324 @mike-hunhoff
-- extractor: fix IDA and vivisect string and bytes features overlap and tests #1327 #1336 @xusheng6
 
 ### capa explorer IDA Pro plugin
-- rule generator plugin now loads faster when jumping between functions @stevemk14ebr
-- fix exception when plugin loaded in IDA hosted under idat #1341 @mike-hunhoff
-- improve embedded PE detection performance and reduce FP potential #1344 @mike-hunhoff
+
+### Development
 
 ### Raw diffs
-- [capa v5.0.0...v5.1.0](https://github.com/mandiant/capa/compare/v5.0.0...v5.1.0)
-- [capa-rules v5.0.0...v5.1.0](https://github.com/mandiant/capa-rules/compare/v5.0.0...v5.1.0)
+- [capa v5.0.0...master](https://github.com/mandiant/capa/compare/v5.0.0...master)
+- [capa-rules v5.0.0...master](https://github.com/mandiant/capa-rules/compare/v5.0.0...master)
 
 
 ## v5.0.0 (2023-02-08)

LICENSE.txt

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (C) 2023 Mandiant, Inc.
+   Copyright (C) 2020 Mandiant, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-832-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-779-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
@@ -170,8 +170,6 @@ capa explorer helps you identify interesting areas of a program and build new ca
 
 ![capa + IDA Pro integration](https://github.com/mandiant/capa/blob/master/doc/img/explorer_expanded.png)
 
-If you use Ghidra, you can use the Python 3 [Ghidra feature extractor](/capa/ghidra/). This integration enables capa to extract features directly from your Ghidra database, which can help you identify capabilities in programs that you analyze using Ghidra.
-
 # further information
 ## capa
 - [Installation](https://github.com/mandiant/capa/blob/master/doc/installation.md)

capa/engine.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,7 +8,7 @@
 
 import copy
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator
+from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator, cast
 
 import capa.perf
 import capa.features.common
@@ -43,12 +43,10 @@ class Statement:
         self.description = description
 
     def __str__(self):
-        name = self.name.lower()
-        children = ",".join(map(str, self.get_children()))
         if self.description:
-            return f"{name}({children} = {self.description})"
+            return "%s(%s = %s)" % (self.name.lower(), ",".join(map(str, self.get_children())), self.description)
         else:
-            return f"{name}({children})"
+            return "%s(%s)" % (self.name.lower(), ",".join(map(str, self.get_children())))
 
     def __repr__(self):
         return str(self)
@@ -71,7 +69,7 @@ class Statement:
             yield child
 
         if hasattr(self, "children"):
-            for child in self.children:
+            for child in getattr(self, "children"):
                 assert isinstance(child, (Statement, Feature))
                 yield child
 
@@ -83,7 +81,7 @@ class Statement:
                 self.child = new
 
         if hasattr(self, "children"):
-            children = self.children
+            children = getattr(self, "children")
             for i, child in enumerate(children):
                 if child is existing:
                     children[i] = new
@@ -234,9 +232,9 @@ class Range(Statement):
 
     def __str__(self):
         if self.max == (1 << 64 - 1):
-            return f"range({str(self.child)}, min={self.min}, max=infinity)"
+            return "range(%s, min=%d, max=infinity)" % (str(self.child), self.min)
         else:
-            return f"range({str(self.child)}, min={self.min}, max={self.max})"
+            return "range(%s, min=%d, max=%d)" % (str(self.child), self.min, self.max)
 
 
 class Subscope(Statement):

capa/exceptions.py

@@ -1,10 +1,3 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 class UnsupportedRuntimeError(RuntimeError):
     pass
 

capa/features/address.py

@@ -1,10 +1,3 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 import abc
 
 

capa/features/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/common.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -100,10 +100,7 @@ class Result:
         return self.success
 
 
-class Feature(abc.ABC):  # noqa: B024
-    # this is an abstract class, since we don't want anyone to instantiate it directly,
-    # but it doesn't have any abstract methods.
-
+class Feature(abc.ABC):
     def __init__(
         self,
         value: Union[str, int, float, bytes],
@@ -127,17 +124,12 @@ class Feature(abc.ABC):  # noqa: B024
         return self.name == other.name and self.value == other.value
 
     def __lt__(self, other):
-        # implementing sorting by serializing to JSON is a huge hack.
-        # its slow, inelegant, and probably doesn't work intuitively;
-        # however, we only use it for deterministic output, so it's good enough for now.
-
-        # circular import
-        # we should fix if this wasn't already a huge hack.
+        # TODO: this is a huge hack!
         import capa.features.freeze.features
 
         return (
-            capa.features.freeze.features.feature_from_capa(self).model_dump_json()
-            < capa.features.freeze.features.feature_from_capa(other).model_dump_json()
+            capa.features.freeze.features.feature_from_capa(self).json()
+            < capa.features.freeze.features.feature_from_capa(other).json()
         )
 
     def get_name_str(self) -> str:
@@ -157,11 +149,11 @@ class Feature(abc.ABC):  # noqa: B024
     def __str__(self):
         if self.value is not None:
             if self.description:
-                return f"{self.get_name_str()}({self.get_value_str()} = {self.description})"
+                return "%s(%s = %s)" % (self.get_name_str(), self.get_value_str(), self.description)
             else:
-                return f"{self.get_name_str()}({self.get_value_str()})"
+                return "%s(%s)" % (self.get_name_str(), self.get_value_str())
         else:
-            return f"{self.get_name_str()}"
+            return "%s" % self.get_name_str()
 
     def __repr__(self):
         return str(self)
@@ -250,7 +242,7 @@ class Substring(String):
 
     def __str__(self):
         assert isinstance(self.value, str)
-        return f"substring({escape_string(self.value)})"
+        return "substring(%s)" % escape_string(self.value)
 
 
 class _MatchedSubstring(Substring):
@@ -275,9 +267,11 @@ class _MatchedSubstring(Substring):
         self.matches = matches
 
     def __str__(self):
-        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
         assert isinstance(self.value, str)
-        return f'substring("{self.value}", matches = {matches})'
+        return 'substring("%s", matches = %s)' % (
+            self.value,
+            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
+        )
 
 
 class Regex(String):
@@ -296,7 +290,7 @@ class Regex(String):
             if value.endswith("/i"):
                 value = value[: -len("i")]
             raise ValueError(
-                f"invalid regular expression: {value} it should use Python syntax, try it at https://pythex.org"
+                "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value
             ) from exc
 
     def evaluate(self, ctx, short_circuit=True):
@@ -342,7 +336,7 @@ class Regex(String):
 
     def __str__(self):
         assert isinstance(self.value, str)
-        return f"regex(string =~ {self.value})"
+        return "regex(string =~ %s)" % self.value
 
 
 class _MatchedRegex(Regex):
@@ -367,9 +361,11 @@ class _MatchedRegex(Regex):
         self.matches = matches
 
     def __str__(self):
-        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
         assert isinstance(self.value, str)
-        return f"regex(string =~ {self.value}, matches = {matches})"
+        return "regex(string =~ %s, matches = %s)" % (
+            self.value,
+            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
+        )
 
 
 class StringFactory:
@@ -425,8 +421,6 @@ OS_MACOS = "macos"
 OS_ANY = "any"
 VALID_OS = {os.value for os in capa.features.extractors.elf.OS}
 VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS, OS_ANY})
-# internal only, not to be used in rules
-OS_AUTO = "auto"
 
 
 class OS(Feature):
@@ -434,20 +428,6 @@ class OS(Feature):
         super().__init__(value, description=description)
         self.name = "os"
 
-    def evaluate(self, ctx, **kwargs):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature." + self.name] += 1
-
-        for feature, locations in ctx.items():
-            if not isinstance(feature, (OS,)):
-                continue
-
-            assert isinstance(feature.value, str)
-            if OS_ANY in (self.value, feature.value) or self.value == feature.value:
-                return Result(True, self, [], locations=locations)
-
-        return Result(False, self, [])
-
 
 FORMAT_PE = "pe"
 FORMAT_ELF = "elf"
@@ -458,7 +438,6 @@ FORMAT_AUTO = "auto"
 FORMAT_SC32 = "sc32"
 FORMAT_SC64 = "sc64"
 FORMAT_FREEZE = "freeze"
-FORMAT_RESULT = "result"
 FORMAT_UNKNOWN = "unknown"
 
 

capa/features/extractors/base_extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/binja/basicblock.py

@@ -1,184 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import string
-import struct
-from typing import Tuple, Iterator
-
-from binaryninja import Function, Settings
-from binaryninja import BasicBlock as BinjaBasicBlock
-from binaryninja import (
-    BinaryView,
-    SymbolType,
-    RegisterValueType,
-    VariableSourceType,
-    MediumLevelILSetVar,
-    MediumLevelILOperation,
-    MediumLevelILBasicBlock,
-    MediumLevelILInstruction,
-)
-
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address
-from capa.features.basicblock import BasicBlock
-from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
-
-use_const_outline: bool = False
-settings: Settings = Settings()
-if settings.contains("analysis.outlining.builtins") and settings.get_bool("analysis.outlining.builtins"):
-    use_const_outline = True
-
-
-def get_printable_len_ascii(s: bytes) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
-    count = 0
-    for c in s:
-        if c == 0:
-            return count
-        if c < 127 and chr(c) in string.printable:
-            count += 1
-    return count
-
-
-def get_printable_len_wide(s: bytes) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
-    if all(c == 0x00 for c in s[1::2]):
-        return get_printable_len_ascii(s[::2])
-    return 0
-
-
-def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
-    bv: BinaryView = f.view
-
-    if il.operation != MediumLevelILOperation.MLIL_CALL:
-        return 0
-
-    target = il.dest
-    if target.operation not in [MediumLevelILOperation.MLIL_CONST, MediumLevelILOperation.MLIL_CONST_PTR]:
-        return 0
-
-    addr = target.value.value
-    sym = bv.get_symbol_at(addr)
-    if not sym or sym.type != SymbolType.LibraryFunctionSymbol:
-        return 0
-
-    if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
-        return 0
-
-    if len(il.params) < 2:
-        return 0
-
-    dest = il.params[0]
-    if dest.operation in [MediumLevelILOperation.MLIL_ADDRESS_OF, MediumLevelILOperation.MLIL_VAR]:
-        var = dest.src
-    else:
-        return 0
-
-    if var.source_type != VariableSourceType.StackVariableSourceType:
-        return 0
-
-    src = il.params[1]
-    if src.value.type != RegisterValueType.ConstantDataAggregateValue:
-        return 0
-
-    s = f.get_constant_data(RegisterValueType.ConstantDataAggregateValue, src.value.value)
-    return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
-
-
-def get_printable_len(il: MediumLevelILSetVar) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
-    width = il.dest.type.width
-    value = il.src.value.value
-
-    if width == 1:
-        chars = struct.pack("<B", value & 0xFF)
-    elif width == 2:
-        chars = struct.pack("<H", value & 0xFFFF)
-    elif width == 4:
-        chars = struct.pack("<I", value & 0xFFFFFFFF)
-    elif width == 8:
-        chars = struct.pack("<Q", value & 0xFFFFFFFFFFFFFFFF)
-    else:
-        return 0
-
-    def is_printable_ascii(chars_: bytes):
-        return all(c < 127 and chr(c) in string.printable for c in chars_)
-
-    def is_printable_utf16le(chars_: bytes):
-        if all(c == 0x00 for c in chars_[1::2]):
-            return is_printable_ascii(chars_[::2])
-
-    if is_printable_ascii(chars):
-        return width
-
-    if is_printable_utf16le(chars):
-        return width // 2
-
-    return 0
-
-
-def is_mov_imm_to_stack(il: MediumLevelILInstruction) -> bool:
-    """verify instruction moves immediate onto stack"""
-    if il.operation != MediumLevelILOperation.MLIL_SET_VAR:
-        return False
-
-    if il.src.operation != MediumLevelILOperation.MLIL_CONST:
-        return False
-
-    if il.dest.source_type != VariableSourceType.StackVariableSourceType:
-        return False
-
-    return True
-
-
-def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
-    """check basic block for stackstring indicators
-
-    true if basic block contains enough moves of constant bytes to the stack
-    """
-    count = 0
-    for il in bb:
-        if use_const_outline:
-            count += get_stack_string_len(f, il)
-        else:
-            if is_mov_imm_to_stack(il):
-                count += get_printable_len(il)
-
-    if count > MIN_STACKSTRING_LEN:
-        return True
-    return False
-
-
-def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract stackstring indicators from basic block"""
-    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
-    if bb[1] is not None and bb_contains_stackstring(fh.inner, bb[1]):
-        yield Characteristic("stack string"), bbh.address
-
-
-def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract tight loop indicators from a basic block"""
-    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
-    for edge in bb[0].outgoing_edges:
-        if edge.target.start == bb[0].start:
-            yield Characteristic("tight loop"), bbh.address
-
-
-def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract basic block features"""
-    for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, addr in bb_handler(fh, bbh):
-            yield feature, addr
-    yield BasicBlock(), bbh.address
-
-
-BASIC_BLOCK_HANDLERS = (
-    extract_bb_tight_loop,
-    extract_bb_stackstring,
-)

capa/features/extractors/binja/extractor.py

@@ -1,75 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import List, Tuple, Iterator
-
-import binaryninja as binja
-
-import capa.features.extractors.elf
-import capa.features.extractors.binja.file
-import capa.features.extractors.binja.insn
-import capa.features.extractors.binja.global_
-import capa.features.extractors.binja.function
-import capa.features.extractors.binja.basicblock
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
-
-
-class BinjaFeatureExtractor(FeatureExtractor):
-    def __init__(self, bv: binja.BinaryView):
-        super().__init__()
-        self.bv = bv
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv))
-        self.global_features.extend(capa.features.extractors.binja.global_.extract_os(self.bv))
-        self.global_features.extend(capa.features.extractors.binja.global_.extract_arch(self.bv))
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(self.bv.start)
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.binja.file.extract_features(self.bv)
-
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        for f in self.bv.functions:
-            yield FunctionHandle(address=AbsoluteVirtualAddress(f.start), inner=f)
-
-    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.binja.function.extract_features(fh)
-
-    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
-        f: binja.Function = fh.inner
-        # Set up a MLIL basic block dict look up to associate the disassembly basic block with its MLIL basic block
-        mlil_lookup = {}
-        for mlil_bb in f.mlil.basic_blocks:
-            mlil_lookup[mlil_bb.source_block.start] = mlil_bb
-
-        for bb in f.basic_blocks:
-            mlil_bb = mlil_lookup.get(bb.start)
-
-            yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=(bb, mlil_bb))
-
-    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.binja.basicblock.extract_features(fh, bbh)
-
-    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
-        import capa.features.extractors.binja.helpers as binja_helpers
-
-        bb: Tuple[binja.BasicBlock, binja.MediumLevelILBasicBlock] = bbh.inner
-        addr = bb[0].start
-
-        for text, length in bb[0]:
-            insn = binja_helpers.DisassemblyInstruction(addr, length, text)
-            yield InsnHandle(address=AbsoluteVirtualAddress(addr), inner=insn)
-            addr += length
-
-    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
-        yield from capa.features.extractors.binja.insn.extract_features(fh, bbh, ih)

capa/features/extractors/binja/file.py

@@ -1,187 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import struct
-from typing import Tuple, Iterator
-
-from binaryninja import Segment, BinaryView, SymbolType, SymbolBinding
-
-import capa.features.extractors.common
-import capa.features.extractors.helpers
-import capa.features.extractors.strings
-from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
-from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.binja.helpers import read_c_string, unmangle_c_name
-
-
-def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[int, int]]:
-    """check segment for embedded PE
-
-    adapted for binja from:
-    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
-    """
-    mz_xor = [
-        (
-            capa.features.extractors.helpers.xor_static(b"MZ", i),
-            capa.features.extractors.helpers.xor_static(b"PE", i),
-            i,
-        )
-        for i in range(256)
-    ]
-
-    todo = []
-    # If this is the first segment of the binary, skip the first bytes. Otherwise, there will always be a matched
-    # PE at the start of the binaryview.
-    start = seg.start
-    if bv.view_type == "PE" and start == bv.start:
-        start += 1
-
-    for mzx, pex, i in mz_xor:
-        for off, _ in bv.find_all_data(start, seg.end, mzx):
-            todo.append((off, mzx, pex, i))
-
-    while len(todo):
-        off, mzx, pex, i = todo.pop()
-
-        # The MZ header has one field we will check e_lfanew is at 0x3c
-        e_lfanew = off + 0x3C
-
-        if seg.end < (e_lfanew + 4):
-            continue
-
-        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(bv.read(e_lfanew, 4), i))[0]
-
-        peoff = off + newoff
-        if seg.end < (peoff + 2):
-            continue
-
-        if bv.read(peoff, 2) == pex:
-            yield off, i
-
-
-def extract_file_embedded_pe(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract embedded PE features"""
-    for seg in bv.segments:
-        for ea, _ in check_segment_for_pe(bv, seg):
-            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
-
-
-def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract function exports"""
-    for sym in bv.get_symbols_of_type(SymbolType.FunctionSymbol):
-        if sym.binding in [SymbolBinding.GlobalBinding, SymbolBinding.WeakBinding]:
-            name = sym.short_name
-            yield Export(name), AbsoluteVirtualAddress(sym.address)
-            unmangled_name = unmangle_c_name(name)
-            if name != unmangled_name:
-                yield Export(unmangled_name), AbsoluteVirtualAddress(sym.address)
-
-    for sym in bv.get_symbols_of_type(SymbolType.DataSymbol):
-        if sym.binding not in [SymbolBinding.GlobalBinding]:
-            continue
-
-        name = sym.short_name
-        if not name.startswith("__forwarder_name"):
-            continue
-
-        # Due to https://github.com/Vector35/binaryninja-api/issues/4641, in binja version 3.5, the symbol's name
-        # does not contain the DLL name. As a workaround, we read the C string at the symbol's address, which contains
-        # both the DLL name and the function name.
-        # Once the above issue is closed in the next binjs stable release, we can update the code here to use the
-        # symbol name directly.
-        name = read_c_string(bv, sym.address, 1024)
-        forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(name)
-        yield Export(forwarded_name), AbsoluteVirtualAddress(sym.address)
-        yield Characteristic("forwarded export"), AbsoluteVirtualAddress(sym.address)
-
-
-def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract function imports
-
-    1. imports by ordinal:
-     - modulename.#ordinal
-
-    2. imports by name, results in two features to support importname-only
-       matching:
-     - modulename.importname
-     - importname
-    """
-    for sym in bv.get_symbols_of_type(SymbolType.ImportAddressSymbol):
-        lib_name = str(sym.namespace)
-        addr = AbsoluteVirtualAddress(sym.address)
-        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym.short_name):
-            yield Import(name), addr
-
-        ordinal = sym.ordinal
-        if ordinal != 0 and (lib_name != ""):
-            ordinal_name = f"#{ordinal}"
-            for name in capa.features.extractors.helpers.generate_symbols(lib_name, ordinal_name):
-                yield Import(name), addr
-
-
-def extract_file_section_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract section names"""
-    for name, section in bv.sections.items():
-        yield Section(name), AbsoluteVirtualAddress(section.start)
-
-
-def extract_file_strings(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract ASCII and UTF-16 LE strings"""
-    for s in bv.strings:
-        yield String(s.value), FileOffsetAddress(s.start)
-
-
-def extract_file_function_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract the names of statically-linked library functions.
-    """
-    for sym_name in bv.symbols:
-        for sym in bv.symbols[sym_name]:
-            if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
-                continue
-
-            name = sym.short_name
-            yield FunctionName(name), sym.address
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                yield FunctionName(name[1:]), sym.address
-
-
-def extract_file_format(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    view_type = bv.view_type
-    if view_type in ["PE", "COFF"]:
-        yield Format(FORMAT_PE), NO_ADDRESS
-    elif view_type == "ELF":
-        yield Format(FORMAT_ELF), NO_ADDRESS
-    elif view_type == "Raw":
-        # no file type to return when processing a binary file, but we want to continue processing
-        return
-    else:
-        raise NotImplementedError(f"unexpected file format: {view_type}")
-
-
-def extract_features(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract file features"""
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler(bv):
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_strings,
-    extract_file_section_names,
-    extract_file_embedded_pe,
-    extract_file_function_names,
-    extract_file_format,
-)

capa/features/extractors/binja/find_binja_api.py

@@ -1,35 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import subprocess
-from pathlib import Path
-
-# When the script gets executed as a standalone executable (via PyInstaller), `import binaryninja` does not work because
-# we have excluded the binaryninja module in `pyinstaller.spec`. The trick here is to call the system Python and try
-# to find out the path of the binaryninja module that has been installed.
-# Note, including the binaryninja module in the `pyintaller.spec` would not work, since the binaryninja module tries to
-# find the binaryninja core e.g., `libbinaryninjacore.dylib`, using a relative path. And this does not work when the
-# binaryninja module is extracted by the PyInstaller.
-code = r"""
-from pathlib import Path
-from importlib import util
-spec = util.find_spec('binaryninja')
-if spec is not None:
-    if len(spec.submodule_search_locations) > 0:
-            path = Path(spec.submodule_search_locations[0])
-            # encode the path with utf8 then convert to hex, make sure it can be read and restored properly
-            print(str(path.parent).encode('utf8').hex())
-"""
-
-
-def find_binja_path() -> Path:
-    raw_output = subprocess.check_output(["python", "-c", code]).decode("ascii").strip()
-    return Path(bytes.fromhex(raw_output).decode("utf8"))
-
-
-if __name__ == "__main__":
-    print(find_binja_path())

capa/features/extractors/binja/function.py

@@ -1,104 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Tuple, Iterator
-
-from binaryninja import Function, BinaryView, SymbolType, RegisterValueType, LowLevelILOperation
-
-from capa.features.file import FunctionName
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors import loops
-from capa.features.extractors.base_extractor import FunctionHandle
-
-
-def extract_function_calls_to(fh: FunctionHandle):
-    """extract callers to a function"""
-    func: Function = fh.inner
-
-    for caller in func.caller_sites:
-        # Everything that is a code reference to the current function is considered a caller, which actually includes
-        # many other references that are NOT a caller. For example, an instruction `push function_start` will also be
-        # considered a caller to the function
-        llil = caller.llil
-        if (llil is None) or llil.operation not in [
-            LowLevelILOperation.LLIL_CALL,
-            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-            LowLevelILOperation.LLIL_JUMP,
-            LowLevelILOperation.LLIL_TAILCALL,
-        ]:
-            continue
-
-        if llil.dest.value.type not in [
-            RegisterValueType.ImportedAddressValue,
-            RegisterValueType.ConstantValue,
-            RegisterValueType.ConstantPointerValue,
-        ]:
-            continue
-
-        address = llil.dest.value.value
-        if address != func.start:
-            continue
-
-        yield Characteristic("calls to"), AbsoluteVirtualAddress(caller.address)
-
-
-def extract_function_loop(fh: FunctionHandle):
-    """extract loop indicators from a function"""
-    func: Function = fh.inner
-
-    edges = []
-
-    # construct control flow graph
-    for bb in func.basic_blocks:
-        for edge in bb.outgoing_edges:
-            edges.append((bb.start, edge.target.start))
-
-    if loops.has_loop(edges):
-        yield Characteristic("loop"), fh.address
-
-
-def extract_recursive_call(fh: FunctionHandle):
-    """extract recursive function call"""
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-    if bv is None:
-        return
-
-    for ref in bv.get_code_refs(func.start):
-        if ref.function == func:
-            yield Characteristic("recursive call"), fh.address
-
-
-def extract_function_name(fh: FunctionHandle):
-    """extract function names (e.g., symtab names)"""
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-    if bv is None:
-        return
-
-    for sym in bv.get_symbols(func.start):
-        if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
-            continue
-
-        name = sym.short_name
-        yield FunctionName(name), sym.address
-        if name.startswith("_"):
-            # some linkers may prefix linked routines with a `_` to avoid name collisions.
-            # extract features for both the mangled and un-mangled representations.
-            # e.g. `_fwrite` -> `fwrite`
-            # see: https://stackoverflow.com/a/2628384/87207
-            yield FunctionName(name[1:]), sym.address
-
-
-def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    for func_handler in FUNCTION_HANDLERS:
-        for feature, addr in func_handler(fh):
-            yield feature, addr
-
-
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call, extract_function_name)

capa/features/extractors/binja/global_.py

@@ -1,60 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-from typing import Tuple, Iterator
-
-from binaryninja import BinaryView
-
-from capa.features.common import OS, OS_MACOS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
-from capa.features.address import NO_ADDRESS, Address
-
-logger = logging.getLogger(__name__)
-
-
-def extract_os(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    name = bv.platform.name
-    if "-" in name:
-        name = name.split("-")[0]
-
-    if name == "windows":
-        yield OS(OS_WINDOWS), NO_ADDRESS
-
-    elif name == "macos":
-        yield OS(OS_MACOS), NO_ADDRESS
-
-    elif name in ["linux", "freebsd", "decree"]:
-        yield OS(name), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a further CLI argument to specify the OS,
-        # but i think this would be rarely used.
-        # rules that rely on OS conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess OS", name)
-        return
-
-
-def extract_arch(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    arch = bv.arch.name
-    if arch == "x86_64":
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    elif arch == "x86":
-        yield Arch(ARCH_I386), NO_ADDRESS
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", arch)
-        return

capa/features/extractors/binja/helpers.py

@@ -1,69 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import re
-from typing import List, Callable
-from dataclasses import dataclass
-
-from binaryninja import BinaryView, LowLevelILInstruction
-from binaryninja.architecture import InstructionTextToken
-
-
-@dataclass
-class DisassemblyInstruction:
-    address: int
-    length: int
-    text: List[InstructionTextToken]
-
-
-LLIL_VISITOR = Callable[[LowLevelILInstruction, LowLevelILInstruction, int], bool]
-
-
-def visit_llil_exprs(il: LowLevelILInstruction, func: LLIL_VISITOR):
-    # BN does not really support operand index at the disassembly level, so use the LLIL operand index as a substitute.
-    # Note, this is NOT always guaranteed to be the same as disassembly operand.
-    for i, op in enumerate(il.operands):
-        if isinstance(op, LowLevelILInstruction) and func(op, il, i):
-            visit_llil_exprs(op, func)
-
-
-def unmangle_c_name(name: str) -> str:
-    # https://learn.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170#FormatC
-    # Possible variations for BaseThreadInitThunk:
-    # @BaseThreadInitThunk@12
-    # _BaseThreadInitThunk
-    # _BaseThreadInitThunk@12
-    # It is also possible for a function to have a `Stub` appended to its name:
-    # _lstrlenWStub@4
-
-    # A small optimization to avoid running the regex too many times
-    # this still increases the unit test execution time from 170s to 200s, should be able to accelerate it
-    #
-    # TODO(xusheng): performance optimizations to improve test execution time
-    # https://github.com/mandiant/capa/issues/1610
-    if name[0] in ["@", "_"]:
-        match = re.match(r"^[@|_](.*?)(Stub)?(@\d+)?$", name)
-        if match:
-            return match.group(1)
-
-    return name
-
-
-def read_c_string(bv: BinaryView, offset: int, max_len: int) -> str:
-    s: List[str] = []
-    while len(s) < max_len:
-        try:
-            c = bv.read(offset + len(s), 1)[0]
-        except Exception:
-            break
-
-        if c == 0:
-            break
-
-        s.append(chr(c))
-
-    return "".join(s)

capa/features/extractors/binja/insn.py

@@ -1,586 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Any, List, Tuple, Iterator, Optional
-
-from binaryninja import Function
-from binaryninja import BasicBlock as BinjaBasicBlock
-from binaryninja import (
-    BinaryView,
-    ILRegister,
-    SymbolType,
-    BinaryReader,
-    RegisterValueType,
-    LowLevelILOperation,
-    LowLevelILInstruction,
-)
-
-import capa.features.extractors.helpers
-from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
-from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.binja.helpers import DisassemblyInstruction, visit_llil_exprs
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-# security cookie checks may perform non-zeroing XORs, these are expected within a certain
-# byte range within the first and returning basic blocks, this helps to reduce FP features
-SECURITY_COOKIE_BYTES_DELTA = 0x40
-
-
-# check if a function is a stub function to another function/symbol. The criteria is:
-# 1. The function must only have one basic block
-# 2. The function must only make one call/jump to another address
-# If the function being checked is a stub function, returns the target address. Otherwise, return None.
-def is_stub_function(bv: BinaryView, addr: int) -> Optional[int]:
-    funcs = bv.get_functions_at(addr)
-    for func in funcs:
-        if len(func.basic_blocks) != 1:
-            continue
-
-        call_count = 0
-        call_target = None
-        for il in func.llil.instructions:
-            if il.operation in [
-                LowLevelILOperation.LLIL_CALL,
-                LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-                LowLevelILOperation.LLIL_JUMP,
-                LowLevelILOperation.LLIL_TAILCALL,
-            ]:
-                call_count += 1
-                if il.dest.value.type in [
-                    RegisterValueType.ImportedAddressValue,
-                    RegisterValueType.ConstantValue,
-                    RegisterValueType.ConstantPointerValue,
-                ]:
-                    call_target = il.dest.value.value
-
-        if call_count == 1 and call_target is not None:
-            return call_target
-
-    return None
-
-
-def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction API features
-
-    example:
-       call dword [0x00473038]
-    """
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    for llil in func.get_llils_at(ih.address):
-        if llil.operation in [
-            LowLevelILOperation.LLIL_CALL,
-            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-            LowLevelILOperation.LLIL_JUMP,
-            LowLevelILOperation.LLIL_TAILCALL,
-        ]:
-            if llil.dest.value.type not in [
-                RegisterValueType.ImportedAddressValue,
-                RegisterValueType.ConstantValue,
-                RegisterValueType.ConstantPointerValue,
-            ]:
-                continue
-            address = llil.dest.value.value
-            candidate_addrs = [address]
-            stub_addr = is_stub_function(bv, address)
-            if stub_addr is not None:
-                candidate_addrs.append(stub_addr)
-
-            for address in candidate_addrs:
-                for sym in func.view.get_symbols(address):
-                    if sym is None or sym.type not in [
-                        SymbolType.ImportAddressSymbol,
-                        SymbolType.ImportedFunctionSymbol,
-                        SymbolType.FunctionSymbol,
-                    ]:
-                        continue
-
-                    sym_name = sym.short_name
-
-                    lib_name = ""
-                    import_lib = bv.lookup_imported_object_library(sym.address)
-                    if import_lib is not None:
-                        lib_name = import_lib[0].name
-                        if lib_name.endswith(".dll"):
-                            lib_name = lib_name[:-4]
-                        elif lib_name.endswith(".so"):
-                            lib_name = lib_name[:-3]
-
-                    for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name):
-                        yield API(name), ih.address
-
-                    if sym_name.startswith("_"):
-                        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name[1:]):
-                            yield API(name), ih.address
-
-
-def extract_insn_number_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction number features
-    example:
-        push    3136B0h         ; dwControlCode
-    """
-    func: Function = fh.inner
-
-    results: List[Tuple[Any[Number, OperandNumber], Address]] = []
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        if il.operation == LowLevelILOperation.LLIL_LOAD:
-            return False
-
-        if il.operation not in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-            return True
-
-        for op in parent.operands:
-            if isinstance(op, ILRegister) and op.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
-                return False
-            elif isinstance(op, LowLevelILInstruction) and op.operation == LowLevelILOperation.LLIL_REG:
-                if op.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
-                    return False
-
-        raw_value = il.value.value
-        if parent.operation == LowLevelILOperation.LLIL_SUB:
-            raw_value = -raw_value
-
-        results.append((Number(raw_value), ih.address))
-        results.append((OperandNumber(index, raw_value), ih.address))
-
-        return False
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse referenced byte sequences
-    example:
-        push    offset iid_004118d4_IShellLinkA ; riid
-    """
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    candidate_addrs = set()
-
-    llil = func.get_llil_at(ih.address)
-    if llil is None or llil.operation in [LowLevelILOperation.LLIL_CALL, LowLevelILOperation.LLIL_CALL_STACK_ADJUST]:
-        return
-
-    for ref in bv.get_code_refs_from(ih.address):
-        if ref == ih.address:
-            continue
-
-        if len(bv.get_functions_containing(ref)) > 0:
-            continue
-
-        candidate_addrs.add(ref)
-
-    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-            value = il.value.value
-            if value > 0:
-                candidate_addrs.add(value)
-            return False
-
-        return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    for addr in candidate_addrs:
-        extracted_bytes = bv.read(addr, MAX_BYTES_FEATURE_SIZE)
-        if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-            if bv.get_string_at(addr) is None:
-                # don't extract byte features for obvious strings
-                yield Bytes(extracted_bytes), ih.address
-
-
-def extract_insn_string_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction string features
-
-    example:
-        push offset aAcr     ; "ACR  > "
-    """
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    candidate_addrs = set()
-
-    # collect candidate address from code refs directly
-    for ref in bv.get_code_refs_from(ih.address):
-        if ref == ih.address:
-            continue
-
-        if len(bv.get_functions_containing(ref)) > 0:
-            continue
-
-        candidate_addrs.add(ref)
-
-    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-            value = il.value.value
-            if value > 0:
-                candidate_addrs.add(value)
-            return False
-
-        return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    # Now we have all the candidate address, check them for string or pointer to string
-    br = BinaryReader(bv)
-    for addr in candidate_addrs:
-        found = bv.get_string_at(addr)
-        if found:
-            yield String(found.value), ih.address
-
-        br.seek(addr)
-        pointer = None
-        if bv.arch.address_size == 4:
-            pointer = br.read32()
-        elif bv.arch.address_size == 8:
-            pointer = br.read64()
-
-        if pointer is not None:
-            found = bv.get_string_at(pointer)
-            if found:
-                yield String(found.value), ih.address
-
-
-def extract_insn_offset_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction structure offset features
-
-    example:
-        .text:0040112F cmp [esi+4], ebx
-    """
-    func: Function = fh.inner
-
-    results: List[Tuple[Any[Offset, OperandOffset], Address]] = []
-    address_size = func.view.arch.address_size * 8
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        #  The most common case, read/write dereference to something like `dword [eax+0x28]`
-        if il.operation in [LowLevelILOperation.LLIL_ADD, LowLevelILOperation.LLIL_SUB]:
-            left = il.left
-            right = il.right
-            # Exclude offsets based on stack/franme pointers
-            if left.operation == LowLevelILOperation.LLIL_REG and left.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
-                return True
-
-            if right.operation != LowLevelILOperation.LLIL_CONST:
-                return True
-
-            raw_value = right.value.value
-            # If this is not a dereference, then this must be an add and the offset must be in the range \
-            # [0, MAX_STRUCTURE_SIZE]. For example,
-            # add eax, 0x10,
-            # lea ebx, [eax + 1]
-            if parent.operation not in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
-                if il.operation != LowLevelILOperation.LLIL_ADD or (not 0 < raw_value < MAX_STRUCTURE_SIZE):
-                    return False
-
-            if address_size > 0:
-                # BN also encodes the constant value as two's complement, we need to restore its original value
-                value = capa.features.extractors.helpers.twos_complement(raw_value, address_size)
-            else:
-                value = raw_value
-
-            results.append((Offset(value), ih.address))
-            results.append((OperandOffset(index, value), ih.address))
-            return False
-
-        # An edge case: for code like `push dword [esi]`, we need to generate a feature for offset 0x0
-        elif il.operation in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
-            if il.operands[0].operation == LowLevelILOperation.LLIL_REG:
-                results.append((Offset(0), ih.address))
-                results.append((OperandOffset(index, 0), ih.address))
-                return False
-
-        return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def is_nzxor_stack_cookie(f: Function, bb: BinjaBasicBlock, llil: LowLevelILInstruction) -> bool:
-    """check if nzxor exists within stack cookie delta"""
-    # TODO(xusheng): use LLIL SSA to do more accurate analysis
-    # https://github.com/mandiant/capa/issues/1609
-
-    reg_names = []
-    if llil.left.operation == LowLevelILOperation.LLIL_REG:
-        reg_names.append(llil.left.src.name)
-
-    if llil.right.operation == LowLevelILOperation.LLIL_REG:
-        reg_names.append(llil.right.src.name)
-
-    # stack cookie reg should be stack/frame pointer
-    if not any(reg in ["ebp", "esp", "rbp", "rsp", "sp"] for reg in reg_names):
-        return False
-
-    # expect security cookie init in first basic block within first bytes (instructions)
-    if len(bb.incoming_edges) == 0 and llil.address < (bb.start + SECURITY_COOKIE_BYTES_DELTA):
-        return True
-
-    # ... or within last bytes (instructions) before a return
-    if len(bb.outgoing_edges) == 0 and llil.address > (bb.end - SECURITY_COOKIE_BYTES_DELTA):
-        return True
-
-    return False
-
-
-def extract_insn_nzxor_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction non-zeroing XOR instruction
-    ignore expected non-zeroing XORs, e.g. security cookies
-    """
-    func: Function = fh.inner
-
-    results = []
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        # If the two operands of the xor instruction are the same, the LLIL will be translated to other instructions,
-        # e.g., <llil: eax = 0>, (LLIL_SET_REG). So we do not need to check whether the two operands are the same.
-        if il.operation == LowLevelILOperation.LLIL_XOR:
-            # Exclude cases related to the stack cookie
-            if is_nzxor_stack_cookie(fh.inner, bbh.inner[0], il):
-                return False
-            results.append((Characteristic("nzxor"), ih.address))
-            return False
-        else:
-            return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def extract_insn_mnemonic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction mnemonic features"""
-    insn: DisassemblyInstruction = ih.inner
-    yield Mnemonic(insn.text[0].text), ih.address
-
-
-def extract_insn_obfs_call_plus_5_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse call $+5 instruction from the given instruction.
-    """
-    insn: DisassemblyInstruction = ih.inner
-    if insn.text[0].text == "call" and insn.text[2].text == "$+5" and insn.length == 5:
-        yield Characteristic("call $+5"), ih.address
-
-
-def extract_insn_peb_access_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction peb access
-
-    fs:[0x30] on x86, gs:[0x60] on x64
-    """
-    func: Function = fh.inner
-
-    results = []
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILOperation, index: int) -> bool:
-        if il.operation != LowLevelILOperation.LLIL_LOAD:
-            return True
-
-        src = il.src
-        if src.operation != LowLevelILOperation.LLIL_ADD:
-            return True
-
-        left = src.left
-        right = src.right
-
-        if left.operation != LowLevelILOperation.LLIL_REG:
-            return True
-
-        reg = left.src.name
-
-        if right.operation != LowLevelILOperation.LLIL_CONST:
-            return True
-
-        value = right.value.value
-        if (reg, value) not in (("fsbase", 0x30), ("gsbase", 0x60)):
-            return True
-
-        results.append((Characteristic("peb access"), ih.address))
-        return False
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def extract_insn_segment_access_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction fs or gs access"""
-    func: Function = fh.inner
-
-    results = []
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        if il.operation == LowLevelILOperation.LLIL_REG:
-            reg = il.src.name
-            if reg == "fsbase":
-                results.append((Characteristic("fs access"), ih.address))
-                return False
-            elif reg == "gsbase":
-                results.append((Characteristic("gs access"), ih.address))
-                return False
-            return False
-
-        return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def extract_insn_cross_section_cflow(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    if bv is None:
-        return
-
-    seg1 = bv.get_segment_at(ih.address)
-    sections1 = bv.get_sections_at(ih.address)
-    for ref in bv.get_code_refs_from(ih.address):
-        if len(bv.get_functions_at(ref)) == 0:
-            continue
-
-        seg2 = bv.get_segment_at(ref)
-        sections2 = bv.get_sections_at(ref)
-        if seg1 != seg2 or sections1 != sections2:
-            yield Characteristic("cross section flow"), ih.address
-
-
-def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract functions calls from features
-
-    most relevant at the function scope, however, its most efficient to extract at the instruction scope
-    """
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    if bv is None:
-        return
-
-    for il in func.get_llils_at(ih.address):
-        if il.operation not in [
-            LowLevelILOperation.LLIL_CALL,
-            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-            LowLevelILOperation.LLIL_TAILCALL,
-        ]:
-            continue
-
-        dest = il.dest
-        if dest.operation == LowLevelILOperation.LLIL_CONST_PTR:
-            value = dest.value.value
-            yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
-        elif dest.operation == LowLevelILOperation.LLIL_CONST:
-            yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value)
-        elif dest.operation == LowLevelILOperation.LLIL_LOAD:
-            indirect_src = dest.src
-            if indirect_src.operation == LowLevelILOperation.LLIL_CONST_PTR:
-                value = indirect_src.value.value
-                yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
-            elif indirect_src.operation == LowLevelILOperation.LLIL_CONST:
-                yield Characteristic("calls from"), AbsoluteVirtualAddress(indirect_src.value)
-        elif dest.operation == LowLevelILOperation.LLIL_REG:
-            if dest.value.type in [
-                RegisterValueType.ImportedAddressValue,
-                RegisterValueType.ConstantValue,
-                RegisterValueType.ConstantPointerValue,
-            ]:
-                yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value.value)
-
-
-def extract_function_indirect_call_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
-    does not include calls like => call ds:dword_ABD4974
-
-    most relevant at the function or basic block scope;
-    however, its most efficient to extract at the instruction scope
-    """
-    func: Function = fh.inner
-
-    llil = func.get_llil_at(ih.address)
-    if llil is None or llil.operation not in [
-        LowLevelILOperation.LLIL_CALL,
-        LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-        LowLevelILOperation.LLIL_TAILCALL,
-    ]:
-        return
-
-    if llil.dest.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-        return
-
-    if llil.dest.operation == LowLevelILOperation.LLIL_LOAD:
-        src = llil.dest.src
-        if src.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-            return
-
-    yield Characteristic("indirect call"), ih.address
-
-
-def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract instruction features"""
-    for inst_handler in INSTRUCTION_HANDLERS:
-        for feature, ea in inst_handler(f, bbh, insn):
-            yield feature, ea
-
-
-INSTRUCTION_HANDLERS = (
-    extract_insn_api_features,
-    extract_insn_number_features,
-    extract_insn_bytes_features,
-    extract_insn_string_features,
-    extract_insn_offset_features,
-    extract_insn_nzxor_characteristic_features,
-    extract_insn_mnemonic_features,
-    extract_insn_obfs_call_plus_5_characteristic_features,
-    extract_insn_peb_access_characteristic_features,
-    extract_insn_cross_section_cflow,
-    extract_insn_segment_access_features,
-    extract_function_calls_from,
-    extract_function_indirect_call_characteristic_features,
-)

capa/features/extractors/common.py

@@ -1,10 +1,3 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 import io
 import logging
 import binascii
@@ -17,31 +10,12 @@ import capa.features
 import capa.features.extractors.elf
 import capa.features.extractors.pefile
 import capa.features.extractors.strings
-from capa.features.common import (
-    OS,
-    OS_ANY,
-    OS_AUTO,
-    ARCH_ANY,
-    FORMAT_PE,
-    FORMAT_ELF,
-    OS_WINDOWS,
-    FORMAT_FREEZE,
-    FORMAT_RESULT,
-    Arch,
-    Format,
-    String,
-    Feature,
-)
+from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, FORMAT_FREEZE, Arch, Format, String, Feature
 from capa.features.freeze import is_freeze
 from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress
 
 logger = logging.getLogger(__name__)
 
-# match strings for formats
-MATCH_PE = b"MZ"
-MATCH_ELF = b"\x7fELF"
-MATCH_RESULT = b'{"meta":'
-
 
 def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[String, Address]]:
     """
@@ -55,14 +29,12 @@ def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[String, Address]]:
 
 
 def extract_format(buf) -> Iterator[Tuple[Feature, Address]]:
-    if buf.startswith(MATCH_PE):
+    if buf.startswith(b"MZ"):
         yield Format(FORMAT_PE), NO_ADDRESS
-    elif buf.startswith(MATCH_ELF):
+    elif buf.startswith(b"\x7fELF"):
         yield Format(FORMAT_ELF), NO_ADDRESS
     elif is_freeze(buf):
         yield Format(FORMAT_FREEZE), NO_ADDRESS
-    elif buf.startswith(MATCH_RESULT):
-        yield Format(FORMAT_RESULT), NO_ADDRESS
     else:
         # we likely end up here:
         #  1. handling a file format (e.g. macho)
@@ -73,13 +45,10 @@ def extract_format(buf) -> Iterator[Tuple[Feature, Address]]:
 
 
 def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
-    if buf.startswith(MATCH_PE):
+    if buf.startswith(b"MZ"):
         yield from capa.features.extractors.pefile.extract_file_arch(pe=pefile.PE(data=buf))
 
-    elif buf.startswith(MATCH_RESULT):
-        yield Arch(ARCH_ANY), NO_ADDRESS
-
-    elif buf.startswith(MATCH_ELF):
+    elif buf.startswith(b"\x7fELF"):
         with contextlib.closing(io.BytesIO(buf)) as f:
             arch = capa.features.extractors.elf.detect_elf_arch(f)
 
@@ -104,15 +73,10 @@ def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
         return
 
 
-def extract_os(buf, os=OS_AUTO) -> Iterator[Tuple[Feature, Address]]:
-    if os != OS_AUTO:
-        yield OS(os), NO_ADDRESS
-
-    if buf.startswith(MATCH_PE):
+def extract_os(buf) -> Iterator[Tuple[Feature, Address]]:
+    if buf.startswith(b"MZ"):
         yield OS(OS_WINDOWS), NO_ADDRESS
-    elif buf.startswith(MATCH_RESULT):
-        yield OS(OS_ANY), NO_ADDRESS
-    elif buf.startswith(MATCH_ELF):
+    elif buf.startswith(b"\x7fELF"):
         with contextlib.closing(io.BytesIO(buf)) as f:
             os = capa.features.extractors.elf.detect_elf_os(f)
 
@@ -128,6 +92,8 @@ def extract_os(buf, os=OS_AUTO) -> Iterator[Tuple[Feature, Address]]:
         #  2. handling a new file format (e.g. macho)
         #
         # for (1) we can't do much - its shellcode and all bets are off.
+        # we could maybe accept a further CLI argument to specify the OS,
+        # but i think this would be rarely used.
         # rules that rely on OS conditions will fail to match on shellcode.
         #
         # for (2), this logic will need to be updated as the format is implemented.

capa/features/extractors/dnfile/basicblock.py

@@ -0,0 +1,45 @@
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from typing import Tuple, Iterator
+
+from dncil.cil.instruction import Instruction
+
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
+
+
+def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract stackstring indicators from basic block"""
+    raise NotImplementedError
+
+
+def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract tight loop indicators from a basic block"""
+    first: Instruction = bbh.inner.instructions[0]
+    last: Instruction = bbh.inner.instructions[-1]
+
+    if any((last.is_br(), last.is_cond_br(), last.is_leave())):
+        if last.operand == first.offset:
+            yield Characteristic("tight loop"), bbh.address
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract basic block features"""
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, addr in bb_handler(fh, bbh):
+            yield feature, addr
+    yield BasicBlock(), bbh.address
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    # extract_bb_stackstring,
+)

capa/features/extractors/dnfile/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,20 +8,21 @@
 
 from __future__ import annotations
 
-from typing import Dict, List, Tuple, Union, Iterator, Optional
-from pathlib import Path
+from typing import Set, Dict, List, Tuple, Union, Iterator, Optional
 
 import dnfile
 from dncil.cil.opcode import OpCodes
+from dncil.cil.instruction import Instruction
 
 import capa.features.extractors
 import capa.features.extractors.dotnetfile
 import capa.features.extractors.dnfile.file
 import capa.features.extractors.dnfile.insn
 import capa.features.extractors.dnfile.function
+import capa.features.extractors.dnfile.basicblock
 from capa.features.common import Feature
 from capa.features.address import NO_ADDRESS, Address, DNTokenAddress, DNTokenOffsetAddress
-from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
+from capa.features.extractors.dnfile.types import DnType, DnBasicBlock, DnUnmanagedMethod
 from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
 from capa.features.extractors.dnfile.helpers import (
     get_dotnet_types,
@@ -53,25 +54,25 @@ class DnFileFeatureExtractorCache:
             self.types[type_.token] = type_
 
     def get_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.imports.get(token)
+        return self.imports.get(token, None)
 
     def get_native_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.native_imports.get(token)
+        return self.native_imports.get(token, None)
 
     def get_method(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.methods.get(token)
+        return self.methods.get(token, None)
 
     def get_field(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.fields.get(token)
+        return self.fields.get(token, None)
 
     def get_type(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.types.get(token)
+        return self.types.get(token, None)
 
 
 class DnfileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: Path):
+    def __init__(self, path: str):
         super().__init__()
-        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+        self.pe: dnfile.dnPE = dnfile.dnPE(path)
 
         # pre-compute .NET token lookup tables; each .NET method has access to this cache for feature extraction
         # most relevant at instruction scope
@@ -99,7 +100,13 @@ class DnfileFeatureExtractor(FeatureExtractor):
             fh: FunctionHandle = FunctionHandle(
                 address=DNTokenAddress(token),
                 inner=method,
-                ctx={"pe": self.pe, "calls_from": set(), "calls_to": set(), "cache": self.token_cache},
+                ctx={
+                    "pe": self.pe,
+                    "calls_from": set(),
+                    "calls_to": set(),
+                    "blocks": list(),
+                    "cache": self.token_cache,
+                },
             )
 
             # method tokens should be unique
@@ -120,7 +127,7 @@ class DnfileFeatureExtractor(FeatureExtractor):
                 address: DNTokenAddress = DNTokenAddress(insn.operand.value)
 
                 # record call to destination method; note: we only consider MethodDef methods for destinations
-                dest: Optional[FunctionHandle] = methods.get(address)
+                dest: Optional[FunctionHandle] = methods.get(address, None)
                 if dest is not None:
                     dest.ctx["calls_to"].add(fh.address)
 
@@ -128,26 +135,99 @@ class DnfileFeatureExtractor(FeatureExtractor):
                 # those calls to other MethodDef methods e.g. calls to imported MemberRef methods
                 fh.ctx["calls_from"].add(address)
 
+        # calculate basic blocks
+        for fh in methods.values():
+            # calculate basic block leaders where,
+            #   1. The first instruction of the intermediate code is a leader
+            #   2. Instructions that are targets of unconditional or conditional jump/goto statements are leaders
+            #   3. Instructions that immediately follow unconditional or conditional jump/goto statements are considered leaders
+            #   https://www.geeksforgeeks.org/basic-blocks-in-compiler-design/
+
+            leaders: Set[int] = set()
+            for idx, insn in enumerate(fh.inner.instructions):
+                if idx == 0:
+                    # add #1
+                    leaders.add(insn.offset)
+
+                if any((insn.is_br(), insn.is_cond_br(), insn.is_leave())):
+                    # add #2
+                    leaders.add(insn.operand)
+                    # add #3
+                    try:
+                        leaders.add(fh.inner.instructions[idx + 1].offset)
+                    except IndexError:
+                        # may encounter branch at end of method
+                        continue
+
+            # build basic blocks using leaders
+            bb_curr: Optional[DnBasicBlock] = None
+            for idx, insn in enumerate(fh.inner.instructions):
+                if insn.offset in leaders:
+                    # new leader, new basic block
+                    bb_curr = DnBasicBlock(instructions=[insn])
+                    fh.ctx["blocks"].append(bb_curr)
+                    continue
+
+                assert bb_curr is not None
+                bb_curr.instructions.append(insn)
+
+            # create mapping of first instruction to basic block
+            bb_map: Dict[int, DnBasicBlock] = {}
+            for bb in fh.ctx["blocks"]:
+                if len(bb.instructions) == 0:
+                    # TODO: consider error?
+                    continue
+                bb_map[bb.instructions[0].offset] = bb
+
+            # connect basic blocks
+            for idx, bb in enumerate(fh.ctx["blocks"]):
+                if len(bb.instructions) == 0:
+                    # TODO: consider error?
+                    continue
+
+                last = bb.instructions[-1]
+
+                # connect branches to other basic blocks
+                if any((last.is_br(), last.is_cond_br(), last.is_leave())):
+                    bb_branch: Optional[DnBasicBlock] = bb_map.get(last.operand, None)
+                    if bb_branch is not None:
+                        # TODO: consider None error?
+                        bb.succs.append(bb_branch)
+                        bb_branch.preds.append(bb)
+
+                if any((last.is_br(), last.is_leave())):
+                    # no fallthrough
+                    continue
+
+                # connect fallthrough
+                try:
+                    bb_next: DnBasicBlock = fh.ctx["blocks"][idx + 1]
+                    bb.succs.append(bb_next)
+                    bb_next.preds.append(bb)
+                except IndexError:
+                    continue
+
         yield from methods.values()
 
     def extract_function_features(self, fh) -> Iterator[Tuple[Feature, Address]]:
         yield from capa.features.extractors.dnfile.function.extract_features(fh)
 
-    def get_basic_blocks(self, f) -> Iterator[BBHandle]:
-        # each dotnet method is considered 1 basic block
-        yield BBHandle(
-            address=f.address,
-            inner=f.inner,
-        )
+    def get_basic_blocks(self, fh) -> Iterator[BBHandle]:
+        for bb in fh.ctx["blocks"]:
+            yield BBHandle(
+                address=DNTokenOffsetAddress(
+                    fh.address, bb.instructions[0].offset - (fh.inner.offset + fh.inner.header_size)
+                ),
+                inner=bb,
+            )
 
     def extract_basic_block_features(self, fh, bbh):
-        # we don't support basic block features
-        yield from []
+        yield from capa.features.extractors.dnfile.basicblock.extract_features(fh, bbh)
 
     def get_instructions(self, fh, bbh):
         for insn in bbh.inner.instructions:
             yield InsnHandle(
-                address=DNTokenOffsetAddress(bbh.address, insn.offset - (fh.inner.offset + fh.inner.header_size)),
+                address=DNTokenOffsetAddress(fh.address, insn.offset - (fh.inner.offset + fh.inner.header_size)),
                 inner=insn,
             )
 

capa/features/extractors/dnfile/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/dnfile/function.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -13,6 +13,7 @@ from typing import Tuple, Iterator
 
 from capa.features.common import Feature, Characteristic
 from capa.features.address import Address
+from capa.features.extractors import loops
 from capa.features.extractors.base_extractor import FunctionHandle
 
 logger = logging.getLogger(__name__)
@@ -38,7 +39,13 @@ def extract_recursive_call(fh: FunctionHandle) -> Iterator[Tuple[Characteristic,
 
 def extract_function_loop(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
     """extract loop indicators from a function"""
-    raise NotImplementedError()
+    edges = []
+    for bb in fh.ctx["blocks"]:
+        for succ in bb.succs:
+            edges.append((bb.instructions[0].offset, succ.instructions[0].offset))
+
+    if loops.has_loop(edges):
+        yield Characteristic("loop"), fh.address
 
 
 def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
@@ -47,4 +54,9 @@ def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
             yield feature, addr
 
 
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_calls_from, extract_recursive_call)
+FUNCTION_HANDLERS = (
+    extract_function_calls_to,
+    extract_function_calls_from,
+    extract_recursive_call,
+    extract_function_loop,
+)

capa/features/extractors/dnfile/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -52,7 +52,7 @@ def resolve_dotnet_token(pe: dnfile.dnPE, token: Token) -> Union[dnfile.base.MDT
             return InvalidToken(token.value)
         return user_string
 
-    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(token.table)
+    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(token.table, None)
     if table is None:
         # table index is not valid
         return InvalidToken(token.value)
@@ -204,7 +204,7 @@ def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
                 continue
 
             token: int = calculate_dotnet_token_value(method.table.number, method.row_index)
-            access: Optional[str] = accessor_map.get(token)
+            access: Optional[str] = accessor_map.get(token, None)
 
             method_name: str = method.row.Name
             if method_name.startswith(("get_", "set_")):

capa/features/extractors/dnfile/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -9,7 +9,7 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Tuple, Union, Iterator, Optional
+from typing import TYPE_CHECKING, Any, Dict, Tuple, Union, Iterator, Optional
 
 if TYPE_CHECKING:
     from capa.features.extractors.dnfile.extractor import DnFileFeatureExtractorCache

capa/features/extractors/dnfile/types.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,10 +6,13 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from typing import Optional
+from typing import TYPE_CHECKING, Dict, List, Optional
+
+if TYPE_CHECKING:
+    from dncil.cil.instruction import Instruction
 
 
-class DnType:
+class DnType(object):
     def __init__(self, token: int, class_: str, namespace: str = "", member: str = "", access: Optional[str] = None):
         self.token: int = token
         self.access: Optional[str] = access
@@ -72,3 +75,10 @@ class DnUnmanagedMethod:
     @staticmethod
     def format_name(module, method):
         return f"{module}.{method}"
+
+
+class DnBasicBlock:
+    def __init__(self, preds=None, succs=None, instructions=None):
+        self.succs: List[DnBasicBlock] = succs or []
+        self.preds: List[DnBasicBlock] = preds or []
+        self.instructions: List[Instruction] = instructions or []

capa/features/extractors/dnfile_.py

@@ -1,13 +1,5 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 import logging
 from typing import Tuple, Iterator
-from pathlib import Path
 
 import dnfile
 import pefile
@@ -82,10 +74,10 @@ GLOBAL_HANDLERS = (
 
 
 class DnfileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: Path):
+    def __init__(self, path: str):
         super().__init__()
-        self.path: Path = path
-        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+        self.path: str = path
+        self.pe: dnfile.dnPE = dnfile.dnPE(path)
 
     def get_base_address(self) -> AbsoluteVirtualAddress:
         return AbsoluteVirtualAddress(0x0)

capa/features/extractors/dotnetfile.py

@@ -1,13 +1,5 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 import logging
-from typing import Tuple, Iterator
-from pathlib import Path
+from typing import Tuple, Iterator, cast
 
 import dnfile
 import pefile
@@ -166,10 +158,10 @@ GLOBAL_HANDLERS = (
 
 
 class DotnetFileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: Path):
+    def __init__(self, path: str):
         super().__init__()
-        self.path: Path = path
-        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
+        self.path: str = path
+        self.pe: dnfile.dnPE = dnfile.dnPE(path)
 
     def get_base_address(self):
         return NO_ADDRESS

capa/features/extractors/elf.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -13,8 +13,6 @@ from enum import Enum
 from typing import Set, Dict, List, Tuple, BinaryIO, Iterator, Optional
 from dataclasses import dataclass
 
-import Elf  # from vivisect
-
 logger = logging.getLogger(__name__)
 
 
@@ -26,7 +24,7 @@ def align(v, alignment):
         return v + (alignment - remainder)
 
 
-def read_cstr(buf, offset) -> str:
+def read_cstr(buf, offset):
     s = buf[offset:]
     s, _, _ = s.partition(b"\x00")
     return s.decode("utf-8")
@@ -56,7 +54,6 @@ class OS(str, Enum):
     CLOUD = "cloud"
     SYLLABLE = "syllable"
     NACL = "nacl"
-    ANDROID = "android"
 
 
 # via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658
@@ -91,23 +88,8 @@ class Shdr:
     offset: int
     size: int
     link: int
-    entsize: int
     buf: bytes
 
-    @classmethod
-    def from_viv(cls, section, buf: bytes) -> "Shdr":
-        return cls(
-            section.sh_name,
-            section.sh_type,
-            section.sh_flags,
-            section.sh_addr,
-            section.sh_offset,
-            section.sh_size,
-            section.sh_link,
-            section.sh_entsize,
-            buf,
-        )
-
 
 class ELF:
     def __init__(self, f: BinaryIO):
@@ -139,14 +121,14 @@ class ELF:
         elif ei_class == 2:
             self.bitness = 64
         else:
-            raise CorruptElfFile(f"invalid ei_class: 0x{ei_class:02x}")
+            raise CorruptElfFile("invalid ei_class: 0x%02x" % ei_class)
 
         if ei_data == 1:
             self.endian = "<"
         elif ei_data == 2:
             self.endian = ">"
         else:
-            raise CorruptElfFile(f"not an ELF file: invalid ei_data: 0x{ei_data:02x}")
+            raise CorruptElfFile("not an ELF file: invalid ei_data: 0x%02x" % ei_data)
 
         if self.bitness == 32:
             e_phoff, e_shoff = struct.unpack_from(self.endian + "II", self.file_header, 0x1C)
@@ -338,12 +320,12 @@ class ELF:
         shent = self.shbuf[shent_offset : shent_offset + self.e_shentsize]
 
         if self.bitness == 32:
-            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link, _, _, sh_entsize = struct.unpack_from(
-                self.endian + "IIIIIIIIII", shent, 0x0
+            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link = struct.unpack_from(
+                self.endian + "IIIIIII", shent, 0x0
             )
         elif self.bitness == 64:
-            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link, _, _, sh_entsize = struct.unpack_from(
-                self.endian + "IIQQQQIIQQ", shent, 0x0
+            sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link = struct.unpack_from(
+                self.endian + "IIQQQQI", shent, 0x0
             )
         else:
             raise NotImplementedError()
@@ -355,7 +337,7 @@ class ELF:
         if len(buf) != sh_size:
             raise ValueError("failed to read section header content")
 
-        return Shdr(sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link, sh_entsize, buf)
+        return Shdr(sh_name, sh_type, sh_flags, sh_addr, sh_offset, sh_size, sh_link, buf)
 
     @property
     def section_headers(self):
@@ -414,7 +396,7 @@ class ELF:
                 # there should be vn_cnt of these.
                 # each entry describes an ABI name required by the shared object.
                 vna_offset = vn_offset + vn_aux
-                for _ in range(vn_cnt):
+                for i in range(vn_cnt):
                     # ElfXX_Vernaux layout is the same on 32 and 64 bit
                     _, _, _, vna_name, vna_next = struct.unpack_from(self.endian + "IHHII", shdr.buf, vna_offset)
 
@@ -475,12 +457,10 @@ class ELF:
         for d_tag, d_val in self.dynamic_entries:
             if d_tag == DT_STRTAB:
                 strtab_addr = d_val
-                break
 
         for d_tag, d_val in self.dynamic_entries:
             if d_tag == DT_STRSZ:
                 strtab_size = d_val
-                break
 
         if strtab_addr is None:
             return None
@@ -490,10 +470,8 @@ class ELF:
 
         strtab_offset = None
         for shdr in self.section_headers:
-            # the section header address should be defined
-            if shdr.addr and shdr.addr <= strtab_addr < shdr.addr + shdr.size:
+            if shdr.addr <= strtab_addr < shdr.addr + shdr.size:
                 strtab_offset = shdr.offset + (strtab_addr - shdr.addr)
-                break
 
         if strtab_offset is None:
             return None
@@ -522,27 +500,7 @@ class ELF:
             if d_tag != DT_NEEDED:
                 continue
 
-            try:
-                yield read_cstr(strtab, d_val)
-            except UnicodeDecodeError as e:
-                logger.warning("failed to read DT_NEEDED entry: %s", str(e))
-
-    @property
-    def symtab(self) -> Optional[Tuple[Shdr, Shdr]]:
-        """
-        fetch the Shdr for the symtab and the associated strtab.
-        """
-        SHT_SYMTAB = 0x2
-        for shdr in self.section_headers:
-            if shdr.type != SHT_SYMTAB:
-                continue
-
-            # the linked section contains strings referenced by the symtab structures.
-            strtab_shdr = self.parse_section_header(shdr.link)
-
-            return shdr, strtab_shdr
-
-        return None
+            yield read_cstr(strtab, d_val)
 
 
 @dataclass
@@ -645,101 +603,11 @@ class SHNote:
         return ABITag(os, kmajor, kminor, kpatch)
 
 
-@dataclass
-class Symbol:
-    name_offset: int
-    value: int
-    size: int
-    info: int
-    other: int
-    shndx: int
-
-
-class SymTab:
-    def __init__(
-        self,
-        endian: str,
-        bitness: int,
-        symtab: Shdr,
-        strtab: Shdr,
-    ) -> None:
-        self.symbols: List[Symbol] = []
-
-        self.symtab = symtab
-        self.strtab = strtab
-
-        self._parse(endian, bitness, symtab.buf)
-
-    def _parse(self, endian: str, bitness: int, symtab_buf: bytes) -> None:
-        """
-        return the symbol's information in
-        the order specified by sys/elf32.h
-        """
-        if self.symtab.entsize == 0:
-            return
-
-        for i in range(int(len(self.symtab.buf) / self.symtab.entsize)):
-            if bitness == 32:
-                name_offset, value, size, info, other, shndx = struct.unpack_from(
-                    endian + "IIIBBH", symtab_buf, i * self.symtab.entsize
-                )
-            elif bitness == 64:
-                name_offset, info, other, shndx, value, size = struct.unpack_from(
-                    endian + "IBBHQQ", symtab_buf, i * self.symtab.entsize
-                )
-
-            self.symbols.append(Symbol(name_offset, value, size, info, other, shndx))
-
-    def get_name(self, symbol: Symbol) -> str:
-        """
-        fetch a symbol's name from symtab's
-        associated strings' section (SHT_STRTAB)
-        """
-        if not self.strtab:
-            raise ValueError("no strings found")
-
-        for i in range(symbol.name_offset, self.strtab.size):
-            if self.strtab.buf[i] == 0:
-                return self.strtab.buf[symbol.name_offset : i].decode("utf-8")
-
-        raise ValueError("symbol name not found")
-
-    def get_symbols(self) -> Iterator[Symbol]:
-        """
-        return a tuple: (name, value, size, info, other, shndx)
-        for each symbol contained in the symbol table
-        """
-        yield from self.symbols
-
-    @classmethod
-    def from_viv(cls, elf: Elf.Elf) -> Optional["SymTab"]:
-        endian = "<" if elf.getEndian() == 0 else ">"
-        bitness = elf.bits
-
-        SHT_SYMTAB = 0x2
-        for section in elf.sections:
-            if section.sh_type == SHT_SYMTAB:
-                strtab_section = elf.sections[section.sh_link]
-                sh_symtab = Shdr.from_viv(section, elf.readAtOffset(section.sh_offset, section.sh_size))
-                sh_strtab = Shdr.from_viv(
-                    strtab_section, elf.readAtOffset(strtab_section.sh_offset, strtab_section.sh_size)
-                )
-
-        try:
-            return cls(endian, bitness, sh_symtab, sh_strtab)
-        except NameError:
-            return None
-        except Exception:
-            # all exceptions that could be encountered by
-            # cls._parse() imply a faulty symbol's table.
-            raise CorruptElfFile("malformed symbol's table")
-
-
-def guess_os_from_osabi(elf: ELF) -> Optional[OS]:
+def guess_os_from_osabi(elf) -> Optional[OS]:
     return elf.ei_osabi
 
 
-def guess_os_from_ph_notes(elf: ELF) -> Optional[OS]:
+def guess_os_from_ph_notes(elf) -> Optional[OS]:
     # search for PT_NOTE sections that specify an OS
     # for example, on Linux there is a GNU section with minimum kernel version
     PT_NOTE = 0x4
@@ -767,11 +635,6 @@ def guess_os_from_ph_notes(elf: ELF) -> Optional[OS]:
         elif note.name == "FreeBSD":
             logger.debug("note owner: %s", "FREEBSD")
             return OS.FREEBSD
-        elif note.name == "Android":
-            logger.debug("note owner: %s", "Android")
-            # see the following for parsing the structure:
-            # https://android.googlesource.com/platform/ndk/+/master/parse_elfnote.py
-            return OS.ANDROID
         elif note.name == "GNU":
             abi_tag = note.abi_tag
             if abi_tag:
@@ -783,7 +646,7 @@ def guess_os_from_ph_notes(elf: ELF) -> Optional[OS]:
     return None
 
 
-def guess_os_from_sh_notes(elf: ELF) -> Optional[OS]:
+def guess_os_from_sh_notes(elf) -> Optional[OS]:
     # search for notes stored in sections that aren't visible in program headers.
     # e.g. .note.Linux in Linux kernel modules.
     SHT_NOTE = 0x7
@@ -816,7 +679,7 @@ def guess_os_from_sh_notes(elf: ELF) -> Optional[OS]:
     return None
 
 
-def guess_os_from_linker(elf: ELF) -> Optional[OS]:
+def guess_os_from_linker(elf) -> Optional[OS]:
     # search for recognizable dynamic linkers (interpreters)
     # for example, on linux, we see file paths like: /lib64/ld-linux-x86-64.so.2
     linker = elf.linker
@@ -826,12 +689,12 @@ def guess_os_from_linker(elf: ELF) -> Optional[OS]:
     return None
 
 
-def guess_os_from_abi_versions_needed(elf: ELF) -> Optional[OS]:
+def guess_os_from_abi_versions_needed(elf) -> Optional[OS]:
     # then lets look for GLIBC symbol versioning requirements.
     # this will let us guess about linux/hurd in some cases.
 
     versions_needed = elf.versions_needed
-    if any(abi.startswith("GLIBC") for abi in itertools.chain(*versions_needed.values())):
+    if any(map(lambda abi: abi.startswith("GLIBC"), itertools.chain(*versions_needed.values()))):
         # there are any GLIBC versions needed
 
         if elf.e_machine != "i386":
@@ -857,103 +720,39 @@ def guess_os_from_abi_versions_needed(elf: ELF) -> Optional[OS]:
     return None
 
 
-def guess_os_from_needed_dependencies(elf: ELF) -> Optional[OS]:
+def guess_os_from_needed_dependencies(elf) -> Optional[OS]:
     for needed in elf.needed:
         if needed.startswith("libmachuser.so"):
             return OS.HURD
         if needed.startswith("libhurduser.so"):
             return OS.HURD
-        if needed.startswith("libandroid.so"):
-            return OS.ANDROID
-
-    return None
-
-
-def guess_os_from_symtab(elf: ELF) -> Optional[OS]:
-    shdrs = elf.symtab
-    if not shdrs:
-        # executable does not contain a symbol table
-        # or the symbol's names are stripped
-        return None
-
-    symtab_shdr, strtab_shdr = shdrs
-    symtab = SymTab(elf.endian, elf.bitness, symtab_shdr, strtab_shdr)
-
-    keywords = {
-        OS.LINUX: [
-            "linux",
-            "/linux/",
-        ],
-    }
-
-    for symbol in symtab.get_symbols():
-        sym_name = symtab.get_name(symbol)
-
-        for os, hints in keywords.items():
-            if any(hint in sym_name for hint in hints):
-                return os
 
     return None
 
 
 def detect_elf_os(f) -> str:
     """
-    f: type Union[BinaryIO, IDAIO, GHIDRAIO]
+    f: type Union[BinaryIO, IDAIO]
     """
-    try:
-        elf = ELF(f)
-    except Exception as e:
-        logger.warning("Error parsing ELF file: %s", e)
-        return "unknown"
+    elf = ELF(f)
 
-    try:
-        osabi_guess = guess_os_from_osabi(elf)
-        logger.debug("guess: osabi: %s", osabi_guess)
-    except Exception as e:
-        logger.warning("Error guessing OS from OSABI: %s", e)
-        osabi_guess = None
+    osabi_guess = guess_os_from_osabi(elf)
+    logger.debug("guess: osabi: %s", osabi_guess)
 
-    try:
-        ph_notes_guess = guess_os_from_ph_notes(elf)
-        logger.debug("guess: ph notes: %s", ph_notes_guess)
-    except Exception as e:
-        logger.warning("Error guessing OS from program header notes: %s", e)
-        ph_notes_guess = None
+    ph_notes_guess = guess_os_from_ph_notes(elf)
+    logger.debug("guess: ph notes: %s", ph_notes_guess)
 
-    try:
-        sh_notes_guess = guess_os_from_sh_notes(elf)
-        logger.debug("guess: sh notes: %s", sh_notes_guess)
-    except Exception as e:
-        logger.warning("Error guessing OS from section header notes: %s", e)
-        sh_notes_guess = None
+    sh_notes_guess = guess_os_from_sh_notes(elf)
+    logger.debug("guess: sh notes: %s", sh_notes_guess)
 
-    try:
-        linker_guess = guess_os_from_linker(elf)
-        logger.debug("guess: linker: %s", linker_guess)
-    except Exception as e:
-        logger.warning("Error guessing OS from linker: %s", e)
-        linker_guess = None
+    linker_guess = guess_os_from_linker(elf)
+    logger.debug("guess: linker: %s", linker_guess)
 
-    try:
-        abi_versions_needed_guess = guess_os_from_abi_versions_needed(elf)
-        logger.debug("guess: ABI versions needed: %s", abi_versions_needed_guess)
-    except Exception as e:
-        logger.warning("Error guessing OS from ABI versions needed: %s", e)
-        abi_versions_needed_guess = None
+    abi_versions_needed_guess = guess_os_from_abi_versions_needed(elf)
+    logger.debug("guess: ABI versions needed: %s", abi_versions_needed_guess)
 
-    try:
-        needed_dependencies_guess = guess_os_from_needed_dependencies(elf)
-        logger.debug("guess: needed dependencies: %s", needed_dependencies_guess)
-    except Exception as e:
-        logger.warning("Error guessing OS from needed dependencies: %s", e)
-        needed_dependencies_guess = None
-
-    try:
-        symtab_guess = guess_os_from_symtab(elf)
-        logger.debug("guess: pertinent symbol name: %s", symtab_guess)
-    except Exception as e:
-        logger.warning("Error guessing OS from symbol table: %s", e)
-        symtab_guess = None
+    needed_dependencies_guess = guess_os_from_needed_dependencies(elf)
+    logger.debug("guess: needed dependencies: %s", needed_dependencies_guess)
 
     ret = None
 
@@ -975,9 +774,6 @@ def detect_elf_os(f) -> str:
     elif needed_dependencies_guess:
         ret = needed_dependencies_guess
 
-    elif symtab_guess:
-        ret = symtab_guess
-
     return ret.value if ret is not None else "unknown"
 
 

capa/features/extractors/elffile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,13 +8,11 @@
 import io
 import logging
 from typing import Tuple, Iterator
-from pathlib import Path
 
 from elftools.elf.elffile import ELFFile, SymbolTableSection
-from elftools.elf.relocation import RelocationSection
 
 import capa.features.extractors.common
-from capa.features.file import Export, Import, Section
+from capa.features.file import Import, Section
 from capa.features.common import OS, FORMAT_ELF, Arch, Format, Feature
 from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
 from capa.features.extractors.base_extractor import FeatureExtractor
@@ -22,8 +20,11 @@ from capa.features.extractors.base_extractor import FeatureExtractor
 logger = logging.getLogger(__name__)
 
 
-def extract_file_export_names(elf: ELFFile, **kwargs):
-    for section in elf.iter_sections():
+def extract_file_import_names(elf, **kwargs):
+    # see https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/scripts/readelf.py#L372
+    symbol_tables = [(idx, s) for idx, s in enumerate(elf.iter_sections()) if isinstance(s, SymbolTableSection)]
+
+    for _, section in symbol_tables:
         if not isinstance(section, SymbolTableSection):
             continue
 
@@ -33,64 +34,14 @@ def extract_file_export_names(elf: ELFFile, **kwargs):
 
         logger.debug("Symbol table '%s' contains %s entries:", section.name, section.num_symbols())
 
-        for symbol in section.iter_symbols():
-            # The following conditions are based on the following article
-            # http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
-            if not symbol.name:
-                continue
-            if symbol.entry.st_info.type not in ["STT_FUNC", "STT_OBJECT", "STT_IFUNC"]:
-                continue
-            if symbol.entry.st_value == 0:
-                continue
-            if symbol.entry.st_shndx == "SHN_UNDEF":
-                continue
-
-            yield Export(symbol.name), AbsoluteVirtualAddress(symbol.entry.st_value)
-
-
-def extract_file_import_names(elf: ELFFile, **kwargs):
-    # Create a dictionary to store symbol names by their index
-    symbol_names = {}
-
-    # Extract symbol names and store them in the dictionary
-    for section in elf.iter_sections():
-        if not isinstance(section, SymbolTableSection):
-            continue
-
         for _, symbol in enumerate(section.iter_symbols()):
-            # The following conditions are based on the following article
-            # http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
-            if not symbol.name:
-                continue
-            if symbol.entry.st_info.type not in ["STT_FUNC", "STT_OBJECT", "STT_IFUNC"]:
-                continue
-            if symbol.entry.st_value != 0:
-                continue
-            if symbol.entry.st_shndx != "SHN_UNDEF":
-                continue
-            if symbol.entry.st_name == 0:
-                continue
-
-            symbol_names[_] = symbol.name
-
-    for section in elf.iter_sections():
-        if not isinstance(section, RelocationSection):
-            continue
-
-        if section["sh_entsize"] == 0:
-            logger.debug("Symbol table '%s' has a sh_entsize of zero!", section.name)
-            continue
-
-        logger.debug("Symbol table '%s' contains %s entries:", section.name, section.num_relocations())
-
-        for relocation in section.iter_relocations():
-            # Extract the symbol name from the symbol table using the symbol index in the relocation
-            if relocation["r_info_sym"] not in symbol_names:
-                continue
-            yield Import(symbol_names[relocation["r_info_sym"]]), FileOffsetAddress(relocation["r_offset"])
+            if symbol.name and symbol.entry.st_info.type == "STT_FUNC":
+                # TODO symbol address
+                # TODO symbol version info?
+                yield Import(symbol.name), FileOffsetAddress(0x0)
 
 
-def extract_file_section_names(elf: ELFFile, **kwargs):
+def extract_file_section_names(elf, **kwargs):
     for section in elf.iter_sections():
         if section.name:
             yield Section(section.name), AbsoluteVirtualAddress(section.header.sh_addr)
@@ -102,7 +53,7 @@ def extract_file_strings(buf, **kwargs):
     yield from capa.features.extractors.common.extract_file_strings(buf)
 
 
-def extract_file_os(elf: ELFFile, buf, **kwargs):
+def extract_file_os(elf, buf, **kwargs):
     # our current approach does not always get an OS value, e.g. for packed samples
     # for file limitation purposes, we're more lax here
     try:
@@ -116,7 +67,8 @@ def extract_file_format(**kwargs):
     yield Format(FORMAT_ELF), NO_ADDRESS
 
 
-def extract_file_arch(elf: ELFFile, **kwargs):
+def extract_file_arch(elf, **kwargs):
+    # TODO merge with capa.features.extractors.elf.detect_elf_arch()
     arch = elf.get_machine_arch()
     if arch == "x86":
         yield Arch("i386"), NO_ADDRESS
@@ -133,7 +85,7 @@ def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, i
 
 
 FILE_HANDLERS = (
-    extract_file_export_names,
+    # TODO extract_file_export_names,
     extract_file_import_names,
     extract_file_section_names,
     extract_file_strings,
@@ -155,10 +107,11 @@ GLOBAL_HANDLERS = (
 
 
 class ElfFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: Path):
+    def __init__(self, path: str):
         super().__init__()
-        self.path: Path = path
-        self.elf = ELFFile(io.BytesIO(path.read_bytes()))
+        self.path = path
+        with open(self.path, "rb") as f:
+            self.elf = ELFFile(io.BytesIO(f.read()))
 
     def get_base_address(self):
         # virtual address of the first segment with type LOAD
@@ -167,13 +120,15 @@ class ElfFeatureExtractor(FeatureExtractor):
                 return AbsoluteVirtualAddress(segment.header.p_vaddr)
 
     def extract_global_features(self):
-        buf = self.path.read_bytes()
+        with open(self.path, "rb") as f:
+            buf = f.read()
 
         for feature, addr in extract_global_features(self.elf, buf):
             yield feature, addr
 
     def extract_file_features(self):
-        buf = self.path.read_bytes()
+        with open(self.path, "rb") as f:
+            buf = f.read()
 
         for feature, addr in extract_file_features(self.elf, buf):
             yield feature, addr

capa/features/extractors/ghidra/basicblock.py

@@ -1,152 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import string
-import struct
-from typing import Tuple, Iterator
-
-import ghidra
-from ghidra.program.model.lang import OperandType
-
-import capa.features.extractors.ghidra.helpers
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address
-from capa.features.basicblock import BasicBlock
-from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
-
-
-def get_printable_len(op: ghidra.program.model.scalar.Scalar) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
-    op_bit_len = op.bitLength()
-    op_byte_len = op_bit_len // 8
-    op_val = op.getValue()
-
-    if op_bit_len == 8:
-        chars = struct.pack("<B", op_val & 0xFF)
-    elif op_bit_len == 16:
-        chars = struct.pack("<H", op_val & 0xFFFF)
-    elif op_bit_len == 32:
-        chars = struct.pack("<I", op_val & 0xFFFFFFFF)
-    elif op_bit_len == 64:
-        chars = struct.pack("<Q", op_val & 0xFFFFFFFFFFFFFFFF)
-    else:
-        raise ValueError(f"Unhandled operand data type 0x{op_bit_len:x}.")
-
-    def is_printable_ascii(chars_: bytes):
-        return all(c < 127 and chr(c) in string.printable for c in chars_)
-
-    def is_printable_utf16le(chars_: bytes):
-        if all(c == 0x00 for c in chars_[1::2]):
-            return is_printable_ascii(chars_[::2])
-
-    if is_printable_ascii(chars):
-        return op_byte_len
-
-    if is_printable_utf16le(chars):
-        return op_byte_len
-
-    return 0
-
-
-def is_mov_imm_to_stack(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    """verify instruction moves immediate onto stack"""
-
-    # Ghidra will Bitwise OR the OperandTypes to assign multiple
-    # i.e., the first operand is a stackvar (dynamically allocated),
-    # and the second is a scalar value (single int/char/float/etc.)
-    mov_its_ops = [(OperandType.ADDRESS | OperandType.DYNAMIC), OperandType.SCALAR]
-    found = False
-
-    # MOV dword ptr [EBP + local_*], 0x65
-    if insn.getMnemonicString().startswith("MOV"):
-        found = all(insn.getOperandType(i) == mov_its_ops[i] for i in range(2))
-
-    return found
-
-
-def bb_contains_stackstring(bb: ghidra.program.model.block.CodeBlock) -> bool:
-    """check basic block for stackstring indicators
-
-    true if basic block contains enough moves of constant bytes to the stack
-    """
-    count = 0
-    for insn in currentProgram().getListing().getInstructions(bb, True):  # type: ignore [name-defined] # noqa: F821
-        if is_mov_imm_to_stack(insn):
-            count += get_printable_len(insn.getScalar(1))
-        if count > MIN_STACKSTRING_LEN:
-            return True
-    return False
-
-
-def _bb_has_tight_loop(bb: ghidra.program.model.block.CodeBlock):
-    """
-    parse tight loops, true if last instruction in basic block branches to bb start
-    """
-    # Reverse Ordered, first InstructionDB
-    last_insn = currentProgram().getListing().getInstructions(bb, False).next()  # type: ignore [name-defined] # noqa: F821
-
-    if last_insn.getFlowType().isJump():
-        return last_insn.getAddress(0) == bb.getMinAddress()
-
-    return False
-
-
-def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract stackstring indicators from basic block"""
-    bb: ghidra.program.model.block.CodeBlock = bbh.inner
-
-    if bb_contains_stackstring(bb):
-        yield Characteristic("stack string"), bbh.address
-
-
-def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """check basic block for tight loop indicators"""
-    bb: ghidra.program.model.block.CodeBlock = bbh.inner
-
-    if _bb_has_tight_loop(bb):
-        yield Characteristic("tight loop"), bbh.address
-
-
-BASIC_BLOCK_HANDLERS = (
-    extract_bb_tight_loop,
-    extract_bb_stackstring,
-)
-
-
-def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract features from the given basic block.
-
-    args:
-        bb: the basic block to process.
-
-    yields:
-      Tuple[Feature, int]: the features and their location found in this basic block.
-    """
-    yield BasicBlock(), bbh.address
-    for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, addr in bb_handler(fh, bbh):
-            yield feature, addr
-
-
-def main():
-    features = []
-    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
-
-    for fh in GhidraFeatureExtractor().get_functions():
-        for bbh in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
-            features.extend(list(extract_features(fh, bbh)))
-
-    import pprint
-
-    pprint.pprint(features)  # noqa: T203
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ghidra/extractor.py

@@ -1,75 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import List, Tuple, Iterator
-
-import capa.features.extractors.ghidra.file
-import capa.features.extractors.ghidra.insn
-import capa.features.extractors.ghidra.global_
-import capa.features.extractors.ghidra.function
-import capa.features.extractors.ghidra.basicblock
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
-
-
-class GhidraFeatureExtractor(FeatureExtractor):
-    def __init__(self):
-        super().__init__()
-        import capa.features.extractors.ghidra.helpers as ghidra_helpers
-
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.ghidra.file.extract_file_format())
-        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_os())
-        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_arch())
-        self.imports = ghidra_helpers.get_file_imports()
-        self.externs = ghidra_helpers.get_file_externs()
-        self.fakes = ghidra_helpers.map_fake_import_addrs()
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(currentProgram().getImageBase().getOffset())  # type: ignore [name-defined] # noqa: F821
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.ghidra.file.extract_features()
-
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        import capa.features.extractors.ghidra.helpers as ghidra_helpers
-
-        for fhandle in ghidra_helpers.get_function_symbols():
-            fh: FunctionHandle = FunctionHandle(
-                address=AbsoluteVirtualAddress(fhandle.getEntryPoint().getOffset()),
-                inner=fhandle,
-                ctx={"imports_cache": self.imports, "externs_cache": self.externs, "fakes_cache": self.fakes},
-            )
-            yield fh
-
-    @staticmethod
-    def get_function(addr: int) -> FunctionHandle:
-        func = getFunctionContaining(toAddr(addr))  # type: ignore [name-defined] # noqa: F821
-        return FunctionHandle(address=AbsoluteVirtualAddress(func.getEntryPoint().getOffset()), inner=func)
-
-    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.ghidra.function.extract_features(fh)
-
-    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
-        import capa.features.extractors.ghidra.helpers as ghidra_helpers
-
-        yield from ghidra_helpers.get_function_blocks(fh)
-
-    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.ghidra.basicblock.extract_features(fh, bbh)
-
-    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
-        import capa.features.extractors.ghidra.helpers as ghidra_helpers
-
-        yield from ghidra_helpers.get_insn_in_range(bbh)
-
-    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
-        yield from capa.features.extractors.ghidra.insn.extract_features(fh, bbh, ih)

capa/features/extractors/ghidra/file.py

@@ -1,202 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import re
-import struct
-from typing import List, Tuple, Iterator
-
-from ghidra.program.model.symbol import SourceType, SymbolType
-
-import capa.features.extractors.common
-import capa.features.extractors.helpers
-import capa.features.extractors.strings
-import capa.features.extractors.ghidra.helpers
-from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
-from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
-
-MAX_OFFSET_PE_AFTER_MZ = 0x200
-
-
-def find_embedded_pe(block_bytez: bytes, mz_xor: List[Tuple[bytes, bytes, int]]) -> Iterator[Tuple[int, int]]:
-    """check segment for embedded PE
-
-    adapted for Ghidra from:
-    https://github.com/vivisect/vivisect/blob/91e8419a861f4977https://github.com/vivisect/vivisect/blob/91e8419a861f49779f18316f155311967e696836/PE/carve.py#L259f18316f155311967e696836/PE/carve.py#L25
-    """
-    todo = []
-
-    for mzx, pex, i in mz_xor:
-        for match in re.finditer(re.escape(mzx), block_bytez):
-            todo.append((match.start(), mzx, pex, i))
-
-    seg_max = len(block_bytez)  # type: ignore [name-defined] # noqa: F821
-    while len(todo):
-        off, mzx, pex, i = todo.pop()
-
-        # MZ header has one field we will check e_lfanew is at 0x3c
-        e_lfanew = off + 0x3C
-
-        if seg_max < e_lfanew + 4:
-            continue
-
-        e_lfanew_bytes = block_bytez[e_lfanew : e_lfanew + 4]
-        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(e_lfanew_bytes, i))[0]
-
-        # assume XOR'd "PE" bytes exist within threshold
-        if newoff > MAX_OFFSET_PE_AFTER_MZ:
-            continue
-
-        peoff = off + newoff
-        if seg_max < peoff + 2:
-            continue
-
-        pe_bytes = block_bytez[peoff : peoff + 2]
-        if pe_bytes == pex:
-            yield off, i
-
-
-def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
-    """extract embedded PE features"""
-
-    # pre-compute XOR pairs
-    mz_xor: List[Tuple[bytes, bytes, int]] = [
-        (
-            capa.features.extractors.helpers.xor_static(b"MZ", i),
-            capa.features.extractors.helpers.xor_static(b"PE", i),
-            i,
-        )
-        for i in range(256)
-    ]
-
-    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
-        if not all((block.isLoaded(), block.isInitialized(), "Headers" not in block.getName())):
-            continue
-
-        for off, _ in find_embedded_pe(capa.features.extractors.ghidra.helpers.get_block_bytes(block), mz_xor):
-            # add offset back to block start
-            ea: int = block.getStart().add(off).getOffset()
-
-            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
-
-
-def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract function exports"""
-    st = currentProgram().getSymbolTable()  # type: ignore [name-defined] # noqa: F821
-    for addr in st.getExternalEntryPointIterator():
-        yield Export(st.getPrimarySymbol(addr).getName()), AbsoluteVirtualAddress(addr.getOffset())
-
-
-def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract function imports
-
-    1. imports by ordinal:
-     - modulename.#ordinal
-
-    2. imports by name, results in two features to support importname-only
-       matching:
-     - modulename.importname
-     - importname
-    """
-
-    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
-        for r in f.getSymbol().getReferences():
-            if r.getReferenceType().isData():
-                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
-
-        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_*
-        if "Ordinal_" in fstr[1]:
-            fstr[1] = f"#{fstr[1].split('_')[1]}"
-
-        for name in capa.features.extractors.helpers.generate_symbols(fstr[0][:-4], fstr[1]):
-            yield Import(name), AbsoluteVirtualAddress(addr)
-
-
-def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract section names"""
-
-    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
-        yield Section(block.getName()), AbsoluteVirtualAddress(block.getStart().getOffset())
-
-
-def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
-    """extract ASCII and UTF-16 LE strings"""
-
-    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
-        if block.isInitialized():
-            p_bytes = capa.features.extractors.ghidra.helpers.get_block_bytes(block)
-
-        for s in capa.features.extractors.strings.extract_ascii_strings(p_bytes):
-            offset = block.getStart().getOffset() + s.offset
-            yield String(s.s), FileOffsetAddress(offset)
-
-        for s in capa.features.extractors.strings.extract_unicode_strings(p_bytes):
-            offset = block.getStart().getOffset() + s.offset
-            yield String(s.s), FileOffsetAddress(offset)
-
-
-def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract the names of statically-linked library functions.
-    """
-
-    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
-        # .isExternal() misses more than this config for the function symbols
-        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
-            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
-            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
-                name = name[13:]
-            addr = AbsoluteVirtualAddress(sym.getAddress().getOffset())
-            yield FunctionName(name), addr
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                yield FunctionName(name[1:]), addr
-
-
-def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
-    ef = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
-    if "PE" in ef:
-        yield Format(FORMAT_PE), NO_ADDRESS
-    elif "ELF" in ef:
-        yield Format(FORMAT_ELF), NO_ADDRESS
-    elif "Raw" in ef:
-        # no file type to return when processing a binary file, but we want to continue processing
-        return
-    else:
-        raise NotImplementedError(f"unexpected file format: {ef}")
-
-
-def extract_features() -> Iterator[Tuple[Feature, Address]]:
-    """extract file features"""
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler():
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_file_embedded_pe,
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_section_names,
-    extract_file_strings,
-    extract_file_function_names,
-    extract_file_format,
-)
-
-
-def main():
-    """ """
-    import pprint
-
-    pprint.pprint(list(extract_features()))  # noqa: T203
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ghidra/function.py

@@ -1,73 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Tuple, Iterator
-
-import ghidra
-from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
-
-import capa.features.extractors.ghidra.helpers
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors import loops
-from capa.features.extractors.base_extractor import FunctionHandle
-
-
-def extract_function_calls_to(fh: FunctionHandle):
-    """extract callers to a function"""
-    f: ghidra.program.database.function.FunctionDB = fh.inner
-    for ref in f.getSymbol().getReferences():
-        if ref.getReferenceType().isCall():
-            yield Characteristic("calls to"), AbsoluteVirtualAddress(ref.getFromAddress().getOffset())
-
-
-def extract_function_loop(fh: FunctionHandle):
-    f: ghidra.program.database.function.FunctionDB = fh.inner
-
-    edges = []
-    for block in SimpleBlockIterator(BasicBlockModel(currentProgram()), f.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
-        dests = block.getDestinations(monitor())  # type: ignore [name-defined] # noqa: F821
-        s_addrs = block.getStartAddresses()
-
-        while dests.hasNext():  # For loop throws Python TypeError
-            for addr in s_addrs:
-                edges.append((addr.getOffset(), dests.next().getDestinationAddress().getOffset()))
-
-    if loops.has_loop(edges):
-        yield Characteristic("loop"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
-
-
-def extract_recursive_call(fh: FunctionHandle):
-    f: ghidra.program.database.function.FunctionDB = fh.inner
-
-    for func in f.getCalledFunctions(monitor()):  # type: ignore [name-defined] # noqa: F821
-        if func.getEntryPoint().getOffset() == f.getEntryPoint().getOffset():
-            yield Characteristic("recursive call"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
-
-
-def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    for func_handler in FUNCTION_HANDLERS:
-        for feature, addr in func_handler(fh):
-            yield feature, addr
-
-
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
-
-
-def main():
-    """ """
-    features = []
-    for fhandle in capa.features.extractors.ghidra.helpers.get_function_symbols():
-        features.extend(list(extract_features(fhandle)))
-
-    import pprint
-
-    pprint.pprint(features)  # noqa: T203
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ghidra/global_.py

@@ -1,67 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-import contextlib
-from typing import Tuple, Iterator
-
-import capa.ghidra.helpers
-import capa.features.extractors.elf
-import capa.features.extractors.ghidra.helpers
-from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
-from capa.features.address import NO_ADDRESS, Address
-
-logger = logging.getLogger(__name__)
-
-
-def extract_os() -> Iterator[Tuple[Feature, Address]]:
-    format_name: str = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
-
-    if "PE" in format_name:
-        yield OS(OS_WINDOWS), NO_ADDRESS
-
-    elif "ELF" in format_name:
-        with contextlib.closing(capa.ghidra.helpers.GHIDRAIO()) as f:
-            os = capa.features.extractors.elf.detect_elf_os(f)
-
-        yield OS(os), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a further CLI argument to specify the OS,
-        # but i think this would be rarely used.
-        # rules that rely on OS conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess OS", format_name)
-        return
-
-
-def extract_arch() -> Iterator[Tuple[Feature, Address]]:
-    lang_id = currentProgram().getMetadata().get("Language ID")  # type: ignore [name-defined] # noqa: F821
-
-    if "x86" in lang_id and "64" in lang_id:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-
-    elif "x86" in lang_id and "32" in lang_id:
-        yield Arch(ARCH_I386), NO_ADDRESS
-
-    elif "x86" not in lang_id:
-        logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel")
-        return
-
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", lang_id)
-        return

capa/features/extractors/ghidra/helpers.py

@@ -1,277 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Dict, List, Iterator
-
-import ghidra
-import java.lang
-from ghidra.program.model.lang import OperandType
-from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
-from ghidra.program.model.symbol import SourceType, SymbolType
-from ghidra.program.model.address import AddressSpace
-
-import capa.features.extractors.helpers
-from capa.features.common import THUNK_CHAIN_DEPTH_DELTA
-from capa.features.address import AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-
-def ints_to_bytes(bytez: List[int]) -> bytes:
-    """convert Java signed ints to Python bytes
-
-    args:
-        bytez: list of Java signed ints
-    """
-    return bytes([b & 0xFF for b in bytez])
-
-
-def find_byte_sequence(addr: ghidra.program.model.address.Address, seq: bytes) -> Iterator[int]:
-    """yield all ea of a given byte sequence
-
-    args:
-        addr: start address
-        seq: bytes to search e.g. b"\x01\x03"
-    """
-    seqstr = "".join([f"\\x{b:02x}" for b in seq])
-    eas = findBytes(addr, seqstr, java.lang.Integer.MAX_VALUE, 1)  # type: ignore [name-defined] # noqa: F821
-
-    yield from eas
-
-
-def get_bytes(addr: ghidra.program.model.address.Address, length: int) -> bytes:
-    """yield length bytes at addr
-
-    args:
-        addr: Address to begin pull from
-        length: length of bytes to pull
-    """
-    try:
-        return ints_to_bytes(getBytes(addr, length))  # type: ignore [name-defined] # noqa: F821
-    except RuntimeError:
-        return b""
-
-
-def get_block_bytes(block: ghidra.program.model.mem.MemoryBlock) -> bytes:
-    """yield all bytes in a given block
-
-    args:
-        block: MemoryBlock to pull from
-    """
-    return get_bytes(block.getStart(), block.getSize())
-
-
-def get_function_symbols():
-    """yield all non-external function symbols"""
-    yield from currentProgram().getFunctionManager().getFunctionsNoStubs(True)  # type: ignore [name-defined] # noqa: F821
-
-
-def get_function_blocks(fh: FunctionHandle) -> Iterator[BBHandle]:
-    """yield BBHandle for each bb in a given function"""
-
-    func: ghidra.program.database.function.FunctionDB = fh.inner
-    for bb in SimpleBlockIterator(BasicBlockModel(currentProgram()), func.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
-        yield BBHandle(address=AbsoluteVirtualAddress(bb.getMinAddress().getOffset()), inner=bb)
-
-
-def get_insn_in_range(bbh: BBHandle) -> Iterator[InsnHandle]:
-    """yield InshHandle for each insn in a given basicblock"""
-    for insn in currentProgram().getListing().getInstructions(bbh.inner, True):  # type: ignore [name-defined] # noqa: F821
-        yield InsnHandle(address=AbsoluteVirtualAddress(insn.getAddress().getOffset()), inner=insn)
-
-
-def get_file_imports() -> Dict[int, List[str]]:
-    """get all import names & addrs"""
-
-    import_dict: Dict[int, List[str]] = {}
-
-    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
-        for r in f.getSymbol().getReferences():
-            if r.getReferenceType().isData():
-                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
-
-        ex_loc = f.getExternalLocation().getAddress()  # map external locations as well (offset into module files)
-
-        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_* / <EXTERNAL>::import
-        if "Ordinal_" in fstr[1]:
-            fstr[1] = f"#{fstr[1].split('_')[1]}"
-
-        # <EXTERNAL> mostly shows up in ELF files, otherwise, strip '.dll' w/ [:-4]
-        fstr[0] = "*" if "<EXTERNAL>" in fstr[0] else fstr[0][:-4]
-
-        for name in capa.features.extractors.helpers.generate_symbols(fstr[0], fstr[1]):
-            import_dict.setdefault(addr, []).append(name)
-            if ex_loc:
-                import_dict.setdefault(ex_loc.getOffset(), []).append(name)
-
-    return import_dict
-
-
-def get_file_externs() -> Dict[int, List[str]]:
-    """
-    Gets function names & addresses of statically-linked library functions
-
-    Ghidra's external namespace is mostly reserved for dynamically-linked
-    imports. Statically-linked functions are part of the global namespace.
-    Filtering on the type, source, and namespace of the symbols yield more
-    statically-linked library functions.
-
-    Example: (PMA Lab 16-01.exe_) 7faafc7e4a5c736ebfee6abbbc812d80:0x407490
-    - __aulldiv
-    - Note: See Symbol Table labels
-    """
-
-    extern_dict: Dict[int, List[str]] = {}
-
-    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
-        # .isExternal() misses more than this config for the function symbols
-        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
-            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
-            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
-                name = name[13:]
-            extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name)
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name[1:])
-
-    return extern_dict
-
-
-def map_fake_import_addrs() -> Dict[int, List[int]]:
-    """
-    Map ghidra's fake import entrypoints to their
-    real addresses
-
-    Helps as many Ghidra Scripting API calls end up returning
-    these external (fake) addresses.
-
-    Undocumented but intended Ghidra behavior:
-     - Import entryPoint fields are stored in the 'EXTERNAL:' AddressSpace.
-       'getEntryPoint()' returns the entryPoint field, which is an offset
-       from the beginning of the assigned AddressSpace. In the case of externals,
-       they start from 1 and increment.
-    https://github.com/NationalSecurityAgency/ghidra/blob/26d4bd9104809747c21f2528cab8aba9aef9acd5/Ghidra/Features/Base/src/test.slow/java/ghidra/program/database/function/ExternalFunctionDBTest.java#L90
-
-    Example: (mimikatz.exe_) 5f66b82558ca92e54e77f216ef4c066c:0x473090
-    - 0x473090 -> PTR_CreateServiceW_00473090
-    - 'EXTERNAL:00000025' -> External Address (ghidra.program.model.address.SpecialAddress)
-    """
-    fake_dict: Dict[int, List[int]] = {}
-
-    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
-        for r in f.getSymbol().getReferences():
-            if r.getReferenceType().isData():
-                fake_dict.setdefault(f.getEntryPoint().getOffset(), []).append(r.getFromAddress().getOffset())
-
-    return fake_dict
-
-
-def check_addr_for_api(
-    addr: ghidra.program.model.address.Address,
-    fakes: Dict[int, List[int]],
-    imports: Dict[int, List[str]],
-    externs: Dict[int, List[str]],
-) -> bool:
-    offset = addr.getOffset()
-
-    fake = fakes.get(offset)
-    if fake:
-        return True
-
-    imp = imports.get(offset)
-    if imp:
-        return True
-
-    extern = externs.get(offset)
-    if extern:
-        return True
-
-    return False
-
-
-def is_call_or_jmp(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    return any(mnem in insn.getMnemonicString() for mnem in ["CALL", "J"])  # JMP, JNE, JNZ, etc
-
-
-def is_sp_modified(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    for i in range(insn.getNumOperands()):
-        if insn.getOperandType(i) == OperandType.REGISTER:
-            return "SP" in insn.getRegister(i).getName() and insn.getOperandRefType(i).isWrite()
-    return False
-
-
-def is_stack_referenced(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    """generic catch-all for stack references"""
-    for i in range(insn.getNumOperands()):
-        if insn.getOperandType(i) == OperandType.REGISTER:
-            if "BP" in insn.getRegister(i).getName():
-                return True
-            else:
-                continue
-
-    return any(ref.isStackReference() for ref in insn.getReferencesFrom())
-
-
-def is_zxor(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    # assume XOR insn
-    # XOR's against the same operand zero out
-    ops = []
-    operands = []
-    for i in range(insn.getNumOperands()):
-        ops.append(insn.getOpObjects(i))
-
-    # Operands stored in a 2D array
-    for j in range(len(ops)):
-        for k in range(len(ops[j])):
-            operands.append(ops[j][k])
-
-    return all(n == operands[0] for n in operands)
-
-
-def handle_thunk(addr: ghidra.program.model.address.Address):
-    """Follow thunk chains down to a reasonable depth"""
-    ref = addr
-    for _ in range(THUNK_CHAIN_DEPTH_DELTA):
-        thunk_jmp = getInstructionAt(ref)  # type: ignore [name-defined] # noqa: F821
-        if thunk_jmp and is_call_or_jmp(thunk_jmp):
-            if OperandType.isAddress(thunk_jmp.getOperandType(0)):
-                ref = thunk_jmp.getAddress(0)
-        else:
-            thunk_dat = getDataContaining(ref)  # type: ignore [name-defined] # noqa: F821
-            if thunk_dat and thunk_dat.isDefined() and thunk_dat.isPointer():
-                ref = thunk_dat.getValue()
-                break  # end of thunk chain reached
-    return ref
-
-
-def dereference_ptr(insn: ghidra.program.database.code.InstructionDB):
-    addr_code = OperandType.ADDRESS | OperandType.CODE
-    to_deref = insn.getAddress(0)
-    dat = getDataContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
-
-    if insn.getOperandType(0) == addr_code:
-        thfunc = getFunctionContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
-        if thfunc and thfunc.isThunk():
-            return handle_thunk(to_deref)
-        else:
-            # if it doesn't poin to a thunk, it's usually a jmp to a label
-            return to_deref
-    if not dat:
-        return to_deref
-    if dat.isDefined() and dat.isPointer():
-        addr = dat.getValue()
-        # now we need to check the addr space to see if it is truly resolvable
-        # ghidra sometimes likes to hand us direct RAM addrs, which typically point
-        # to api calls that we can't actually resolve as such
-        if addr.getAddressSpace().getType() == AddressSpace.TYPE_RAM:
-            return to_deref
-        else:
-            return addr
-    else:
-        return to_deref

capa/features/extractors/ghidra/insn.py

@@ -1,521 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Any, Dict, Tuple, Iterator
-
-import ghidra
-from ghidra.program.model.lang import OperandType
-from ghidra.program.model.block import SimpleBlockModel
-
-import capa.features.extractors.helpers
-import capa.features.extractors.ghidra.helpers
-from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
-from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-# security cookie checks may perform non-zeroing XORs, these are expected within a certain
-# byte range within the first and returning basic blocks, this helps to reduce FP features
-SECURITY_COOKIE_BYTES_DELTA = 0x40
-
-
-def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
-    """Populate the import cache for this context"""
-    if "imports_cache" not in ctx:
-        ctx["imports_cache"] = capa.features.extractors.ghidra.helpers.get_file_imports()
-    return ctx["imports_cache"]
-
-
-def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
-    """Populate the externs cache for this context"""
-    if "externs_cache" not in ctx:
-        ctx["externs_cache"] = capa.features.extractors.ghidra.helpers.get_file_externs()
-    return ctx["externs_cache"]
-
-
-def get_fakes(ctx: Dict[str, Any]) -> Dict[int, Any]:
-    """Populate the fake import addrs cache for this context"""
-    if "fakes_cache" not in ctx:
-        ctx["fakes_cache"] = capa.features.extractors.ghidra.helpers.map_fake_import_addrs()
-    return ctx["fakes_cache"]
-
-
-def check_for_api_call(
-    insn, externs: Dict[int, Any], fakes: Dict[int, Any], imports: Dict[int, Any], imp_or_ex: bool
-) -> Iterator[Any]:
-    """check instruction for API call
-
-    params:
-        externs - external library functions cache
-        fakes - mapped fake import addresses cache
-        imports - imported functions cache
-        imp_or_ex - flag to check imports or externs
-
-    yields:
-        matched api calls
-    """
-    info = ()
-    funcs = imports if imp_or_ex else externs
-
-    # assume only CALLs or JMPs are passed
-    ref_type = insn.getOperandType(0)
-    addr_data = OperandType.ADDRESS | OperandType.DATA  # needs dereferencing
-    addr_code = OperandType.ADDRESS | OperandType.CODE  # needs dereferencing
-
-    if OperandType.isRegister(ref_type):
-        if OperandType.isAddress(ref_type):
-            # If it's an address in a register, check the mapped fake addrs
-            # since they're dereferenced to their fake addrs
-            op_ref = insn.getAddress(0).getOffset()
-            ref = fakes.get(op_ref)  # obtain the real addr
-            if not ref:
-                return
-        else:
-            return
-    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
-        # we must dereference and check if the addr is a pointer to an api function
-        addr_ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
-        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
-            return
-        ref = addr_ref.getOffset()
-    elif ref_type == OperandType.DYNAMIC | OperandType.ADDRESS or ref_type == OperandType.DYNAMIC:
-        return  # cannot resolve dynamics statically
-    else:
-        # pure address does not need to get dereferenced/ handled
-        addr_ref = insn.getAddress(0)
-        if not addr_ref:
-            # If it returned null, it was an indirect
-            # that had no address reference.
-            # This check is faster than checking for (indirect and not address)
-            return
-        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
-            return
-        ref = addr_ref.getOffset()
-
-    if isinstance(ref, list):  # ref from REG | ADDR
-        for r in ref:
-            info = funcs.get(r)  # type: ignore
-            if info:
-                yield info
-    else:
-        info = funcs.get(ref)  # type: ignore
-        if info:
-            yield info
-
-
-def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
-        return
-
-    externs = get_externs(fh.ctx)
-    fakes = get_fakes(fh.ctx)
-    imports = get_imports(fh.ctx)
-
-    # check calls to imported functions
-    for api in check_for_api_call(insn, externs, fakes, imports, True):
-        for imp in api:
-            yield API(imp), ih.address
-
-    # check calls to extern functions
-    for api in check_for_api_call(insn, externs, fakes, imports, False):
-        for ext in api:
-            yield API(ext), ih.address
-
-
-def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction number features
-    example:
-        push    3136B0h         ; dwControlCode
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if insn.getMnemonicString().startswith("RET"):
-        # skip things like:
-        #   .text:0042250E retn 8
-        return
-
-    if capa.features.extractors.ghidra.helpers.is_sp_modified(insn):
-        # skip things like:
-        #   .text:00401145 add esp, 0Ch
-        return
-
-    for i in range(insn.getNumOperands()):
-        # Exceptions for LEA insn:
-        # invalid operand encoding, considered numbers instead of offsets
-        # see: mimikatz.exe_:0x4018C0
-        if insn.getOperandType(i) == OperandType.DYNAMIC and insn.getMnemonicString().startswith("LEA"):
-            # Additional check, avoid yielding "wide" values (ex. mimikatz.exe:0x471EE6 LEA EBX, [ECX + EAX*0x4])
-            op_objs = insn.getOpObjects(i)
-            if len(op_objs) == 3:  # ECX, EAX, 0x4
-                continue
-
-            if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
-                const = op_objs[-1].getUnsignedValue()
-                addr = ih.address
-
-                yield Number(const), addr
-                yield OperandNumber(i, const), addr
-        elif not OperandType.isScalar(insn.getOperandType(i)):
-            # skip things like:
-            #   references, void types
-            continue
-        else:
-            const = insn.getScalar(i).getUnsignedValue()
-            addr = ih.address
-
-            yield Number(const), addr
-            yield OperandNumber(i, const), addr
-
-            if insn.getMnemonicString().startswith("ADD") and 0 < const < MAX_STRUCTURE_SIZE:
-                # for pattern like:
-                #
-                #     add eax, 0x10
-                #
-                # assume 0x10 is also an offset (imagine eax is a pointer).
-                yield Offset(const), addr
-                yield OperandOffset(i, const), addr
-
-
-def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction structure offset features
-
-    example:
-        .text:0040112F cmp [esi+4], ebx
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if insn.getMnemonicString().startswith("LEA"):
-        return
-
-    # ignore any stack references
-    if not capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
-        # Ghidra stores operands in 2D arrays if they contain offsets
-        for i in range(insn.getNumOperands()):
-            if insn.getOperandType(i) == OperandType.DYNAMIC:  # e.g. [esi + 4]
-                # manual extraction, since the default api calls only work on the 1st dimension of the array
-                op_objs = insn.getOpObjects(i)
-                if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
-                    op_off = op_objs[-1].getValue()
-                    yield Offset(op_off), ih.address
-                    yield OperandOffset(i, op_off), ih.address
-                else:
-                    yield Offset(0), ih.address
-                    yield OperandOffset(i, 0), ih.address
-
-
-def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse referenced byte sequences
-    example:
-        push    offset iid_004118d4_IShellLinkA ; riid
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
-        return
-
-    ref = insn.getAddress()  # init to insn addr
-    for i in range(insn.getNumOperands()):
-        if OperandType.isAddress(insn.getOperandType(i)):
-            ref = insn.getAddress(i)  # pulls pointer if there is one
-
-    if ref != insn.getAddress():  # bail out if there's no pointer
-        ghidra_dat = getDataAt(ref)  # type: ignore [name-defined] # noqa: F821
-        if (
-            ghidra_dat and not ghidra_dat.hasStringValue() and not ghidra_dat.isPointer()
-        ):  # avoid if the data itself is a pointer
-            extracted_bytes = capa.features.extractors.ghidra.helpers.get_bytes(ref, MAX_BYTES_FEATURE_SIZE)
-            if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-                # don't extract byte features for obvious strings
-                yield Bytes(extracted_bytes), ih.address
-
-
-def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction string features
-
-    example:
-        push offset aAcr     ; "ACR  > "
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-    dyn_addr = OperandType.DYNAMIC | OperandType.ADDRESS
-
-    ref = insn.getAddress()
-    for i in range(insn.getNumOperands()):
-        if OperandType.isScalarAsAddress(insn.getOperandType(i)):
-            ref = insn.getAddress(i)
-        # strings are also referenced dynamically via pointers & arrays, so we need to deref them
-        if insn.getOperandType(i) == dyn_addr:
-            ref = insn.getAddress(i)
-            dat = getDataAt(ref)  # type: ignore [name-defined] # noqa: F821
-            if dat and dat.isPointer():
-                ref = dat.getValue()
-
-    if ref != insn.getAddress():
-        ghidra_dat = getDataAt(ref)  # type: ignore [name-defined] # noqa: F821
-        if ghidra_dat and ghidra_dat.hasStringValue():
-            yield String(ghidra_dat.getValue()), ih.address
-
-
-def extract_insn_mnemonic_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction mnemonic features"""
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    yield Mnemonic(insn.getMnemonicString().lower()), ih.address
-
-
-def extract_insn_obfs_call_plus_5_characteristic_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse call $+5 instruction from the given instruction.
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
-        return
-
-    code_ref = OperandType.ADDRESS | OperandType.CODE
-    ref = insn.getAddress()
-    for i in range(insn.getNumOperands()):
-        if insn.getOperandType(i) == code_ref:
-            ref = insn.getAddress(i)
-
-    if insn.getAddress().add(5) == ref:
-        yield Characteristic("call $+5"), ih.address
-
-
-def extract_insn_segment_access_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction fs or gs access"""
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    insn_str = insn.toString()
-
-    if "FS:" in insn_str:
-        yield Characteristic("fs access"), ih.address
-
-    if "GS:" in insn_str:
-        yield Characteristic("gs access"), ih.address
-
-
-def extract_insn_peb_access_characteristic_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction peb access
-
-    fs:[0x30] on x86, gs:[0x60] on x64
-
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    insn_str = insn.toString()
-    if insn_str.startswith(("PUSH", "MOV")):
-        if "FS:[0x30]" in insn_str or "GS:[0x60]" in insn_str:
-            yield Characteristic("peb access"), ih.address
-
-
-def extract_insn_cross_section_cflow(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
-        return
-
-    externs = get_externs(fh.ctx)
-    fakes = get_fakes(fh.ctx)
-    imports = get_imports(fh.ctx)
-
-    # OperandType to dereference
-    addr_data = OperandType.ADDRESS | OperandType.DATA
-    addr_code = OperandType.ADDRESS | OperandType.CODE
-
-    ref_type = insn.getOperandType(0)
-
-    # both OperandType flags must be present
-    # bail on REGISTER alone
-    if OperandType.isRegister(ref_type):
-        if OperandType.isAddress(ref_type):
-            ref = insn.getAddress(0)  # Ghidra dereferences REG | ADDR
-            if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
-                return
-        else:
-            return
-    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
-        # we must dereference and check if the addr is a pointer to an api function
-        ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
-        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
-            return
-    elif ref_type == OperandType.DYNAMIC | OperandType.ADDRESS or ref_type == OperandType.DYNAMIC:
-        return  # cannot resolve dynamics statically
-    else:
-        # pure address does not need to get dereferenced/ handled
-        ref = insn.getAddress(0)
-        if not ref:
-            # If it returned null, it was an indirect
-            # that had no address reference.
-            # This check is faster than checking for (indirect and not address)
-            return
-        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
-            return
-
-    this_mem_block = getMemoryBlock(insn.getAddress())  # type: ignore [name-defined] # noqa: F821
-    ref_block = getMemoryBlock(ref)  # type: ignore [name-defined] # noqa: F821
-    if ref_block != this_mem_block:
-        yield Characteristic("cross section flow"), ih.address
-
-
-def extract_function_calls_from(
-    fh: FunctionHandle,
-    bb: BBHandle,
-    ih: InsnHandle,
-) -> Iterator[Tuple[Feature, Address]]:
-    """extract functions calls from features
-
-    most relevant at the function scope, however, its most efficient to extract at the instruction scope
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if insn.getMnemonicString().startswith("CALL"):
-        # This method of "dereferencing" addresses/ pointers
-        # is not as robust as methods in other functions,
-        # but works just fine for this one
-        reference = 0
-        for ref in insn.getReferencesFrom():
-            addr = ref.getToAddress()
-
-            # avoid returning fake addrs
-            if not addr.isExternalAddress():
-                reference = addr.getOffset()
-
-            # if a reference is < 0, then ghidra pulled an offset from a DYNAMIC | ADDR (usually a stackvar)
-            # these cannot be resolved to actual addrs
-            if reference > 0:
-                yield Characteristic("calls from"), AbsoluteVirtualAddress(reference)
-
-
-def extract_function_indirect_call_characteristic_features(
-    fh: FunctionHandle,
-    bb: BBHandle,
-    ih: InsnHandle,
-) -> Iterator[Tuple[Feature, Address]]:
-    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
-    does not include calls like => call ds:dword_ABD4974
-
-    most relevant at the function or basic block scope;
-    however, its most efficient to extract at the instruction scope
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if insn.getMnemonicString().startswith("CALL"):
-        if OperandType.isRegister(insn.getOperandType(0)):
-            yield Characteristic("indirect call"), ih.address
-        if OperandType.isIndirect(insn.getOperandType(0)):
-            yield Characteristic("indirect call"), ih.address
-
-
-def check_nzxor_security_cookie_delta(
-    fh: ghidra.program.database.function.FunctionDB, insn: ghidra.program.database.code.InstructionDB
-):
-    """Get the function containing the insn
-    Get the last block of the function that contains the insn
-
-    Check the bb containing the insn
-    Check the last bb of the function containing the insn
-    """
-
-    model = SimpleBlockModel(currentProgram())  # type: ignore [name-defined] # noqa: F821
-    insn_addr = insn.getAddress()
-    func_asv = fh.getBody()
-    first_addr = func_asv.getMinAddress()
-    last_addr = func_asv.getMaxAddress()
-
-    if model.getFirstCodeBlockContaining(
-        first_addr, monitor()  # type: ignore [name-defined] # noqa: F821
-    ) == model.getFirstCodeBlockContaining(
-        last_addr, monitor()  # type: ignore [name-defined] # noqa: F821
-    ):
-        if insn_addr < first_addr.add(SECURITY_COOKIE_BYTES_DELTA):
-            return True
-        else:
-            return insn_addr > last_addr.add(SECURITY_COOKIE_BYTES_DELTA * -1)
-    else:
-        return False
-
-
-def extract_insn_nzxor_characteristic_features(
-    fh: FunctionHandle,
-    bb: BBHandle,
-    ih: InsnHandle,
-) -> Iterator[Tuple[Feature, Address]]:
-    f: ghidra.program.database.function.FunctionDB = fh.inner
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if "XOR" not in insn.getMnemonicString():
-        return
-    if capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
-        return
-    if capa.features.extractors.ghidra.helpers.is_zxor(insn):
-        return
-    if check_nzxor_security_cookie_delta(f, insn):
-        return
-    yield Characteristic("nzxor"), ih.address
-
-
-def extract_features(
-    fh: FunctionHandle,
-    bb: BBHandle,
-    insn: InsnHandle,
-) -> Iterator[Tuple[Feature, Address]]:
-    for insn_handler in INSTRUCTION_HANDLERS:
-        for feature, addr in insn_handler(fh, bb, insn):
-            yield feature, addr
-
-
-INSTRUCTION_HANDLERS = (
-    extract_insn_api_features,
-    extract_insn_number_features,
-    extract_insn_bytes_features,
-    extract_insn_string_features,
-    extract_insn_offset_features,
-    extract_insn_nzxor_characteristic_features,
-    extract_insn_mnemonic_features,
-    extract_insn_obfs_call_plus_5_characteristic_features,
-    extract_insn_peb_access_characteristic_features,
-    extract_insn_cross_section_cflow,
-    extract_insn_segment_access_features,
-    extract_function_calls_from,
-    extract_function_indirect_call_characteristic_features,
-)
-
-
-def main():
-    """ """
-    features = []
-    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
-
-    for fh in GhidraFeatureExtractor().get_functions():
-        for bb in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
-            for insn in capa.features.extractors.ghidra.helpers.get_insn_in_range(bb):
-                features.extend(list(extract_features(fh, bb, insn)))
-
-    import pprint
-
-    pprint.pprint(features)  # noqa: T203
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -55,7 +55,7 @@ def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
     dll = dll.lower()
 
     # kernel32.CreateFileA
-    yield f"{dll}.{symbol}"
+    yield "%s.%s" % (dll, symbol)
 
     if not is_ordinal(symbol):
         # CreateFileA
@@ -63,30 +63,13 @@ def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
 
     if is_aw_function(symbol):
         # kernel32.CreateFile
-        yield f"{dll}.{symbol[:-1]}"
+        yield "%s.%s" % (dll, symbol[:-1])
 
         if not is_ordinal(symbol):
             # CreateFile
             yield symbol[:-1]
 
 
-def reformat_forwarded_export_name(forwarded_name: str) -> str:
-    """
-    a forwarded export has a DLL name/path an symbol name.
-    we want the former to be lowercase, and the latter to be verbatim.
-    """
-
-    # use rpartition so we can split on separator between dll and name.
-    # the dll name can be a full path, like in the case of
-    # ef64d6d7c34250af8e21a10feb931c9b
-    # which i assume means the path can have embedded periods.
-    # so we don't want the first period, we want the last.
-    forwarded_dll, _, forwarded_symbol = forwarded_name.rpartition(".")
-    forwarded_dll = forwarded_dll.lower()
-
-    return f"{forwarded_dll}.{forwarded_symbol}"
-
-
 def all_zeros(bytez: bytes) -> bool:
     return all(b == 0 for b in builtins.bytes(bytez))
 

capa/features/extractors/ida/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -34,7 +34,7 @@ def get_printable_len(op: idaapi.op_t) -> int:
     elif op.dtype == idaapi.dt_qword:
         chars = struct.pack("<Q", op_val)
     else:
-        raise ValueError(f"Unhandled operand data type 0x{op.dtype:x}.")
+        raise ValueError("Unhandled operand data type 0x%x." % op.dtype)
 
     def is_printable_ascii(chars_: bytes):
         return all(c < 127 and chr(c) in string.printable for c in chars_)
@@ -104,3 +104,19 @@ BASIC_BLOCK_HANDLERS = (
     extract_bb_tight_loop,
     extract_bb_stackstring,
 )
+
+
+def main():
+    features = []
+    for fhandle in helpers.get_functions(skip_thunks=True, skip_libs=True):
+        f: idaapi.func_t = fhandle.inner
+        for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
+            features.extend(list(extract_features(fhandle, bb)))
+
+    import pprint
+
+    pprint.pprint(features)
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ida/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/ida/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -12,7 +12,6 @@ from typing import Tuple, Iterator
 import idc
 import idaapi
 import idautils
-import ida_entry
 
 import capa.features.extractors.common
 import capa.features.extractors.helpers
@@ -22,14 +21,12 @@ from capa.features.file import Export, Import, Section, FunctionName
 from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
 from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
 
-MAX_OFFSET_PE_AFTER_MZ = 0x200
-
 
 def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
     """check segment for embedded PE
 
     adapted for IDA from:
-    https://github.com/vivisect/vivisect/blob/91e8419a861f49779f18316f155311967e696836/PE/carve.py#L25
+    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
     """
     seg_max = seg.end_ea
     mz_xor = [
@@ -43,14 +40,13 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
 
     todo = []
     for mzx, pex, i in mz_xor:
-        # find all segment offsets containing XOR'd "MZ" bytes
         for off in capa.features.extractors.ida.helpers.find_byte_sequence(seg.start_ea, seg.end_ea, mzx):
             todo.append((off, mzx, pex, i))
 
     while len(todo):
         off, mzx, pex, i = todo.pop()
 
-        # MZ header has one field we will check e_lfanew is at 0x3c
+        # The MZ header has one field we will check e_lfanew is at 0x3c
         e_lfanew = off + 0x3C
 
         if seg_max < (e_lfanew + 4):
@@ -58,10 +54,6 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
 
         newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(idc.get_bytes(e_lfanew, 4), i))[0]
 
-        # assume XOR'd "PE" bytes exist within threshold
-        if newoff > MAX_OFFSET_PE_AFTER_MZ:
-            continue
-
         peoff = off + newoff
         if seg_max < (peoff + 2):
             continue
@@ -69,6 +61,9 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
         if idc.get_bytes(peoff, 2) == pex:
             yield off, i
 
+        for nextres in capa.features.extractors.ida.helpers.find_byte_sequence(off + 1, seg.end_ea, mzx):
+            todo.append((nextres, mzx, pex, i))
+
 
 def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
     """extract embedded PE features
@@ -84,14 +79,8 @@ def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
 
 def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
     """extract function exports"""
-    for _, ordinal, ea, name in idautils.Entries():
-        forwarded_name = ida_entry.get_entry_forwarder(ordinal)
-        if forwarded_name is None:
-            yield Export(name), AbsoluteVirtualAddress(ea)
-        else:
-            forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
-            yield Export(forwarded_name), AbsoluteVirtualAddress(ea)
-            yield Characteristic("forwarded export"), AbsoluteVirtualAddress(ea)
+    for _, _, ea, name in idautils.Entries():
+        yield Export(name), AbsoluteVirtualAddress(ea)
 
 
 def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
@@ -113,13 +102,13 @@ def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
             for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1]):
                 yield Import(name), addr
             dll = info[0]
-            symbol = f"#{info[2]}"
+            symbol = "#%d" % (info[2])
         elif info[1]:
             dll = info[0]
             symbol = info[1]
         elif info[2]:
             dll = info[0]
-            symbol = f"#{info[2]}"
+            symbol = "#%d" % (info[2])
         else:
             continue
 
@@ -187,7 +176,7 @@ def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
         # no file type to return when processing a binary file, but we want to continue processing
         return
     else:
-        raise NotImplementedError(f"unexpected file format: {file_info.filetype}")
+        raise NotImplementedError("unexpected file format: %d" % file_info.filetype)
 
 
 def extract_features() -> Iterator[Tuple[Feature, Address]]:
@@ -206,3 +195,14 @@ FILE_HANDLERS = (
     extract_file_function_names,
     extract_file_format,
 )
+
+
+def main():
+    """ """
+    import pprint
+
+    pprint.pprint(list(extract_features()))
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ida/function.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -50,3 +50,18 @@ def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
 
 
 FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
+
+
+def main():
+    """ """
+    features = []
+    for fhandle in capa.features.extractors.ida.helpers.get_functions(skip_thunks=True, skip_libs=True):
+        features.extend(list(extract_features(fhandle)))
+
+    import pprint
+
+    pprint.pprint(features)
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ida/global_.py

@@ -1,10 +1,3 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 import logging
 import contextlib
 from typing import Tuple, Iterator

capa/features/extractors/ida/helpers.py

@@ -1,11 +1,10 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import functools
 from typing import Any, Dict, Tuple, Iterator, Optional
 
 import idc
@@ -26,10 +25,9 @@ def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
         end: max virtual address
         seq: bytes to search e.g. b"\x01\x03"
     """
-    seqstr = " ".join([f"{b:02x}" for b in seq])
+    seqstr = " ".join(["%02x" % b for b in seq])
     while True:
-        # TODO(mike-hunhoff): find_binary is deprecated. Please use ida_bytes.bin_search() instead.
-        # https://github.com/mandiant/capa/issues/1606
+        # TODO find_binary: Deprecated. Please use ida_bytes.bin_search() instead.
         ea = idaapi.find_binary(start, end, seqstr, 0, idaapi.SEARCH_DOWN)
         if ea == idaapi.BADADDR:
             break
@@ -82,22 +80,9 @@ def get_segment_buffer(seg: idaapi.segment_t) -> bytes:
     return buff if buff else b""
 
 
-def inspect_import(imports, library, ea, function, ordinal):
-    if function and function.startswith("__imp_"):
-        # handle mangled PE imports
-        function = function[len("__imp_") :]
-
-    if function and "@@" in function:
-        # handle mangled ELF imports, like "fopen@@glibc_2.2.5"
-        function, _, _ = function.partition("@@")
-
-    imports[ea] = (library.lower(), function, ordinal)
-    return True
-
-
 def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
     """get file imports"""
-    imports: Dict[int, Tuple[str, str, int]] = {}
+    imports = {}
 
     for idx in range(idaapi.get_import_module_qty()):
         library = idaapi.get_import_module_name(idx)
@@ -105,15 +90,22 @@ def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
         if not library:
             continue
 
-        # IDA uses section names for the library of ELF imports, like ".dynsym".
-        # These are not useful to us, we may need to expand this list over time
-        # TODO(williballenthin): find all section names used by IDA
-        # https://github.com/mandiant/capa/issues/1419
-        if library == ".dynsym":
-            library = ""
+        # IDA uses section names for the library of ELF imports, like ".dynsym"
+        library = library.lstrip(".")
 
-        cb = functools.partial(inspect_import, imports, library)
-        idaapi.enum_import_names(idx, cb)
+        def inspect_import(ea, function, ordinal):
+            if function and function.startswith("__imp_"):
+                # handle mangled PE imports
+                function = function[len("__imp_") :]
+
+            if function and "@@" in function:
+                # handle mangled ELF imports, like "fopen@@glibc_2.2.5"
+                function, _, _ = function.partition("@@")
+
+            imports[ea] = (library.lower(), function, ordinal)
+            return True
+
+        idaapi.enum_import_names(idx, inspect_import)
 
     return imports
 
@@ -122,7 +114,7 @@ def get_file_externs() -> Dict[int, Tuple[str, str, int]]:
     externs = {}
 
     for seg in get_segments(skip_header_segments=True):
-        if seg.type != ida_segment.SEG_XTRN:
+        if not (seg.type == ida_segment.SEG_XTRN):
             continue
 
         for ea in idautils.Functions(seg.start_ea, seg.end_ea):
@@ -275,18 +267,20 @@ def is_op_offset(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
 
 def is_sp_modified(insn: idaapi.insn_t) -> bool:
     """determine if instruction modifies SP, ESP, RSP"""
-    return any(
-        op.reg == idautils.procregs.sp.reg and is_op_write(insn, op)
-        for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,))
-    )
+    for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
+        if op.reg == idautils.procregs.sp.reg and is_op_write(insn, op):
+            # register is stack and written
+            return True
+    return False
 
 
 def is_bp_modified(insn: idaapi.insn_t) -> bool:
     """check if instruction modifies BP, EBP, RBP"""
-    return any(
-        op.reg == idautils.procregs.bp.reg and is_op_write(insn, op)
-        for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,))
-    )
+    for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
+        if op.reg == idautils.procregs.bp.reg and is_op_write(insn, op):
+            # register is base and written
+            return True
+    return False
 
 
 def is_frame_register(reg: int) -> bool:
@@ -332,7 +326,10 @@ def mask_op_val(op: idaapi.op_t) -> int:
 
 def is_function_recursive(f: idaapi.func_t) -> bool:
     """check if function is recursive"""
-    return any(f.contains(ref) for ref in idautils.CodeRefsTo(f.start_ea, True))
+    for ref in idautils.CodeRefsTo(f.start_ea, True):
+        if f.contains(ref):
+            return True
+    return False
 
 
 def is_basic_block_tight_loop(bb: idaapi.BasicBlock) -> bool:
@@ -381,7 +378,8 @@ def find_data_reference_from_insn(insn: idaapi.insn_t, max_depth: int = 10) -> i
 def get_function_blocks(f: idaapi.func_t) -> Iterator[idaapi.BasicBlock]:
     """yield basic blocks contained in specified function"""
     # leverage idaapi.FC_NOEXT flag to ignore useless external blocks referenced by the function
-    yield from idaapi.FlowChart(f, flags=(idaapi.FC_PREDS | idaapi.FC_NOEXT))
+    for block in idaapi.FlowChart(f, flags=(idaapi.FC_PREDS | idaapi.FC_NOEXT)):
+        yield block
 
 
 def is_basic_block_return(bb: idaapi.BasicBlock) -> bool:

capa/features/extractors/ida/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -73,7 +73,7 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
     """
     insn: idaapi.insn_t = ih.inner
 
-    if insn.get_canon_mnem() not in ("call", "jmp"):
+    if not insn.get_canon_mnem() in ("call", "jmp"):
         return
 
     # check calls to imported functions
@@ -172,7 +172,7 @@ def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
     if ref != insn.ea:
         extracted_bytes = capa.features.extractors.ida.helpers.read_bytes_at(ref, MAX_BYTES_FEATURE_SIZE)
         if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-            if not capa.features.extractors.ida.helpers.find_string_at(ref):
+            if not capa.features.extractors.ida.helpers.find_string_at(insn.ea):
                 # don't extract byte features for obvious strings
                 yield Bytes(extracted_bytes), ih.address
 
@@ -216,7 +216,7 @@ def extract_insn_offset_features(
 
         p_info = capa.features.extractors.ida.helpers.get_op_phrase_info(op)
 
-        op_off = p_info.get("offset")
+        op_off = p_info.get("offset", None)
         if op_off is None:
             continue
 
@@ -398,16 +398,14 @@ def extract_insn_peb_access_characteristic_features(
     if insn.itype not in (idaapi.NN_push, idaapi.NN_mov):
         return
 
-    if all(op.type != idaapi.o_mem for op in insn.ops):
+    if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
         # try to optimize for only memory references
         return
 
     disasm = idc.GetDisasm(insn.ea)
 
     if " fs:30h" in disasm or " gs:60h" in disasm:
-        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
-        # scanning the disassembly text is a hack.
-        # https://github.com/mandiant/capa/issues/1605
+        # TODO: replace above with proper IDA
         yield Characteristic("peb access"), ih.address
 
 
@@ -421,22 +419,18 @@ def extract_insn_segment_access_features(
     """
     insn: idaapi.insn_t = ih.inner
 
-    if all(op.type != idaapi.o_mem for op in insn.ops):
+    if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
         # try to optimize for only memory references
         return
 
     disasm = idc.GetDisasm(insn.ea)
 
     if " fs:" in disasm:
-        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
-        # scanning the disassembly text is a hack.
-        # https://github.com/mandiant/capa/issues/1605
+        # TODO: replace above with proper IDA
         yield Characteristic("fs access"), ih.address
 
     if " gs:" in disasm:
-        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
-        # scanning the disassembly text is a hack.
-        # https://github.com/mandiant/capa/issues/1605
+        # TODO: replace above with proper IDA
         yield Characteristic("gs access"), ih.address
 
 
@@ -447,7 +441,7 @@ def extract_insn_cross_section_cflow(
     insn: idaapi.insn_t = ih.inner
 
     for ref in idautils.CodeRefsFrom(insn.ea, False):
-        if ref in get_imports(fh.ctx):
+        if ref in get_imports(fh.ctx).keys():
             # ignore API calls
             continue
         if not idaapi.getseg(ref):
@@ -507,3 +501,20 @@ INSTRUCTION_HANDLERS = (
     extract_function_calls_from,
     extract_function_indirect_call_characteristic_features,
 )
+
+
+def main():
+    """ """
+    features = []
+    for f in capa.features.extractors.ida.helpers.get_functions(skip_thunks=True, skip_libs=True):
+        for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
+            for insn in capa.features.extractors.ida.helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
+                features.extend(list(extract_features(f, bb, insn)))
+
+    import pprint
+
+    pprint.pprint(features)
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/loops.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/null.py

@@ -1,10 +1,3 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 from typing import Dict, List, Tuple
 from dataclasses import dataclass
 

capa/features/extractors/pefile.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,7 +7,6 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import logging
-from pathlib import Path
 
 import pefile
 
@@ -40,20 +39,8 @@ def extract_file_export_names(pe, **kwargs):
                 name = export.name.partition(b"\x00")[0].decode("ascii")
             except UnicodeDecodeError:
                 continue
-
-            if export.forwarder is None:
-                va = base_address + export.address
-                yield Export(name), AbsoluteVirtualAddress(va)
-
-            else:
-                try:
-                    forwarded_name = export.forwarder.partition(b"\x00")[0].decode("ascii")
-                except UnicodeDecodeError:
-                    continue
-                forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
-                va = base_address + export.address
-                yield Export(forwarded_name), AbsoluteVirtualAddress(va)
-                yield Characteristic("forwarded export"), AbsoluteVirtualAddress(va)
+            va = base_address + export.address
+            yield Export(name), AbsoluteVirtualAddress(va)
 
 
 def extract_file_import_names(pe, **kwargs):
@@ -77,7 +64,7 @@ def extract_file_import_names(pe, **kwargs):
 
             for imp in dll.imports:
                 if imp.import_by_ordinal:
-                    impname = f"#{imp.ordinal}"
+                    impname = "#%s" % imp.ordinal
                 else:
                     try:
                         impname = imp.name.partition(b"\x00")[0].decode("ascii")
@@ -186,21 +173,23 @@ GLOBAL_HANDLERS = (
 
 
 class PefileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: Path):
+    def __init__(self, path: str):
         super().__init__()
-        self.path: Path = path
-        self.pe = pefile.PE(str(path))
+        self.path = path
+        self.pe = pefile.PE(path)
 
     def get_base_address(self):
         return AbsoluteVirtualAddress(self.pe.OPTIONAL_HEADER.ImageBase)
 
     def extract_global_features(self):
-        buf = Path(self.path).read_bytes()
+        with open(self.path, "rb") as f:
+            buf = f.read()
 
         yield from extract_global_features(self.pe, buf)
 
     def extract_file_features(self):
-        buf = Path(self.path).read_bytes()
+        with open(self.path, "rb") as f:
+            buf = f.read()
 
         yield from extract_file_features(self.pe, buf)
 

capa/features/extractors/strings.py

@@ -1,6 +1,6 @@
 # strings code from FLOSS, https://github.com/mandiant/flare-floss
 #
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -9,7 +9,6 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import re
-import contextlib
 from collections import namedtuple
 
 ASCII_BYTE = r" !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t".encode(
@@ -82,5 +81,24 @@ def extract_unicode_strings(buf, n=4):
         reg = b"((?:[%s]\x00){%d,})" % (ASCII_BYTE, n)
         r = re.compile(reg)
     for match in r.finditer(buf):
-        with contextlib.suppress(UnicodeDecodeError):
+        try:
             yield String(match.group().decode("utf-16"), match.start())
+        except UnicodeDecodeError:
+            pass
+
+
+def main():
+    import sys
+
+    with open(sys.argv[1], "rb") as f:
+        b = f.read()
+
+    for s in extract_ascii_strings(b):
+        print("0x{:x}: {:s}".format(s.offset, s.s))
+
+    for s in extract_unicode_strings(b):
+        print("0x{:x}: {:s}".format(s.offset, s.s))
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/viv/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -92,6 +92,7 @@ def is_mov_imm_to_stack(instr: envi.archs.i386.disasm.i386Opcode) -> bool:
     if not src.isImmed():
         return False
 
+    # TODO what about 64-bit operands?
     if not isinstance(dst, envi.archs.i386.disasm.i386SibOper) and not isinstance(
         dst, envi.archs.i386.disasm.i386RegMemOper
     ):
@@ -120,7 +121,7 @@ def get_printable_len(oper: envi.archs.i386.disasm.i386ImmOper) -> int:
     elif oper.tsize == 8:
         chars = struct.pack("<Q", oper.imm)
     else:
-        raise ValueError(f"unexpected oper.tsize: {oper.tsize}")
+        raise ValueError("unexpected oper.tsize: %d" % (oper.tsize))
 
     if is_printable_ascii(chars):
         return oper.tsize

capa/features/extractors/viv/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,8 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import logging
-from typing import Any, Dict, List, Tuple, Iterator
-from pathlib import Path
+from typing import List, Tuple, Iterator
 
 import viv_utils
 import viv_utils.flirt
@@ -26,16 +25,17 @@ logger = logging.getLogger(__name__)
 
 
 class VivisectFeatureExtractor(FeatureExtractor):
-    def __init__(self, vw, path: Path, os):
+    def __init__(self, vw, path):
         super().__init__()
         self.vw = vw
         self.path = path
-        self.buf = path.read_bytes()
+        with open(self.path, "rb") as f:
+            self.buf = f.read()
 
         # pre-compute these because we'll yield them at *every* scope.
         self.global_features: List[Tuple[Feature, Address]] = []
         self.global_features.extend(capa.features.extractors.viv.file.extract_file_format(self.buf))
-        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf, os))
+        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf))
         self.global_features.extend(capa.features.extractors.viv.global_.extract_arch(self.vw))
 
     def get_base_address(self):
@@ -49,11 +49,8 @@ class VivisectFeatureExtractor(FeatureExtractor):
         yield from capa.features.extractors.viv.file.extract_features(self.vw, self.buf)
 
     def get_functions(self) -> Iterator[FunctionHandle]:
-        cache: Dict[str, Any] = {}
         for va in sorted(self.vw.getFunctions()):
-            yield FunctionHandle(
-                address=AbsoluteVirtualAddress(va), inner=viv_utils.Function(self.vw, va), ctx={"cache": cache}
-            )
+            yield FunctionHandle(address=AbsoluteVirtualAddress(va), inner=viv_utils.Function(self.vw, va))
 
     def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
         yield from capa.features.extractors.viv.function.extract_features(fh)

capa/features/extractors/viv/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,7 +8,6 @@
 from typing import Tuple, Iterator
 
 import PE.carve as pe_carve  # vivisect PE
-import vivisect
 import viv_utils
 import viv_utils.flirt
 
@@ -17,7 +16,7 @@ import capa.features.extractors.common
 import capa.features.extractors.helpers
 import capa.features.extractors.strings
 from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import Feature, Characteristic
+from capa.features.common import String, Feature, Characteristic
 from capa.features.address import Address, FileOffsetAddress, AbsoluteVirtualAddress
 
 
@@ -26,35 +25,10 @@ def extract_file_embedded_pe(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]
         yield Characteristic("embedded pe"), FileOffsetAddress(offset)
 
 
-def get_first_vw_filename(vw: vivisect.VivWorkspace):
-    # vivisect associates metadata with each file that its loaded into the workspace.
-    # capa only loads a single file into each workspace.
-    # so to access the metadata for the file in question, we can just take the first one.
-    # otherwise, we'd have to pass around the module name of the file we're analyzing,
-    # which is a pain.
-    #
-    # so this is a simplifying assumption.
-    return next(iter(vw.filemeta.keys()))
-
-
-def extract_file_export_names(vw: vivisect.VivWorkspace, **kwargs) -> Iterator[Tuple[Feature, Address]]:
+def extract_file_export_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     for va, _, name, _ in vw.getExports():
         yield Export(name), AbsoluteVirtualAddress(va)
 
-    if vw.getMeta("Format") == "pe":
-        pe = vw.parsedbin
-        baseaddr = pe.IMAGE_NT_HEADERS.OptionalHeader.ImageBase
-        for rva, _, forwarded_name in vw.getFileMeta(get_first_vw_filename(vw), "forwarders"):
-            try:
-                forwarded_name = forwarded_name.partition(b"\x00")[0].decode("ascii")
-            except UnicodeDecodeError:
-                continue
-
-            forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
-            va = baseaddr + rva
-            yield Export(forwarded_name), AbsoluteVirtualAddress(va)
-            yield Characteristic("forwarded export"), AbsoluteVirtualAddress(va)
-
 
 def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
     """
@@ -70,7 +44,7 @@ def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]
         modname, impname = tinfo.split(".", 1)
         if is_viv_ord_impname(impname):
             # replace ord prefix with #
-            impname = "#" + impname[len("ord") :]
+            impname = "#%s" % impname[len("ord") :]
 
         addr = AbsoluteVirtualAddress(va)
         for name in capa.features.extractors.helpers.generate_symbols(modname, impname):

capa/features/extractors/viv/function.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -11,11 +11,9 @@ import envi
 import viv_utils
 import vivisect.const
 
-from capa.features.file import FunctionName
 from capa.features.common import Feature, Characteristic
 from capa.features.address import Address, AbsoluteVirtualAddress
 from capa.features.extractors import loops
-from capa.features.extractors.elf import SymTab
 from capa.features.extractors.base_extractor import FunctionHandle
 
 
@@ -32,28 +30,6 @@ def interface_extract_function_XXX(fh: FunctionHandle) -> Iterator[Tuple[Feature
     raise NotImplementedError
 
 
-def extract_function_symtab_names(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    if fh.inner.vw.metadata["Format"] == "elf":
-        # the file's symbol table gets added to the metadata of the vivisect workspace.
-        # this is in order to eliminate the computational overhead of refetching symtab each time.
-        if "symtab" not in fh.ctx["cache"]:
-            try:
-                fh.ctx["cache"]["symtab"] = SymTab.from_viv(fh.inner.vw.parsedbin)
-            except Exception:
-                fh.ctx["cache"]["symtab"] = None
-
-        symtab = fh.ctx["cache"]["symtab"]
-        if symtab:
-            for symbol in symtab.get_symbols():
-                sym_name = symtab.get_name(symbol)
-                sym_value = symbol.value
-                sym_info = symbol.info
-
-                STT_FUNC = 0x2
-                if sym_value == fh.address and sym_info & STT_FUNC != 0:
-                    yield FunctionName(sym_name), fh.address
-
-
 def extract_function_calls_to(fhandle: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
     f: viv_utils.Function = fhandle.inner
     for src, _, _, _ in f.vw.getXrefsTo(f.va, rtype=vivisect.const.REF_CODE):
@@ -103,8 +79,4 @@ def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
             yield feature, addr
 
 
-FUNCTION_HANDLERS = (
-    extract_function_symtab_names,
-    extract_function_calls_to,
-    extract_function_loop,
-)
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)

capa/features/extractors/viv/global_.py

@@ -1,13 +1,9 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 import logging
 from typing import Tuple, Iterator
 
+import envi.archs.i386
+import envi.archs.amd64
+
 from capa.features.common import ARCH_I386, ARCH_AMD64, Arch, Feature
 from capa.features.address import NO_ADDRESS, Address
 
@@ -15,11 +11,10 @@ logger = logging.getLogger(__name__)
 
 
 def extract_arch(vw) -> Iterator[Tuple[Feature, Address]]:
-    arch = vw.getMeta("Architecture")
-    if arch == "amd64":
+    if isinstance(vw.arch, envi.archs.amd64.Amd64Module):
         yield Arch(ARCH_AMD64), NO_ADDRESS
 
-    elif arch == "i386":
+    elif isinstance(vw.arch, envi.archs.i386.i386Module):
         yield Arch(ARCH_I386), NO_ADDRESS
 
     else:

capa/features/extractors/viv/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/viv/indirect_calls.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,7 +7,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import collections
-from typing import Set, List, Deque, Tuple, Optional
+from typing import Set, List, Deque, Tuple, Union, Optional
 
 import envi
 import vivisect.const
@@ -71,7 +71,7 @@ class NotFoundError(Exception):
     pass
 
 
-def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Optional[int]]:
+def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int, None]]:
     """
     scan backwards from the given address looking for assignments to the given register.
     if a constant, return that value.
@@ -87,8 +87,8 @@ def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Optional[
     raises:
       NotFoundError: when the definition cannot be found.
     """
-    q: Deque[int] = collections.deque()
-    seen: Set[int] = set()
+    q = collections.deque()  # type: Deque[int]
+    seen = set([])  # type: Set[int]
 
     q.extend(get_previous_instructions(vw, va))
     while q:

capa/features/extractors/viv/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -22,7 +22,6 @@ import capa.features.extractors.viv.helpers
 from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
 from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Feature, Characteristic
 from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.elf import SymTab
 from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
 from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_indirect_call
 
@@ -110,26 +109,6 @@ def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterato
         if not target:
             return
 
-        if f.vw.metadata["Format"] == "elf":
-            if "symtab" not in fh.ctx["cache"]:
-                # the symbol table gets stored as a function's attribute in order to avoid running
-                # this code everytime the call is made, thus preventing the computational overhead.
-                try:
-                    fh.ctx["cache"]["symtab"] = SymTab.from_viv(f.vw.parsedbin)
-                except Exception:
-                    fh.ctx["cache"]["symtab"] = None
-
-            symtab = fh.ctx["cache"]["symtab"]
-            if symtab:
-                for symbol in symtab.get_symbols():
-                    sym_name = symtab.get_name(symbol)
-                    sym_value = symbol.value
-                    sym_info = symbol.info
-
-                    STT_FUNC = 0x2
-                    if sym_value == target and sym_info & STT_FUNC != 0:
-                        yield API(sym_name), ih.address
-
         if viv_utils.flirt.is_library_function(f.vw, target):
             name = viv_utils.get_function_name(f.vw, target)
             yield API(name), ih.address
@@ -196,13 +175,8 @@ def derefs(vw, p):
     while True:
         if not vw.isValidPointer(p):
             return
-
         yield p
 
-        if vw.isProbablyString(p) or vw.isProbablyUnicode(p):
-            # don't deref strings that coincidentally are pointers
-            return
-
         try:
             next = vw.readMemoryPtr(p)
         except Exception:
@@ -288,16 +262,16 @@ def extract_insn_bytes_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera
         else:
             continue
 
-        for vv in derefs(f.vw, v):
+        for v in derefs(f.vw, v):
             try:
-                buf = read_bytes(f.vw, vv)
+                buf = read_bytes(f.vw, v)
             except envi.exc.SegmentationViolation:
                 continue
 
             if capa.features.extractors.helpers.all_zeros(buf):
                 continue
 
-            if f.vw.isProbablyString(vv) or f.vw.isProbablyUnicode(vv):
+            if f.vw.isProbablyString(v):
                 # don't extract byte features for obvious strings
                 continue
 
@@ -351,6 +325,7 @@ def is_security_cookie(f, bb, insn) -> bool:
     if oper.isReg() and oper.reg not in [
         envi.archs.i386.regs.REG_ESP,
         envi.archs.i386.regs.REG_EBP,
+        # TODO: do x64 support for real.
         envi.archs.amd64.regs.REG_RBP,
         envi.archs.amd64.regs.REG_RSP,
     ]:
@@ -410,7 +385,9 @@ def extract_insn_obfs_call_plus_5_characteristic_features(f, bb, ih: InsnHandle)
         if insn.va + 5 == insn.opers[0].getOperValue(insn):
             yield Characteristic("call $+5"), ih.address
 
-    if isinstance(insn.opers[0], (envi.archs.i386.disasm.i386ImmMemOper, envi.archs.amd64.disasm.Amd64RipRelOper)):
+    if isinstance(insn.opers[0], envi.archs.i386.disasm.i386ImmMemOper) or isinstance(
+        insn.opers[0], envi.archs.amd64.disasm.Amd64RipRelOper
+    ):
         if insn.va + 5 == insn.opers[0].getOperAddr(insn):
             yield Characteristic("call $+5"), ih.address
 
@@ -419,6 +396,7 @@ def extract_insn_peb_access_characteristic_features(f, bb, ih: InsnHandle) -> It
     """
     parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
     """
+    # TODO handle where fs/gs are loaded into a register or onto the stack and used later
     insn: envi.Opcode = ih.inner
 
     if insn.mnem not in ["push", "mov"]:
@@ -642,6 +620,7 @@ def extract_op_offset_features(
         if oper.reg == envi.archs.i386.regs.REG_EBP:
             return
 
+        # TODO: do x64 support for real.
         if oper.reg == envi.archs.amd64.regs.REG_RBP:
             return
 
@@ -695,9 +674,9 @@ def extract_op_string_features(
     else:
         return
 
-    for vv in derefs(f.vw, v):
+    for v in derefs(f.vw, v):
         try:
-            s = read_string(f.vw, vv).rstrip("\x00")
+            s = read_string(f.vw, v).rstrip("\x00")
         except ValueError:
             continue
         else:

capa/features/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/freeze/__init__.py

@@ -1,7 +1,7 @@
 """
 capa freeze file format: `| capa0000 | + zlib(utf-8(json(...)))`
 
-Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
 You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -12,9 +12,9 @@ See the License for the specific language governing permissions and limitations
 import zlib
 import logging
 from enum import Enum
-from typing import List, Tuple, Union
+from typing import Any, List, Tuple, Union
 
-from pydantic import Field, BaseModel, ConfigDict
+from pydantic import Field, BaseModel
 
 import capa.helpers
 import capa.version
@@ -31,7 +31,8 @@ logger = logging.getLogger(__name__)
 
 
 class HashableModel(BaseModel):
-    model_config = ConfigDict(frozen=True)
+    class Config:
+        frozen = True
 
 
 class AddressType(str, Enum):
@@ -45,7 +46,7 @@ class AddressType(str, Enum):
 
 class Address(HashableModel):
     type: AddressType
-    value: Union[int, Tuple[int, int], None] = None  # None default value to support deserialization of NO_ADDRESS
+    value: Union[int, Tuple[int, int], None]
 
     @classmethod
     def from_capa(cls, a: capa.features.address.Address) -> "Address":
@@ -158,7 +159,9 @@ class BasicBlockFeature(HashableModel):
     basic_block: Address = Field(alias="basic block")
     address: Address
     feature: Feature
-    model_config = ConfigDict(populate_by_name=True)
+
+    class Config:
+        allow_population_by_field_name = True
 
 
 class InstructionFeature(HashableModel):
@@ -191,20 +194,26 @@ class FunctionFeatures(BaseModel):
     address: Address
     features: Tuple[FunctionFeature, ...]
     basic_blocks: Tuple[BasicBlockFeatures, ...] = Field(alias="basic blocks")
-    model_config = ConfigDict(populate_by_name=True)
+
+    class Config:
+        allow_population_by_field_name = True
 
 
 class Features(BaseModel):
     global_: Tuple[GlobalFeature, ...] = Field(alias="global")
     file: Tuple[FileFeature, ...]
     functions: Tuple[FunctionFeatures, ...]
-    model_config = ConfigDict(populate_by_name=True)
+
+    class Config:
+        allow_population_by_field_name = True
 
 
 class Extractor(BaseModel):
     name: str
     version: str = capa.version.__version__
-    model_config = ConfigDict(populate_by_name=True)
+
+    class Config:
+        allow_population_by_field_name = True
 
 
 class Freeze(BaseModel):
@@ -212,7 +221,9 @@ class Freeze(BaseModel):
     base_address: Address = Field(alias="base address")
     extractor: Extractor
     features: Features
-    model_config = ConfigDict(populate_by_name=True)
+
+    class Config:
+        allow_population_by_field_name = True
 
 
 def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> str:
@@ -257,8 +268,7 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
                     basic_block=bbaddr,
                     address=Address.from_capa(addr),
                     feature=feature_from_capa(feature),
-                )  # type: ignore
-                # Mypy is unable to recognise `basic_block` as a argument due to alias
+                )
                 for feature, addr in extractor.extract_basic_block_features(f, bb)
             ]
 
@@ -277,52 +287,49 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
                 instructions.append(
                     InstructionFeatures(
                         address=iaddr,
-                        features=tuple(ifeatures),
+                        features=ifeatures,
                     )
                 )
 
             basic_blocks.append(
                 BasicBlockFeatures(
                     address=bbaddr,
-                    features=tuple(bbfeatures),
-                    instructions=tuple(instructions),
+                    features=bbfeatures,
+                    instructions=instructions,
                 )
             )
 
         function_features.append(
             FunctionFeatures(
                 address=faddr,
-                features=tuple(ffeatures),
+                features=ffeatures,
                 basic_blocks=basic_blocks,
-            )  # type: ignore
-            # Mypy is unable to recognise `basic_blocks` as a argument due to alias
+            )
         )
 
     features = Features(
         global_=global_features,
-        file=tuple(file_features),
-        functions=tuple(function_features),
-    )  # type: ignore
-    # Mypy is unable to recognise `global_` as a argument due to alias
+        file=file_features,
+        functions=function_features,
+    )
 
     freeze = Freeze(
         version=2,
         base_address=Address.from_capa(extractor.get_base_address()),
         extractor=Extractor(name=extractor.__class__.__name__),
         features=features,
-    )  # type: ignore
-    # Mypy is unable to recognise `base_address` as a argument due to alias
+    )
 
-    return freeze.model_dump_json()
+    return freeze.json()
 
 
 def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
     """deserialize a set of features (as a NullFeatureExtractor) from a string."""
     import capa.features.extractors.null as null
 
-    freeze = Freeze.model_validate_json(s)
+    freeze = Freeze.parse_raw(s)
     if freeze.version != 2:
-        raise ValueError(f"unsupported freeze format version: {freeze.version}")
+        raise ValueError("unsupported freeze format version: %d", freeze.version)
 
     return null.NullFeatureExtractor(
         base_address=freeze.base_address.to_capa(),
@@ -371,7 +378,6 @@ def load(buf: bytes) -> capa.features.extractors.base_extractor.FeatureExtractor
 def main(argv=None):
     import sys
     import argparse
-    from pathlib import Path
 
     import capa.main
 
@@ -379,16 +385,17 @@ def main(argv=None):
         argv = sys.argv[1:]
 
     parser = argparse.ArgumentParser(description="save capa features to a file")
-    capa.main.install_common_args(parser, {"sample", "format", "backend", "os", "signatures"})
+    capa.main.install_common_args(parser, {"sample", "format", "backend", "signatures"})
     parser.add_argument("output", type=str, help="Path to output file")
     args = parser.parse_args(args=argv)
     capa.main.handle_common_args(args)
 
     sigpaths = capa.main.get_signatures(args.signatures)
 
-    extractor = capa.main.get_extractor(args.sample, args.format, args.os, args.backend, sigpaths, False)
+    extractor = capa.main.get_extractor(args.sample, args.format, args.backend, sigpaths, False)
 
-    Path(args.output).write_bytes(dump(extractor))
+    with open(args.output, "wb") as f:
+        f.write(dump(extractor))
 
     return 0
 

capa/features/freeze/features.py

@@ -1,14 +1,7 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 import binascii
 from typing import Union, Optional
 
-from pydantic import Field, BaseModel, ConfigDict
+from pydantic import Field, BaseModel
 
 import capa.features.file
 import capa.features.insn
@@ -17,7 +10,9 @@ import capa.features.basicblock
 
 
 class FeatureModel(BaseModel):
-    model_config = ConfigDict(frozen=True, populate_by_name=True)
+    class Config:
+        frozen = True
+        allow_population_by_field_name = True
 
     def to_capa(self) -> capa.features.common.Feature:
         if isinstance(self, OSFeature):
@@ -106,79 +101,59 @@ class FeatureModel(BaseModel):
 
 def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
     if isinstance(f, capa.features.common.OS):
-        assert isinstance(f.value, str)
         return OSFeature(os=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Arch):
-        assert isinstance(f.value, str)
         return ArchFeature(arch=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Format):
-        assert isinstance(f.value, str)
         return FormatFeature(format=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.MatchedRule):
-        assert isinstance(f.value, str)
         return MatchFeature(match=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Characteristic):
-        assert isinstance(f.value, str)
         return CharacteristicFeature(characteristic=f.value, description=f.description)
 
     elif isinstance(f, capa.features.file.Export):
-        assert isinstance(f.value, str)
         return ExportFeature(export=f.value, description=f.description)
 
     elif isinstance(f, capa.features.file.Import):
-        assert isinstance(f.value, str)
-        return ImportFeature(import_=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `import_` as a argument due to alias
+        return ImportFeature(import_=f.value, description=f.description)
 
     elif isinstance(f, capa.features.file.Section):
-        assert isinstance(f.value, str)
         return SectionFeature(section=f.value, description=f.description)
 
     elif isinstance(f, capa.features.file.FunctionName):
-        assert isinstance(f.value, str)
-        return FunctionNameFeature(function_name=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `function_name` as a argument due to alias
+        return FunctionNameFeature(function_name=f.value, description=f.description)
 
     # must come before check for String due to inheritance
     elif isinstance(f, capa.features.common.Substring):
-        assert isinstance(f.value, str)
         return SubstringFeature(substring=f.value, description=f.description)
 
     # must come before check for String due to inheritance
     elif isinstance(f, capa.features.common.Regex):
-        assert isinstance(f.value, str)
         return RegexFeature(regex=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.String):
-        assert isinstance(f.value, str)
         return StringFeature(string=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Class):
-        assert isinstance(f.value, str)
-        return ClassFeature(class_=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `class_` as a argument due to alias
+        return ClassFeature(class_=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Namespace):
-        assert isinstance(f.value, str)
         return NamespaceFeature(namespace=f.value, description=f.description)
 
     elif isinstance(f, capa.features.basicblock.BasicBlock):
         return BasicBlockFeature(description=f.description)
 
     elif isinstance(f, capa.features.insn.API):
-        assert isinstance(f.value, str)
         return APIFeature(api=f.value, description=f.description)
 
     elif isinstance(f, capa.features.insn.Property):
-        assert isinstance(f.value, str)
         return PropertyFeature(property=f.value, access=f.access, description=f.description)
 
     elif isinstance(f, capa.features.insn.Number):
-        assert isinstance(f.value, (int, float))
         return NumberFeature(number=f.value, description=f.description)
 
     elif isinstance(f, capa.features.common.Bytes):
@@ -187,22 +162,16 @@ def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
         return BytesFeature(bytes=binascii.hexlify(buf).decode("ascii"), description=f.description)
 
     elif isinstance(f, capa.features.insn.Offset):
-        assert isinstance(f.value, int)
         return OffsetFeature(offset=f.value, description=f.description)
 
     elif isinstance(f, capa.features.insn.Mnemonic):
-        assert isinstance(f.value, str)
         return MnemonicFeature(mnemonic=f.value, description=f.description)
 
     elif isinstance(f, capa.features.insn.OperandNumber):
-        assert isinstance(f.value, int)
-        return OperandNumberFeature(index=f.index, operand_number=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `operand_number` as a argument due to alias
+        return OperandNumberFeature(index=f.index, operand_number=f.value, description=f.description)
 
     elif isinstance(f, capa.features.insn.OperandOffset):
-        assert isinstance(f.value, int)
-        return OperandOffsetFeature(index=f.index, operand_offset=f.value, description=f.description)  # type: ignore
-        # Mypy is unable to recognise `operand_offset` as a argument due to alias
+        return OperandOffsetFeature(index=f.index, operand_offset=f.value, description=f.description)
 
     else:
         raise NotImplementedError(f"feature_from_capa({type(f)}) not implemented")
@@ -211,141 +180,141 @@ def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
 class OSFeature(FeatureModel):
     type: str = "os"
     os: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class ArchFeature(FeatureModel):
     type: str = "arch"
     arch: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class FormatFeature(FeatureModel):
     type: str = "format"
     format: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class MatchFeature(FeatureModel):
     type: str = "match"
     match: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class CharacteristicFeature(FeatureModel):
     type: str = "characteristic"
     characteristic: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class ExportFeature(FeatureModel):
     type: str = "export"
     export: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class ImportFeature(FeatureModel):
     type: str = "import"
     import_: str = Field(alias="import")
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class SectionFeature(FeatureModel):
     type: str = "section"
     section: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class FunctionNameFeature(FeatureModel):
     type: str = "function name"
     function_name: str = Field(alias="function name")
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class SubstringFeature(FeatureModel):
     type: str = "substring"
     substring: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class RegexFeature(FeatureModel):
     type: str = "regex"
     regex: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class StringFeature(FeatureModel):
     type: str = "string"
     string: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class ClassFeature(FeatureModel):
     type: str = "class"
     class_: str = Field(alias="class")
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class NamespaceFeature(FeatureModel):
     type: str = "namespace"
     namespace: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class BasicBlockFeature(FeatureModel):
     type: str = "basic block"
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class APIFeature(FeatureModel):
     type: str = "api"
     api: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class PropertyFeature(FeatureModel):
     type: str = "property"
-    access: Optional[str] = None
+    access: Optional[str]
     property: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class NumberFeature(FeatureModel):
     type: str = "number"
     number: Union[int, float]
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class BytesFeature(FeatureModel):
     type: str = "bytes"
     bytes: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class OffsetFeature(FeatureModel):
     type: str = "offset"
     offset: int
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class MnemonicFeature(FeatureModel):
     type: str = "mnemonic"
     mnemonic: str
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class OperandNumberFeature(FeatureModel):
     type: str = "operand number"
     index: int
     operand_number: int = Field(alias="operand number")
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 class OperandOffsetFeature(FeatureModel):
     type: str = "operand offset"
     index: int
     operand_offset: int = Field(alias="operand offset")
-    description: Optional[str] = None
+    description: Optional[str]
 
 
 Feature = Union[

capa/features/insn.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -15,9 +15,9 @@ from capa.features.common import VALID_FEATURE_ACCESS, Feature
 def hex(n: int) -> str:
     """render the given number using upper case hex, like: 0x123ABC"""
     if n < 0:
-        return f"-0x{(-n):X}"
+        return "-0x%X" % (-n)
     else:
-        return f"0x{(n):X}"
+        return "0x%X" % n
 
 
 class API(Feature):
@@ -31,7 +31,7 @@ class _AccessFeature(Feature, abc.ABC):
         super().__init__(value, description=description)
         if access is not None:
             if access not in VALID_FEATURE_ACCESS:
-                raise ValueError(f"{self.name} access type {access} not valid")
+                raise ValueError("%s access type %s not valid" % (self.name, access))
         self.access = access
 
     def __hash__(self):
@@ -53,15 +53,6 @@ class Property(_AccessFeature):
 
 class Number(Feature):
     def __init__(self, value: Union[int, float], description=None):
-        """
-        args:
-          value (int or float): positive or negative integer, or floating point number.
-
-        the range of the value is:
-          - if positive, the range of u64
-          - if negative, the range of i64
-          - if floating, the range and precision of double
-        """
         super().__init__(value, description=description)
 
     def get_value_str(self):
@@ -70,7 +61,7 @@ class Number(Feature):
         elif isinstance(self.value, float):
             return str(self.value)
         else:
-            raise ValueError(f"invalid value type {type(self.value)}")
+            raise ValueError("invalid value type")
 
 
 # max recognized structure size (and therefore, offset size)
@@ -79,14 +70,6 @@ MAX_STRUCTURE_SIZE = 0x10000
 
 class Offset(Feature):
     def __init__(self, value: int, description=None):
-        """
-        args:
-          value (int): the offset, which can be positive or negative.
-
-        the range of the value is:
-          - if positive, the range of u64
-          - if negative, the range of i64
-        """
         super().__init__(value, description=description)
 
     def get_value_str(self):
@@ -109,7 +92,7 @@ MAX_OPERAND_INDEX = MAX_OPERAND_COUNT - 1
 class _Operand(Feature, abc.ABC):
     # superclass: don't use directly
     # subclasses should set self.name and provide the value string formatter
-    def __init__(self, index: int, value: Union[int, float], description=None):
+    def __init__(self, index: int, value: int, description=None):
         super().__init__(value, description=description)
         self.index = index
 
@@ -122,45 +105,24 @@ class _Operand(Feature, abc.ABC):
 
 class OperandNumber(_Operand):
     # cached names so we don't do extra string formatting every ctor
-    NAMES = [f"operand[{i}].number" for i in range(MAX_OPERAND_COUNT)]
+    NAMES = ["operand[%d].number" % i for i in range(MAX_OPERAND_COUNT)]
 
     # operand[i].number: 0x12
-    def __init__(self, index: int, value: Union[int, float], description=None):
-        """
-        args:
-          value (int or float): positive or negative integer, or floating point number.
-
-        the range of the value is:
-          - if positive, the range of u64
-          - if negative, the range of i64
-          - if floating, the range and precision of double
-        """
-        super().__init__(index, value, description=description)
-        self.name = self.NAMES[index]
-
-    def get_value_str(self) -> str:
-        if isinstance(self.value, int):
-            return capa.helpers.hex(self.value)
-        elif isinstance(self.value, float):
-            return str(self.value)
-        else:
-            raise ValueError("invalid value type")
-
-
-class OperandOffset(_Operand):
-    # cached names so we don't do extra string formatting every ctor
-    NAMES = [f"operand[{i}].offset" for i in range(MAX_OPERAND_COUNT)]
-
-    # operand[i].offset: 0x12
     def __init__(self, index: int, value: int, description=None):
-        """
-        args:
-          value (int): the offset, which can be positive or negative.
-
-        the range of the value is:
-          - if positive, the range of u64
-          - if negative, the range of i64
-        """
+        super().__init__(index, value, description=description)
+        self.name = self.NAMES[index]
+
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, int)
+        return hex(self.value)
+
+
+class OperandOffset(_Operand):
+    # cached names so we don't do extra string formatting every ctor
+    NAMES = ["operand[%d].offset" % i for i in range(MAX_OPERAND_COUNT)]
+
+    # operand[i].offset: 0x12
+    def __init__(self, index: int, value: int, description=None):
         super().__init__(index, value, description=description)
         self.name = self.NAMES[index]
 

capa/ghidra/README.md

@@ -1,172 +0,0 @@
-<div align="center">
-    <img src="/doc/img/ghidra_backend_logo.png" width=300 height=175>
-</div>
-
-The Ghidra feature extractor is an application of the FLARE team's open-source project, Ghidrathon, to integrate capa with Ghidra using Python 3. capa is a framework that uses a well-defined collection of rules to identify capabilities in a program. You can run capa against a PE file, ELF file, or shellcode and it tells you what it thinks the program can do. For example, it might suggest that the program is a backdoor, can install services, or relies on HTTP to communicate. The Ghidra feature extractor can be used to run capa analysis on your Ghidra databases without needing access to the original binary file.
-
-<img src="/doc/img/ghidra_script_mngr_output.png">
-
-## Getting Started
-
-### Installation
-
-Please ensure that you have the following dependencies installed before continuing:
-
-| Dependency | Version | Source |
-|------------|---------|--------|
-| Ghidrathon | `>= 3.0.0` | https://github.com/mandiant/Ghidrathon |
-| Python | `>= 3.8` | https://www.python.org/downloads |
-| Ghidra | `>= 10.2` | https://ghidra-sre.org |
-
-In order to run capa using using Ghidra, you must install capa as a library, obtain the official capa rules that match the capa version you have installed, and configure the Python 3 script [capa_ghidra.py](/capa/ghidra/capa_ghidra.py). You can do this by completing the following steps using the Python 3 interpreter that you have configured for your Ghidrathon installation:
-
-1. Install capa and its dependencies from PyPI using the following command:
-```bash
-$ pip install flare-capa
-```
-
-2. Download and extract the [official capa rules](https://github.com/mandiant/capa-rules/releases) that match the capa version you have installed. Use the following command to view the version of capa you have installed:
-```bash
-$ pip show flare-capa
-OR
-$ capa --version
-```
-
-3. Copy [capa_ghidra.py](/capa/ghidra/capa_ghidra.py) to your `$USER_HOME/ghidra_scripts` directory or manually add `</path/to/ghidra_capa.py/>` to the Ghidra Script Manager.
-
-## Usage
-
-After completing the installation steps you can execute `capa_ghidra.py` using the Ghidra Script Manager or Headless Analyzer.
-
-### Ghidra Script Manager
-
-To execute `capa_ghidra.py` using the Ghidra Script Manager, first open the Ghidra Script Manager by navigating to `Window > Script Manager` in the Ghidra Code Browser. Next, locate `capa_ghidra.py` by selecting the `Python 3 > capa` category or using the Ghidra Script Manager search funtionality. Finally, double-click `capa_ghidra.py` to execute the script. If you don't see `capa_ghidra.py`, make sure you have copied the script to your `$USER_HOME/ghidra_scripts` directory or manually added `</path/to/ghidra_capa.py/>` to the Ghidra Script Manager
-
-When executed, `capa_ghidra.py` asks you to provide your capa rules directory and preferred output format. `capa_ghidra.py` supports `default`, `verbose`, and `vverbose` output formats when executed from the Ghidra Script Manager. `capa_ghidra.py` writes output to the Ghidra Console Window.
-
-#### Example
-
-The following is an example of running `capa_ghidra.py` using the Ghidra Script Manager:
-
-Selecting capa rules:
-<img src="/doc/img/ghidra_script_mngr_rules.png">
-
-Choosing output format:
-<img src="/doc/img/ghidra_script_mngr_verbosity.png">
-
-Viewing results in Ghidra Console Window:
-<img src="/doc/img/ghidra_script_mngr_output.png">
-
-### Ghidra Headless Analyzer
-
-To execute `capa_ghidra.py` using the Ghidra Headless Analyzer, you can use the Ghidra `analyzeHeadless` script located in your `$GHIDRA_HOME/support` directory. You will need to provide the following arguments to the Ghidra `analyzeHeadless` script:
-
-1. `</path/to/ghidra/project/>`: path to Ghidra project
-2. `<ghidra_project_name>`: name of Ghidra Project
-3. `-process <sample_name>`: name of sample `<sample_name>`
-4. `-ScriptPath </path/to/capa_ghidra/>`: OPTIONAL argument specifying path `</path/to/capa_ghidra/>` to `capa_ghidra.py`
-5. `-PostScript capa_ghidra.py`: executes `capa_ghidra.py` as post-analysis script
-6. `"<capa_args>"`: single, quoted string containing capa arguments that must specify capa rules directory and output format, e.g. `"<path/to/capa/rules> --verbose"`. `capa_ghidra.py` supports `default`, `verbose`, `vverbose` and `json` formats when executed using the Ghidra Headless Analyzer. `capa_ghidra.py` writes output to the console window used to execute the Ghidra `analyzeHeadless` script.
-7. `-processor <languageID>`: required ONLY if sample `<sample_name>` is shellcode. More information on specifying the `<languageID>` can be found in the `$GHIDRA_HOME/support/analyzeHeadlessREADME.html` documentation.
-
-The following is an example of combining these arguments into a single `analyzeHeadless` script command:
-
-```
-$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -process <sample_name> -PostScript capa_ghidra.py "/path/to/capa/rules/ --verbose"
-```
-
-You may also want to run capa against a sample that you have not yet imported into your Ghidra project. The following is an example of importing a sample and running `capa_ghidra.py` using a single `analyzeHeadless` script command:
-
-```
-$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -Import </path/to/sample> -PostScript capa_ghidra.py "/path/to/capa/rules/ --verbose"
-```
-
-You can also provide `capa_ghidra.py` the single argument `"help"` to view supported arguments when running the script using the Ghidra Headless Analyzer:
-```
-$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -process <sample_name> -PostScript capa_ghidra.py "help"
-```
-
-#### Example
-
-The following is an example of running `capa_ghidra.py` against a shellcode sample using the Ghidra `analyzeHeadless` script:
-```
-$ analyzeHeadless /home/wumbo/Desktop/ghidra_projects/ capa_test -process 499c2a85f6e8142c3f48d4251c9c7cd6.raw32 -processor x86:LE:32:default -PostScript capa_ghidra.py "/home/wumbo/capa/rules -vv"
-[...]
-
-INFO  REPORT: Analysis succeeded for file: /499c2a85f6e8142c3f48d4251c9c7cd6.raw32 (HeadlessAnalyzer)  
-INFO  SCRIPT: /home/wumbo/ghidra_scripts/capa_ghidra.py (HeadlessAnalyzer)  
-md5                     499c2a85f6e8142c3f48d4251c9c7cd6                                                                                                                                                                                                    
-sha1
-sha256                  e8e02191c1b38c808d27a899ac164b3675eb5cadd3a8907b0ffa863714000e72
-path                    /home/wumbo/capa/tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32
-timestamp               2023-08-29 17:57:00.946588
-capa version            6.1.0
-os                      unknown os
-format                  Raw Binary
-arch                    x86
-extractor               ghidra
-base address            global
-rules                   /home/wumbo/capa/rules
-function count          42
-library function count  0
-total feature count     1970
-
-contain loop (24 matches, only showing first match of library rule)
-author  moritz.raabe@mandiant.com
-scope   function
-function @ 0x0
-  or:
-    characteristic: loop @ 0x0
-    characteristic: tight loop @ 0x278
-
-contain obfuscated stackstrings
-namespace  anti-analysis/obfuscation/string/stackstring
-author     moritz.raabe@mandiant.com
-scope      basic block
-att&ck     Defense Evasion::Obfuscated Files or Information::Indicator Removal from Tools [T1027.005]
-mbc        Anti-Static Analysis::Executable Code Obfuscation::Argument Obfuscation [B0032.020], Anti-Static Analysis::Executable Code Obfuscation::Stack Strings [B0032.017]
-basic block @ 0x0 in function 0x0
-  characteristic: stack string @ 0x0
-
-encode data using XOR
-namespace  data-manipulation/encoding/xor
-author     moritz.raabe@mandiant.com
-scope      basic block
-att&ck     Defense Evasion::Obfuscated Files or Information [T1027]
-mbc        Defense Evasion::Obfuscated Files or Information::Encoding-Standard Algorithm [E1027.m02], Data::Encode Data::XOR [C0026.002]
-basic block @ 0x8AF in function 0x8A1
-  and:
-    characteristic: tight loop @ 0x8AF
-    characteristic: nzxor @ 0x8C0
-    not: = filter for potential false positives
-      or:
-        or: = unsigned bitwise negation operation (~i)
-          number: 0xFFFFFFFF = bitwise negation for unsigned 32 bits
-          number: 0xFFFFFFFFFFFFFFFF = bitwise negation for unsigned 64 bits
-        or: = signed bitwise negation operation (~i)
-          number: 0xFFFFFFF = bitwise negation for signed 32 bits
-          number: 0xFFFFFFFFFFFFFFF = bitwise negation for signed 64 bits
-        or: = Magic constants used in the implementation of strings functions.
-          number: 0x7EFEFEFF = optimized string constant for 32 bits
-          number: 0x81010101 = -0x81010101 = 0x7EFEFEFF
-          number: 0x81010100 = 0x81010100 = ~0x7EFEFEFF
-          number: 0x7EFEFEFEFEFEFEFF = optimized string constant for 64 bits
-          number: 0x8101010101010101 = -0x8101010101010101 = 0x7EFEFEFEFEFEFEFF
-          number: 0x8101010101010100 = 0x8101010101010100 = ~0x7EFEFEFEFEFEFEFF
-
-get OS information via KUSER_SHARED_DATA
-namespace   host-interaction/os/version
-author      @mr-tz
-scope       function
-att&ck      Discovery::System Information Discovery [T1082]
-references  https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/ntexapi_x/kuser_shared_data/index.htm
-function @ 0x1CA6
-  or:
-    number: 0x7FFE026C = NtMajorVersion @ 0x1D18
-
-
-
-Script /home/wumbo/ghidra_scripts/capa_ghidra.py called exit with code 0
-
-[...]
-```

capa/ghidra/capa_ghidra.py

@@ -1,166 +0,0 @@
-# Run capa against loaded Ghidra database
-# @author Mike Hunhoff (mehunhoff@google.com)
-# @category Python 3.capa
-
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import sys
-import logging
-import pathlib
-import argparse
-
-import capa
-import capa.main
-import capa.rules
-import capa.ghidra.helpers
-import capa.render.default
-import capa.features.extractors.ghidra.extractor
-
-logger = logging.getLogger("capa_ghidra")
-
-
-def run_headless():
-    parser = argparse.ArgumentParser(description="The FLARE team's open-source tool to integrate capa with Ghidra.")
-
-    parser.add_argument(
-        "rules",
-        type=str,
-        help="path to rule file or directory",
-    )
-    parser.add_argument(
-        "-v", "--verbose", action="store_true", help="enable verbose result document (no effect with --json)"
-    )
-    parser.add_argument(
-        "-vv", "--vverbose", action="store_true", help="enable very verbose result document (no effect with --json)"
-    )
-    parser.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR")
-    parser.add_argument("-q", "--quiet", action="store_true", help="disable all output but errors")
-    parser.add_argument("-j", "--json", action="store_true", help="emit JSON instead of text")
-
-    script_args = list(getScriptArgs())  # type: ignore [name-defined] # noqa: F821
-    if not script_args or len(script_args) > 1:
-        script_args = []
-    else:
-        script_args = script_args[0].split()
-        for idx, arg in enumerate(script_args):
-            if arg.lower() == "help":
-                script_args[idx] = "--help"
-
-    args = parser.parse_args(args=script_args)
-
-    if args.quiet:
-        logging.basicConfig(level=logging.WARNING)
-        logging.getLogger().setLevel(logging.WARNING)
-    elif args.debug:
-        logging.basicConfig(level=logging.DEBUG)
-        logging.getLogger().setLevel(logging.DEBUG)
-    else:
-        logging.basicConfig(level=logging.INFO)
-        logging.getLogger().setLevel(logging.INFO)
-
-    logger.debug("running in Ghidra headless mode")
-
-    rules_path = pathlib.Path(args.rules)
-
-    logger.debug("rule path: %s", rules_path)
-    rules = capa.main.get_rules([rules_path])
-
-    meta = capa.ghidra.helpers.collect_metadata([rules_path])
-    extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
-
-    capabilities, counts = capa.main.find_capabilities(rules, extractor, False)
-
-    meta.analysis.feature_counts = counts["feature_counts"]
-    meta.analysis.library_functions = counts["library_functions"]
-    meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
-
-    if capa.main.has_file_limitation(rules, capabilities, is_standalone=True):
-        logger.info("capa encountered warnings during analysis")
-
-    if args.json:
-        print(capa.render.json.render(meta, rules, capabilities))  # noqa: T201
-    elif args.vverbose:
-        print(capa.render.vverbose.render(meta, rules, capabilities))  # noqa: T201
-    elif args.verbose:
-        print(capa.render.verbose.render(meta, rules, capabilities))  # noqa: T201
-    else:
-        print(capa.render.default.render(meta, rules, capabilities))  # noqa: T201
-
-    return 0
-
-
-def run_ui():
-    logging.basicConfig(level=logging.INFO)
-    logging.getLogger().setLevel(logging.INFO)
-
-    rules_dir: str = ""
-    try:
-        selected_dir = askDirectory("Choose capa rules directory", "Ok")  # type: ignore [name-defined] # noqa: F821
-        if selected_dir:
-            rules_dir = selected_dir.getPath()
-    except RuntimeError:
-        # RuntimeError thrown when user selects "Cancel"
-        pass
-
-    if not rules_dir:
-        logger.info("You must choose a capa rules directory before running capa.")
-        return capa.main.E_MISSING_RULES
-
-    verbose = askChoice(  # type: ignore [name-defined] # noqa: F821
-        "capa output verbosity", "Choose capa output verbosity", ["default", "verbose", "vverbose"], "default"
-    )
-
-    rules_path: pathlib.Path = pathlib.Path(rules_dir)
-    logger.info("running capa using rules from %s", str(rules_path))
-
-    rules = capa.main.get_rules([rules_path])
-
-    meta = capa.ghidra.helpers.collect_metadata([rules_path])
-    extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
-
-    capabilities, counts = capa.main.find_capabilities(rules, extractor, True)
-
-    meta.analysis.feature_counts = counts["feature_counts"]
-    meta.analysis.library_functions = counts["library_functions"]
-    meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
-
-    if capa.main.has_file_limitation(rules, capabilities, is_standalone=False):
-        logger.info("capa encountered warnings during analysis")
-
-    if verbose == "vverbose":
-        print(capa.render.vverbose.render(meta, rules, capabilities))  # noqa: T201
-    elif verbose == "verbose":
-        print(capa.render.verbose.render(meta, rules, capabilities))  # noqa: T201
-    else:
-        print(capa.render.default.render(meta, rules, capabilities))  # noqa: T201
-
-    return 0
-
-
-def main():
-    if not capa.ghidra.helpers.is_supported_ghidra_version():
-        return capa.main.E_UNSUPPORTED_GHIDRA_VERSION
-
-    if not capa.ghidra.helpers.is_supported_file_type():
-        return capa.main.E_INVALID_FILE_TYPE
-
-    if not capa.ghidra.helpers.is_supported_arch_type():
-        return capa.main.E_INVALID_FILE_ARCH
-
-    if isRunningHeadless():  # type: ignore [name-defined] # noqa: F821
-        return run_headless()
-    else:
-        return run_ui()
-
-
-if __name__ == "__main__":
-    if sys.version_info < (3, 8):
-        from capa.exceptions import UnsupportedRuntimeError
-
-        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+")
-    sys.exit(main())

capa/ghidra/helpers.py

@@ -1,159 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-import datetime
-import contextlib
-from typing import List
-from pathlib import Path
-
-import capa
-import capa.version
-import capa.features.common
-import capa.features.freeze
-import capa.render.result_document as rdoc
-import capa.features.extractors.ghidra.helpers
-
-logger = logging.getLogger("capa")
-
-# file type as returned by Ghidra
-SUPPORTED_FILE_TYPES = ("Executable and Linking Format (ELF)", "Portable Executable (PE)", "Raw Binary")
-
-
-class GHIDRAIO:
-    """
-    An object that acts as a file-like object,
-    using bytes from the current Ghidra listing.
-    """
-
-    def __init__(self):
-        super().__init__()
-
-        self.offset = 0
-        self.bytes_ = self.get_bytes()
-
-    def seek(self, offset, whence=0):
-        assert whence == 0
-        self.offset = offset
-
-    def read(self, size):
-        logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, currentProgram().getImageBase().add(self.offset).getOffset())  # type: ignore [name-defined] # noqa: F821
-
-        if size > len(self.bytes_) - self.offset:
-            logger.debug("cannot read 0x%x bytes at 0x%x (ea: BADADDR)", size, self.offset)
-            return b""
-        else:
-            return self.bytes_[self.offset : self.offset + size]
-
-    def close(self):
-        return
-
-    def get_bytes(self):
-        file_bytes = currentProgram().getMemory().getAllFileBytes()[0]  # type: ignore [name-defined] # noqa: F821
-
-        # getOriginalByte() allows for raw file parsing on the Ghidra side
-        # other functions will fail as Ghidra will think that it's reading uninitialized memory
-        bytes_ = [file_bytes.getOriginalByte(i) for i in range(file_bytes.getSize())]
-
-        return capa.features.extractors.ghidra.helpers.ints_to_bytes(bytes_)
-
-
-def is_supported_ghidra_version():
-    version = float(getGhidraVersion()[:4])  # type: ignore [name-defined] # noqa: F821
-    if version < 10.2:
-        warning_msg = "capa does not support this Ghidra version"
-        logger.warning(warning_msg)
-        logger.warning("Your Ghidra version is: %s. Supported versions are: Ghidra >= 10.2", version)
-        return False
-    return True
-
-
-def is_running_headless():
-    return isRunningHeadless()  # type: ignore [name-defined] # noqa: F821
-
-
-def is_supported_file_type():
-    file_info = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
-    if file_info not in SUPPORTED_FILE_TYPES:
-        logger.error("-" * 80)
-        logger.error(" Input file does not appear to be a supported file type.")
-        logger.error(" ")
-        logger.error(
-            " capa currently only supports analyzing PE, ELF, or binary files containing x86 (32- and 64-bit) shellcode."
-        )
-        logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
-        logger.error("-" * 80)
-        return False
-    return True
-
-
-def is_supported_arch_type():
-    lang_id = str(currentProgram().getLanguageID()).lower()  # type: ignore [name-defined] # noqa: F821
-
-    if not all((lang_id.startswith("x86"), any(arch in lang_id for arch in ("32", "64")))):
-        logger.error("-" * 80)
-        logger.error(" Input file does not appear to target a supported architecture.")
-        logger.error(" ")
-        logger.error(" capa currently only supports analyzing x86 (32- and 64-bit).")
-        logger.error("-" * 80)
-        return False
-    return True
-
-
-def get_file_md5():
-    return currentProgram().getExecutableMD5()  # type: ignore [name-defined] # noqa: F821
-
-
-def get_file_sha256():
-    return currentProgram().getExecutableSHA256()  # type: ignore [name-defined] # noqa: F821
-
-
-def collect_metadata(rules: List[Path]):
-    md5 = get_file_md5()
-    sha256 = get_file_sha256()
-
-    info = currentProgram().getLanguageID().toString()  # type: ignore [name-defined] # noqa: F821
-    if "x86" in info and "64" in info:
-        arch = "x86_64"
-    elif "x86" in info and "32" in info:
-        arch = "x86"
-    else:
-        arch = "unknown arch"
-
-    format_name: str = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
-    if "PE" in format_name:
-        os = "windows"
-    elif "ELF" in format_name:
-        with contextlib.closing(capa.ghidra.helpers.GHIDRAIO()) as f:
-            os = capa.features.extractors.elf.detect_elf_os(f)
-    else:
-        os = "unknown os"
-
-    return rdoc.Metadata(
-        timestamp=datetime.datetime.now(),
-        version=capa.version.__version__,
-        argv=(),
-        sample=rdoc.Sample(
-            md5=md5,
-            sha1="",
-            sha256=sha256,
-            path=currentProgram().getExecutablePath(),  # type: ignore [name-defined] # noqa: F821
-        ),
-        analysis=rdoc.Analysis(
-            format=currentProgram().getExecutableFormat(),  # type: ignore [name-defined] # noqa: F821
-            arch=arch,
-            os=os,
-            extractor="ghidra",
-            rules=tuple(r.resolve().absolute().as_posix() for r in rules),
-            base_address=capa.features.freeze.Address.from_capa(currentProgram().getImageBase().getOffset()),  # type: ignore [name-defined] # noqa: F821
-            layout=rdoc.Layout(
-                functions=(),
-            ),
-            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
-            library_functions=(),
-        ),
-    )

capa/helpers.py

@@ -1,18 +1,13 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import inspect
+import os
 import logging
-import contextlib
-import importlib.util
 from typing import NoReturn
-from pathlib import Path
-
-import tqdm
 
 from capa.exceptions import UnsupportedFormatError
 from capa.features.common import FORMAT_PE, FORMAT_SC32, FORMAT_SC64, FORMAT_DOTNET, FORMAT_UNKNOWN, Format
@@ -27,45 +22,41 @@ logger = logging.getLogger("capa")
 def hex(n: int) -> str:
     """render the given number using upper case hex, like: 0x123ABC"""
     if n < 0:
-        return f"-0x{(-n):X}"
+        return "-0x%X" % (-n)
     else:
-        return f"0x{(n):X}"
+        return "0x%X" % n
 
 
-def get_file_taste(sample_path: Path) -> bytes:
-    if not sample_path.exists():
-        raise IOError(f"sample path {sample_path} does not exist or cannot be accessed")
-    taste = sample_path.open("rb").read(8)
+def get_file_taste(sample_path: str) -> bytes:
+    if not os.path.exists(sample_path):
+        raise IOError("sample path %s does not exist or cannot be accessed" % sample_path)
+    with open(sample_path, "rb") as f:
+        taste = f.read(8)
     return taste
 
 
 def is_runtime_ida():
-    return importlib.util.find_spec("idc") is not None
-
-
-def is_runtime_ghidra():
     try:
-        currentProgram  # type: ignore [name-defined] # noqa: F821
-    except NameError:
+        import idc
+    except ImportError:
         return False
-    return True
+    else:
+        return True
 
 
-def assert_never(value) -> NoReturn:
-    # careful: python -O will remove this assertion.
-    # but this is only used for type checking, so it's ok.
-    assert False, f"Unhandled value: {value} ({type(value).__name__})"  # noqa: B011
+def assert_never(value: NoReturn) -> NoReturn:
+    assert False, f"Unhandled value: {value} ({type(value).__name__})"
 
 
-def get_format_from_extension(sample: Path) -> str:
-    if sample.name.endswith(EXTENSIONS_SHELLCODE_32):
+def get_format_from_extension(sample: str) -> str:
+    if sample.endswith(EXTENSIONS_SHELLCODE_32):
         return FORMAT_SC32
-    elif sample.name.endswith(EXTENSIONS_SHELLCODE_64):
+    elif sample.endswith(EXTENSIONS_SHELLCODE_64):
         return FORMAT_SC64
     return FORMAT_UNKNOWN
 
 
-def get_auto_format(path: Path) -> str:
+def get_auto_format(path: str) -> str:
     format_ = get_format(path)
     if format_ == FORMAT_UNKNOWN:
         format_ = get_format_from_extension(path)
@@ -74,12 +65,13 @@ def get_auto_format(path: Path) -> str:
     return format_
 
 
-def get_format(sample: Path) -> str:
+def get_format(sample: str) -> str:
     # imported locally to avoid import cycle
     from capa.features.extractors.common import extract_format
     from capa.features.extractors.dnfile_ import DnfileFeatureExtractor
 
-    buf = sample.read_bytes()
+    with open(sample, "rb") as f:
+        buf = f.read()
 
     for feature, _ in extract_format(buf):
         if feature == Format(FORMAT_PE):
@@ -93,39 +85,6 @@ def get_format(sample: Path) -> str:
     return FORMAT_UNKNOWN
 
 
-@contextlib.contextmanager
-def redirecting_print_to_tqdm(disable_progress):
-    """
-    tqdm (progress bar) expects to have fairly tight control over console output.
-    so calls to `print()` will break the progress bar and make things look bad.
-    so, this context manager temporarily replaces the `print` implementation
-    with one that is compatible with tqdm.
-    via: https://stackoverflow.com/a/42424890/87207
-    """
-    old_print = print  # noqa: T202 [reserved word print used]
-
-    def new_print(*args, **kwargs):
-        # If tqdm.tqdm.write raises error, use builtin print
-        if disable_progress:
-            old_print(*args, **kwargs)
-        else:
-            try:
-                tqdm.tqdm.write(*args, **kwargs)
-            except Exception:
-                old_print(*args, **kwargs)
-
-    try:
-        # Globally replace print with new_print.
-        # Verified this works manually on Python 3.11:
-        #     >>> import inspect
-        #     >>> inspect.builtins
-        #     <module 'builtins' (built-in)>
-        inspect.builtins.print = new_print  # type: ignore
-        yield
-    finally:
-        inspect.builtins.print = old_print  # type: ignore
-
-
 def log_unsupported_format_error():
     logger.error("-" * 80)
     logger.error(" Input file does not appear to be a PE or ELF file.")
@@ -159,7 +118,7 @@ def log_unsupported_runtime_error():
     logger.error("-" * 80)
     logger.error(" Unsupported runtime or Python interpreter.")
     logger.error(" ")
-    logger.error(" capa supports running under Python 3.8 and higher.")
+    logger.error(" capa supports running under Python 3.7 and higher.")
     logger.error(" ")
     logger.error(
         " If you're seeing this message on the command line, please ensure you're running a supported Python version."

capa/ida/helpers.py

@@ -1,15 +1,15 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import json
 import logging
 import datetime
 import contextlib
-from typing import List, Optional
-from pathlib import Path
+from typing import Optional
 
 import idc
 import idaapi
@@ -22,8 +22,7 @@ import capa
 import capa.version
 import capa.render.utils as rutils
 import capa.features.common
-import capa.features.freeze
-import capa.render.result_document as rdoc
+import capa.render.result_document
 from capa.features.address import AbsoluteVirtualAddress
 
 logger = logging.getLogger("capa")
@@ -46,8 +45,7 @@ NETNODE_RULES_CACHE_ID = "rules-cache-id"
 
 
 def inform_user_ida_ui(message):
-    # this isn't a logger, this is IDA's logging facility
-    idaapi.info(f"{message}. Please refer to IDA Output window for more information.")  # noqa: G004
+    idaapi.info("%s. Please refer to IDA Output window for more information." % message)
 
 
 def is_supported_ida_version():
@@ -55,7 +53,7 @@ def is_supported_ida_version():
     if version < 7.4 or version >= 9:
         warning_msg = "This plugin does not support your IDA Pro version"
         logger.warning(warning_msg)
-        logger.warning("Your IDA Pro version is: %s. Supported versions are: IDA >= 7.4 and IDA < 9.0.", version)
+        logger.warning("Your IDA Pro version is: %s. Supported versions are: IDA >= 7.4 and IDA < 9.0." % version)
         return False
     return True
 
@@ -120,7 +118,7 @@ def get_file_sha256():
     return sha256
 
 
-def collect_metadata(rules: List[Path]):
+def collect_metadata(rules):
     """ """
     md5 = get_file_md5()
     sha256 = get_file_sha256()
@@ -142,35 +140,37 @@ def collect_metadata(rules: List[Path]):
     else:
         os = "unknown os"
 
-    return rdoc.Metadata(
-        timestamp=datetime.datetime.now(),
-        version=capa.version.__version__,
-        argv=(),
-        sample=rdoc.Sample(
-            md5=md5,
-            sha1="",  # not easily accessible
-            sha256=sha256,
-            path=idaapi.get_input_file_path(),
-        ),
-        analysis=rdoc.Analysis(
-            format=idaapi.get_file_type_name(),
-            arch=arch,
-            os=os,
-            extractor="ida",
-            rules=tuple(r.resolve().absolute().as_posix() for r in rules),
-            base_address=capa.features.freeze.Address.from_capa(idaapi.get_imagebase()),
-            layout=rdoc.Layout(
-                functions=(),
+    return {
+        "timestamp": datetime.datetime.now().isoformat(),
+        "argv": [],
+        "sample": {
+            "md5": md5,
+            "sha1": "",  # not easily accessible
+            "sha256": sha256,
+            "path": idaapi.get_input_file_path(),
+        },
+        "analysis": {
+            "format": idaapi.get_file_type_name(),
+            "arch": arch,
+            "os": os,
+            "extractor": "ida",
+            "rules": rules,
+            "base_address": idaapi.get_imagebase(),
+            "layout": {
                 # this is updated after capabilities have been collected.
                 # will look like:
                 #
                 # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
-            ),
+            },
             # ignore these for now - not used by IDA plugin.
-            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
-            library_functions=(),
-        ),
-    )
+            "feature_counts": {
+                "file": {},
+                "functions": {},
+            },
+            "library_functions": {},
+        },
+        "version": capa.version.__version__,
+    }
 
 
 class IDAIO:
@@ -213,16 +213,16 @@ def idb_contains_cached_results() -> bool:
         n = netnode.Netnode(CAPA_NETNODE)
         return bool(n.get(NETNODE_RESULTS))
     except netnode.NetnodeCorruptError as e:
-        logger.exception(str(e))
+        logger.error("%s", e, exc_info=True)
         return False
 
 
-def load_and_verify_cached_results() -> Optional[rdoc.ResultDocument]:
+def load_and_verify_cached_results() -> Optional[capa.render.result_document.ResultDocument]:
     """verifies that cached results have valid (mapped) addresses for the current database"""
     logger.debug("loading cached capa results from netnode '%s'", CAPA_NETNODE)
 
     n = netnode.Netnode(CAPA_NETNODE)
-    doc = rdoc.ResultDocument.model_validate_json(n[NETNODE_RESULTS])
+    doc = capa.render.result_document.ResultDocument.parse_obj(json.loads(n[NETNODE_RESULTS]))
 
     for rule in rutils.capability_rules(doc):
         for location_, _ in rule.matches:

capa/ida/plugin/README.md

@@ -95,7 +95,7 @@ can update using the `Settings` button.
 
 ### Requirements
 
-capa explorer supports Python versions >= 3.8.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:
+capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:
 
 * IDA 7.4
 * IDA 7.5
@@ -105,7 +105,7 @@ capa explorer supports Python versions >= 3.8.x and IDA Pro versions >= 7.4. The
 * IDA 8.1
 * IDA 8.2
 
-capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.8.x).
+capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x).
 
 If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues).
 

capa/ida/plugin/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -38,12 +38,6 @@ class CapaExplorerPlugin(idaapi.plugin_t):
         """called when IDA is loading the plugin"""
         logging.basicConfig(level=logging.INFO)
 
-        # do not load plugin unless hosted in idaq (IDA Qt)
-        if not idaapi.is_idaq():
-            # note: it does not appear that IDA calls "init" by default when hosted in idat; we keep this
-            # check here for good measure
-            return idaapi.PLUGIN_SKIP
-
         import capa.ida.helpers
 
         # do not load plugin if IDA version/file type not supported
@@ -67,16 +61,7 @@ class CapaExplorerPlugin(idaapi.plugin_t):
           arg (int): bitflag. Setting LSB enables automatic analysis upon
           loading. The other bits are currently undefined. See `form.Options`.
         """
-        if not self.form:
-            self.form = CapaExplorerForm(self.PLUGIN_NAME, arg)
-        else:
-            widget = idaapi.find_widget(self.form.form_title)
-            if widget:
-                idaapi.activate_widget(widget, True)
-            else:
-                self.form.Show()
-                self.form.load_capa_results(False, True)
-
+        self.form = CapaExplorerForm(self.PLUGIN_NAME, arg)
         return True
 
 

capa/ida/plugin/cache.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -10,7 +10,7 @@ from __future__ import annotations
 
 import itertools
 import collections
-from typing import Set, Dict, Tuple, Union, Optional
+from typing import Set, Dict, List, Tuple, Union, Optional
 
 import capa.engine
 from capa.rules import Scope, RuleSet
@@ -37,21 +37,18 @@ class CapaRuleGenFeatureCacheNode:
         self.children: Set[CapaRuleGenFeatureCacheNode] = set()
 
     def __hash__(self):
-        # TODO(mike-hunhoff): confirm this is unique enough
-        # https://github.com/mandiant/capa/issues/1604
+        # TODO: unique enough?
         return hash((self.address,))
 
     def __eq__(self, other):
         if not isinstance(other, type(self)):
             return NotImplemented
-        # TODO(mike-hunhoff): confirm this is unique enough
-        # https://github.com/mandiant/capa/issues/1604
+        # TODO: unique enough?
         return self.address == other.address
 
 
 class CapaRuleGenFeatureCache:
-    def __init__(self, extractor: CapaExplorerFeatureExtractor):
-        self.extractor = extractor
+    def __init__(self, fh_list: List[FunctionHandle], extractor: CapaExplorerFeatureExtractor):
         self.global_features: FeatureSet = collections.defaultdict(set)
 
         self.file_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(None, None)
@@ -59,11 +56,12 @@ class CapaRuleGenFeatureCache:
         self.bb_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
         self.insn_nodes: Dict[Address, CapaRuleGenFeatureCacheNode] = {}
 
-        self._find_global_features()
-        self._find_file_features()
+        self._find_global_features(extractor)
+        self._find_file_features(extractor)
+        self._find_function_and_below_features(fh_list, extractor)
 
-    def _find_global_features(self):
-        for feature, addr in self.extractor.extract_global_features():
+    def _find_global_features(self, extractor: CapaExplorerFeatureExtractor):
+        for feature, addr in extractor.extract_global_features():
             # not all global features may have virtual addresses.
             # if not, then at least ensure the feature shows up in the index.
             # the set of addresses will still be empty.
@@ -73,45 +71,46 @@ class CapaRuleGenFeatureCache:
                 if feature not in self.global_features:
                     self.global_features[feature] = set()
 
-    def _find_file_features(self):
+    def _find_file_features(self, extractor: CapaExplorerFeatureExtractor):
         # not all file features may have virtual addresses.
         # if not, then at least ensure the feature shows up in the index.
         # the set of addresses will still be empty.
-        for feature, addr in self.extractor.extract_file_features():
+        for feature, addr in extractor.extract_file_features():
             if addr is not None:
                 self.file_node.features[feature].add(addr)
             else:
                 if feature not in self.file_node.features:
                     self.file_node.features[feature] = set()
 
-    def _find_function_and_below_features(self, fh: FunctionHandle):
-        f_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(fh, self.file_node)
+    def _find_function_and_below_features(self, fh_list: List[FunctionHandle], extractor: CapaExplorerFeatureExtractor):
+        for fh in fh_list:
+            f_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(fh, self.file_node)
 
-        # extract basic block and below features
-        for bbh in self.extractor.get_basic_blocks(fh):
-            bb_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(bbh, f_node)
+            # extract basic block and below features
+            for bbh in extractor.get_basic_blocks(fh):
+                bb_node: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(bbh, f_node)
 
-            # extract instruction features
-            for ih in self.extractor.get_instructions(fh, bbh):
-                inode: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(ih, bb_node)
+                # extract instruction features
+                for ih in extractor.get_instructions(fh, bbh):
+                    inode: CapaRuleGenFeatureCacheNode = CapaRuleGenFeatureCacheNode(ih, bb_node)
 
-                for feature, addr in self.extractor.extract_insn_features(fh, bbh, ih):
-                    inode.features[feature].add(addr)
+                    for feature, addr in extractor.extract_insn_features(fh, bbh, ih):
+                        inode.features[feature].add(addr)
 
-                self.insn_nodes[inode.address] = inode
+                    self.insn_nodes[inode.address] = inode
 
-            # extract basic block features
-            for feature, addr in self.extractor.extract_basic_block_features(fh, bbh):
-                bb_node.features[feature].add(addr)
+                # extract basic block features
+                for feature, addr in extractor.extract_basic_block_features(fh, bbh):
+                    bb_node.features[feature].add(addr)
 
-            # store basic block features in cache and function parent
-            self.bb_nodes[bb_node.address] = bb_node
+                # store basic block features in cache and function parent
+                self.bb_nodes[bb_node.address] = bb_node
 
-        # extract function features
-        for feature, addr in self.extractor.extract_function_features(fh):
-            f_node.features[feature].add(addr)
+            # extract function features
+            for feature, addr in extractor.extract_function_features(fh):
+                f_node.features[feature].add(addr)
 
-        self.func_nodes[f_node.address] = f_node
+            self.func_nodes[f_node.address] = f_node
 
     def _find_instruction_capabilities(
         self, ruleset: RuleSet, insn: CapaRuleGenFeatureCacheNode
@@ -156,7 +155,7 @@ class CapaRuleGenFeatureCache:
     def find_code_capabilities(
         self, ruleset: RuleSet, fh: FunctionHandle
     ) -> Tuple[FeatureSet, MatchResults, MatchResults, MatchResults]:
-        f_node: Optional[CapaRuleGenFeatureCacheNode] = self._get_cached_func_node(fh)
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self.func_nodes.get(fh.address, None)
         if f_node is None:
             return {}, {}, {}, {}
 
@@ -196,16 +195,8 @@ class CapaRuleGenFeatureCache:
         _, matches = ruleset.match(Scope.FILE, features, NO_ADDRESS)
         return features, matches
 
-    def _get_cached_func_node(self, fh: FunctionHandle) -> Optional[CapaRuleGenFeatureCacheNode]:
-        f_node: Optional[CapaRuleGenFeatureCacheNode] = self.func_nodes.get(fh.address)
-        if f_node is None:
-            # function is not in our cache, do extraction now
-            self._find_function_and_below_features(fh)
-            f_node = self.func_nodes.get(fh.address)
-        return f_node
-
     def get_all_function_features(self, fh: FunctionHandle) -> FeatureSet:
-        f_node: Optional[CapaRuleGenFeatureCacheNode] = self._get_cached_func_node(fh)
+        f_node: Optional[CapaRuleGenFeatureCacheNode] = self.func_nodes.get(fh.address, None)
         if f_node is None:
             return {}
 

capa/ida/plugin/capa_explorer.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/error.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/extractor.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -26,7 +26,7 @@ class CapaExplorerProgressIndicator(QtCore.QObject):
         """
         if ida_kernwin.user_cancelled():
             raise UserCancelledError("user cancelled")
-        self.progress.emit(f"extracting features from {text}")
+        self.progress.emit("extracting features from %s" % text)
 
 
 class CapaExplorerFeatureExtractor(IdaFeatureExtractor):
@@ -40,5 +40,5 @@ class CapaExplorerFeatureExtractor(IdaFeatureExtractor):
         self.indicator = CapaExplorerProgressIndicator()
 
     def extract_function_features(self, fh: FunctionHandle):
-        self.indicator.update(f"function at {hex(fh.inner.start_ea)}")
+        self.indicator.update("function at 0x%X" % fh.inner.start_ea)
         return super().extract_function_features(fh)

capa/ida/plugin/form.py

@@ -1,17 +1,16 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import os
 import copy
 import logging
 import itertools
 import collections
-from enum import IntFlag
 from typing import Any, List, Optional
-from pathlib import Path
 
 import idaapi
 import ida_kernwin
@@ -30,7 +29,7 @@ import capa.features.extractors.ida.extractor
 from capa.rules import Rule
 from capa.engine import FeatureSet
 from capa.rules.cache import compute_ruleset_cache_identifier
-from capa.ida.plugin.icon import ICON
+from capa.ida.plugin.icon import QICON
 from capa.ida.plugin.view import (
     CapaExplorerQtreeView,
     CapaExplorerRulegenEditor,
@@ -58,6 +57,9 @@ CAPA_OFFICIAL_RULESET_URL = f"https://github.com/mandiant/capa-rules/releases/ta
 CAPA_RULESET_DOC_URL = "https://github.com/mandiant/capa/blob/master/doc/rules.md"
 
 
+from enum import IntFlag
+
+
 class Options(IntFlag):
     NO_ANALYSIS = 0  # No auto analysis
     ANALYZE_AUTO = 1  # Runs the analysis when starting the explorer, see details below
@@ -71,22 +73,23 @@ AnalyzeOptionsText = {
 }
 
 
-def write_file(path: Path, data):
+def write_file(path, data):
     """ """
-    path.write_bytes(data)
+    with open(path, "wb") as save_file:
+        save_file.write(data)
 
 
 def trim_function_name(f, max_length=25):
     """ """
     n = idaapi.get_name(f.start_ea)
     if len(n) > max_length:
-        n = f"{n[:max_length]}..."
+        n = "%s..." % n[:max_length]
     return n
 
 
 def update_wait_box(text):
     """update the IDA wait box"""
-    ida_kernwin.replace_wait_box(f"capa explorer...{text}")
+    ida_kernwin.replace_wait_box("capa explorer...%s" % text)
 
 
 class QLineEditClicked(QtWidgets.QLineEdit):
@@ -189,10 +192,8 @@ class CapaExplorerForm(idaapi.PluginForm):
         # caches used to speed up capa explorer analysis - these must be init to None
         self.resdoc_cache: Optional[capa.render.result_document.ResultDocument] = None
         self.program_analysis_ruleset_cache: Optional[capa.rules.RuleSet] = None
-        self.feature_extractor: Optional[CapaExplorerFeatureExtractor] = None
-        self.rulegen_feature_extractor: Optional[CapaExplorerFeatureExtractor] = None
-        self.rulegen_feature_cache: Optional[CapaRuleGenFeatureCache] = None
         self.rulegen_ruleset_cache: Optional[capa.rules.RuleSet] = None
+        self.rulegen_feature_cache: Optional[CapaRuleGenFeatureCache] = None
         self.rulegen_current_function: Optional[FunctionHandle] = None
 
         # models
@@ -237,11 +238,7 @@ class CapaExplorerForm(idaapi.PluginForm):
         load interface and install hooks but do not analyze database
         """
         self.parent = self.FormToPyQtWidget(form)
-
-        pixmap = QtGui.QPixmap()
-        pixmap.loadFromData(ICON)
-
-        self.parent.setWindowIcon(QtGui.QIcon(pixmap))
+        self.parent.setWindowIcon(QICON)
 
         self.load_interface()
         self.load_ida_hooks()
@@ -535,7 +532,7 @@ class CapaExplorerForm(idaapi.PluginForm):
         @param new_ea: destination ea
         @param old_ea: source ea
         """
-        if self.view_tabs.currentIndex() not in (0, 1):
+        if not self.view_tabs.currentIndex() in (0, 1):
             return
 
         if idaapi.get_widget_type(widget) != idaapi.BWN_DISASM:
@@ -576,8 +573,7 @@ class CapaExplorerForm(idaapi.PluginForm):
             path: str = settings.user.get(CAPA_SETTINGS_RULE_PATH, "")
 
             # resolve rules directory - check self and settings first, then ask user
-            # pathlib.Path considers "" equivalent to "." so we first check if rule path is an empty string
-            if not path or not Path(path).exists():
+            if not os.path.exists(path):
                 # configure rules selection messagebox
                 rules_message = QtWidgets.QMessageBox()
                 rules_message.setIcon(QtWidgets.QMessageBox.Information)
@@ -585,7 +581,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                 rules_message.setText("You must specify a directory containing capa rules before running analysis.")
                 rules_message.setInformativeText(
                     "Click 'Ok' to specify a local directory of rules or you can download and extract the official "
-                    + "rules from the URL listed in the details."
+                    f"rules from the URL listed in the details."
                 )
                 rules_message.setDetailedText(f"{CAPA_OFFICIAL_RULESET_URL}")
                 rules_message.setStandardButtons(QtWidgets.QMessageBox.Ok | QtWidgets.QMessageBox.Cancel)
@@ -599,21 +595,20 @@ class CapaExplorerForm(idaapi.PluginForm):
                 if not path:
                     raise UserCancelledError()
 
-                if not Path(path).exists():
-                    logger.error("rule path %s does not exist or cannot be accessed", path)
+                if not os.path.exists(path):
+                    logger.error("rule path %s does not exist or cannot be accessed" % path)
                     return False
 
                 settings.user[CAPA_SETTINGS_RULE_PATH] = path
-        except UserCancelledError:
+        except UserCancelledError as e:
             capa.ida.helpers.inform_user_ida_ui("Analysis requires capa rules")
             logger.warning(
-                "You must specify a directory containing capa rules before running analysis.%s",
-                f"Download and extract the official rules from {CAPA_OFFICIAL_RULESET_URL} (recommended).",
+                f"You must specify a directory containing capa rules before running analysis. Download and extract the official rules from {CAPA_OFFICIAL_RULESET_URL} (recommended)."
             )
             return False
         except Exception as e:
             capa.ida.helpers.inform_user_ida_ui("Failed to load capa rules")
-            logger.exception("Failed to load capa rules (error: %s).", e)
+            logger.error("Failed to load capa rules (error: %s).", e, exc_info=True)
             return False
 
         if ida_kernwin.user_cancelled():
@@ -627,11 +622,11 @@ class CapaExplorerForm(idaapi.PluginForm):
         if not self.ensure_capa_settings_rule_path():
             return False
 
-        rule_path: Path = Path(settings.user.get(CAPA_SETTINGS_RULE_PATH, ""))
+        rule_path: str = settings.user.get(CAPA_SETTINGS_RULE_PATH, "")
         try:
 
             def on_load_rule(_, i, total):
-                update_wait_box(f"loading capa rules from {rule_path} ({i+1} of {total})")
+                update_wait_box("loading capa rules from %s (%d of %d)" % (rule_path, i + 1, total))
                 if ida_kernwin.user_cancelled():
                     raise UserCancelledError("user cancelled")
 
@@ -641,14 +636,14 @@ class CapaExplorerForm(idaapi.PluginForm):
             return None
         except Exception as e:
             capa.ida.helpers.inform_user_ida_ui(
-                f"Failed to load capa rules from {settings.user[CAPA_SETTINGS_RULE_PATH]}"
+                "Failed to load capa rules from %s" % settings.user[CAPA_SETTINGS_RULE_PATH]
             )
 
             logger.error("Failed to load capa rules from %s (error: %s).", settings.user[CAPA_SETTINGS_RULE_PATH], e)
             logger.error(
-                "Make sure your file directory contains properly "  # noqa: G003 [logging statement uses +]
-                + "formatted capa rules. You can download and extract the official rules from %s. "
-                + "Or, for more details, see the rules documentation here: %s",
+                "Make sure your file directory contains properly "
+                "formatted capa rules. You can download and extract the official rules from %s. "
+                "Or, for more details, see the rules documentation here: %s",
                 CAPA_OFFICIAL_RULESET_URL,
                 CAPA_RULESET_DOC_URL,
             )
@@ -692,9 +687,10 @@ class CapaExplorerForm(idaapi.PluginForm):
 
                     update_wait_box("verifying cached results")
 
-                    count_source_rules = self.program_analysis_ruleset_cache.source_rule_count
-                    user_settings = settings.user[CAPA_SETTINGS_RULE_PATH]
-                    view_status_rules: str = f"{user_settings} ({count_source_rules} rules)"
+                    view_status_rules: str = "%s (%d rules)" % (
+                        settings.user[CAPA_SETTINGS_RULE_PATH],
+                        self.program_analysis_ruleset_cache.source_rule_count,
+                    )
 
                     # warn user about potentially outdated rules, depending on the use-case this may be expected
                     if (
@@ -706,15 +702,16 @@ class CapaExplorerForm(idaapi.PluginForm):
 
                         capa.ida.helpers.inform_user_ida_ui("Cached results were generated using different capas rules")
                         logger.warning(
-                            "capa is showing you cached results from a previous analysis run.%s ",
-                            "Your rules have changed since and you should reanalyze the program to see new results.",
+                            "capa is showing you cached results from a previous analysis run. Your rules have changed since and you should reanalyze the program to see new results."
                         )
                         view_status_rules = "no rules matched for cache"
 
-                    cached_results_time = self.resdoc_cache.meta.timestamp.strftime("%Y-%m-%d %H:%M:%S")
-                    new_view_status = f"capa rules: {view_status_rules}, cached results (created {cached_results_time})"
+                    new_view_status = "capa rules: %s, cached results (created %s)" % (
+                        view_status_rules,
+                        self.resdoc_cache.meta.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
+                    )
                 except Exception as e:
-                    logger.exception("Failed to load cached capa results (error: %s).", e)
+                    logger.error("Failed to load cached capa results (error: %s).", e, exc_info=True)
                     return False
             else:
                 # load results from fresh anlaysis
@@ -724,14 +721,16 @@ class CapaExplorerForm(idaapi.PluginForm):
 
                 def slot_progress_feature_extraction(text):
                     """slot function to handle feature extraction progress updates"""
-                    update_wait_box(f"{text} ({self.process_count} of {self.process_total})")
+                    update_wait_box("%s (%d of %d)" % (text, self.process_count, self.process_total))
                     self.process_count += 1
 
+                update_wait_box("initializing feature extractor")
+
                 try:
-                    self.feature_extractor = CapaExplorerFeatureExtractor()
-                    self.feature_extractor.indicator.progress.connect(slot_progress_feature_extraction)
+                    extractor = CapaExplorerFeatureExtractor()
+                    extractor.indicator.progress.connect(slot_progress_feature_extraction)
                 except Exception as e:
-                    logger.exception("Failed to initialize feature extractor (error: %s)", e)
+                    logger.error("Failed to initialize feature extractor (error: %s).", e, exc_info=True)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -741,9 +740,9 @@ class CapaExplorerForm(idaapi.PluginForm):
                 update_wait_box("calculating analysis")
 
                 try:
-                    self.process_total += len(tuple(self.feature_extractor.get_functions()))
+                    self.process_total += len(tuple(extractor.get_functions()))
                 except Exception as e:
-                    logger.exception("Failed to calculate analysis (error: %s).", e)
+                    logger.error("Failed to calculate analysis (error: %s).", e, exc_info=True)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -767,19 +766,15 @@ class CapaExplorerForm(idaapi.PluginForm):
                 update_wait_box("extracting features")
 
                 try:
-                    meta = capa.ida.helpers.collect_metadata([Path(settings.user[CAPA_SETTINGS_RULE_PATH])])
-                    capabilities, counts = capa.main.find_capabilities(
-                        ruleset, self.feature_extractor, disable_progress=True
-                    )
-
-                    meta.analysis.feature_counts = counts["feature_counts"]
-                    meta.analysis.library_functions = counts["library_functions"]
-                    meta.analysis.layout = capa.main.compute_layout(ruleset, self.feature_extractor, capabilities)
+                    meta = capa.ida.helpers.collect_metadata([settings.user[CAPA_SETTINGS_RULE_PATH]])
+                    capabilities, counts = capa.main.find_capabilities(ruleset, extractor, disable_progress=True)
+                    meta["analysis"].update(counts)
+                    meta["analysis"]["layout"] = capa.main.compute_layout(ruleset, extractor, capabilities)
                 except UserCancelledError:
                     logger.info("User cancelled analysis.")
                     return False
                 except Exception as e:
-                    logger.exception("Failed to extract capabilities from database (error: %s)", e)
+                    logger.error("Failed to extract capabilities from database (error: %s)", e, exc_info=True)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -791,8 +786,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                 try:
                     # support binary files specifically for x86/AMD64 shellcode
                     # warn user binary file is loaded but still allow capa to process it
-                    # TODO(mike-hunhoff): check specific architecture of binary files based on how user configured IDA processors
-                    # https://github.com/mandiant/capa/issues/1603
+                    # TODO: check specific architecture of binary files based on how user configured IDA processors
                     if idaapi.get_file_type_name() == "Binary file":
                         logger.warning("-" * 80)
                         logger.warning(" Input file appears to be a binary file.")
@@ -813,7 +807,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                     if capa.main.has_file_limitation(ruleset, capabilities, is_standalone=False):
                         capa.ida.helpers.inform_user_ida_ui("capa encountered file limitation warnings during analysis")
                 except Exception as e:
-                    logger.exception("Failed to check for file limitations (error: %s)", e)
+                    logger.error("Failed to check for file limitations (error: %s)", e, exc_info=True)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -827,7 +821,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                         meta, ruleset, capabilities
                     )
                 except Exception as e:
-                    logger.exception("Failed to collect results (error: %s)", e)
+                    logger.error("Failed to collect results (error: %s)", e, exc_info=True)
                     return False
 
                 if ida_kernwin.user_cancelled():
@@ -843,11 +837,14 @@ class CapaExplorerForm(idaapi.PluginForm):
                     capa.ida.helpers.save_rules_cache_id(ruleset_id)
                     logger.info("Saved cached results to database")
                 except Exception as e:
-                    logger.exception("Failed to save results to database (error: %s)", e)
+                    logger.error("Failed to save results to database (error: %s)", e, exc_info=True)
                     return False
-                user_settings = settings.user[CAPA_SETTINGS_RULE_PATH]
-                count_source_rules = self.program_analysis_ruleset_cache.source_rule_count
-                new_view_status = f"capa rules: {user_settings} ({count_source_rules} rules)"
+
+                new_view_status = "capa rules: %s (%d rules)" % (
+                    settings.user[CAPA_SETTINGS_RULE_PATH],
+                    self.program_analysis_ruleset_cache.source_rule_count,
+                )
+
         # regardless of new analysis, render results - e.g. we may only want to render results after checking
         # show results by function
 
@@ -864,7 +861,7 @@ class CapaExplorerForm(idaapi.PluginForm):
 
             self.model_data.render_capa_doc(self.resdoc_cache, self.view_show_results_by_function.isChecked())
         except Exception as e:
-            logger.exception("Failed to render results (error: %s)", e)
+            logger.error("Failed to render results (error: %s)", e, exc_info=True)
             return False
 
         self.set_view_status_label(new_view_status)
@@ -916,7 +913,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                 has_cache: bool = capa.ida.helpers.idb_contains_cached_results()
             except Exception as e:
                 capa.ida.helpers.inform_user_ida_ui("Failed to check for cached results, reanalyzing program")
-                logger.exception("Failed to check for cached results (error: %s)", e)
+                logger.error("Failed to check for cached results (error: %s)", e, exc_info=True)
                 return False
 
             if ida_kernwin.user_cancelled():
@@ -936,7 +933,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                         ] = capa.ida.helpers.load_and_verify_cached_results()
                     except Exception as e:
                         capa.ida.helpers.inform_user_ida_ui("Failed to verify cached results, reanalyzing program")
-                        logger.exception("Failed to verify cached results (error: %s)", e)
+                        logger.error("Failed to verify cached results (error: %s)", e, exc_info=True)
                         return False
 
                     if results is None:
@@ -949,9 +946,9 @@ class CapaExplorerForm(idaapi.PluginForm):
                         "Reanalyze program",
                         "",
                         ida_kernwin.ASKBTN_YES,
-                        "This database contains capa results generated on "
-                        + results.meta.timestamp.strftime("%Y-%m-%d at %H:%M:%S")
-                        + ".\nLoad existing data or analyze program again?",
+                        f"This database contains capa results generated on "
+                        f"{results.meta.timestamp.strftime('%Y-%m-%d at %H:%M:%S')}.\n"
+                        f"Load existing data or analyze program again?",
                     )
 
                     if btn_id == ida_kernwin.ASKBTN_CANCEL:
@@ -978,21 +975,26 @@ class CapaExplorerForm(idaapi.PluginForm):
         # so we'll work with a local copy of the ruleset.
         ruleset = copy.deepcopy(self.rulegen_ruleset_cache)
 
+        # clear feature cache
+        if self.rulegen_feature_cache is not None:
+            self.rulegen_feature_cache = None
+
         # clear cached function
         if self.rulegen_current_function is not None:
             self.rulegen_current_function = None
 
-        # these are init once objects, create on tab change
-        if self.rulegen_feature_cache is None or self.rulegen_feature_extractor is None:
-            try:
-                update_wait_box("performing one-time file analysis")
-                self.rulegen_feature_extractor = CapaExplorerFeatureExtractor()
-                self.rulegen_feature_cache = CapaRuleGenFeatureCache(self.rulegen_feature_extractor)
-            except Exception as e:
-                logger.exception("Failed to initialize feature extractor (error: %s)", e)
-                return False
-        else:
-            logger.info("Reusing prior rulegen cache")
+        if ida_kernwin.user_cancelled():
+            logger.info("User cancelled analysis.")
+            return False
+
+        update_wait_box("Initializing feature extractor")
+
+        try:
+            # must use extractor to get function, as capa analysis requires casted object
+            extractor = CapaExplorerFeatureExtractor()
+        except Exception as e:
+            logger.error("Failed to initialize feature extractor (error: %s)", e, exc_info=True)
+            return False
 
         if ida_kernwin.user_cancelled():
             logger.info("User cancelled analysis.")
@@ -1004,9 +1006,24 @@ class CapaExplorerForm(idaapi.PluginForm):
         try:
             f = idaapi.get_func(idaapi.get_screen_ea())
             if f is not None:
-                self.rulegen_current_function = self.rulegen_feature_extractor.get_function(f.start_ea)
+                self.rulegen_current_function = extractor.get_function(f.start_ea)
         except Exception as e:
-            logger.exception("Failed to resolve function at address 0x%X (error: %s)", f.start_ea, e)
+            logger.error("Failed to resolve function at address 0x%X (error: %s)", f.start_ea, e, exc_info=True)
+            return False
+
+        if ida_kernwin.user_cancelled():
+            logger.info("User cancelled analysis.")
+            return False
+
+        # extract features
+        try:
+            fh_list: List[FunctionHandle] = []
+            if self.rulegen_current_function is not None:
+                fh_list.append(self.rulegen_current_function)
+
+            self.rulegen_feature_cache = CapaRuleGenFeatureCache(fh_list, extractor)
+        except Exception as e:
+            logger.error("Failed to extract features (error: %s)", e, exc_info=True)
             return False
 
         if ida_kernwin.user_cancelled():
@@ -1032,7 +1049,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                     for addr, _ in result:
                         all_function_features[capa.features.common.MatchedRule(name)].add(addr)
         except Exception as e:
-            logger.exception("Failed to generate rule matches (error: %s)", e)
+            logger.error("Failed to generate rule matches (error: %s)", e, exc_info=True)
             return False
 
         if ida_kernwin.user_cancelled():
@@ -1053,7 +1070,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                 for addr, _ in result:
                     all_file_features[capa.features.common.MatchedRule(name)].add(addr)
         except Exception as e:
-            logger.exception("Failed to generate file rule matches (error: %s)", e)
+            logger.error("Failed to generate file rule matches (error: %s)", e, exc_info=True)
             return False
 
         if ida_kernwin.user_cancelled():
@@ -1073,10 +1090,10 @@ class CapaExplorerForm(idaapi.PluginForm):
             self.view_rulegen_features.load_features(all_file_features, all_function_features)
 
             self.set_view_status_label(
-                f"capa rules: {settings.user[CAPA_SETTINGS_RULE_PATH]} ({settings.user[CAPA_SETTINGS_RULE_PATH]} rules)"
+                "capa rules: %s (%d rules)" % (settings.user[CAPA_SETTINGS_RULE_PATH], ruleset.source_rule_count)
             )
         except Exception as e:
-            logger.exception("Failed to render views (error: %s)", e)
+            logger.error("Failed to render views (error: %s)", e, exc_info=True)
             return False
 
         return True
@@ -1161,7 +1178,7 @@ class CapaExplorerForm(idaapi.PluginForm):
             assert self.rulegen_ruleset_cache is not None
             assert self.rulegen_feature_cache is not None
         except Exception as e:
-            logger.exception("Failed to access cache (error: %s)", e)
+            logger.error("Failed to access cache (error: %s)", e, exc_info=True)
             self.set_rulegen_status("Error: see console output for more details")
             return
 
@@ -1205,11 +1222,11 @@ class CapaExplorerForm(idaapi.PluginForm):
                 self.set_rulegen_status(f"Failed to create function rule matches from rule set ({e})")
                 return
 
-            if rule.scope == capa.rules.Scope.FUNCTION and rule.name in func_matches:
+            if rule.scope == capa.rules.Scope.FUNCTION and rule.name in func_matches.keys():
                 is_match = True
-            elif rule.scope == capa.rules.Scope.BASIC_BLOCK and rule.name in bb_matches:
+            elif rule.scope == capa.rules.Scope.BASIC_BLOCK and rule.name in bb_matches.keys():
                 is_match = True
-            elif rule.scope == capa.rules.Scope.INSTRUCTION and rule.name in insn_matches:
+            elif rule.scope == capa.rules.Scope.INSTRUCTION and rule.name in insn_matches.keys():
                 is_match = True
         elif rule.scope == capa.rules.Scope.FILE:
             try:
@@ -1217,7 +1234,7 @@ class CapaExplorerForm(idaapi.PluginForm):
             except Exception as e:
                 self.set_rulegen_status(f"Failed to create file rule matches from rule set ({e})")
                 return
-            if rule.name in file_matches:
+            if rule.name in file_matches.keys():
                 is_match = True
         else:
             is_match = False
@@ -1244,6 +1261,7 @@ class CapaExplorerForm(idaapi.PluginForm):
         elif index == 1:
             self.set_view_status_label(self.view_status_label_rulegen_cache)
             self.view_status_label_analysis_cache = status_prev
+
             self.view_reset_button.setText("Clear")
 
     def slot_rulegen_editor_update(self):
@@ -1305,10 +1323,10 @@ class CapaExplorerForm(idaapi.PluginForm):
             idaapi.info("No program analysis to save.")
             return
 
-        s = self.resdoc_cache.model_dump_json().encode("utf-8")
+        s = self.resdoc_cache.json().encode("utf-8")
 
-        path = Path(self.ask_user_capa_json_file())
-        if not path.exists():
+        path = self.ask_user_capa_json_file()
+        if not path:
             return
 
         write_file(path, s)
@@ -1320,8 +1338,8 @@ class CapaExplorerForm(idaapi.PluginForm):
             idaapi.info("No rule to save.")
             return
 
-        path = Path(self.ask_user_capa_rule_file())
-        if not path.exists():
+        path = self.ask_user_capa_rule_file()
+        if not path:
             return
 
         write_file(path, s)

capa/ida/plugin/hooks.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -30,7 +30,7 @@ class CapaExplorerIdaHooks(idaapi.UI_Hooks):
 
         @retval must be 0
         """
-        self.process_action_handle = self.process_action_hooks.get(name)
+        self.process_action_handle = self.process_action_hooks.get(name, None)
 
         if self.process_action_handle:
             self.process_action_handle(self.process_action_meta)

capa/ida/plugin/item.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -30,7 +30,7 @@ def info_to_name(display):
 
 def ea_to_hex(ea):
     """convert effective address (ea) to hex for display"""
-    return f"{hex(ea)}"
+    return "%08X" % ea
 
 
 class CapaExplorerDataItem:
@@ -130,7 +130,8 @@ class CapaExplorerDataItem:
 
     def children(self) -> Iterator["CapaExplorerDataItem"]:
         """yield children"""
-        yield from self._children
+        for child in self._children:
+            yield child
 
     def removeChildren(self):
         """remove children"""

capa/ida/plugin/model.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -369,35 +369,34 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             if statement.type != rd.CompoundStatementType.NOT:
                 display = statement.type
                 if statement.description:
-                    display += f" ({statement.description})"
+                    display += " (%s)" % statement.description
                 return CapaExplorerDefaultItem(parent, display)
         elif isinstance(statement, rd.CompoundStatement) and statement.type == rd.CompoundStatementType.NOT:
-            # TODO(mike-hunhoff): verify that we can display NOT statements
-            # https://github.com/mandiant/capa/issues/1602
+            # TODO: do we display 'not'
             pass
         elif isinstance(statement, rd.SomeStatement):
-            display = f"{statement.count} or more"
+            display = "%d or more" % statement.count
             if statement.description:
-                display += f" ({statement.description})"
+                display += " (%s)" % statement.description
             return CapaExplorerDefaultItem(parent, display)
         elif isinstance(statement, rd.RangeStatement):
             # `range` is a weird node, its almost a hybrid of statement + feature.
             # it is a specific feature repeated multiple times.
             # there's no additional logic in the feature part, just the existence of a feature.
             # so, we have to inline some of the feature rendering here.
-            display = f"count({self.capa_doc_feature_to_display(statement.child)}): "
+            display = "count(%s): " % self.capa_doc_feature_to_display(statement.child)
 
             if statement.max == statement.min:
-                display += f"{statement.min}"
+                display += "%d" % (statement.min)
             elif statement.min == 0:
-                display += f"{statement.max} or fewer"
+                display += "%d or fewer" % (statement.max)
             elif statement.max == (1 << 64 - 1):
-                display += f"{statement.min} or more"
+                display += "%d or more" % (statement.min)
             else:
-                display += f"between {statement.min} and {statement.max}"
+                display += "between %d and %d" % (statement.min, statement.max)
 
             if statement.description:
-                display += f" ({statement.description})"
+                display += " (%s)" % statement.description
 
             parent2 = CapaExplorerFeatureItem(parent, display=display)
 
@@ -409,7 +408,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         elif isinstance(statement, rd.SubscopeStatement):
             display = str(statement.scope)
             if statement.description:
-                display += f" ({statement.description})"
+                display += " (%s)" % statement.description
             return CapaExplorerSubscopeItem(parent, display)
         else:
             raise RuntimeError("unexpected match statement type: " + str(statement))
@@ -422,13 +421,12 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         @param doc: result doc
         """
         if not match.success:
-            # TODO(mike-hunhoff): display failed branches at some point? Help with debugging rules?
-            # https://github.com/mandiant/capa/issues/1601
+            # TODO: display failed branches at some point? Help with debugging rules?
             return
 
         # optional statement with no successful children is empty
         if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
-            if not any(m.success for m in match.children):
+            if not any(map(lambda m: m.success, match.children)):
                 return
 
         if isinstance(match.node, rd.StatementNode):
@@ -539,7 +537,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
         if value:
             if isinstance(feature, frzf.StringFeature):
-                value = f'"{capa.features.common.escape_string(value)}"'
+                value = '"%s"' % capa.features.common.escape_string(value)
 
             if isinstance(feature, frzf.PropertyFeature) and feature.access is not None:
                 key = f"property/{feature.access}"
@@ -549,11 +547,11 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                 key = f"operand[{feature.index}].offset"
 
             if feature.description:
-                return f"{key}({value} = {feature.description})"
+                return "%s(%s = %s)" % (key, value, feature.description)
             else:
-                return f"{key}({value})"
+                return "%s(%s)" % (key, value)
         else:
-            return f"{key}"
+            return "%s" % key
 
     def render_capa_doc_feature_node(
         self,
@@ -628,7 +626,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             matched_rule_source = ""
 
             # check if match is a matched rule
-            matched_rule = doc.rules.get(feature.match)
+            matched_rule = doc.rules.get(feature.match, None)
             if matched_rule is not None:
                 matched_rule_source = matched_rule.source
 
@@ -671,7 +669,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         elif isinstance(feature, frzf.StringFeature):
             # display string preview
             return CapaExplorerStringViewItem(
-                parent, display, location, f'"{capa.features.common.escape_string(feature.string)}"'
+                parent, display, location, '"%s"' % capa.features.common.escape_string(feature.string)
             )
 
         elif isinstance(

capa/ida/plugin/proxy.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/ida/plugin/view.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,7 +6,6 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import re
-from typing import Dict, Optional
 from collections import Counter
 
 import idc
@@ -59,12 +58,12 @@ def parse_yaml_line(feature):
         if m:
             # reconstruct count without description
             feature, value, description, count = m.groups()
-            feature = f"- count({feature}({value})): {count}"
+            feature = "- count(%s(%s)): %s" % (feature, value, count)
     elif not feature.startswith("#"):
         feature, _, comment = feature.partition("#")
         feature, _, description = feature.partition("=")
 
-    return (o.strip() for o in (feature, description, comment))
+    return map(lambda o: o.strip(), (feature, description, comment))
 
 
 def parse_node_for_feature(feature, description, comment, depth):
@@ -73,18 +72,18 @@ def parse_node_for_feature(feature, description, comment, depth):
     display = ""
 
     if feature.startswith("#"):
-        display += f"{' '*depth}{feature}\n"
+        display += "%s%s\n" % (" " * depth, feature)
     elif description:
         if feature.startswith(("- and", "- or", "- optional", "- basic block", "- not", "- instruction:")):
-            display += f"{' '*depth}{feature}\n"
+            display += "%s%s" % (" " * depth, feature)
             if comment:
-                display += f" # {comment}"
-            display += f"\n{' '*(depth+2)}- description: {description}\n"
+                display += " # %s" % comment
+            display += "\n%s- description: %s\n" % (" " * (depth + 2), description)
         elif feature.startswith("- string"):
-            display += f"{' '*depth}{feature}\n"
+            display += "%s%s" % (" " * depth, feature)
             if comment:
-                display += f" # {comment}"
-            display += f"\n{' '*(depth+2)}description: {description}\n"
+                display += " # %s" % comment
+            display += "\n%sdescription: %s\n" % (" " * (depth + 2), description)
         elif feature.startswith("- count"):
             # count is weird, we need to format description based on feature type, so we parse with regex
             # assume format - count(<feature_name>(<feature_value>)): <count>
@@ -92,22 +91,28 @@ def parse_node_for_feature(feature, description, comment, depth):
             if m:
                 name, value, count = m.groups()
                 if name in ("string",):
-                    display += f"{' '*depth}{feature}"
+                    display += "%s%s" % (" " * depth, feature)
                     if comment:
-                        display += f" # {comment}"
-                    display += f"\n{' '*(depth+2)}description: {description}\n"
+                        display += " # %s" % comment
+                    display += "\n%sdescription: %s\n" % (" " * (depth + 2), description)
                 else:
-                    display += f"{' '*depth}- count({name}({value} = {description})): {count}"
+                    display += "%s- count(%s(%s = %s)): %s" % (
+                        " " * depth,
+                        name,
+                        value,
+                        description,
+                        count,
+                    )
                     if comment:
-                        display += f" # {comment}\n"
+                        display += " # %s\n" % comment
         else:
-            display += f"{' '*depth}{feature} = {description}"
+            display += "%s%s = %s" % (" " * depth, feature, description)
             if comment:
-                display += f" # {comment}\n"
+                display += " # %s\n" % comment
     else:
-        display += f"{' '*depth}{feature}"
+        display += "%s%s" % (" " * depth, feature)
         if comment:
-            display += f" # {comment}\n"
+            display += " # %s\n" % comment
 
     return display if display.endswith("\n") else display + "\n"
 
@@ -193,14 +198,14 @@ class CapaExplorerRulegenPreview(QtWidgets.QTextEdit):
             "    name: <insert_name>",
             "    namespace: <insert_namespace>",
             "    authors:",
-            f"      - {author}",
-            f"    scope: {scope}",
+            "      - %s" % author,
+            "    scope: %s" % scope,
             "    references:",
             "      - <insert_references>",
             "    examples:",
-            f"      - {capa.ida.helpers.get_file_md5().upper()}:{hex(ea)}"
+            "      - %s:0x%X" % (capa.ida.helpers.get_file_md5().upper(), ea)
             if ea
-            else f"      - {capa.ida.helpers.get_file_md5().upper()}",
+            else "      - %s" % (capa.ida.helpers.get_file_md5().upper()),
             "  features:",
         ]
         self.setText("\n".join(metadata_default))
@@ -499,13 +504,12 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
             rule_text += "\n  features:\n"
 
         for o in iterate_tree(self):
-            feature, description, comment = (o.strip() for o in tuple(o.text(i) for i in range(3)))
+            feature, description, comment = map(lambda o: o.strip(), tuple(o.text(i) for i in range(3)))
             rule_text += parse_node_for_feature(feature, description, comment, calc_item_depth(o))
 
-        # TODO(mike-hunhoff): we avoid circular update by disabling signals when updating
+        # FIXME we avoid circular update by disabling signals when updating
         # the preview. Preferably we would refactor the code to avoid this
-        # in the first place.
-        # https://github.com/mandiant/capa/issues/1600
+        # in the first place
         self.preview.blockSignals(True)
         self.preview.setPlainText(rule_text)
         self.preview.blockSignals(False)
@@ -535,7 +539,7 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
 
         # build submenu with modify actions
         sub_menu = build_context_menu(self.parent(), sub_actions)
-        sub_menu.setTitle(f"Nest feature{'' if len(tuple(self.get_features(selected=True))) == 1 else 's'}")
+        sub_menu.setTitle("Nest feature%s" % ("" if len(tuple(self.get_features(selected=True))) == 1 else "s"))
 
         # build main menu with submenu + main actions
         menu = build_context_menu(self.parent(), (sub_menu,) + actions)
@@ -648,23 +652,23 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
         counted = list(zip(Counter(features).keys(), Counter(features).values()))
 
         # single features
-        for k, _ in filter(lambda t: t[1] == 1, counted):
+        for k, v in filter(lambda t: t[1] == 1, counted):
             if isinstance(k, (capa.features.common.String,)):
-                value = f'"{capa.features.common.escape_string(k.get_value_str())}"'
+                value = '"%s"' % capa.features.common.escape_string(k.get_value_str())
             else:
                 value = k.get_value_str()
-            self.new_feature_node(top_node, (f"- {k.name.lower()}: {value}", ""))
+            self.new_feature_node(top_node, ("- %s: %s" % (k.name.lower(), value), ""))
 
         # n > 1 features
         for k, v in filter(lambda t: t[1] > 1, counted):
             if k.value:
                 if isinstance(k, (capa.features.common.String,)):
-                    value = f'"{capa.features.common.escape_string(k.get_value_str())}"'
+                    value = '"%s"' % capa.features.common.escape_string(k.get_value_str())
                 else:
                     value = k.get_value_str()
-                display = f"- count({k.name.lower()}({value})): {v}"
+                display = "- count(%s(%s)): %d" % (k.name.lower(), value, v)
             else:
-                display = f"- count({k.name.lower()}): {v}"
+                display = "- count(%s): %d" % (k.name.lower(), v)
             self.new_feature_node(top_node, (display, ""))
 
         self.update_preview()
@@ -684,12 +688,10 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
 
             # we don't add a new node for description; either set description column of parent's last child
             # or the parent itself
-            if feature.startswith("description:"):
-                description = feature[len("description:") :].lstrip()
-                if parent.childCount():
-                    parent.child(parent.childCount() - 1).setText(1, description)
-                else:
-                    parent.setText(1, description)
+            if parent.childCount():
+                parent.child(parent.childCount() - 1).setText(1, feature.lstrip("description:").lstrip())
+            else:
+                parent.setText(1, feature.lstrip("description:").lstrip())
             return None
         elif feature.startswith("- description:"):
             if not parent:
@@ -697,8 +699,7 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
                 return None
 
             # we don't add a new node for description; set the description column of the parent instead
-            description = feature[len("- description:") :].lstrip()
-            parent.setText(1, description)
+            parent.setText(1, feature.lstrip("- description:").lstrip())
             return None
 
         node = QtWidgets.QTreeWidgetItem(parent)
@@ -879,7 +880,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
             if isinstance(self.selectedItems()[0].data(0, 0x100), capa.features.common.Bytes):
                 actions.append(("Add n bytes...", (), self.slot_add_n_bytes_feature))
         else:
-            action_add_features_fmt = f"Add {selected_items_count} features"
+            action_add_features_fmt = "Add %d features" % selected_items_count
 
         actions.append((action_add_features_fmt, (), self.slot_add_selected_features))
 
@@ -1015,7 +1016,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
 
         return o
 
-    def load_features(self, file_features, func_features: Optional[Dict] = None):
+    def load_features(self, file_features, func_features={}):
         """ """
         self.parse_features_for_tree(self.new_parent_node(self, ("File Scope",)), file_features)
         if func_features:
@@ -1028,7 +1029,7 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
 
         def format_address(e):
             if isinstance(e, AbsoluteVirtualAddress):
-                return f"{hex(int(e))}"
+                return "%X" % int(e)
             else:
                 return ""
 
@@ -1037,8 +1038,8 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
             name = feature.name.lower()
             value = feature.get_value_str()
             if isinstance(feature, (capa.features.common.String,)):
-                value = f'"{capa.features.common.escape_string(value)}"'
-            return f"{name}({value})"
+                value = '"%s"' % capa.features.common.escape_string(value)
+            return "%s(%s)" % (name, value)
 
         for feature, addrs in sorted(features.items(), key=lambda k: sorted(k[1])):
             if isinstance(feature, capa.features.basicblock.BasicBlock):
@@ -1224,7 +1225,8 @@ class CapaExplorerQtreeView(QtWidgets.QTreeView):
             yield self.new_action(*action)
 
         # add default actions
-        yield from self.load_default_context_menu_actions(data)
+        for action in self.load_default_context_menu_actions(data):
+            yield action
 
     def load_default_context_menu(self, pos, item, model_index):
         """create default custom context menu

capa/optimizer.py

@@ -1,10 +1,3 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 import logging
 
 import capa.engine as ceng
@@ -29,7 +22,7 @@ def get_node_cost(node):
         # substring and regex features require a full scan of each string
         # which we anticipate is more expensive then a hash lookup feature (e.g. mnemonic or count).
         #
-        # fun research: compute the average cost of these feature relative to hash feature
+        # TODO: compute the average cost of these feature relative to hash feature
         # and adjust the factor accordingly.
         return 2
 

capa/perf.py

@@ -1,10 +1,3 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
 import typing
 import collections
 

capa/render/default.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -11,6 +11,7 @@ import collections
 import tabulate
 
 import capa.render.utils as rutils
+import capa.features.freeze as frz
 import capa.render.result_document as rd
 import capa.features.freeze.features as frzf
 from capa.rules import RuleSet
@@ -39,7 +40,7 @@ def render_meta(doc: rd.ResultDocument, ostream: StringIO):
         ("path", doc.meta.sample.path),
     ]
 
-    ostream.write(tabulate.tabulate(rows, tablefmt="mixed_outline"))
+    ostream.write(tabulate.tabulate(rows, tablefmt="psql"))
     ostream.write("\n")
 
 
@@ -48,7 +49,7 @@ def find_subrule_matches(doc: rd.ResultDocument):
     collect the rule names that have been matched as a subrule match.
     this way we can avoid displaying entries for things that are too specific.
     """
-    matches = set()
+    matches = set([])
 
     def rec(match: rd.Match):
         if not match.success:
@@ -64,7 +65,7 @@ def find_subrule_matches(doc: rd.ResultDocument):
             matches.add(match.node.feature.match)
 
     for rule in rutils.capability_rules(doc):
-        for _, match in rule.matches:
+        for address, match in rule.matches:
             rec(match)
 
     return matches
@@ -96,12 +97,12 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
         if count == 1:
             capability = rutils.bold(rule.meta.name)
         else:
-            capability = f"{rutils.bold(rule.meta.name)} ({count} matches)"
+            capability = "%s (%d matches)" % (rutils.bold(rule.meta.name), count)
         rows.append((capability, rule.meta.namespace))
 
     if rows:
         ostream.write(
-            tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline")
+            tabulate.tabulate(rows, headers=[width("CAPABILITY", 50), width("NAMESPACE", 50)], tablefmt="psql")
         )
         ostream.write("\n")
     else:
@@ -134,9 +135,9 @@ def render_attack(doc: rd.ResultDocument, ostream: StringIO):
         inner_rows = []
         for technique, subtechnique, id in sorted(techniques):
             if not subtechnique:
-                inner_rows.append(f"{rutils.bold(technique)} {id}")
+                inner_rows.append("%s %s" % (rutils.bold(technique), id))
             else:
-                inner_rows.append(f"{rutils.bold(technique)}::{subtechnique} {id}")
+                inner_rows.append("%s::%s %s" % (rutils.bold(technique), subtechnique, id))
         rows.append(
             (
                 rutils.bold(tactic.upper()),
@@ -147,7 +148,7 @@ def render_attack(doc: rd.ResultDocument, ostream: StringIO):
     if rows:
         ostream.write(
             tabulate.tabulate(
-                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="mixed_grid"
+                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="psql"
             )
         )
         ostream.write("\n")
@@ -177,9 +178,9 @@ def render_mbc(doc: rd.ResultDocument, ostream: StringIO):
         inner_rows = []
         for behavior, method, id in sorted(behaviors):
             if not method:
-                inner_rows.append(f"{rutils.bold(behavior)} [{id}]")
+                inner_rows.append("%s [%s]" % (rutils.bold(behavior), id))
             else:
-                inner_rows.append(f"{rutils.bold(behavior)}::{method} [{id}]")
+                inner_rows.append("%s::%s [%s]" % (rutils.bold(behavior), method, id))
         rows.append(
             (
                 rutils.bold(objective.upper()),
@@ -189,9 +190,7 @@ def render_mbc(doc: rd.ResultDocument, ostream: StringIO):
 
     if rows:
         ostream.write(
-            tabulate.tabulate(
-                rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="mixed_grid"
-            )
+            tabulate.tabulate(rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="psql")
         )
         ostream.write("\n")