extractors: Do not use generate_api_features

`generate_api_features` was merged with the implementation of
`generate_import_features` and replaced by `generate_symbol`:
2b2656c2a3
 Use the new function in the miasm backend implementation.

.devcontainer/Dockerfile

@@ -1,21 +0,0 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3/.devcontainer/base.Dockerfile
-
-# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
-ARG VARIANT="3.10-bullseye"
-FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
-
-# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
-ARG NODE_VERSION="none"
-RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
-
-# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
-# COPY requirements.txt /tmp/pip-tmp/
-# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
-#    && rm -rf /tmp/pip-tmp
-
-# [Optional] Uncomment this section to install additional OS packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>
-
-# [Optional] Uncomment this line to install global node packages.
-# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.devcontainer/devcontainer.json

@@ -1,51 +0,0 @@
-// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
-// https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3
-{
-	"name": "Python 3",
-	"build": {
-		"dockerfile": "Dockerfile",
-		"context": "..",
-		"args": { 
-			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
-			// Append -bullseye or -buster to pin to an OS version.
-			// Use -bullseye variants on local on arm64/Apple Silicon.
-			"VARIANT": "3.10",
-			// Options
-			"NODE_VERSION": "none"
-		}
-	},
-
-	// Set *default* container specific settings.json values on container create.
-	"settings": { 
-		"python.defaultInterpreterPath": "/usr/local/bin/python",
-		"python.linting.enabled": true,
-		"python.linting.pylintEnabled": true,
-		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
-		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
-		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
-	},
-
-	// Add the IDs of extensions you want installed when the container is created.
-	"extensions": [
-		"ms-python.python",
-		"ms-python.vscode-pylance"
-	],
-
-	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	// "forwardPorts": [],
-
-	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "git submodule update --init && pip3 install --user -e .[dev] && pre-commit install",
-
-	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-	"remoteUser": "vscode",
-	"features": {
-		"git": "latest"
-	}
-}

.gitattributes

@@ -1,9 +0,0 @@
-# Set the default behavior, in case people don't have core.autocrlf set.
-* text=auto
-
-# Explicitly declare text files you want to always be normalized and converted
-# to native line endings on checkout.
-*.py text
-*.yml text
-*.md text
-*.txt text

.github/CODE_OF_CONDUCT.md

@@ -1,46 +1,46 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version]
-
-[homepage]: https://contributor-covenant.org
-[version]: https://contributor-covenant.org/version/1/4/
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

.github/CONTRIBUTING.md

@@ -1,210 +1,197 @@
-# Contributing to Capa
-
-First off, thanks for taking the time to contribute!
-
-The following is a set of guidelines for contributing to capa and its packages, which are hosted in the [Mandiant Organization](https://github.com/mandiant) on GitHub. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
-
-#### Table Of Contents
-
-[Code of Conduct](#code-of-conduct)
-
-[What should I know before I get started?](#what-should-i-know-before-i-get-started)
-  * [Capa and its Repositories](#capa-and-its-repositories)
-  * [Capa Design Decisions](#design-decisions)
-
-[How Can I Contribute?](#how-can-i-contribute)
-  * [Reporting Bugs](#reporting-bugs)
-  * [Suggesting Enhancements](#suggesting-enhancements)
-  * [Your First Code Contribution](#your-first-code-contribution)
-  * [Pull Requests](#pull-requests)
-
-[Styleguides](#styleguides)
-  * [Git Commit Messages](#git-commit-messages)
-  * [Python Styleguide](#python-styleguide)
-  * [Rules Styleguide](#rules-styleguide)
-
-## Code of Conduct
-
-This project and everyone participating in it is governed by the [Capa Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers.
-
-## What should I know before I get started?
-
-### Capa and its repositories
-
-We host the capa project as three GitHub repositories:
-  - [capa](https://github.com/mandiant/capa)
-  - [capa-rules](https://github.com/mandiant/capa-rules)
-  - [capa-testfiles](https://github.com/mandiant/capa-testfiles)
-  
-The command line tools, logic engine, and other Python source code are found in the `capa` repository.
-This is the repository to fork when you want to enhance the features, performance, or user interface of capa.
-Do *not* push rules directly to this repository, instead...
-
-The standard rules contributed by the community are found in the `capa-rules` repository.
-When you have an idea for a new rule, you should open a PR against `capa-rules`.
-We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and discussions should happen.
-If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`.
-Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way.
-
-Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository.
-These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites;
- however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate,
- a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth.
-
-### Design Decisions
-
-When we make a significant decision in how we maintain the project and what we can or cannot support,
- we will document it in the [capa issues tracker](https://github.com/mandiant/capa/issues).
-This is the best place review our discussions about what/how/why we do things in the project.
-If you have a question, check to see if it is documented there.
-If it is *not* documented there, or you can't find an answer, please open a issue.
-We'll link to existing issues when appropriate to keep discussions in one place.
-
-## How Can I Contribute?
-
-### Reporting Bugs
-
-This section guides you through submitting a bug report for capa.
-Following these guidelines helps maintainers and the community understand your report, reproduce the behavior, and find related reports.
-
-Before creating bug reports, please check [this list](#before-submitting-a-bug-report)
- as you might find out that you don't need to create one.
-When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report).
-Fill out [the required template](./ISSUE_TEMPLATE/bug_report.md),
- the information it asks for helps us resolve issues faster.
-
-> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one.
-
-#### Before Submitting A Bug Report
-
-* **Determine [which repository the problem should be reported in](#capa-and-its-repositories)**.
-* **Perform a [cursory search](https://github.com/mandiant/capa/issues?q=is%3Aissue)** to see if the problem has already been reported. If it has **and the issue is still open**, add a comment to the existing issue instead of opening a new one.
-
-#### How Do I Submit A (Good) Bug Report?
-
-Bugs are tracked as [GitHub issues](https://guides.github.com/features/issues/).
-After you've determined [which repository](#capa-and-its-repositories) your bug is related to,
- create an issue on that repository and provide the following information by filling in 
- [the template](./ISSUE_TEMPLATE/bug_report.md).
-
-Explain the problem and include additional details to help maintainers reproduce the problem:
-
-* **Use a clear and descriptive title** for the issue to identify the problem.
-* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started capa, e.g. which command exactly you used in the terminal, or how you started capa otherwise.
-* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
-* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior.
-* **Explain which behavior you expected to see instead and why.**
-* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
-* **If you're reporting that capa crashed**, include the stack trace from the terminal. Include the stack trace in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist.
-* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below.
-
-Provide more context by answering these questions:
-
-* **Did the problem start happening recently** (e.g. after updating to a new version of capa) or was this always a problem?
-* If the problem started happening recently, **can you reproduce the problem in an older version of capa?** What's the most recent version in which the problem doesn't happen? You can download older versions of capa from [the releases page](https://github.com/mandiant/capa/releases).
-* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens.
-* If the problem is related to working with files (e.g. opening and editing files), **does the problem happen for all files and projects or only some?** Does the problem happen only when working with local or remote files (e.g. on network drives), with files of a specific type (e.g. only JavaScript or Python files), with large files or files with very long lines, or with files in a specific encoding? Is there anything else special about the files you are using?
-
-Include details about your configuration and environment:
-
-* **Which version of capa are you using?** You can get the exact version by running `capa --version` in your terminal.
-* **What's the name and version of the OS you're using**?
-
-### Suggesting Enhancements
-
-This section guides you through submitting an enhancement suggestion for capa, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion and find related suggestions.
-
-Before creating enhancement suggestions, please check [this list](#before-submitting-an-enhancement-suggestion) as you might find out that you don't need to create one. When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](./ISSUE_TEMPLATE/feature_request.md), including the steps that you imagine you would take if the feature you're requesting existed.
-
-#### Before Submitting An Enhancement Suggestion
-
-* **Determine [which repository the enhancement should be suggested in](#capa-and-its-repositories).**
-* **Perform a [cursory search](https://github.com/mandiant/capa/issues?q=is%3Aissue)** to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one.
-
-#### How Do I Submit A (Good) Enhancement Suggestion?
-
-Enhancement suggestions are tracked as [GitHub issues](https://guides.github.com/features/issues/). After you've determined [which repository](#capa-and-its-repositories) your enhancement suggestion is related to, create an issue on that repository and provide the following information:
-
-* **Use a clear and descriptive title** for the issue to identify the suggestion.
-* **Provide a step-by-step description of the suggested enhancement** in as many details as possible.
-* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
-* **Describe the current behavior** and **explain which behavior you expected to see instead** and why.
-* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of capa which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
-* **Explain why this enhancement would be useful** to most capa users and isn't something that can or should be implemented as an external tool that uses capa as a library.
-* **Specify which version of capa you're using.** You can get the exact version by running `capa --version` in your terminal.
-* **Specify the name and version of the OS you're using.**
-
-### Your First Code Contribution
-
-Unsure where to begin contributing to capa? You can start by looking through these `good-first-issue` and `rule-idea` issues:
-
-* [good-first-issue](https://github.com/mandiant/capa/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code, and a test or two.
-* [rule-idea](https://github.com/mandiant/capa-rules/issues?q=is%3Aissue+is%3Aopen+label%3A%22rule+idea%22) - issues that describe potential new rule ideas.
-
-Both issue lists are sorted by total number of comments. While not perfect, number of comments is a reasonable proxy for impact a given change will have.
-
-#### Local development
-
-capa and all its resources can be developed locally. 
-For instructions on how to do this, see the "Method 3" section of the [installation guide](https://github.com/mandiant/capa/blob/master/doc/installation.md).
-
-### Pull Requests
-
-The process described here has several goals:
-
-- Maintain capa's quality
-- Fix problems that are important to users
-- Engage the community in working toward the best possible capa
-- Enable a sustainable system for capa's maintainers to review contributions
-
-Please follow these steps to have your contribution considered by the maintainers:
-
-0. Sign the [Contributor License Agreement](#contributor-license-agreement)
-1. Follow the [styleguides](#styleguides)
-2. Update the CHANGELOG and add tests and documentation. In case they are not needed, indicate it in [the PR template](pull_request_template.md).
-3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing? </summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
-
-While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
-
-### Contributor License Agreement
-
-Contributions to this project must be accompanied by a Contributor License
-Agreement. You (or your employer) retain the copyright to your contribution,
-this simply gives us permission to use and redistribute your contributions as
-part of the project. Head over to <https://cla.developers.google.com/> to see
-your current agreements on file or to sign a new one.
-
-You generally only need to submit a CLA once, so if you've already submitted one
-(even if it was for a different project), you probably don't need to do it
-again.
-
-## Styleguides
-
-### Git Commit Messages
-
-* Use the present tense ("Add feature" not "Added feature")
-* Use the imperative mood ("Move cursor to..." not "Moves cursor to...")
-* Prefix the first line with the component in question ("rules: ..." or "render: ...")
-* Reference issues and pull requests liberally after the first line
-
-### Python Styleguide
-
-All Python code must adhere to the style guide used by capa: 
-
-  1. [PEP8](https://www.python.org/dev/peps/pep-0008/), with clarifications from
-  2. [Willi's style guide](https://docs.google.com/document/d/1iRpeg-w4DtibwytUyC_dDT7IGhNGBP25-nQfuBa-Fyk/edit?usp=sharing), formatted with
-  3. [isort](https://pypi.org/project/isort/) (with line width 120 and ordered by line length), and formatted with
-  4. [black](https://github.com/psf/black) (with line width 120), and formatted with
-  5. [dos2unix](https://linux.die.net/man/1/dos2unix)
-  
-Our CI pipeline will reformat and enforce the Python styleguide.
- 
-### Rules Styleguide
-
-All (non-nursery) capa rules must:
-
-  1. pass the [linter](https://github.com/mandiant/capa/blob/master/scripts/lint.py), and
-  2. be formatted with [capafmt](https://github.com/mandiant/capa/blob/master/scripts/capafmt.py)
-  
-This ensures that all rules meet the same minimum level of quality and are structured in a consistent way.
-Our CI pipeline will reformat and enforce the capa rules styleguide.
+# Contributing to Capa
+
+First off, thanks for taking the time to contribute!
+
+The following is a set of guidelines for contributing to capa and its packages, which are hosted in the [FireEye Organization](https://github.com/fireeye) on GitHub. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
+
+#### Table Of Contents
+
+[Code of Conduct](#code-of-conduct)
+
+[What should I know before I get started?](#what-should-i-know-before-i-get-started)
+  * [Capa and its Repositories](#capa-and-its-repositories)
+  * [Capa Design Decisions](#design-decisions)
+
+[How Can I Contribute?](#how-can-i-contribute)
+  * [Reporting Bugs](#reporting-bugs)
+  * [Suggesting Enhancements](#suggesting-enhancements)
+  * [Your First Code Contribution](#your-first-code-contribution)
+  * [Pull Requests](#pull-requests)
+
+[Styleguides](#styleguides)
+  * [Git Commit Messages](#git-commit-messages)
+  * [Python Styleguide](#python-styleguide)
+  * [Rules Styleguide](#rules-styleguide)
+
+## Code of Conduct
+
+This project and everyone participating in it is governed by the [Capa Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers.
+
+## What should I know before I get started?
+
+### Capa and its repositories
+
+We host the capa project as three Github repositories:
+  - [capa](https://github.com/fireeye/capa)
+  - [capa-rules](https://github.com/fireeye/capa-rules)
+  - [capa-testfiles](https://github.com/fireeye/capa-testfiles)
+  
+The command line tools, logic engine, and other Python source code are found in the `capa` repository.
+This is the repository to fork when you want to enhance the features, performance, or user interface of capa.
+Do *not* push rules directly to this repository, instead...
+
+The standard rules contributed by the community are found in the `capa-rules` repository.
+When you have an idea for a new rule, you should open a PR against `capa-rules`.
+We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and discussions should happen.
+If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`.
+Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way.
+
+Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository.
+These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites;
+ however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate,
+ a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth.
+
+### Design Decisions
+
+When we make a significant decision in how we maintain the project and what we can or cannot support,
+ we will document it in the [capa issues tracker](https://github.com/fireeye/capa/issues).
+This is the best place review our discussions about what/how/why we do things in the project.
+If you have a question, check to see if it is documented there.
+If it is *not* documented there, or you can't find an answer, please open a issue.
+We'll link to existing issues when appropriate to keep discussions in one place.
+
+## How Can I Contribute?
+
+### Reporting Bugs
+
+This section guides you through submitting a bug report for capa.
+Following these guidelines helps maintainers and the community understand your report, reproduce the behavior, and find related reports.
+
+Before creating bug reports, please check [this list](#before-submitting-a-bug-report)
+ as you might find out that you don't need to create one.
+When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report).
+Fill out [the required template](./ISSUE_TEMPLATE/bug_report.md),
+ the information it asks for helps us resolve issues faster.
+
+> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one.
+
+#### Before Submitting A Bug Report
+
+* **Determine [which repository the problem should be reported in](#capa-and-its-repositories)**.
+* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the problem has already been reported. If it has **and the issue is still open**, add a comment to the existing issue instead of opening a new one.
+
+#### How Do I Submit A (Good) Bug Report?
+
+Bugs are tracked as [GitHub issues](https://guides.github.com/features/issues/).
+After you've determined [which repository](#capa-and-its-repositories) your bug is related to,
+ create an issue on that repository and provide the following information by filling in 
+ [the template](./ISSUE_TEMPLATE/bug_report.md).
+
+Explain the problem and include additional details to help maintainers reproduce the problem:
+
+* **Use a clear and descriptive title** for the issue to identify the problem.
+* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started capa, e.g. which command exactly you used in the terminal, or how you started capa otherwise.
+* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
+* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior.
+* **Explain which behavior you expected to see instead and why.**
+* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
+* **If you're reporting that capa crashed**, include the stack trace from the terminal. Include the stack trace in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist.
+* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below.
+
+Provide more context by answering these questions:
+
+* **Did the problem start happening recently** (e.g. after updating to a new version of capa) or was this always a problem?
+* If the problem started happening recently, **can you reproduce the problem in an older version of capa?** What's the most recent version in which the problem doesn't happen? You can download older versions of capa from [the releases page](https://github.com/fireeye/capa/releases).
+* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens.
+* If the problem is related to working with files (e.g. opening and editing files), **does the problem happen for all files and projects or only some?** Does the problem happen only when working with local or remote files (e.g. on network drives), with files of a specific type (e.g. only JavaScript or Python files), with large files or files with very long lines, or with files in a specific encoding? Is there anything else special about the files you are using?
+
+Include details about your configuration and environment:
+
+* **Which version of capa are you using?** You can get the exact version by running `capa --version` in your terminal.
+* **What's the name and version of the OS you're using**?
+
+### Suggesting Enhancements
+
+This section guides you through submitting an enhancement suggestion for capa, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion and find related suggestions.
+
+Before creating enhancement suggestions, please check [this list](#before-submitting-an-enhancement-suggestion) as you might find out that you don't need to create one. When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](./ISSUE_TEMPLATE/feature_request.md), including the steps that you imagine you would take if the feature you're requesting existed.
+
+#### Before Submitting An Enhancement Suggestion
+
+* **Determine [which repository the enhancement should be suggested in](#capa-and-its-repositories).**
+* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one.
+
+#### How Do I Submit A (Good) Enhancement Suggestion?
+
+Enhancement suggestions are tracked as [GitHub issues](https://guides.github.com/features/issues/). After you've determined [which repository](#capa-and-its-repositories) your enhancement suggestion is related to, create an issue on that repository and provide the following information:
+
+* **Use a clear and descriptive title** for the issue to identify the suggestion.
+* **Provide a step-by-step description of the suggested enhancement** in as many details as possible.
+* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
+* **Describe the current behavior** and **explain which behavior you expected to see instead** and why.
+* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of capa which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
+* **Explain why this enhancement would be useful** to most capa users and isn't something that can or should be implemented as an external tool that uses capa as a library.
+* **Specify which version of capa you're using.** You can get the exact version by running `capa --version` in your terminal.
+* **Specify the name and version of the OS you're using.**
+
+### Your First Code Contribution
+
+Unsure where to begin contributing to capa? You can start by looking through these `good-first-issue` and `rule-idea` issues:
+
+* [good-first-issue](https://github.com/fireeye/capa/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code, and a test or two.
+* [rule-idea](https://github.com/fireeye/capa-rules/issues?q=is%3Aissue+is%3Aopen+label%3A%22rule+idea%22) - issues that describe potential new rule ideas.
+
+Both issue lists are sorted by total number of comments. While not perfect, number of comments is a reasonable proxy for impact a given change will have.
+
+#### Local development
+
+capa and all its resources can be developed locally. 
+For instructions on how to do this, see the "Method 3" section of the [installation guide](https://github.com/fireeye/capa/blob/master/doc/installation.md).
+
+### Pull Requests
+
+The process described here has several goals:
+
+- Maintain capa's quality
+- Fix problems that are important to users
+- Engage the community in working toward the best possible capa
+- Enable a sustainable system for capa's maintainers to review contributions
+
+Please follow these steps to have your contribution considered by the maintainers:
+
+1. Follow all instructions in [the template](PULL_REQUEST_TEMPLATE.md)
+2. Follow the [styleguides](#styleguides)
+3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing? </summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
+
+While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
+
+## Styleguides
+
+### Git Commit Messages
+
+* Use the present tense ("Add feature" not "Added feature")
+* Use the imperative mood ("Move cursor to..." not "Moves cursor to...")
+* Prefix the first line with the component in question ("rules: ..." or "render: ...")
+* Reference issues and pull requests liberally after the first line
+
+### Python Styleguide
+
+All Python code must adhere to the style guide used by capa: 
+
+  1. [PEP8](https://www.python.org/dev/peps/pep-0008/), with clarifications from
+  2. [Willi's style guide](https://docs.google.com/document/d/1iRpeg-w4DtibwytUyC_dDT7IGhNGBP25-nQfuBa-Fyk/edit?usp=sharing), formatted with
+  3. [isort](https://pypi.org/project/isort/) (with line width 120 and ordered by line length), and formatted with
+  4. [black](https://github.com/psf/black) (with line width 120), and formatted with
+  5. [dos2unix](https://linux.die.net/man/1/dos2unix)
+  
+Our CI pipeline will reformat and enforce the Python styleguide.
+ 
+### Rules Styleguide
+
+All (non-nursery) capa rules must:
+
+  1. pass the [linter](https://github.com/fireeye/capa/blob/master/scripts/lint.py), and
+  2. be formatted with [capafmt](https://github.com/fireeye/capa/blob/master/scripts/capafmt.py)
+  
+This ensures that all rules meet the same minimum level of quality and are structured in a consistent way.
+Our CI pipeline will reformat and enforce the capa rules styleguide.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,47 +1,47 @@
----
-name: Bug report
-about: Create a report to help us improve
-
----
-<!--
-# Is your bug report related to capa rules (for example a false positive)?
-We use submodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/mandiant/capa-rules/issues.
-
-# Have you checked that your issue isn't already filed?
-Please search if there is a similar issue at https://github.com/mandiant/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
-
-# Have you read capa's Code of Conduct?
-By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/mandiant/capa/blob/master/.github/CODE_OF_CONDUCT.md
-
-# Have you read capa's CONTRIBUTING guide?
-It contains helpful information about how to contribute to capa. Check https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md#reporting-bugs
--->
-
-### Description
-
-<!-- Description of the issue -->
-
-### Steps to Reproduce
-
-<!-- 1. First Step -->
-<!-- 2. Second Step -->
-<!-- 3. and so on… -->
-
-**Expected behavior:**
-
-<!-- What you expect to happen -->
-
-**Actual behavior:**
-
-<!-- What actually happens -->
-
-### Versions
-
-<!-- You can get this information from copy and pasting the output of `capa --version` from the command line.
- Please specify the component you're using (e.g. standalone tool or IDA Pro integration) and your Python version.
- Also, please include the OS and what version of the OS you're running. -->
-
-### Additional Information
-
-<!-- Any additional information, configuration or data that might be necessary to reproduce the issue. -->
-
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+<!--
+# Is your bug report related to capa rules (for example a false positive)?
+We use sybmodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/fireeye/capa-rules/issues.
+
+# Have you checked that your issue isn't already filed?
+Please search if there is a similar issue at https://github.com/fireeye/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
+
+# Have you read capa's Code of Conduct?
+By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/fireeye/capa/blob/master/.github/CODE_OF_CONDUCT.md
+
+# Have you read capa's CONTRIBUTING guide?
+It contains helpful information about how to contribute to capa. Check https://github.com/fireeye/capa/blob/master/.github/CONTRIBUTING.md#reporting-bugs
+-->
+
+### Description
+
+<!-- Description of the issue -->
+
+### Steps to Reproduce
+
+<!-- 1. First Step -->
+<!-- 2. Second Step -->
+<!-- 3. and so on… -->
+
+**Expected behavior:**
+
+<!-- What you expect to happen -->
+
+**Actual behavior:**
+
+<!-- What actually happens -->
+
+### Versions
+
+<!-- You can get this information from copy and pasting the output of `capa --version` from the command line.
+ Please specify the component you're using (e.g. standalone tool or IDA Pro integration) and your Python version.
+ Also, please include the OS and what version of the OS you're running. -->
+
+### Additional Information
+
+<!-- Any additional information, configuration or data that might be necessary to reproduce the issue. -->
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,35 +1,35 @@
----
-name: Feature request
-about: Suggest an idea for capa
-
----
-<!--
-# Is your issue related to capa rules (for example an idea for a new rule)?
-We use submodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/mandiant/capa-rules/issues.
-
-# Have you checked that your issue isn't already filed?
-Please search if there is a similar issue at https://github.com/mandiant/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
-
-# Have you read capa's Code of Conduct?
-By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/mandiant/capa/blob/master/.github/CODE_OF_CONDUCT.md
-
-# Have you read capa's CONTRIBUTING guide?
-It contains helpful information about how to contribute to capa. Check https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md#suggesting-enhancements
--->
-
-### Summary
-
-<!-- One paragraph explanation of the feature. -->
-
-### Motivation
-
-<!-- Why are we doing this? What use cases does it support? What is the expected outcome? -->
-
-### Describe alternatives you've considered
-
-<!-- A clear and concise description of the alternative solutions you've considered. -->
-
-## Additional context
-
-<!-- Add any other context or screenshots about the feature request here. -->
-
+---
+name: Feature request
+about: Suggest an idea for capa
+
+---
+<!--
+# Is your issue related to capa rules (for example an idea for a new rule)?
+We use sybmodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/fireeye/capa-rules/issues.
+
+# Have you checked that your issue isn't already filed?
+Please search if there is a similar issue at https://github.com/fireeye/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
+
+# Have you read capa's Code of Conduct?
+By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/fireeye/capa/blob/master/.github/CODE_OF_CONDUCT.md
+
+# Have you read capa's CONTRIBUTING guide?
+It contains helpful information about how to contribute to capa. Check https://github.com/fireeye/capa/blob/master/.github/CONTRIBUTING.md#suggesting-enhancements
+-->
+
+### Summary
+
+<!-- One paragraph explanation of the feature. -->
+
+### Motivation
+
+<!-- Why are we doing this? What use cases does it support? What is the expected outcome? -->
+
+### Describe alternatives you've considered
+
+<!-- A clear and concise description of the alternative solutions you've considered. -->
+
+## Additional context
+
+<!-- Add any other context or screenshots about the feature request here. -->
+

.github/flake8.ini

@@ -1,43 +0,0 @@
-[flake8]
-max-line-length = 120
-
-extend-ignore =
-    # E203: whitespace before ':'  (black does this)
-    E203,
-    # F401: `foo` imported but unused  (prefer ruff)
-    F401,
-    # F811 Redefinition of unused `foo`  (prefer ruff)
-    F811,
-    # E501 line too long  (prefer black)
-    E501,
-    # E701 multiple statements on one line (colon)  (prefer black, see https://github.com/psf/black/issues/4173)
-    E701,
-    # B010 Do not call setattr with a constant attribute value
-    B010,
-    # G200 Logging statement uses exception in arguments
-    G200,
-    # SIM102 Use a single if-statement instead of nested if-statements
-    # doesn't provide a space for commenting or logical separation of conditions
-    SIM102,
-    # SIM114 Use logical or and a single body
-    # makes logic trees too complex
-    SIM114,
-    # SIM117 Use 'with Foo, Bar:' instead of multiple with statements
-    # makes lines too long
-    SIM117
-
-per-file-ignores =
-    # T201 print found.
-    #
-    # scripts are meant to print output
-    scripts/*: T201
-    # capa.exe is meant to print output
-    capa/main.py: T201
-    # IDA tests emit results to output window so need to print
-    tests/test_ida_features.py: T201
-    # utility used to find the Binary Ninja API via invoking python.exe
-    capa/features/extractors/binja/find_binja_api.py: T201
-
-copyright-check = True
-copyright-min-file-size = 1 
-copyright-regexp = Copyright \(C\) 2023 Mandiant, Inc. All Rights Reserved.

.github/mypy/mypy.ini

@@ -1,91 +0,0 @@
-[mypy]
-
-[mypy-halo.*]
-ignore_missing_imports = True
-
-[mypy-tqdm.*]
-ignore_missing_imports = True
-
-[mypy-ruamel.*]
-ignore_missing_imports = True
-
-[mypy-networkx.*]
-ignore_missing_imports = True
-
-[mypy-pefile.*]
-ignore_missing_imports = True
-
-[mypy-viv_utils.*]
-ignore_missing_imports = True
-
-[mypy-flirt.*]
-ignore_missing_imports = True
-
-[mypy-lief.*]
-ignore_missing_imports = True
-
-[mypy-idc.*]
-ignore_missing_imports = True
-
-[mypy-vivisect.*]
-ignore_missing_imports = True
-
-[mypy-envi.*]
-ignore_missing_imports = True
-
-[mypy-PE.*]
-ignore_missing_imports = True
-
-[mypy-idaapi.*]
-ignore_missing_imports = True
-
-[mypy-idautils.*]
-ignore_missing_imports = True
-
-[mypy-ida_auto.*]
-ignore_missing_imports = True
-
-[mypy-ida_bytes.*]
-ignore_missing_imports = True
-
-[mypy-ida_nalt.*]
-ignore_missing_imports = True
-
-[mypy-ida_kernwin.*]
-ignore_missing_imports = True
-
-[mypy-ida_settings.*]
-ignore_missing_imports = True
-
-[mypy-ida_funcs.*]
-ignore_missing_imports = True
-
-[mypy-ida_loader.*]
-ignore_missing_imports = True
-
-[mypy-ida_segment.*]
-ignore_missing_imports = True
-
-[mypy-PyQt5.*]
-ignore_missing_imports = True
-
-[mypy-binaryninja.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-devtools.*]
-ignore_missing_imports = True
-
-[mypy-elftools.*]
-ignore_missing_imports = True
-
-[mypy-dncil.*]
-ignore_missing_imports = True
-
-[mypy-netnode.*]
-ignore_missing_imports = True
-
-[mypy-ghidra.*]
-ignore_missing_imports = True

.github/pull_request_template.md

@@ -1,22 +0,0 @@
-
-<!--
-Thank you for contributing to capa! <3
-
-Please read capa's CONTRIBUTING guide if you haven't done so already.
-It contains helpful information about how to contribute to capa. Check https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md
-
-Please describe the changes in this pull request (PR). Include your motivation and context to help us review.
-
-Please mention the issue your PR addresses (if any):
-closes #issue_number
--->
-
-
-### Checklist
-
-<!-- CHANGELOG.md has a `master (unreleased)` section. Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning in the release notes to this file. -->
-- [ ] No CHANGELOG update needed
-<!-- Tests prove that your fix/work as expected and ensure it doesn't break on the feature. -->
-- [ ] No new tests needed
-<!-- Please help us keeping capa documentation up-to-date -->
-- [ ] No documentation update needed

.github/pyinstaller/hooks/hook-vivisect.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 
 from PyInstaller.utils.hooks import copy_metadata
 
@@ -13,173 +13,3 @@ from PyInstaller.utils.hooks import copy_metadata
 #
 # ref: https://github.com/pyinstaller/pyinstaller/issues/1713#issuecomment-162682084
 datas = copy_metadata("vivisect")
-
-excludedimports = [
-    # viv gui requires these heavy libraries,
-    # but viv as a library doesn't.
-    # they shouldn't be installed in our configuration,
-    # but we'll ensure they don't slip in here (such as on developers' systems).
-    "PyQt5",
-    "qt5",
-    "pyqtwebengine",
-    # the above are imported by these viv modules.
-    # so really, we'd want to exclude these submodules of viv.
-    # but i dont think this works.
-    "vqt",
-    "vdb.qt",
-    "envi.qt",
-    # unused by capa
-    "pyasn1",
-]
-
-hiddenimports = [
-    # vivisect does manual/runtime importing of its modules,
-    # so declare the things that could be imported here.
-    "vivisect",
-    "vivisect.analysis",
-    "vivisect.analysis.amd64",
-    "vivisect.analysis.amd64.emulation",
-    "vivisect.analysis.amd64.golang",
-    "vivisect.analysis.crypto",
-    "vivisect.analysis.crypto.constants",
-    "vivisect.analysis.elf",
-    "vivisect.analysis.elf.elfplt",
-    "vivisect.analysis.elf.elfplt_late",
-    "vivisect.analysis.elf.libc_start_main",
-    "vivisect.analysis.generic",
-    "vivisect.analysis.generic.codeblocks",
-    "vivisect.analysis.generic.emucode",
-    "vivisect.analysis.generic.entrypoints",
-    "vivisect.analysis.generic.funcentries",
-    "vivisect.analysis.generic.impapi",
-    "vivisect.analysis.generic.linker",
-    "vivisect.analysis.generic.mkpointers",
-    "vivisect.analysis.generic.noret",
-    "vivisect.analysis.generic.pointers",
-    "vivisect.analysis.generic.pointertables",
-    "vivisect.analysis.generic.relocations",
-    "vivisect.analysis.generic.strconst",
-    "vivisect.analysis.generic.switchcase",
-    "vivisect.analysis.generic.symswitchcase",
-    "vivisect.analysis.generic.thunks",
-    "vivisect.analysis.i386",
-    "vivisect.analysis.i386.calling",
-    "vivisect.analysis.i386.golang",
-    "vivisect.analysis.i386.importcalls",
-    "vivisect.analysis.i386.instrhook",
-    "vivisect.analysis.i386.thunk_reg",
-    "vivisect.analysis.ms",
-    "vivisect.analysis.ms.hotpatch",
-    "vivisect.analysis.ms.localhints",
-    "vivisect.analysis.ms.msvc",
-    "vivisect.analysis.ms.msvcfunc",
-    "vivisect.analysis.ms.vftables",
-    "vivisect.analysis.pe",
-    "vivisect.impapi.posix.amd64",
-    "vivisect.impapi.posix.i386",
-    "vivisect.impapi.windows",
-    "vivisect.impapi.windows.advapi_32",
-    "vivisect.impapi.windows.advapi_64",
-    "vivisect.impapi.windows.amd64",
-    "vivisect.impapi.windows.gdi_32",
-    "vivisect.impapi.windows.gdi_64",
-    "vivisect.impapi.windows.i386",
-    "vivisect.impapi.windows.kernel_32",
-    "vivisect.impapi.windows.kernel_64",
-    "vivisect.impapi.windows.msvcr100_32",
-    "vivisect.impapi.windows.msvcr100_64",
-    "vivisect.impapi.windows.msvcr110_32",
-    "vivisect.impapi.windows.msvcr110_64",
-    "vivisect.impapi.windows.msvcr120_32",
-    "vivisect.impapi.windows.msvcr120_64",
-    "vivisect.impapi.windows.msvcr71_32",
-    "vivisect.impapi.windows.msvcr80_32",
-    "vivisect.impapi.windows.msvcr80_64",
-    "vivisect.impapi.windows.msvcr90_32",
-    "vivisect.impapi.windows.msvcr90_64",
-    "vivisect.impapi.windows.msvcrt_32",
-    "vivisect.impapi.windows.msvcrt_64",
-    "vivisect.impapi.windows.ntdll_32",
-    "vivisect.impapi.windows.ntdll_64",
-    "vivisect.impapi.windows.ole_32",
-    "vivisect.impapi.windows.ole_64",
-    "vivisect.impapi.windows.rpcrt4_32",
-    "vivisect.impapi.windows.rpcrt4_64",
-    "vivisect.impapi.windows.shell_32",
-    "vivisect.impapi.windows.shell_64",
-    "vivisect.impapi.windows.user_32",
-    "vivisect.impapi.windows.user_64",
-    "vivisect.impapi.windows.ws2plus_32",
-    "vivisect.impapi.windows.ws2plus_64",
-    "vivisect.impapi.winkern",
-    "vivisect.impapi.winkern.i386",
-    "vivisect.impapi.winkern.amd64",
-    "vivisect.parsers.blob",
-    "vivisect.parsers.elf",
-    "vivisect.parsers.ihex",
-    "vivisect.parsers.macho",
-    "vivisect.parsers.pe",
-    "vivisect.storage",
-    "vivisect.storage.basicfile",
-    "vstruct.constants",
-    "vstruct.constants.ntstatus",
-    "vstruct.defs",
-    "vstruct.defs.arm7",
-    "vstruct.defs.bmp",
-    "vstruct.defs.dns",
-    "vstruct.defs.elf",
-    "vstruct.defs.gif",
-    "vstruct.defs.ihex",
-    "vstruct.defs.inet",
-    "vstruct.defs.java",
-    "vstruct.defs.kdcom",
-    "vstruct.defs.macho",
-    "vstruct.defs.macho.const",
-    "vstruct.defs.macho.fat",
-    "vstruct.defs.macho.loader",
-    "vstruct.defs.macho.stabs",
-    "vstruct.defs.minidump",
-    "vstruct.defs.pcap",
-    "vstruct.defs.pe",
-    "vstruct.defs.pptp",
-    "vstruct.defs.rar",
-    "vstruct.defs.swf",
-    "vstruct.defs.win32",
-    "vstruct.defs.windows",
-    "vstruct.defs.windows.win_5_1_i386",
-    "vstruct.defs.windows.win_5_1_i386.ntdll",
-    "vstruct.defs.windows.win_5_1_i386.ntoskrnl",
-    "vstruct.defs.windows.win_5_1_i386.win32k",
-    "vstruct.defs.windows.win_5_2_i386",
-    "vstruct.defs.windows.win_5_2_i386.ntdll",
-    "vstruct.defs.windows.win_5_2_i386.ntoskrnl",
-    "vstruct.defs.windows.win_5_2_i386.win32k",
-    "vstruct.defs.windows.win_6_1_amd64",
-    "vstruct.defs.windows.win_6_1_amd64.ntdll",
-    "vstruct.defs.windows.win_6_1_amd64.ntoskrnl",
-    "vstruct.defs.windows.win_6_1_amd64.win32k",
-    "vstruct.defs.windows.win_6_1_i386",
-    "vstruct.defs.windows.win_6_1_i386.ntdll",
-    "vstruct.defs.windows.win_6_1_i386.ntoskrnl",
-    "vstruct.defs.windows.win_6_1_i386.win32k",
-    "vstruct.defs.windows.win_6_1_wow64",
-    "vstruct.defs.windows.win_6_1_wow64.ntdll",
-    "vstruct.defs.windows.win_6_2_amd64",
-    "vstruct.defs.windows.win_6_2_amd64.ntdll",
-    "vstruct.defs.windows.win_6_2_amd64.ntoskrnl",
-    "vstruct.defs.windows.win_6_2_amd64.win32k",
-    "vstruct.defs.windows.win_6_2_i386",
-    "vstruct.defs.windows.win_6_2_i386.ntdll",
-    "vstruct.defs.windows.win_6_2_i386.ntoskrnl",
-    "vstruct.defs.windows.win_6_2_i386.win32k",
-    "vstruct.defs.windows.win_6_2_wow64",
-    "vstruct.defs.windows.win_6_2_wow64.ntdll",
-    "vstruct.defs.windows.win_6_3_amd64",
-    "vstruct.defs.windows.win_6_3_amd64.ntdll",
-    "vstruct.defs.windows.win_6_3_amd64.ntoskrnl",
-    "vstruct.defs.windows.win_6_3_i386",
-    "vstruct.defs.windows.win_6_3_i386.ntdll",
-    "vstruct.defs.windows.win_6_3_i386.ntoskrnl",
-    "vstruct.defs.windows.win_6_3_wow64",
-    "vstruct.defs.windows.win_6_3_wow64.ntdll",
-]

.github/pyinstaller/pyinstaller.spec

@@ -1,40 +1,176 @@
 # -*- mode: python -*-
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 import os.path
 import subprocess
 
 import wcwidth
 
 
+# when invoking pyinstaller from the project root,
+# this gets run from the project root.
+with open('./capa/version.py', 'wb') as f:
+    # git output will look like:
+    #
+    #     tags/v1.0.0-0-g3af38dc
+    #         ------- tag
+    #                 - commits since
+    #                   g------- git hash fragment
+    version = (subprocess.check_output(["git", "describe", "--always", "--tags", "--long"])
+               .strip()
+               .replace("tags/", ""))
+    f.write("__version__ = '%s'" % version)
+
 a = Analysis(
     # when invoking pyinstaller from the project root,
     # this gets invoked from the directory of the spec file,
     # i.e. ./.github/pyinstaller
-    ["../../capa/main.py"],
-    pathex=["capa"],
+    ['../../capa/main.py'],
+    pathex=['capa'],
     binaries=None,
     datas=[
         # when invoking pyinstaller from the project root,
         # this gets invoked from the directory of the spec file,
         # i.e. ./.github/pyinstaller
-        ("../../rules", "rules"),
-        ("../../sigs", "sigs"),
-        ("../../cache", "cache"),
+        ('../../rules', 'rules'),
+
         # capa.render.default uses tabulate that depends on wcwidth.
         # it seems wcwidth uses a json file `version.json`
         # and this doesn't get picked up by pyinstaller automatically.
         # so we manually embed the wcwidth resources here.
         #
         # ref: https://stackoverflow.com/a/62278462/87207
-        (os.path.dirname(wcwidth.__file__), "wcwidth"),
+        (os.path.dirname(wcwidth.__file__), 'wcwidth')
+    ],
+    hiddenimports=[
+        # vivisect does manual/runtime importing of its modules,
+        # so declare the things that could be imported here.
+        "vivisect",
+        "vivisect.analysis",
+        "vivisect.analysis.amd64",
+        "vivisect.analysis.amd64",
+        "vivisect.analysis.amd64.emulation",
+        "vivisect.analysis.amd64.golang",
+        "vivisect.analysis.crypto",
+        "vivisect.analysis.crypto",
+        "vivisect.analysis.crypto.constants",
+        "vivisect.analysis.elf",
+        "vivisect.analysis.elf",
+        "vivisect.analysis.elf.elfplt",
+        "vivisect.analysis.elf.libc_start_main",
+        "vivisect.analysis.generic",
+        "vivisect.analysis.generic",
+        "vivisect.analysis.generic.codeblocks",
+        "vivisect.analysis.generic.emucode",
+        "vivisect.analysis.generic.entrypoints",
+        "vivisect.analysis.generic.funcentries",
+        "vivisect.analysis.generic.impapi",
+        "vivisect.analysis.generic.mkpointers",
+        "vivisect.analysis.generic.pointers",
+        "vivisect.analysis.generic.pointertables",
+        "vivisect.analysis.generic.relocations",
+        "vivisect.analysis.generic.strconst",
+        "vivisect.analysis.generic.switchcase",
+        "vivisect.analysis.generic.thunks",
+        "vivisect.analysis.i386",
+        "vivisect.analysis.i386",
+        "vivisect.analysis.i386.calling",
+        "vivisect.analysis.i386.golang",
+        "vivisect.analysis.i386.importcalls",
+        "vivisect.analysis.i386.instrhook",
+        "vivisect.analysis.i386.thunk_bx",
+        "vivisect.analysis.ms",
+        "vivisect.analysis.ms",
+        "vivisect.analysis.ms.hotpatch",
+        "vivisect.analysis.ms.localhints",
+        "vivisect.analysis.ms.msvc",
+        "vivisect.analysis.ms.msvcfunc",
+        "vivisect.analysis.ms.vftables",
+        "vivisect.analysis.pe",
+        "vivisect.impapi.posix.amd64",
+        "vivisect.impapi.posix.i386",
+        "vivisect.impapi.windows",
+        "vivisect.impapi.windows.amd64",
+        "vivisect.impapi.windows.i386",
+        "vivisect.impapi.winkern.i386",
+        "vivisect.impapi.winkern.amd64",
+        "vivisect.parsers.blob",
+        "vivisect.parsers.elf",
+        "vivisect.parsers.ihex",
+        "vivisect.parsers.macho",
+        "vivisect.parsers.pe",
+        "vivisect.parsers.utils",
+        "vivisect.storage",
+        "vivisect.storage.basicfile",
+        "vstruct.constants",
+        "vstruct.constants.ntstatus",
+        "vstruct.defs",
+        "vstruct.defs.arm7",
+        "vstruct.defs.bmp",
+        "vstruct.defs.dns",
+        "vstruct.defs.elf",
+        "vstruct.defs.gif",
+        "vstruct.defs.ihex",
+        "vstruct.defs.inet",
+        "vstruct.defs.java",
+        "vstruct.defs.kdcom",
+        "vstruct.defs.macho",
+        "vstruct.defs.macho.const",
+        "vstruct.defs.macho.fat",
+        "vstruct.defs.macho.loader",
+        "vstruct.defs.macho.stabs",
+        "vstruct.defs.minidump",
+        "vstruct.defs.pcap",
+        "vstruct.defs.pe",
+        "vstruct.defs.pptp",
+        "vstruct.defs.rar",
+        "vstruct.defs.swf",
+        "vstruct.defs.win32",
+        "vstruct.defs.windows",
+        "vstruct.defs.windows.win_5_1_i386",
+        "vstruct.defs.windows.win_5_1_i386.ntdll",
+        "vstruct.defs.windows.win_5_1_i386.ntoskrnl",
+        "vstruct.defs.windows.win_5_1_i386.win32k",
+        "vstruct.defs.windows.win_5_2_i386",
+        "vstruct.defs.windows.win_5_2_i386.ntdll",
+        "vstruct.defs.windows.win_5_2_i386.ntoskrnl",
+        "vstruct.defs.windows.win_5_2_i386.win32k",
+        "vstruct.defs.windows.win_6_1_amd64",
+        "vstruct.defs.windows.win_6_1_amd64.ntdll",
+        "vstruct.defs.windows.win_6_1_amd64.ntoskrnl",
+        "vstruct.defs.windows.win_6_1_amd64.win32k",
+        "vstruct.defs.windows.win_6_1_i386",
+        "vstruct.defs.windows.win_6_1_i386.ntdll",
+        "vstruct.defs.windows.win_6_1_i386.ntoskrnl",
+        "vstruct.defs.windows.win_6_1_i386.win32k",
+        "vstruct.defs.windows.win_6_1_wow64",
+        "vstruct.defs.windows.win_6_1_wow64.ntdll",
+        "vstruct.defs.windows.win_6_2_amd64",
+        "vstruct.defs.windows.win_6_2_amd64.ntdll",
+        "vstruct.defs.windows.win_6_2_amd64.ntoskrnl",
+        "vstruct.defs.windows.win_6_2_amd64.win32k",
+        "vstruct.defs.windows.win_6_2_i386",
+        "vstruct.defs.windows.win_6_2_i386.ntdll",
+        "vstruct.defs.windows.win_6_2_i386.ntoskrnl",
+        "vstruct.defs.windows.win_6_2_i386.win32k",
+        "vstruct.defs.windows.win_6_2_wow64",
+        "vstruct.defs.windows.win_6_2_wow64.ntdll",
+        "vstruct.defs.windows.win_6_3_amd64",
+        "vstruct.defs.windows.win_6_3_amd64.ntdll",
+        "vstruct.defs.windows.win_6_3_amd64.ntoskrnl",
+        "vstruct.defs.windows.win_6_3_i386",
+        "vstruct.defs.windows.win_6_3_i386.ntdll",
+        "vstruct.defs.windows.win_6_3_i386.ntoskrnl",
+        "vstruct.defs.windows.win_6_3_wow64",
+        "vstruct.defs.windows.win_6_3_wow64.ntdll",
     ],
     # when invoking pyinstaller from the project root,
     # this gets run from the project root.
-    hookspath=[".github/pyinstaller/hooks"],
+    hookspath=['.github/pyinstaller/hooks'],
     runtime_hooks=None,
     excludes=[
         # ignore packages that would otherwise be bundled with the .exe.
         # review: build/pyinstaller/xref-pyinstaller.html
+
         # we don't do any GUI stuff, so ignore these modules
         "tkinter",
         "_tkinter",
@@ -44,52 +180,35 @@ a = Analysis(
         # since we don't spawn a notebook, we can safely remove these.
         "IPython",
         "ipywidgets",
-        # these are pulled in by networkx
-        # but we don't need to compute the strongly connected components.
-        "numpy",
-        "scipy",
-        "matplotlib",
-        "pandas",
-        "pytest",
-        # deps from viv that we don't use.
-        # this duplicates the entries in `hook-vivisect`,
-        # but works better this way.
-        "vqt",
-        "vdb.qt",
-        "envi.qt",
-        "PyQt5",
-        "qt5",
-        "pyqtwebengine",
-        "pyasn1",
-        "binaryninja",
-    ],
-)
+    ])
 
-a.binaries = a.binaries - TOC([("tcl85.dll", None, None), ("tk85.dll", None, None), ("_tkinter", None, None)])
+a.binaries = a.binaries - TOC([
+ ('tcl85.dll', None, None),
+ ('tk85.dll', None, None),
+ ('_tkinter', None, None)])
 
 pyz = PYZ(a.pure, a.zipped_data)
 
-exe = EXE(
-    pyz,
-    a.scripts,
-    a.binaries,
-    a.zipfiles,
-    a.datas,
-    exclude_binaries=False,
-    name="capa",
-    icon="logo.ico",
-    debug=False,
-    strip=False,
-    upx=True,
-    console=True,
-)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          exclude_binaries=False,
+          name='capa',
+          icon='logo.ico',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )
 
 # enable the following to debug the contents of the .exe
 #
-# coll = COLLECT(exe,
+#coll = COLLECT(exe,
 #               a.binaries,
 #               a.zipfiles,
 #               a.datas,
 #               strip=None,
 #               upx=True,
 #               name='capa-dat')
+

.github/ruff.toml

@@ -1,43 +0,0 @@
-# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
-# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
-# McCabe complexity (`C901`) by default.
-select = ["E", "F"]
-
-# Allow autofix for all enabled rules (when `--fix`) is provided.
-fixable = ["ALL"]
-unfixable = []
-
-# E402 module level import not at top of file
-# E722 do not use bare 'except'
-# E501 line too long
-ignore = ["E402", "E722", "E501"]
-
-line-length = 120
-
-exclude = [
-    # Exclude a variety of commonly ignored directories.
-    ".bzr",
-    ".direnv",
-    ".eggs",
-    ".git",
-    ".git-rewrite",
-    ".hg",
-    ".mypy_cache",
-    ".nox",
-    ".pants.d",
-    ".pytype",
-    ".ruff_cache",
-    ".svn",
-    ".tox",
-    ".venv",
-    "__pypackages__",
-    "_build",
-    "buck-out",
-    "build",
-    "dist",
-    "node_modules",
-    "venv",
-    # protobuf generated files
-    "*_pb2.py", 
-    "*_pb2.pyi"
-]

.github/tox.ini

@@ -0,0 +1,10 @@
+[pycodestyle]
+; E402: module level import not at top of file
+; W503: line break before binary operator
+; E231 missing whitespace after ',' (emitted by black)
+; E203 whitespace before ':' (emitted by black)
+ignore = E402,W503,E203,E231
+max-line-length = 160
+statistics = True
+count = True
+exclude = .*

.github/workflows/build.yml

@@ -1,135 +1,82 @@
-name: build
-
-on:
-  pull_request:
-    branches: [ master ]
-  release:
-    types: [edited, published]
-
-permissions:
-  contents: write
-
-jobs:
-  build:
-    name: PyInstaller for ${{ matrix.os }} / Py ${{ matrix.python_version }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      # set to false for debugging
-      fail-fast: true
-      matrix:
-        # using Python 3.8 to support running across multiple operating systems including Windows 7
-        include:
-          - os: ubuntu-20.04
-            # use old linux so that the shared library versioning is more portable
-            artifact_name: capa
-            asset_name: linux
-            python_version: 3.8
-          - os: ubuntu-20.04
-            artifact_name: capa
-            asset_name: linux-py311
-            python_version: 3.11
-          - os: windows-2019
-            artifact_name: capa.exe
-            asset_name: windows
-            python_version: 3.8
-          - os: macos-11
-            # use older macOS for assumed better portability
-            artifact_name: capa
-            asset_name: macos
-            python_version: 3.8
-    steps:
-      - name: Checkout capa
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-        with:
-          submodules: true
-      - name: Set up Python ${{ matrix.python_version }}
-        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
-        with:
-          python-version: ${{ matrix.python_version }}
-      - if: matrix.os == 'ubuntu-20.04'
-        run: sudo apt-get install -y libyaml-dev
-      - name: Upgrade pip, setuptools
-        run: python -m pip install --upgrade pip setuptools
-      - name: Install capa with build requirements
-        run: pip install -e .[build]
-      - name: Cache the rule set
-        run: python ./scripts/cache-ruleset.py ./rules/ ./cache/
-      - name: Build standalone executable
-        run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
-      - name: Does it run (PE)?
-        run: dist/capa -d "tests/data/Practical Malware Analysis Lab 01-01.dll_"
-      - name: Does it run (Shellcode)?
-        run: dist/capa -d "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
-      - name: Does it run (ELF)?
-        run: dist/capa -d "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
-      - name: Does it run (CAPE)?
-        run: |
-          7z e "tests/data/dynamic/cape/v2.2/d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json.gz"
-          dist/capa -d "d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json"
-      - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        with:
-          name: ${{ matrix.asset_name }}
-          path: dist/${{ matrix.artifact_name }}
-
-  test_run:
-    name: Test run on ${{ matrix.os }} / ${{ matrix.asset_name }}
-    runs-on: ${{ matrix.os }}
-    needs: [build]
-    strategy:
-      matrix:
-        include:
-          # OSs not already tested above
-          - os: ubuntu-22.04
-            artifact_name: capa
-            asset_name: linux
-          - os: ubuntu-22.04
-            artifact_name: capa
-            asset_name: linux-py311
-          - os: windows-2022
-            artifact_name: capa.exe
-            asset_name: windows
-    steps:
-      - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
-        with:
-          name: ${{ matrix.asset_name }}
-      - name: Set executable flag
-        if: matrix.os != 'windows-2022'
-        run: chmod +x ${{ matrix.artifact_name }}
-      - name: Run capa
-        run: ./${{ matrix.artifact_name }} -h
-
-  zip_and_upload:
-    # upload zipped binaries to Release page
-    if: github.event_name == 'release'
-    name: zip and upload ${{ matrix.asset_name }}
-    runs-on: ubuntu-20.04
-    needs: [build]
-    strategy:
-      matrix:
-        include:
-          - asset_name: linux
-            artifact_name: capa
-          - asset_name: linux-py311
-            artifact_name: capa
-          - asset_name: windows
-            artifact_name: capa.exe
-          - asset_name: macos
-            artifact_name: capa
-    steps:
-      - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
-        with:
-          name: ${{ matrix.asset_name }}
-      - name: Set executable flag
-        run: chmod +x ${{ matrix.artifact_name }}
-      - name: Set zip name
-        run: echo "zip_name=capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV
-      - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
-        run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
-      - name: Upload ${{ env.zip_name }} to GH Release
-        uses: svenstaro/upload-release-action@2728235f7dc9ff598bd86ce3c274b74f802d2208 # v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN}}
-          file: ${{ env.zip_name }}
-          tag: ${{ github.ref }}
+name: build
+
+on:
+  release:
+    types: [edited, published]
+
+jobs:
+  build:
+    name: PyInstaller for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-16.04
+            # use old linux so that the shared library versioning is more portable
+            artifact_name: capa
+            asset_name: linux
+          - os: windows-latest
+            artifact_name: capa.exe
+            asset_name: windows
+          - os: macos-latest
+            artifact_name: capa
+            asset_name: macos
+    steps:
+      - name: Checkout capa
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Set up Python 2.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 2.7
+      - if: matrix.os == 'ubuntu-latest'
+        run: sudo apt-get install -y libyaml-dev
+      - if: matrix.os == 'windows-latest'
+        run: |
+          choco install vcredist2008
+          choco install --ignore-dependencies vcpython27
+      - name: Install PyInstaller
+        # pyinstaller 4 doesn't support Python 2.7
+        run: pip install 'pyinstaller==3.*'
+      - name: Install capa
+        run: pip install -e .
+      - name: Build standalone executable
+        run: pyinstaller .github/pyinstaller/pyinstaller.spec
+      - name: Does it run?
+        run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_"
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.asset_name }}
+          path: dist/${{ matrix.artifact_name }}
+
+  zip:
+    name: zip ${{ matrix.asset_name }}
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        include:
+          - asset_name: linux
+            artifact_name: capa
+          - asset_name: windows
+            artifact_name: capa.exe
+          - asset_name: macos
+            artifact_name: capa
+    steps:
+      - name: Download ${{ matrix.asset_name }}
+        uses: actions/download-artifact@v2
+        with:
+          name: ${{ matrix.asset_name }}
+      - name: Set executable flag
+        run: chmod +x ${{ matrix.artifact_name }}
+      - name: Set zip name
+        run: echo "zip_name=capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV
+      - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
+        run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
+      - name: Upload ${{ env.zip_name }} to GH Release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN}}
+          file: ${{ env.zip_name }}
+          tag: ${{ github.ref }}

.github/workflows/changelog.yml

@@ -1,43 +0,0 @@
-name: changelog
-
-on:
-  # We need pull_request_target instead of pull_request because a write
-  # repository token is needed to add a review to a PR. DO NOT BUILD
-  # OR RUN UNTRUSTED CODE FROM PRs IN THIS ACTION
-  pull_request_target:
-    types: [opened, edited, synchronize]
-
-permissions: read-all
-
-jobs:
-  check_changelog:
-    # no need to check for dependency updates via dependabot
-    if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]'
-    runs-on: ubuntu-20.04
-    env:
-      NO_CHANGELOG: '[x] No CHANGELOG update needed'
-    steps:
-    - name: Get changed files
-      id: files
-      uses: Ana06/get-changed-files@e0c398b7065a8d84700c471b6afc4116d1ba4e96 # v2.2.0
-    - name: check changelog updated
-      id: changelog_updated
-      env:
-        PR_BODY: ${{ github.event.pull_request.body }}
-        FILES: ${{ steps.files.outputs.modified }}
-      run: |
-        echo $FILES | grep -qF 'CHANGELOG.md' || echo $PR_BODY | grep -qiF "$NO_CHANGELOG"
-    - name: Reject pull request if no CHANGELOG update
-      if: ${{ always() && steps.changelog_updated.outcome == 'failure' }}
-      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        event: REQUEST_CHANGES
-        body: "Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning to the `master (unreleased)` section of CHANGELOG.md. If no CHANGELOG update is needed add the following to the PR description: `${{ env.NO_CHANGELOG }}`"
-        allow_duplicate: false
-    - name: Dismiss previous review if CHANGELOG update
-      uses: Ana06/automatic-pull-request-review@0cf4e8a17ba79344ed3fdd7fed6dd0311d08a9d4 # v0.1.0
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        event: DISMISS
-        body: "CHANGELOG updated or no update needed, thanks! :smile:"

.github/workflows/pip-audit.yml

@@ -1,21 +0,0 @@
-name: PIP audit
-
-on:
-  schedule:
-  - cron: '0 8 * * 1'
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    strategy:
-      matrix:
-        python-version: ["3.11"]
-
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-
-      - uses: pypa/gh-action-pip-audit@v1.0.8
-        with:
-          inputs: .

.github/workflows/publish.yml

@@ -1,41 +1,29 @@
-# use PyPI trusted publishing, as described here: 
-# https://blog.trailofbits.com/2023/05/23/trusted-publishing-a-new-benchmark-for-packaging-security/
-name: publish to pypi
-
-on:
-  release:
-    types: [published]
-
-permissions:
-  contents: write
-
-jobs:
-  pypi-publish:
-    runs-on: ubuntu-latest
-    environment:
-      name: release
-    permissions:
-      id-token: write
-    steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      - name: Set up Python
-        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
-        with:
-          python-version: '3.8'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e .[build]
-      - name: build package
-        run: |
-          python -m build
-      - name: upload package artifacts
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        with:
-          path: dist/*
-      - name: publish package
-        uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0  # release/v1
-        with:
-          skip-existing: true
-          verbose: true
-          print-hash: true
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: publish to pypi
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '2.7'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload --skip-existing dist/*

.github/workflows/scorecard.yml

@@ -1,72 +0,0 @@
-# This workflow uses actions that are not certified by GitHub. They are provided
-# by a third-party and are governed by separate terms of service, privacy
-# policy, and support documentation.
-
-name: Scorecard supply-chain security
-on:
-  # For Branch-Protection check. Only the default branch is supported. See
-  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
-  branch_protection_rule:
-  # To guarantee Maintained check is occasionally updated. See
-  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
-  schedule:
-    - cron: '43 4 * * 3'
-  push:
-    branches: [ "master" ]
-
-# Declare default permissions as read only.
-permissions: read-all
-
-jobs:
-  analysis:
-    name: Scorecard analysis
-    runs-on: ubuntu-latest
-    permissions:
-      # Needed to upload the results to code-scanning dashboard.
-      security-events: write
-      # Needed to publish results and get a badge (see publish_results below).
-      id-token: write
-      # Uncomment the permissions below if installing in a private repository.
-      # contents: read
-      # actions: read
-
-    steps:
-      - name: "Checkout code"
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
-        with:
-          persist-credentials: false
-
-      - name: "Run analysis"
-        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
-        with:
-          results_file: results.sarif
-          results_format: sarif
-          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
-          # - you want to enable the Branch-Protection check on a *public* repository, or
-          # - you are installing Scorecard on a *private* repository
-          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
-          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
-
-          # Public repositories:
-          #   - Publish results to OpenSSF REST API for easy access by consumers
-          #   - Allows the repository to include the Scorecard badge.
-          #   - See https://github.com/ossf/scorecard-action#publishing-results.
-          # For private repositories:
-          #   - `publish_results` will always be set to `false`, regardless
-          #     of the value entered here.
-          publish_results: true
-
-      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
-      # format to the repository Actions tab.
-      - name: "Upload artifact"
-        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
-        with:
-          name: SARIF file
-          path: results.sarif
-          retention-days: 5
-
-      # Upload the results to GitHub's code scanning dashboard.
-      - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27
-        with:
-          sarif_file: results.sarif

.github/workflows/tag.yml

@@ -1,32 +0,0 @@
-name: tag
-
-on:
-  release:
-    types: [published]
-
-permissions: read-all
-
-jobs:
-  tag:
-    name: Tag capa rules
-    runs-on: ubuntu-20.04
-    steps:
-    - name: Checkout capa-rules
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      with:
-        repository: mandiant/capa-rules
-        token: ${{ secrets.CAPA_TOKEN }}
-    - name: Tag capa-rules
-      run: |
-        # user information is needed to create annotated tags (with a message)
-        git config user.email 'capa-dev@mandiant.com'
-        git config user.name 'Capa Bot'
-        name=${{ github.event.release.tag_name }}
-        git tag $name -m "https://github.com/mandiant/capa/releases/$name"
-        # TODO update branch name-major=${name%%.*}
-    - name: Push tag to capa-rules
-      uses: ad-m/github-push-action@0fafdd62b84042d49ec0cb92d9cac7f7ce4ec79e # master
-      with:
-        repository: mandiant/capa-rules
-        github_token: ${{ secrets.CAPA_TOKEN }}
-        tags: true

.github/workflows/tests.yml

@@ -6,199 +6,65 @@ on:
   pull_request:
     branches: [ master ]
 
-permissions: read-all
-
-# save workspaces to speed up testing
-env:
-  CAPA_SAVE_WORKSPACE: "True"
-
 jobs:
-  changelog_format:
-    runs-on: ubuntu-20.04
-    steps:
-    - name: Checkout capa
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-    # The sync GH action in capa-rules relies on a single '- *$' in the CHANGELOG file
-    - name: Ensure CHANGELOG has '- *$'
-      run: |
-        number=$(grep '\- *$' CHANGELOG.md | wc -l)
-        if [ $number != 1 ]; then exit 1; fi
-
   code_style:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout capa
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-    # use latest available python to take advantage of best performance
-    - name: Set up Python 3.11
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
       with:
-        python-version: "3.11"
+        python-version: 3.8
     - name: Install dependencies
-      run: pip install -e .[dev]
-    - name: Lint with ruff
-      run: pre-commit run ruff
+      run: pip install 'isort==5.*' black
     - name: Lint with isort
-      run: pre-commit run isort --show-diff-on-failure
+      run: isort --profile black --length-sort --line-width 120 -c .
     - name: Lint with black
-      run: pre-commit run black --show-diff-on-failure
-    - name: Lint with flake8
-      run: pre-commit run flake8 --hook-stage manual
-    - name: Check types with mypy
-      run:  pre-commit run mypy --hook-stage manual
+      run: black -l 120 --check .
 
   rule_linter:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
-    - name: Checkout capa with submodules
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+    - name: Checkout capa with rules submodule
+      uses: actions/checkout@v2
       with:
-        submodules: recursive
-    - name: Set up Python 3.11
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+        submodules: true
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
       with:
-        python-version: "3.11"
+        python-version: 3.8
+    # We don't need vivisect, so we can install capa using Python3
     - name: Install capa
-      run: pip install -e .[dev]
+      run: pip install -e .
     - name: Run rule linter
       run: python scripts/lint.py rules/
 
   tests:
-    name: Tests in ${{ matrix.python-version }} on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
+    name: Tests in ${{ matrix.python }}
+    runs-on: ubuntu-latest
     needs: [code_style, rule_linter]
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-11]
-        # across all operating systems
-        python-version: ["3.8", "3.11"]
         include:
-          # on Ubuntu run these as well
-          - os: ubuntu-20.04
-            python-version: "3.8"
-          - os: ubuntu-20.04
-            python-version: "3.9"
-          - os: ubuntu-20.04
-            python-version: "3.10"
+          - python: 2.7
+          - python: 3.7
+          - python: 3.8
+          - python: 3.9.1
     steps:
     - name: Checkout capa with submodules
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      with:
-        submodules: recursive
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install pyyaml
-      if: matrix.os == 'ubuntu-20.04'
-      run: sudo apt-get install -y libyaml-dev
-    - name: Install capa
-      run: pip install -e .[dev]
-    - name: Run tests (fast)
-      # this set of tests runs about 80% of the cases in 20% of the time,
-      # and should catch most errors quickly.
-      run:  pre-commit run pytest-fast --all-files --hook-stage manual
-    - name: Run tests
-      run: pytest -v tests/
-
-  binja-tests:
-    name: Binary Ninja tests for ${{ matrix.python-version }}
-    env:
-      BN_SERIAL: ${{ secrets.BN_SERIAL }}
-    runs-on: ubuntu-20.04
-    needs: [tests]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.11"]
-    steps:
-    - name: Checkout capa with submodules
-      # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
-      if: ${{ env.BN_SERIAL != 0 }}
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      with:
-        submodules: recursive
-    - name: Set up Python ${{ matrix.python-version }}
-      if: ${{ env.BN_SERIAL != 0 }}
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install pyyaml
-      if: ${{ env.BN_SERIAL != 0 }}
-      run: sudo apt-get install -y libyaml-dev
-    - name: Install capa
-      if: ${{ env.BN_SERIAL != 0 }}
-      run: pip install -e .[dev]
-    - name: install Binary Ninja
-      if: ${{ env.BN_SERIAL != 0 }}
-      run: |
-        mkdir ./.github/binja
-        curl "https://raw.githubusercontent.com/Vector35/binaryninja-api/6812c97/scripts/download_headless.py" -o ./.github/binja/download_headless.py
-        python ./.github/binja/download_headless.py --serial ${{ env.BN_SERIAL }} --output .github/binja/BinaryNinja-headless.zip
-        unzip .github/binja/BinaryNinja-headless.zip -d .github/binja/
-        python .github/binja/binaryninja/scripts/install_api.py --install-on-root --silent
-    - name: Run tests
-      if: ${{ env.BN_SERIAL != 0 }}
-      env:
-        BN_LICENSE: ${{ secrets.BN_LICENSE }}
-      run: pytest -v tests/test_binja_features.py  # explicitly refer to the binja tests for performance. other tests run above.
-
-  ghidra-tests:
-    name: Ghidra tests for ${{ matrix.python-version }}
-    runs-on: ubuntu-20.04
-    needs: [tests]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.8", "3.11"]
-        java-version: ["17"]
-        gradle-version: ["7.3"]
-        ghidra-version: ["10.3"]
-        public-version: ["PUBLIC_20230510"] # for ghidra releases
-        jep-version: ["4.1.1"]
-        ghidrathon-version: ["3.0.0"]
-    steps:
-    - name: Checkout capa with submodules
-      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      uses: actions/checkout@v2
       with:
         submodules: true
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v2
       with:
-        python-version: ${{ matrix.python-version }}
-    - name: Set up Java ${{ matrix.java-version }}
-      uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3
-      with:
-        distribution: 'temurin'
-        java-version: ${{ matrix.java-version }}
-    - name: Set up Gradle ${{ matrix.gradle-version }} 
-      uses: gradle/gradle-build-action@40b6781dcdec2762ad36556682ac74e31030cfe2 # v2.5.1
-      with:
-        gradle-version: ${{ matrix.gradle-version }}
-    - name: Install Jep ${{ matrix.jep-version }} 
-      run : pip install jep==${{ matrix.jep-version }}
-    - name: Install Ghidra ${{ matrix.ghidra-version }} 
-      run: |
-        mkdir ./.github/ghidra
-        wget "https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_${{ matrix.ghidra-version }}_build/ghidra_${{ matrix.ghidra-version }}_${{ matrix.public-version }}.zip" -O ./.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip
-        unzip .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip -d .github/ghidra/
-    - name: Install Ghidrathon
-      run : |
-        mkdir ./.github/ghidrathon
-        curl -o ./.github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip "https://codeload.github.com/mandiant/Ghidrathon/zip/refs/tags/v${{ matrix.ghidrathon-version }}"
-        unzip .github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/
-        gradle -p ./.github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/ -PGHIDRA_INSTALL_DIR=$(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC
-        unzip .github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/dist/*.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions
+        python-version: ${{ matrix.python }}
     - name: Install pyyaml
       run: sudo apt-get install -y libyaml-dev
     - name: Install capa
-      run: pip install -e .[dev] 
+      run: pip install -e .[dev]
     - name: Run tests
-      run: | 
-        mkdir ./.github/ghidra/project
-        .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/support/analyzeHeadless .github/ghidra/project ghidra_test -Import ./tests/data/mimikatz.exe_ -ScriptPath ./tests/ -PostScript test_ghidra_features.py > ../output.log
-        cat ../output.log
-        exit_code=$(cat ../output.log | grep exit | awk '{print $NF}')
-        exit $exit_code
- 
+      run: pytest tests/
+

.gitignore

@@ -108,21 +108,9 @@ venv.bak/
 *.viv
 *.idb
 *.i64
-.vscode
-
 !rules/lib
 
-scripts/perf/*.txt
-scripts/perf/*.svg
-scripts/perf/*.zip
-
-.direnv
-.envrc
-.DS_Store
-*/.DS_Store
-Pipfile
-Pipfile.lock
-/cache/
-.github/binja/binaryninja
-.github/binja/download_headless.py
-.github/binja/BinaryNinja-headless.zip
+# hooks/ci.sh output
+isort-output.log
+black-output.log
+rule-linter-output.log

.pre-commit-config.yaml

@@ -1,129 +0,0 @@
-# install the pre-commit hooks:
-#
-#    ❯ pre-commit install --hook-type pre-commit
-#    pre-commit installed at .git/hooks/pre-commit
-#
-#    ❯ pre-commit install --hook-type pre-push
-#    pre-commit installed at .git/hooks/pre-push
-#
-# run all linters liks:
-#
-#    ❯ pre-commit run --all-files
-#    isort....................................................................Passed
-#    black....................................................................Passed
-#    ruff.....................................................................Passed
-#    flake8...................................................................Passed
-#    mypy.....................................................................Passed
-#
-# run a single linter like:
-#
-#    ❯ pre-commit run --all-files isort
-#    isort....................................................................Passed
-
-repos:
--   repo: local
-    hooks:
-    -   id: isort
-        name: isort
-        stages: [commit, push, manual]
-        language: system
-        entry: isort
-        args: 
-        -   "--length-sort"
-        -   "--profile"
-        -   "black"
-        -   "--line-length=120"
-        -   "--skip-glob"
-        -   "*_pb2.py"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false
-
--   repo: local
-    hooks:
-    -   id: black
-        name: black
-        stages: [commit, push, manual]
-        language: system
-        entry: black
-        args: 
-        -   "--line-length=120"
-        -   "--extend-exclude"
-        -   ".*_pb2.py"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false
-
--   repo: local
-    hooks:
-    -   id: ruff
-        name: ruff
-        stages: [commit, push, manual]
-        language: system
-        entry: ruff
-        args: 
-        -   "check"
-        -   "--config"
-        -   ".github/ruff.toml"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false
-
--   repo: local
-    hooks:
-    -   id: flake8
-        name: flake8
-        stages: [push, manual]
-        language: system
-        entry: flake8
-        args: 
-        -   "--config"
-        -   ".github/flake8.ini"
-        -   "--extend-exclude"
-        -   "capa/render/proto/capa_pb2.py"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false
-
--   repo: local
-    hooks:
-    -   id: mypy
-        name: mypy
-        stages: [push, manual]
-        language: system
-        entry: mypy
-        args: 
-        -   "--check-untyped-defs"
-        -   "--ignore-missing-imports"
-        -   "--config-file=.github/mypy/mypy.ini"
-        -   "capa/"
-        -   "scripts/"
-        -   "tests/"
-        always_run: true
-        pass_filenames: false
-
--   repo: local
-    hooks:
-    -   id: pytest-fast
-        name: pytest (fast)
-        stages: [manual]
-        language: system
-        entry: pytest
-        args: 
-        -   "tests/"
-        -   "--ignore=tests/test_binja_features.py"
-        -   "--ignore=tests/test_ghidra_features.py"
-        -   "--ignore=tests/test_ida_features.py"
-        -   "--ignore=tests/test_viv_features.py"
-        -   "--ignore=tests/test_main.py"
-        -   "--ignore=tests/test_scripts.py"
-        always_run: true
-        pass_filenames: false

LICENSE.txt

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (C) 2023 Mandiant, Inc.
+   Copyright (C) 2020 FireEye, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,22 +1,14 @@
-![capa](https://github.com/mandiant/capa/blob/master/.github/logo.png)
+![capa](.github/logo.png)
 
-[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
-[![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-866-blue.svg)](https://github.com/mandiant/capa-rules)
-[![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
-[![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
+[![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
+[![Number of rules](https://img.shields.io/badge/rules-455-blue.svg)](https://github.com/fireeye/capa-rules)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
 capa detects capabilities in executable files.
-You run it against a PE, ELF, .NET module, shellcode file, or a sandbox report and it tells you what it thinks the program can do.
+You run it against a PE file or shellcode and it tells you what it thinks the program can do.
 For example, it might suggest that the file is a backdoor, is capable of installing services, or relies on HTTP to communicate.
 
-Check out our capa blog posts:
-- [Dynamic capa: Exploring Executable Run-Time Behavior with the CAPE Sandbox](https://www.mandiant.com/resources/blog/dynamic-capa-executable-behavior-cape-sandbox)
-- [capa v4: casting a wider .NET](https://www.mandiant.com/resources/blog/capa-v4-casting-wider-net) (.NET support)
-- [ELFant in the Room – capa v3](https://www.mandiant.com/resources/elfant-in-the-room-capa-v3) (ELF support)
-- [capa 2.0: Better, Stronger, Faster](https://www.mandiant.com/resources/capa-2-better-stronger-faster)
-- [capa: Automatically Identify Malware Capabilities](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities)
+Check out the overview in our first [capa blog post](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html).
 
 ```
 $ capa.exe suspicious.exe
@@ -68,11 +60,18 @@ $ capa.exe suspicious.exe
 
 # download and usage
 
-Download stable releases of the standalone capa binaries [here](https://github.com/mandiant/capa/releases). You can run the standalone binaries without installation. capa is a command line tool that should be run from the terminal.
+Download stable releases of the standalone capa binaries [here](https://github.com/fireeye/capa/releases). You can run the standalone binaries without installation. capa is a command line tool that should be run from the terminal.
 
-To use capa as a library or integrate with another tool, see [doc/installation.md](https://github.com/mandiant/capa/blob/master/doc/installation.md) for further setup instructions.
+<!--
+Alternatively, you can fetch a nightly build of a standalone binary from one of the following links. These are built using the latest development branch.
+- Windows 64bit: TODO
+- Linux: TODO
+- OSX: TODO
+-->
 
-For more information about how to use capa, see [doc/usage.md](https://github.com/mandiant/capa/blob/master/doc/usage.md).
+To use capa as a library or integrate with another tool, see [doc/installation.md](doc/installation.md) for further setup instructions.
+
+For more information about how to use capa, see [doc/usage.md](doc/usage.md).
 
 # example
 
@@ -89,133 +88,34 @@ This is useful for at least two reasons:
   - it shows where within the binary an experienced analyst might study with IDA Pro
 
 ```
-$ capa.exe suspicious.exe -vv
+λ capa.exe suspicious.exe -vv
 ...
 execute shell command and capture output
 namespace   c2/shell
-author      matthew.williams@mandiant.com
+author      matthew.williams@fireeye.com
 scope       function
 att&ck      Execution::Command and Scripting Interpreter::Windows Command Shell [T1059.003]
 references  https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/ns-processthreadsapi-startupinfoa
-function @ 0x4011C0
+examples    Practical Malware Analysis Lab 14-02.exe_:0x4011C0
+function @ 0x10003A13
   and:
-    match: create a process with modified I/O handles and window @ 0x4011C0
+    match: create a process with modified I/O handles and window @ 0x10003A13
       and:
-        number: 257 = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW @ 0x4012B8
         or:
-          number: 68 = StartupInfo.cb (size) @ 0x401282
-        or: = API functions that accept a pointer to a STARTUPINFO structure
-          api: kernel32.CreateProcess @ 0x401343
-    match: create pipe @ 0x4011C0
+          api: kernel32.CreateProcess @ 0x10003D6D
+        number: 0x101 @ 0x10003B03
+        or:
+          number: 0x44 @ 0x10003ADC
+        optional:
+          api: kernel32.GetStartupInfo @ 0x10003AE4
+    match: create pipe @ 0x10003A13
       or:
-        api: kernel32.CreatePipe @ 0x40126F, 0x401280
-    optional:
-      match: create thread @ 0x40136A, 0x4013BA
-        or:
-          and:
-            os: windows
-            or:
-              api: kernel32.CreateThread @ 0x4013D7
-        or:
-          and:
-            os: windows
-            or:
-              api: kernel32.CreateThread @ 0x401395
+        api: kernel32.CreatePipe @ 0x10003ACB
     or:
-      string: "cmd.exe" @ 0x4012FD
+      string: cmd.exe /c  @ 0x10003AED
 ...
 ```
 
-Additionally, capa also supports analyzing [CAPE](https://github.com/kevoreilly/CAPEv2) sandbox reports for dynamic capabilty extraction.
-In order to use this, you first submit your sample to CAPE for analysis, and then run capa against the generated report (JSON).
-
-Here's an example of running capa against a packed binary, and then running capa against the CAPE report of that binary:
-
-```yaml
-$ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.exe
-WARNING:capa.capabilities.common:--------------------------------------------------------------------------------
-WARNING:capa.capabilities.common: This sample appears to be packed.
-WARNING:capa.capabilities.common: 
-WARNING:capa.capabilities.common: Packed samples have often been obfuscated to hide their logic.
-WARNING:capa.capabilities.common: capa cannot handle obfuscation well using static analysis. This means the results may be misleading or incomplete.
-WARNING:capa.capabilities.common: If possible, you should try to unpack this input file before analyzing it with capa.
-WARNING:capa.capabilities.common: Alternatively, run the sample in a supported sandbox and invoke capa against the report to obtain dynamic analysis results.
-WARNING:capa.capabilities.common: 
-WARNING:capa.capabilities.common: Identified via rule: (internal) packer file limitation
-WARNING:capa.capabilities.common: 
-WARNING:capa.capabilities.common: Use -v or -vv if you really want to see the capabilities identified by capa.
-WARNING:capa.capabilities.common:--------------------------------------------------------------------------------
-
-$ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.json
-
-┍━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
-│ ATT&CK Tactic          │ ATT&CK Technique                                                                   │
-┝━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
-│ CREDENTIAL ACCESS      │ Credentials from Password Stores T1555                                             │
-├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
-│ DEFENSE EVASION        │ File and Directory Permissions Modification T1222                                  │
-│                        │ Modify Registry T1112                                                              │
-│                        │ Obfuscated Files or Information T1027                                              │
-│                        │ Virtualization/Sandbox Evasion::User Activity Based Checks T1497.002               │
-├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
-│ DISCOVERY              │ Account Discovery T1087                                                            │
-│                        │ Application Window Discovery T1010                                                 │
-│                        │ File and Directory Discovery T1083                                                 │
-│                        │ Query Registry T1012                                                               │
-│                        │ System Information Discovery T1082                                                 │
-│                        │ System Location Discovery::System Language Discovery T1614.001                     │
-│                        │ System Owner/User Discovery T1033                                                  │
-├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
-│ EXECUTION              │ System Services::Service Execution T1569.002                                       │
-├────────────────────────┼────────────────────────────────────────────────────────────────────────────────────┤
-│ PERSISTENCE            │ Boot or Logon Autostart Execution::Registry Run Keys / Startup Folder T1547.001    │
-│                        │ Boot or Logon Autostart Execution::Winlogon Helper DLL T1547.004                   │
-│                        │ Create or Modify System Process::Windows Service T1543.003                         │
-┕━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
-
-┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑
-│ Capability                                           │ Namespace                                            │
-┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥
-│ check for unmoving mouse cursor (3 matches)          │ anti-analysis/anti-vm/vm-detection                   │
-│ gather bitkinex information                          │ collection/file-managers                             │
-│ gather classicftp information                        │ collection/file-managers                             │
-│ gather filezilla information                         │ collection/file-managers                             │
-│ gather total-commander information                   │ collection/file-managers                             │
-│ gather ultrafxp information                          │ collection/file-managers                             │
-│ resolve DNS (23 matches)                             │ communication/dns                                    │
-│ initialize Winsock library (7 matches)               │ communication/socket                                 │
-│ act as TCP client (3 matches)                        │ communication/tcp/client                             │
-│ create new key via CryptAcquireContext               │ data-manipulation/encryption                         │
-│ encrypt or decrypt via WinCrypt                      │ data-manipulation/encryption                         │
-│ hash data via WinCrypt                               │ data-manipulation/hashing                            │
-│ initialize hashing via WinCrypt                      │ data-manipulation/hashing                            │
-│ hash data with MD5                                   │ data-manipulation/hashing/md5                        │
-│ generate random numbers via WinAPI                   │ data-manipulation/prng                               │
-│ extract resource via kernel32 functions (2 matches)  │ executable/resource                                  │
-│ interact with driver via control codes (2 matches)   │ host-interaction/driver                              │
-│ get Program Files directory (18 matches)             │ host-interaction/file-system                         │
-│ get common file path (575 matches)                   │ host-interaction/file-system                         │
-│ create directory (2 matches)                         │ host-interaction/file-system/create                  │
-│ delete file                                          │ host-interaction/file-system/delete                  │
-│ get file attributes (122 matches)                    │ host-interaction/file-system/meta                    │
-│ set file attributes (8 matches)                      │ host-interaction/file-system/meta                    │
-│ move file                                            │ host-interaction/file-system/move                    │
-│ find taskbar (3 matches)                             │ host-interaction/gui/taskbar/find                    │
-│ get keyboard layout (12 matches)                     │ host-interaction/hardware/keyboard                   │
-│ get disk size                                        │ host-interaction/hardware/storage                    │
-│ get hostname (4 matches)                             │ host-interaction/os/hostname                         │
-│ allocate or change RWX memory (3 matches)            │ host-interaction/process/inject                      │
-│ query or enumerate registry key (3 matches)          │ host-interaction/registry                            │
-│ query or enumerate registry value (8 matches)        │ host-interaction/registry                            │
-│ delete registry key                                  │ host-interaction/registry/delete                     │
-│ start service                                        │ host-interaction/service/start                       │
-│ get session user name                                │ host-interaction/session                             │
-│ persist via Run registry key                         │ persistence/registry/run                             │
-│ persist via Winlogon Helper DLL registry key         │ persistence/registry/winlogon-helper                 │
-│ persist via Windows service (2 matches)              │ persistence/service                                  │
-┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
-```
-
 capa uses a collection of rules to identify capabilities within a program.
 These rules are easy to write, even for those new to reverse engineering.
 By authoring rules, you can extend the capabilities that capa recognizes.
@@ -226,52 +126,39 @@ Here's an example rule used by capa:
 ```yaml
 rule:
   meta:
-    name: create TCP socket
-    namespace: communication/socket/tcp
-    authors:
-      - william.ballenthin@mandiant.com
-      - joakim@intezer.com
-      - anushka.virgaonkar@mandiant.com
-    scopes:
-      static: basic block
-      dynamic: call
-    mbc:
-      - Communication::Socket Communication::Create TCP Socket [C0001.011]
+    name: hash data with CRC32
+    namespace: data-manipulation/checksum/crc32
+    author: moritz.raabe@fireeye.com
+    scope: function
     examples:
-      - Practical Malware Analysis Lab 01-01.dll_:0x10001010
+      - 2D3EDC218A90F03089CC01715A9F047F:0x403CBD
+      - 7D28CB106CB54876B2A5C111724A07CD:0x402350  # RtlComputeCrc32
   features:
     - or:
       - and:
-        - number: 6 = IPPROTO_TCP
-        - number: 1 = SOCK_STREAM
-        - number: 2 = AF_INET
-        - or:
-          - api: ws2_32.socket
-          - api: ws2_32.WSASocket
-          - api: socket
-      - property/read: System.Net.Sockets.TcpClient::Client
+        - mnemonic: shr
+        - number: 0xEDB88320
+        - number: 8
+        - characteristic: nzxor
+      - api: RtlComputeCrc32
 ```
 
-The [github.com/mandiant/capa-rules](https://github.com/mandiant/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
+The [github.com/fireeye/capa-rules](https://github.com/fireeye/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
 Please learn to write rules and contribute new entries as you find interesting techniques in malware.
 
-If you use IDA Pro, then you can use the [capa explorer](https://github.com/mandiant/capa/tree/master/capa/ida/plugin) plugin.
-capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted directly from your IDA Pro database.
+If you use IDA Pro, then you use can use the [capa explorer IDA plugin](capa/ida/plugin/).
+capa explorer lets you quickly identify and navigate to interesting areas of a program and dissect capa rule matches at
+the assembly level.
 
-![capa + IDA Pro integration](https://github.com/mandiant/capa/blob/master/doc/img/explorer_expanded.png)
-
-If you use Ghidra, you can use the Python 3 [Ghidra feature extractor](/capa/ghidra/). This integration enables capa to extract features directly from your Ghidra database, which can help you identify capabilities in programs that you analyze using Ghidra.
+![capa + IDA Pro integration](doc/img/ida_plugin_intro.gif)
 
 # further information
 ## capa
-- [Installation](https://github.com/mandiant/capa/blob/master/doc/installation.md)
-- [Usage](https://github.com/mandiant/capa/blob/master/doc/usage.md)
-- [Limitations](https://github.com/mandiant/capa/blob/master/doc/limitations.md)
-- [Contributing Guide](https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md)
+- [doc/installation](doc/installation.md)
+- [doc/usage](doc/usage.md)
+- [doc/limitations](doc/limitations.md)
+- [Contributing Guide](.github/CONTRIBUTING.md)
 
 ## capa rules
-- [capa-rules repository](https://github.com/mandiant/capa-rules)
-- [capa-rules rule format](https://github.com/mandiant/capa-rules/blob/master/doc/format.md)
-
-## capa testfiles
-The [capa-testfiles repository](https://github.com/mandiant/capa-testfiles) contains the data we use to test capa's code and rules
+- [capa-rules repository](https://github.com/fireeye/capa-rules)
+- [capa-rules rule format](https://github.com/fireeye/capa-rules/blob/master/doc/format.md)

capa/capabilities/common.py

@@ -1,79 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-import itertools
-import collections
-from typing import Any, Tuple
-
-from capa.rules import Scope, RuleSet
-from capa.engine import FeatureSet, MatchResults
-from capa.features.address import NO_ADDRESS
-from capa.features.extractors.base_extractor import FeatureExtractor, StaticFeatureExtractor, DynamicFeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def find_file_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, function_features: FeatureSet):
-    file_features: FeatureSet = collections.defaultdict(set)
-
-    for feature, va in itertools.chain(extractor.extract_file_features(), extractor.extract_global_features()):
-        # not all file features may have virtual addresses.
-        # if not, then at least ensure the feature shows up in the index.
-        # the set of addresses will still be empty.
-        if va:
-            file_features[feature].add(va)
-        else:
-            if feature not in file_features:
-                file_features[feature] = set()
-
-    logger.debug("analyzed file and extracted %d features", len(file_features))
-
-    file_features.update(function_features)
-
-    _, matches = ruleset.match(Scope.FILE, file_features, NO_ADDRESS)
-    return matches, len(file_features)
-
-
-def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalone=True) -> bool:
-    file_limitation_rules = list(filter(lambda r: r.is_file_limitation_rule(), rules.rules.values()))
-
-    for file_limitation_rule in file_limitation_rules:
-        if file_limitation_rule.name not in capabilities:
-            continue
-
-        logger.warning("-" * 80)
-        for line in file_limitation_rule.meta.get("description", "").split("\n"):
-            logger.warning(" %s", line)
-        logger.warning(" Identified via rule: %s", file_limitation_rule.name)
-        if is_standalone:
-            logger.warning(" ")
-            logger.warning(" Use -v or -vv if you really want to see the capabilities identified by capa.")
-        logger.warning("-" * 80)
-
-        # bail on first file limitation
-        return True
-
-    return False
-
-
-def find_capabilities(
-    ruleset: RuleSet, extractor: FeatureExtractor, disable_progress=None, **kwargs
-) -> Tuple[MatchResults, Any]:
-    from capa.capabilities.static import find_static_capabilities
-    from capa.capabilities.dynamic import find_dynamic_capabilities
-
-    if isinstance(extractor, StaticFeatureExtractor):
-        # for the time being, extractors are either static or dynamic.
-        # Remove this assertion once that has changed
-        assert not isinstance(extractor, DynamicFeatureExtractor)
-        return find_static_capabilities(ruleset, extractor, disable_progress=disable_progress, **kwargs)
-    if isinstance(extractor, DynamicFeatureExtractor):
-        return find_dynamic_capabilities(ruleset, extractor, disable_progress=disable_progress, **kwargs)
-
-    raise ValueError(f"unexpected extractor type: {extractor.__class__.__name__}")

capa/capabilities/dynamic.py

@@ -1,198 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-import itertools
-import collections
-from typing import Any, Tuple
-
-import tqdm
-
-import capa.perf
-import capa.features.freeze as frz
-import capa.render.result_document as rdoc
-from capa.rules import Scope, RuleSet
-from capa.engine import FeatureSet, MatchResults
-from capa.helpers import redirecting_print_to_tqdm
-from capa.capabilities.common import find_file_capabilities
-from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle, DynamicFeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def find_call_capabilities(
-    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
-) -> Tuple[FeatureSet, MatchResults]:
-    """
-    find matches for the given rules for the given call.
-
-    returns: tuple containing (features for call, match results for call)
-    """
-    # all features found for the call.
-    features: FeatureSet = collections.defaultdict(set)
-
-    for feature, addr in itertools.chain(
-        extractor.extract_call_features(ph, th, ch), extractor.extract_global_features()
-    ):
-        features[feature].add(addr)
-
-    # matches found at this thread.
-    _, matches = ruleset.match(Scope.CALL, features, ch.address)
-
-    for rule_name, res in matches.items():
-        rule = ruleset[rule_name]
-        for addr, _ in res:
-            capa.engine.index_rule_matches(features, rule, [addr])
-
-    return features, matches
-
-
-def find_thread_capabilities(
-    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle
-) -> Tuple[FeatureSet, MatchResults, MatchResults]:
-    """
-    find matches for the given rules within the given thread.
-
-    returns: tuple containing (features for thread, match results for thread, match results for calls)
-    """
-    # all features found within this thread,
-    # includes features found within calls.
-    features: FeatureSet = collections.defaultdict(set)
-
-    # matches found at the call scope.
-    # might be found at different calls, thats ok.
-    call_matches: MatchResults = collections.defaultdict(list)
-
-    for ch in extractor.get_calls(ph, th):
-        ifeatures, imatches = find_call_capabilities(ruleset, extractor, ph, th, ch)
-        for feature, vas in ifeatures.items():
-            features[feature].update(vas)
-
-        for rule_name, res in imatches.items():
-            call_matches[rule_name].extend(res)
-
-    for feature, va in itertools.chain(extractor.extract_thread_features(ph, th), extractor.extract_global_features()):
-        features[feature].add(va)
-
-    # matches found within this thread.
-    _, matches = ruleset.match(Scope.THREAD, features, th.address)
-
-    for rule_name, res in matches.items():
-        rule = ruleset[rule_name]
-        for va, _ in res:
-            capa.engine.index_rule_matches(features, rule, [va])
-
-    return features, matches, call_matches
-
-
-def find_process_capabilities(
-    ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle
-) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
-    """
-    find matches for the given rules within the given process.
-
-    returns: tuple containing (match results for process, match results for threads, match results for calls, number of features)
-    """
-    # all features found within this process,
-    # includes features found within threads (and calls).
-    process_features: FeatureSet = collections.defaultdict(set)
-
-    # matches found at the basic threads.
-    # might be found at different threads, thats ok.
-    thread_matches: MatchResults = collections.defaultdict(list)
-
-    # matches found at the call scope.
-    # might be found at different calls, thats ok.
-    call_matches: MatchResults = collections.defaultdict(list)
-
-    for th in extractor.get_threads(ph):
-        features, tmatches, cmatches = find_thread_capabilities(ruleset, extractor, ph, th)
-        for feature, vas in features.items():
-            process_features[feature].update(vas)
-
-        for rule_name, res in tmatches.items():
-            thread_matches[rule_name].extend(res)
-
-        for rule_name, res in cmatches.items():
-            call_matches[rule_name].extend(res)
-
-    for feature, va in itertools.chain(extractor.extract_process_features(ph), extractor.extract_global_features()):
-        process_features[feature].add(va)
-
-    _, process_matches = ruleset.match(Scope.PROCESS, process_features, ph.address)
-    return process_matches, thread_matches, call_matches, len(process_features)
-
-
-def find_dynamic_capabilities(
-    ruleset: RuleSet, extractor: DynamicFeatureExtractor, disable_progress=None
-) -> Tuple[MatchResults, Any]:
-    all_process_matches: MatchResults = collections.defaultdict(list)
-    all_thread_matches: MatchResults = collections.defaultdict(list)
-    all_call_matches: MatchResults = collections.defaultdict(list)
-
-    feature_counts = rdoc.DynamicFeatureCounts(file=0, processes=())
-
-    assert isinstance(extractor, DynamicFeatureExtractor)
-    with redirecting_print_to_tqdm(disable_progress):
-        with tqdm.contrib.logging.logging_redirect_tqdm():
-            pbar = tqdm.tqdm
-            if disable_progress:
-                # do not use tqdm to avoid unnecessary side effects when caller intends
-                # to disable progress completely
-                def pbar(s, *args, **kwargs):
-                    return s
-
-            processes = list(extractor.get_processes())
-
-            pb = pbar(processes, desc="matching", unit=" processes", leave=False)
-            for p in pb:
-                process_matches, thread_matches, call_matches, feature_count = find_process_capabilities(
-                    ruleset, extractor, p
-                )
-                feature_counts.processes += (
-                    rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count),
-                )
-                logger.debug("analyzed %s and extracted %d features", p.address, feature_count)
-
-                for rule_name, res in process_matches.items():
-                    all_process_matches[rule_name].extend(res)
-                for rule_name, res in thread_matches.items():
-                    all_thread_matches[rule_name].extend(res)
-                for rule_name, res in call_matches.items():
-                    all_call_matches[rule_name].extend(res)
-
-    # collection of features that captures the rule matches within process and thread scopes.
-    # mapping from feature (matched rule) to set of addresses at which it matched.
-    process_and_lower_features: FeatureSet = collections.defaultdict(set)
-    for rule_name, results in itertools.chain(
-        all_process_matches.items(), all_thread_matches.items(), all_call_matches.items()
-    ):
-        locations = {p[0] for p in results}
-        rule = ruleset[rule_name]
-        capa.engine.index_rule_matches(process_and_lower_features, rule, locations)
-
-    all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, process_and_lower_features)
-    feature_counts.file = feature_count
-
-    matches = dict(
-        itertools.chain(
-            # each rule exists in exactly one scope,
-            # so there won't be any overlap among these following MatchResults,
-            # and we can merge the dictionaries naively.
-            all_thread_matches.items(),
-            all_process_matches.items(),
-            all_call_matches.items(),
-            all_file_matches.items(),
-        )
-    )
-
-    meta = {
-        "feature_counts": feature_counts,
-    }
-
-    return matches, meta

capa/capabilities/static.py

@@ -1,233 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import time
-import logging
-import itertools
-import collections
-from typing import Any, Tuple
-
-import tqdm.contrib.logging
-
-import capa.perf
-import capa.features.freeze as frz
-import capa.render.result_document as rdoc
-from capa.rules import Scope, RuleSet
-from capa.engine import FeatureSet, MatchResults
-from capa.helpers import redirecting_print_to_tqdm
-from capa.capabilities.common import find_file_capabilities
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, StaticFeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def find_instruction_capabilities(
-    ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
-) -> Tuple[FeatureSet, MatchResults]:
-    """
-    find matches for the given rules for the given instruction.
-
-    returns: tuple containing (features for instruction, match results for instruction)
-    """
-    # all features found for the instruction.
-    features: FeatureSet = collections.defaultdict(set)
-
-    for feature, addr in itertools.chain(
-        extractor.extract_insn_features(f, bb, insn), extractor.extract_global_features()
-    ):
-        features[feature].add(addr)
-
-    # matches found at this instruction.
-    _, matches = ruleset.match(Scope.INSTRUCTION, features, insn.address)
-
-    for rule_name, res in matches.items():
-        rule = ruleset[rule_name]
-        for addr, _ in res:
-            capa.engine.index_rule_matches(features, rule, [addr])
-
-    return features, matches
-
-
-def find_basic_block_capabilities(
-    ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle
-) -> Tuple[FeatureSet, MatchResults, MatchResults]:
-    """
-    find matches for the given rules within the given basic block.
-
-    returns: tuple containing (features for basic block, match results for basic block, match results for instructions)
-    """
-    # all features found within this basic block,
-    # includes features found within instructions.
-    features: FeatureSet = collections.defaultdict(set)
-
-    # matches found at the instruction scope.
-    # might be found at different instructions, thats ok.
-    insn_matches: MatchResults = collections.defaultdict(list)
-
-    for insn in extractor.get_instructions(f, bb):
-        ifeatures, imatches = find_instruction_capabilities(ruleset, extractor, f, bb, insn)
-        for feature, vas in ifeatures.items():
-            features[feature].update(vas)
-
-        for rule_name, res in imatches.items():
-            insn_matches[rule_name].extend(res)
-
-    for feature, va in itertools.chain(
-        extractor.extract_basic_block_features(f, bb), extractor.extract_global_features()
-    ):
-        features[feature].add(va)
-
-    # matches found within this basic block.
-    _, matches = ruleset.match(Scope.BASIC_BLOCK, features, bb.address)
-
-    for rule_name, res in matches.items():
-        rule = ruleset[rule_name]
-        for va, _ in res:
-            capa.engine.index_rule_matches(features, rule, [va])
-
-    return features, matches, insn_matches
-
-
-def find_code_capabilities(
-    ruleset: RuleSet, extractor: StaticFeatureExtractor, fh: FunctionHandle
-) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
-    """
-    find matches for the given rules within the given function.
-
-    returns: tuple containing (match results for function, match results for basic blocks, match results for instructions, number of features)
-    """
-    # all features found within this function,
-    # includes features found within basic blocks (and instructions).
-    function_features: FeatureSet = collections.defaultdict(set)
-
-    # matches found at the basic block scope.
-    # might be found at different basic blocks, thats ok.
-    bb_matches: MatchResults = collections.defaultdict(list)
-
-    # matches found at the instruction scope.
-    # might be found at different instructions, thats ok.
-    insn_matches: MatchResults = collections.defaultdict(list)
-
-    for bb in extractor.get_basic_blocks(fh):
-        features, bmatches, imatches = find_basic_block_capabilities(ruleset, extractor, fh, bb)
-        for feature, vas in features.items():
-            function_features[feature].update(vas)
-
-        for rule_name, res in bmatches.items():
-            bb_matches[rule_name].extend(res)
-
-        for rule_name, res in imatches.items():
-            insn_matches[rule_name].extend(res)
-
-    for feature, va in itertools.chain(extractor.extract_function_features(fh), extractor.extract_global_features()):
-        function_features[feature].add(va)
-
-    _, function_matches = ruleset.match(Scope.FUNCTION, function_features, fh.address)
-    return function_matches, bb_matches, insn_matches, len(function_features)
-
-
-def find_static_capabilities(
-    ruleset: RuleSet, extractor: StaticFeatureExtractor, disable_progress=None
-) -> Tuple[MatchResults, Any]:
-    all_function_matches: MatchResults = collections.defaultdict(list)
-    all_bb_matches: MatchResults = collections.defaultdict(list)
-    all_insn_matches: MatchResults = collections.defaultdict(list)
-
-    feature_counts = rdoc.StaticFeatureCounts(file=0, functions=())
-    library_functions: Tuple[rdoc.LibraryFunction, ...] = ()
-
-    assert isinstance(extractor, StaticFeatureExtractor)
-    with redirecting_print_to_tqdm(disable_progress):
-        with tqdm.contrib.logging.logging_redirect_tqdm():
-            pbar = tqdm.tqdm
-            if capa.helpers.is_runtime_ghidra():
-                # Ghidrathon interpreter cannot properly handle
-                # the TMonitor thread that is created via a monitor_interval
-                # > 0
-                pbar.monitor_interval = 0
-            if disable_progress:
-                # do not use tqdm to avoid unnecessary side effects when caller intends
-                # to disable progress completely
-                def pbar(s, *args, **kwargs):
-                    return s
-
-            functions = list(extractor.get_functions())
-            n_funcs = len(functions)
-
-            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False)
-            for f in pb:
-                t0 = time.time()
-                if extractor.is_library_function(f.address):
-                    function_name = extractor.get_function_name(f.address)
-                    logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
-                    library_functions += (
-                        rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name),
-                    )
-                    n_libs = len(library_functions)
-                    percentage = round(100 * (n_libs / n_funcs))
-                    if isinstance(pb, tqdm.tqdm):
-                        pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
-                    continue
-
-                function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(
-                    ruleset, extractor, f
-                )
-                feature_counts.functions += (
-                    rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count),
-                )
-                t1 = time.time()
-
-                match_count = sum(len(res) for res in function_matches.values())
-                match_count += sum(len(res) for res in bb_matches.values())
-                match_count += sum(len(res) for res in insn_matches.values())
-                logger.debug(
-                    "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
-                    f.address,
-                    feature_count,
-                    match_count,
-                    t1 - t0,
-                )
-
-                for rule_name, res in function_matches.items():
-                    all_function_matches[rule_name].extend(res)
-                for rule_name, res in bb_matches.items():
-                    all_bb_matches[rule_name].extend(res)
-                for rule_name, res in insn_matches.items():
-                    all_insn_matches[rule_name].extend(res)
-
-    # collection of features that captures the rule matches within function, BB, and instruction scopes.
-    # mapping from feature (matched rule) to set of addresses at which it matched.
-    function_and_lower_features: FeatureSet = collections.defaultdict(set)
-    for rule_name, results in itertools.chain(
-        all_function_matches.items(), all_bb_matches.items(), all_insn_matches.items()
-    ):
-        locations = {p[0] for p in results}
-        rule = ruleset[rule_name]
-        capa.engine.index_rule_matches(function_and_lower_features, rule, locations)
-
-    all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, function_and_lower_features)
-    feature_counts.file = feature_count
-
-    matches = dict(
-        itertools.chain(
-            # each rule exists in exactly one scope,
-            # so there won't be any overlap among these following MatchResults,
-            # and we can merge the dictionaries naively.
-            all_insn_matches.items(),
-            all_bb_matches.items(),
-            all_function_matches.items(),
-            all_file_matches.items(),
-        )
-    )
-
-    meta = {
-        "feature_counts": feature_counts,
-        "library_functions": library_functions,
-    }
-
-    return matches, meta

capa/engine.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,29 +8,11 @@
 
 import copy
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator
 
-import capa.perf
-import capa.features.common
-from capa.features.common import Result, Feature
-from capa.features.address import Address
-
-if TYPE_CHECKING:
-    # circular import, otherwise
-    import capa.rules
+import capa.features
 
 
-# a collection of features and the locations at which they are found.
-#
-# used throughout matching as the context in which features are searched:
-# to check if a feature exists, do: `Number(0x10) in features`.
-# to collect the locations of a feature, do: `features[Number(0x10)]`
-#
-# aliased here so that the type can be documented and xref'd.
-FeatureSet = Dict[Feature, Set[Address]]
-
-
-class Statement:
+class Statement(object):
     """
     superclass for structural nodes, such as and/or/not.
     this exists to provide a default impl for `__str__` and `__repr__`,
@@ -38,194 +20,157 @@ class Statement:
     """
 
     def __init__(self, description=None):
-        super().__init__()
+        super(Statement, self).__init__()
         self.name = self.__class__.__name__
         self.description = description
 
     def __str__(self):
-        name = self.name.lower()
-        children = ",".join(map(str, self.get_children()))
         if self.description:
-            return f"{name}({children} = {self.description})"
+            return "%s(%s = %s)" % (self.name.lower(), ",".join(map(str, self.get_children())), self.description)
         else:
-            return f"{name}({children})"
+            return "%s(%s)" % (self.name.lower(), ",".join(map(str, self.get_children())))
 
     def __repr__(self):
         return str(self)
 
-    def evaluate(self, features: FeatureSet, short_circuit=True) -> Result:
+    def evaluate(self, ctx):
         """
         classes that inherit `Statement` must implement `evaluate`
 
         args:
-            short_circuit (bool): if true, then statements like and/or/some may short circuit.
+          ctx (defaultdict[Feature, set[VA]])
+
+        returns:
+          Result
         """
         raise NotImplementedError()
 
-    def get_children(self) -> Iterator[Union["Statement", Feature]]:
+    def get_children(self):
         if hasattr(self, "child"):
-            # this really confuses mypy because the property may not exist
-            # since its defined in the subclasses.
-            child = self.child  # type: ignore
-            assert isinstance(child, (Statement, Feature))
-            yield child
+            yield self.child
 
         if hasattr(self, "children"):
             for child in self.children:
-                assert isinstance(child, (Statement, Feature))
                 yield child
 
     def replace_child(self, existing, new):
         if hasattr(self, "child"):
-            # this really confuses mypy because the property may not exist
-            # since its defined in the subclasses.
-            if self.child is existing:  # type: ignore
+            if self.child is existing:
                 self.child = new
 
         if hasattr(self, "children"):
-            children = self.children
-            for i, child in enumerate(children):
+            for i, child in enumerate(self.children):
                 if child is existing:
-                    children[i] = new
+                    self.children[i] = new
+
+
+class Result(object):
+    """
+    represents the results of an evaluation of statements against features.
+
+    instances of this class should behave like a bool,
+    e.g. `assert Result(True, ...) == True`
+
+    instances track additional metadata about evaluation results.
+    they contain references to the statement node (e.g. an And statement),
+     as well as the children Result instances.
+
+    we need this so that we can render the tree of expressions and their results.
+    """
+
+    def __init__(self, success, statement, children, locations=None):
+        """
+        args:
+          success (bool)
+          statement (capa.engine.Statement or capa.features.Feature)
+          children (list[Result])
+          locations (iterable[VA])
+        """
+        super(Result, self).__init__()
+        self.success = success
+        self.statement = statement
+        self.children = children
+        self.locations = locations if locations is not None else ()
+
+    def __eq__(self, other):
+        if isinstance(other, bool):
+            return self.success == other
+        return False
+
+    def __bool__(self):
+        return self.success
+
+    def __nonzero__(self):
+        return self.success
 
 
 class And(Statement):
-    """
-    match if all of the children evaluate to True.
-
-    the order of evaluation is dictated by the property
-    `And.children` (type: List[Statement|Feature]).
-    a query optimizer may safely manipulate the order of these children.
-    """
+    """match if all of the children evaluate to True."""
 
     def __init__(self, children, description=None):
-        super().__init__(description=description)
+        super(And, self).__init__(description=description)
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.and"] += 1
-
-        if short_circuit:
-            results = []
-            for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
-                results.append(result)
-                if not result:
-                    # short circuit
-                    return Result(False, self, results)
-
-            return Result(True, self, results)
-        else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
-            success = all(results)
-            return Result(success, self, results)
+    def evaluate(self, ctx):
+        results = [child.evaluate(ctx) for child in self.children]
+        success = all(results)
+        return Result(success, self, results)
 
 
 class Or(Statement):
-    """
-    match if any of the children evaluate to True.
-
-    the order of evaluation is dictated by the property
-    `Or.children` (type: List[Statement|Feature]).
-    a query optimizer may safely manipulate the order of these children.
-    """
+    """match if any of the children evaluate to True."""
 
     def __init__(self, children, description=None):
-        super().__init__(description=description)
+        super(Or, self).__init__(description=description)
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.or"] += 1
-
-        if short_circuit:
-            results = []
-            for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
-                results.append(result)
-                if result:
-                    # short circuit as soon as we hit one match
-                    return Result(True, self, results)
-
-            return Result(False, self, results)
-        else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
-            success = any(results)
-            return Result(success, self, results)
+    def evaluate(self, ctx):
+        results = [child.evaluate(ctx) for child in self.children]
+        success = any(results)
+        return Result(success, self, results)
 
 
 class Not(Statement):
     """match only if the child evaluates to False."""
 
     def __init__(self, child, description=None):
-        super().__init__(description=description)
+        super(Not, self).__init__(description=description)
         self.child = child
 
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.not"] += 1
-
-        results = [self.child.evaluate(ctx, short_circuit=short_circuit)]
+    def evaluate(self, ctx):
+        results = [self.child.evaluate(ctx)]
         success = not results[0]
         return Result(success, self, results)
 
 
 class Some(Statement):
-    """
-    match if at least N of the children evaluate to True.
-
-    the order of evaluation is dictated by the property
-    `Some.children` (type: List[Statement|Feature]).
-    a query optimizer may safely manipulate the order of these children.
-    """
+    """match if at least N of the children evaluate to True."""
 
     def __init__(self, count, children, description=None):
-        super().__init__(description=description)
+        super(Some, self).__init__(description=description)
         self.count = count
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.some"] += 1
-
-        if short_circuit:
-            results = []
-            satisfied_children_count = 0
-            for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
-                results.append(result)
-                if result:
-                    satisfied_children_count += 1
-
-                if satisfied_children_count >= self.count:
-                    # short circuit as soon as we hit the threshold
-                    return Result(True, self, results)
-
-            return Result(False, self, results)
-        else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
-            # note that here we cast the child result as a bool
-            # because we've overridden `__bool__` above.
-            #
-            # we can't use `if child is True` because the instance is not True.
-            success = sum([1 for child in results if bool(child) is True]) >= self.count
-            return Result(success, self, results)
+    def evaluate(self, ctx):
+        results = [child.evaluate(ctx) for child in self.children]
+        # note that here we cast the child result as a bool
+        # because we've overridden `__bool__` above.
+        #
+        # we can't use `if child is True` because the instance is not True.
+        success = sum([1 for child in results if bool(child) is True]) >= self.count
+        return Result(success, self, results)
 
 
 class Range(Statement):
     """match if the child is contained in the ctx set with a count in the given range."""
 
     def __init__(self, child, min=None, max=None, description=None):
-        super().__init__(description=description)
+        super(Range, self).__init__(description=description)
         self.child = child
         self.min = min if min is not None else 0
         self.max = max if max is not None else (1 << 64 - 1)
 
-    def evaluate(self, ctx, **kwargs):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.range"] += 1
-
+    def evaluate(self, ctx):
         count = len(ctx.get(self.child, []))
         if self.min == 0 and count == 0:
             return Result(True, self, [])
@@ -234,9 +179,9 @@ class Range(Statement):
 
     def __str__(self):
         if self.max == (1 << 64 - 1):
-            return f"range({str(self.child)}, min={self.min}, max=infinity)"
+            return "range(%s, min=%d, max=infinity)" % (str(self.child), self.min)
         else:
-            return f"range({str(self.child)}, min={self.min}, max={self.max})"
+            return "range(%s, min=%d, max=%d)" % (str(self.child), self.min, self.max)
 
 
 class Subscope(Statement):
@@ -245,66 +190,59 @@ class Subscope(Statement):
     the engine should preprocess rules to extract subscope statements into their own rules.
     """
 
-    def __init__(self, scope, child, description=None):
-        super().__init__(description=description)
+    def __init__(self, scope, child):
+        super(Subscope, self).__init__()
         self.scope = scope
         self.child = child
 
-    def evaluate(self, ctx, **kwargs):
+    def evaluate(self, ctx):
         raise ValueError("cannot evaluate a subscope directly!")
 
 
-# mapping from rule name to list of: (location of match, result object)
-#
-# used throughout matching and rendering to collection the results
-#  of statement evaluation and their locations.
-#
-# to check if a rule matched, do: `"TCP client" in matches`.
-# to find where a rule matched, do: `map(first, matches["TCP client"])`
-# to see how a rule matched, do:
-#
-#     for address, match_details in matches["TCP client"]:
-#         inspect(match_details)
-#
-# aliased here so that the type can be documented and xref'd.
-MatchResults = Mapping[str, List[Tuple[Address, Result]]]
-
-
-def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations: Iterable[Address]):
+def topologically_order_rules(rules):
     """
-    record into the given featureset that the given rule matched at the given locations.
+    order the given rules such that dependencies show up before dependents.
+    this means that as we match rules, we can add features for the matches, and these
+     will be matched by subsequent rules if they follow this order.
 
-    naively, this is just adding a MatchedRule feature;
-    however, we also want to record matches for the rule's namespaces.
-
-    updates `features` in-place. doesn't modify the remaining arguments.
+    assumes that the rule dependency graph is a DAG.
     """
-    features[capa.features.common.MatchedRule(rule.name)].update(locations)
-    namespace = rule.meta.get("namespace")
-    if namespace:
-        while namespace:
-            features[capa.features.common.MatchedRule(namespace)].update(locations)
-            namespace, _, _ = namespace.rpartition("/")
+    # we evaluate `rules` multiple times, so if its a generator, realize it into a list.
+    rules = list(rules)
+    namespaces = capa.rules.index_rules_by_namespace(rules)
+    rules = {rule.name: rule for rule in rules}
+    seen = set([])
+    ret = []
+
+    def rec(rule):
+        if rule.name in seen:
+            return
+
+        for dep in rule.get_dependencies(namespaces):
+            rec(rules[dep])
+
+        ret.append(rule)
+        seen.add(rule.name)
+
+    for rule in rules.values():
+        rec(rule)
+
+    return ret
 
 
-def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -> Tuple[FeatureSet, MatchResults]:
+def match(rules, features, va):
     """
-    match the given rules against the given features,
-    returning an updated set of features and the matches.
+    Args:
+      rules (List[capa.rules.Rule]): these must already be ordered topologically by dependency.
+      features (Mapping[capa.features.Feature, int]):
+      va (int): location of the features
 
-    the updated features are just like the input,
-    but extended to include the match features (e.g. names of rules that matched).
-    the given feature set is not modified; an updated copy is returned.
-
-    the given list of rules must be ordered topologically by dependency,
-    or else `match` statements will not be handled correctly.
-
-    this routine should be fairly optimized, but is not guaranteed to be the fastest matcher possible.
-    it has a particularly convenient signature: (rules, features) -> matches
-    other strategies can be imagined that match differently; implement these elsewhere.
-    specifically, this routine does "top down" matching of the given rules against the feature set.
+    Returns:
+      Tuple[List[capa.features.Feature], Dict[str, Tuple[int, capa.engine.Result]]]: two-tuple with entries:
+        - list of features used for matching (which may be greater than argument, due to rule match features), and
+        - mapping from rule name to (location of match, result object)
     """
-    results: MatchResults = collections.defaultdict(list)
+    results = collections.defaultdict(list)
 
     # copy features so that we can modify it
     # without affecting the caller (keep this function pure)
@@ -313,22 +251,15 @@ def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -
     features = collections.defaultdict(set, copy.copy(features))
 
     for rule in rules:
-        res = rule.evaluate(features, short_circuit=True)
+        res = rule.evaluate(features)
         if res:
-            # we first matched the rule with short circuiting enabled.
-            # this is much faster than without short circuiting.
-            # however, we want to collect all results thoroughly,
-            # so once we've found a match quickly,
-            # go back and capture results without short circuiting.
-            res = rule.evaluate(features, short_circuit=False)
+            results[rule.name].append((va, res))
+            features[capa.features.MatchedRule(rule.name)].add(va)
 
-            # sanity check
-            assert bool(res) is True
-
-            results[rule.name].append((addr, res))
-            # we need to update the current `features`
-            # because subsequent iterations of this loop may use newly added features,
-            # such as rule or namespace matches.
-            index_rule_matches(features, rule, [addr])
+            namespace = rule.meta.get("namespace")
+            if namespace:
+                while namespace:
+                    features[capa.features.MatchedRule(namespace)].add(va)
+                    namespace, _, _ = namespace.rpartition("/")
 
     return (features, results)

capa/exceptions.py

@@ -1,25 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-class UnsupportedRuntimeError(RuntimeError):
-    pass
-
-
-class UnsupportedFormatError(ValueError):
-    pass
-
-
-class UnsupportedArchError(ValueError):
-    pass
-
-
-class UnsupportedOSError(ValueError):
-    pass
-
-
-class EmptyReportError(ValueError):
-    pass

capa/features/__init__.py

@@ -0,0 +1,222 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import re
+import sys
+import codecs
+import logging
+
+import capa.engine
+
+logger = logging.getLogger(__name__)
+MAX_BYTES_FEATURE_SIZE = 0x100
+
+# thunks may be chained so we specify a delta to control the depth to which these chains are explored
+THUNK_CHAIN_DEPTH_DELTA = 5
+
+# identifiers for supported architectures names that tweak a feature
+# for example, offset/x32
+ARCH_X32 = "x32"
+ARCH_X64 = "x64"
+VALID_ARCH = (ARCH_X32, ARCH_X64)
+
+
+def bytes_to_str(b):
+    if sys.version_info[0] >= 3:
+        return str(codecs.encode(b, "hex").decode("utf-8"))
+    else:
+        return codecs.encode(b, "hex")
+
+
+def hex_string(h):
+    """ render hex string e.g. "0a40b1" as "0A 40 B1" """
+    return " ".join(h[i : i + 2] for i in range(0, len(h), 2)).upper()
+
+
+class Feature(object):
+    def __init__(self, value, arch=None, description=None):
+        """
+        Args:
+          value (any): the value of the feature, such as the number or string.
+          arch (str): one of the VALID_ARCH values, or None.
+            When None, then the feature applies to any architecture.
+            Modifies the feature name from `feature` to `feature/arch`, like `offset/x32`.
+          description (str): a human-readable description that explains the feature value.
+        """
+        super(Feature, self).__init__()
+
+        if arch is not None:
+            if arch not in VALID_ARCH:
+                raise ValueError("arch '%s' must be one of %s" % (arch, VALID_ARCH))
+            self.name = self.__class__.__name__.lower() + "/" + arch
+        else:
+            self.name = self.__class__.__name__.lower()
+
+        self.value = value
+        self.arch = arch
+        self.description = description
+
+    def __hash__(self):
+        return hash((self.name, self.value, self.arch))
+
+    def __eq__(self, other):
+        return self.name == other.name and self.value == other.value and self.arch == other.arch
+
+    def get_value_str(self):
+        """
+        render the value of this feature, for use by `__str__` and friends.
+        subclasses should override to customize the rendering.
+
+        Returns: any
+        """
+        return self.value
+
+    def __str__(self):
+        if self.value is not None:
+            if self.description:
+                return "%s(%s = %s)" % (self.name, self.get_value_str(), self.description)
+            else:
+                return "%s(%s)" % (self.name, self.get_value_str())
+        else:
+            return "%s" % self.name
+
+    def __repr__(self):
+        return str(self)
+
+    def evaluate(self, ctx):
+        return capa.engine.Result(self in ctx, self, [], locations=ctx.get(self, []))
+
+    def freeze_serialize(self):
+        if self.arch is not None:
+            return (self.__class__.__name__, [self.value, {"arch": self.arch}])
+        else:
+            return (self.__class__.__name__, [self.value])
+
+    @classmethod
+    def freeze_deserialize(cls, args):
+        # as you can see below in code,
+        # if the last argument is a dictionary,
+        # consider it to be kwargs passed to the feature constructor.
+        if len(args) == 1:
+            return cls(*args)
+        elif isinstance(args[-1], dict):
+            kwargs = args[-1]
+            args = args[:-1]
+            return cls(*args, **kwargs)
+
+
+class MatchedRule(Feature):
+    def __init__(self, value, description=None):
+        super(MatchedRule, self).__init__(value, description=description)
+        self.name = "match"
+
+
+class Characteristic(Feature):
+    def __init__(self, value, description=None):
+        super(Characteristic, self).__init__(value, description=description)
+
+
+class String(Feature):
+    def __init__(self, value, description=None):
+        super(String, self).__init__(value, description=description)
+
+
+class Regex(String):
+    def __init__(self, value, description=None):
+        super(Regex, self).__init__(value, description=description)
+        pat = self.value[len("/") : -len("/")]
+        flags = re.DOTALL
+        if value.endswith("/i"):
+            pat = self.value[len("/") : -len("/i")]
+            flags |= re.IGNORECASE
+        try:
+            self.re = re.compile(pat, flags)
+        except re.error:
+            if value.endswith("/i"):
+                value = value[: -len("i")]
+            raise ValueError(
+                "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value
+            )
+
+    def evaluate(self, ctx):
+        for feature, locations in ctx.items():
+            if not isinstance(feature, (capa.features.String,)):
+                continue
+
+            # `re.search` finds a match anywhere in the given string
+            # which implies leading and/or trailing whitespace.
+            # using this mode cleans is more convenient for rule authors,
+            # so that they don't have to prefix/suffix their terms like: /.*foo.*/.
+            if self.re.search(feature.value):
+                # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
+                # this is because `self` may match on many strings, so we can't stuff the matched value into it.
+                # instead, return a new instance that has a reference to both the regex and the matched value.
+                # see #262.
+                return capa.engine.Result(True, _MatchedRegex(self, feature.value), [], locations=locations)
+
+        return capa.engine.Result(False, _MatchedRegex(self, None), [])
+
+    def __str__(self):
+        return "regex(string =~ %s)" % self.value
+
+
+class _MatchedRegex(Regex):
+    """
+    this represents a specific instance of a regular expression feature match.
+    treat it the same as a `Regex` except it has the `match` field that contains the complete string that matched.
+
+    note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
+    """
+
+    def __init__(self, regex, match):
+        """
+        args:
+          regex (Regex): the regex feature that matches
+          match (string|None): the matching string or None if it doesn't match
+        """
+        super(_MatchedRegex, self).__init__(regex.value, description=regex.description)
+        # we want this to collide with the name of `Regex` above,
+        # so that it works nicely with the renderers.
+        self.name = "regex"
+        # this may be None if the regex doesn't match
+        self.match = match
+
+    def __str__(self):
+        return 'regex(string =~ %s, matched = "%s")' % (self.value, self.match)
+
+
+class StringFactory(object):
+    def __new__(self, value, description=None):
+        if value.startswith("/") and (value.endswith("/") or value.endswith("/i")):
+            return Regex(value, description=description)
+        return String(value, description=description)
+
+
+class Bytes(Feature):
+    def __init__(self, value, description=None):
+        super(Bytes, self).__init__(value, description=description)
+
+    def evaluate(self, ctx):
+        for feature, locations in ctx.items():
+            if not isinstance(feature, (capa.features.Bytes,)):
+                continue
+
+            if feature.value.startswith(self.value):
+                return capa.engine.Result(True, self, [], locations=locations)
+
+        return capa.engine.Result(False, self, [])
+
+    def get_value_str(self):
+        return hex_string(bytes_to_str(self.value))
+
+    def freeze_serialize(self):
+        return (self.__class__.__name__, [bytes_to_str(self.value).upper()])
+
+    @classmethod
+    def freeze_deserialize(cls, args):
+        return cls(*[codecs.decode(x, "hex") for x in args])

capa/features/address.py

@@ -1,193 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import abc
-
-
-class Address(abc.ABC):
-    @abc.abstractmethod
-    def __eq__(self, other): ...
-
-    @abc.abstractmethod
-    def __lt__(self, other):
-        # implement < so that addresses can be sorted from low to high
-        ...
-
-    @abc.abstractmethod
-    def __hash__(self):
-        # implement hash so that addresses can be used in sets and dicts
-        ...
-
-    @abc.abstractmethod
-    def __repr__(self):
-        # implement repr to help during debugging
-        ...
-
-
-class AbsoluteVirtualAddress(int, Address):
-    """an absolute memory address"""
-
-    def __new__(cls, v):
-        assert v >= 0
-        return int.__new__(cls, v)
-
-    def __repr__(self):
-        return f"absolute(0x{self:x})"
-
-    def __hash__(self):
-        return int.__hash__(self)
-
-
-class ProcessAddress(Address):
-    """an address of a process in a dynamic execution trace"""
-
-    def __init__(self, pid: int, ppid: int = 0):
-        assert ppid >= 0
-        assert pid > 0
-        self.ppid = ppid
-        self.pid = pid
-
-    def __repr__(self):
-        return "process(%s%s)" % (
-            f"ppid: {self.ppid}, " if self.ppid > 0 else "",
-            f"pid: {self.pid}",
-        )
-
-    def __hash__(self):
-        return hash((self.ppid, self.pid))
-
-    def __eq__(self, other):
-        assert isinstance(other, ProcessAddress)
-        return (self.ppid, self.pid) == (other.ppid, other.pid)
-
-    def __lt__(self, other):
-        assert isinstance(other, ProcessAddress)
-        return (self.ppid, self.pid) < (other.ppid, other.pid)
-
-
-class ThreadAddress(Address):
-    """addresses a thread in a dynamic execution trace"""
-
-    def __init__(self, process: ProcessAddress, tid: int):
-        assert tid >= 0
-        self.process = process
-        self.tid = tid
-
-    def __repr__(self):
-        return f"{self.process}, thread(tid: {self.tid})"
-
-    def __hash__(self):
-        return hash((self.process, self.tid))
-
-    def __eq__(self, other):
-        assert isinstance(other, ThreadAddress)
-        return (self.process, self.tid) == (other.process, other.tid)
-
-    def __lt__(self, other):
-        assert isinstance(other, ThreadAddress)
-        return (self.process, self.tid) < (other.process, other.tid)
-
-
-class DynamicCallAddress(Address):
-    """addesses a call in a dynamic execution trace"""
-
-    def __init__(self, thread: ThreadAddress, id: int):
-        assert id >= 0
-        self.thread = thread
-        self.id = id
-
-    def __repr__(self):
-        return f"{self.thread}, call(id: {self.id})"
-
-    def __hash__(self):
-        return hash((self.thread, self.id))
-
-    def __eq__(self, other):
-        assert isinstance(other, DynamicCallAddress)
-        return (self.thread, self.id) == (other.thread, other.id)
-
-    def __lt__(self, other):
-        assert isinstance(other, DynamicCallAddress)
-        return (self.thread, self.id) < (other.thread, other.id)
-
-
-class RelativeVirtualAddress(int, Address):
-    """a memory address relative to a base address"""
-
-    def __repr__(self):
-        return f"relative(0x{self:x})"
-
-    def __hash__(self):
-        return int.__hash__(self)
-
-
-class FileOffsetAddress(int, Address):
-    """an address relative to the start of a file"""
-
-    def __new__(cls, v):
-        assert v >= 0
-        return int.__new__(cls, v)
-
-    def __repr__(self):
-        return f"file(0x{self:x})"
-
-    def __hash__(self):
-        return int.__hash__(self)
-
-
-class DNTokenAddress(int, Address):
-    """a .NET token"""
-
-    def __new__(cls, token: int):
-        return int.__new__(cls, token)
-
-    def __repr__(self):
-        return f"token(0x{self:x})"
-
-    def __hash__(self):
-        return int.__hash__(self)
-
-
-class DNTokenOffsetAddress(Address):
-    """an offset into an object specified by a .NET token"""
-
-    def __init__(self, token: int, offset: int):
-        assert offset >= 0
-        self.token = token
-        self.offset = offset
-
-    def __eq__(self, other):
-        return (self.token, self.offset) == (other.token, other.offset)
-
-    def __lt__(self, other):
-        return (self.token, self.offset) < (other.token, other.offset)
-
-    def __hash__(self):
-        return hash((self.token, self.offset))
-
-    def __repr__(self):
-        return f"token(0x{self.token:x})+(0x{self.offset:x})"
-
-    def __index__(self):
-        return self.token + self.offset
-
-
-class _NoAddress(Address):
-    def __eq__(self, other):
-        return True
-
-    def __lt__(self, other):
-        return False
-
-    def __hash__(self):
-        return hash(0)
-
-    def __repr__(self):
-        return "no address"
-
-
-NO_ADDRESS = _NoAddress()

capa/features/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,15 +6,22 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from capa.features.common import Feature
+from capa.features import Feature
 
 
 class BasicBlock(Feature):
-    def __init__(self, description=None):
-        super().__init__(0, description=description)
+    def __init__(self):
+        super(BasicBlock, self).__init__(None)
 
     def __str__(self):
         return "basic block"
 
     def get_value_str(self):
         return ""
+
+    def freeze_serialize(self):
+        return (self.__class__.__name__, [])
+
+    @classmethod
+    def freeze_deserialize(cls, args):
+        return cls()

capa/features/com/__init__.py

@@ -1,36 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from enum import Enum
-from typing import Dict, List
-
-from capa.helpers import assert_never
-
-
-class ComType(Enum):
-    CLASS = "class"
-    INTERFACE = "interface"
-
-
-COM_PREFIXES = {
-    ComType.CLASS: "CLSID_",
-    ComType.INTERFACE: "IID_",
-}
-
-
-def load_com_database(com_type: ComType) -> Dict[str, List[str]]:
-    # lazy load these python files since they are so large.
-    # that is, don't load them unless a COM feature is being handled.
-    import capa.features.com.classes
-    import capa.features.com.interfaces
-
-    if com_type == ComType.CLASS:
-        return capa.features.com.classes.COM_CLASSES
-    elif com_type == ComType.INTERFACE:
-        return capa.features.com.interfaces.COM_INTERFACES
-    else:
-        assert_never(com_type)

capa/features/common.py

@@ -1,491 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import re
-import abc
-import codecs
-import typing
-import logging
-import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Union, Optional
-
-if TYPE_CHECKING:
-    # circular import, otherwise
-    import capa.engine
-
-import capa.perf
-import capa.features
-import capa.features.extractors.elf
-from capa.features.address import Address
-
-logger = logging.getLogger(__name__)
-MAX_BYTES_FEATURE_SIZE = 0x100
-
-# thunks may be chained so we specify a delta to control the depth to which these chains are explored
-THUNK_CHAIN_DEPTH_DELTA = 5
-
-
-class FeatureAccess:
-    READ = "read"
-    WRITE = "write"
-
-
-VALID_FEATURE_ACCESS = (FeatureAccess.READ, FeatureAccess.WRITE)
-
-
-def bytes_to_str(b: bytes) -> str:
-    return str(codecs.encode(b, "hex").decode("utf-8"))
-
-
-def hex_string(h: str) -> str:
-    """render hex string e.g. "0a40b1" as "0A 40 B1" """
-    return " ".join(h[i : i + 2] for i in range(0, len(h), 2)).upper()
-
-
-def escape_string(s: str) -> str:
-    """escape special characters"""
-    s = repr(s)
-    if not s.startswith(('"', "'")):
-        # u'hello\r\nworld' -> hello\\r\\nworld
-        s = s[2:-1]
-    else:
-        # 'hello\r\nworld' -> hello\\r\\nworld
-        s = s[1:-1]
-    s = s.replace("\\'", "'")  # repr() may escape "'" in some edge cases, remove
-    s = s.replace('"', '\\"')  # repr() does not escape '"', add
-    return s
-
-
-class Result:
-    """
-    represents the results of an evaluation of statements against features.
-
-    instances of this class should behave like a bool,
-    e.g. `assert Result(True, ...) == True`
-
-    instances track additional metadata about evaluation results.
-    they contain references to the statement node (e.g. an And statement),
-     as well as the children Result instances.
-
-    we need this so that we can render the tree of expressions and their results.
-    """
-
-    def __init__(
-        self,
-        success: bool,
-        statement: Union["capa.engine.Statement", "Feature"],
-        children: List["Result"],
-        locations: Optional[Set[Address]] = None,
-    ):
-        super().__init__()
-        self.success = success
-        self.statement = statement
-        self.children = children
-        self.locations = locations if locations is not None else set()
-
-    def __eq__(self, other):
-        if isinstance(other, bool):
-            return self.success == other
-        return False
-
-    def __bool__(self):
-        return self.success
-
-    def __nonzero__(self):
-        return self.success
-
-
-class Feature(abc.ABC):  # noqa: B024
-    # this is an abstract class, since we don't want anyone to instantiate it directly,
-    # but it doesn't have any abstract methods.
-
-    def __init__(
-        self,
-        value: Union[str, int, float, bytes],
-        description: Optional[str] = None,
-    ):
-        """
-        Args:
-          value (any): the value of the feature, such as the number or string.
-          description (str): a human-readable description that explains the feature value.
-        """
-        super().__init__()
-
-        self.name = self.__class__.__name__.lower()
-        self.value = value
-        self.description = description
-
-    def __hash__(self):
-        return hash((self.name, self.value))
-
-    def __eq__(self, other):
-        return self.name == other.name and self.value == other.value
-
-    def __lt__(self, other):
-        # implementing sorting by serializing to JSON is a huge hack.
-        # its slow, inelegant, and probably doesn't work intuitively;
-        # however, we only use it for deterministic output, so it's good enough for now.
-
-        # circular import
-        # we should fix if this wasn't already a huge hack.
-        import capa.features.freeze.features
-
-        return (
-            capa.features.freeze.features.feature_from_capa(self).model_dump_json()
-            < capa.features.freeze.features.feature_from_capa(other).model_dump_json()
-        )
-
-    def get_name_str(self) -> str:
-        """
-        render the name of this feature, for use by `__str__` and friends.
-        subclasses should override to customize the rendering.
-        """
-        return self.name
-
-    def get_value_str(self) -> str:
-        """
-        render the value of this feature, for use by `__str__` and friends.
-        subclasses should override to customize the rendering.
-        """
-        return str(self.value)
-
-    def __str__(self):
-        if self.value is not None:
-            if self.description:
-                return f"{self.get_name_str()}({self.get_value_str()} = {self.description})"
-            else:
-                return f"{self.get_name_str()}({self.get_value_str()})"
-        else:
-            return f"{self.get_name_str()}"
-
-    def __repr__(self):
-        return str(self)
-
-    def evaluate(self, ctx: Dict["Feature", Set[Address]], **kwargs) -> Result:
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature." + self.name] += 1
-        return Result(self in ctx, self, [], locations=ctx.get(self, set()))
-
-
-class MatchedRule(Feature):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-        self.name = "match"
-
-
-class Characteristic(Feature):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-
-
-class String(Feature):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-
-    def get_value_str(self) -> str:
-        assert isinstance(self.value, str)
-        return escape_string(self.value)
-
-
-class Class(Feature):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-
-
-class Namespace(Feature):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-
-
-class Substring(String):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-        self.value = value
-
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.substring"] += 1
-
-        # mapping from string value to list of locations.
-        # will unique the locations later on.
-        matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
-
-        assert isinstance(self.value, str)
-        for feature, locations in ctx.items():
-            if not isinstance(feature, (String,)):
-                continue
-
-            if not isinstance(feature.value, str):
-                # this is a programming error: String should only contain str
-                raise ValueError("unexpected feature value type")
-
-            if self.value in feature.value:
-                matches[feature.value].update(locations)
-                if short_circuit:
-                    # we found one matching string, thats sufficient to match.
-                    # don't collect other matching strings in this mode.
-                    break
-
-        if matches:
-            # collect all locations
-            locations = set()
-            for locs in matches.values():
-                locations.update(locs)
-
-            # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
-            # this is because `self` may match on many strings, so we can't stuff the matched value into it.
-            # instead, return a new instance that has a reference to both the substring and the matched values.
-            return Result(True, _MatchedSubstring(self, dict(matches)), [], locations=locations)
-        else:
-            return Result(False, _MatchedSubstring(self, {}), [])
-
-    def get_value_str(self) -> str:
-        assert isinstance(self.value, str)
-        return escape_string(self.value)
-
-    def __str__(self):
-        assert isinstance(self.value, str)
-        return f"substring({escape_string(self.value)})"
-
-
-class _MatchedSubstring(Substring):
-    """
-    this represents specific match instances of a substring feature.
-    treat it the same as a `Substring` except it has the `matches` field that contains the complete strings that matched.
-
-    note: this type should only ever be constructed by `Substring.evaluate()`. it is not part of the public API.
-    """
-
-    def __init__(self, substring: Substring, matches: Dict[str, Set[Address]]):
-        """
-        args:
-          substring: the substring feature that matches.
-          match: mapping from matching string to its locations.
-        """
-        super().__init__(str(substring.value), description=substring.description)
-        # we want this to collide with the name of `Substring` above,
-        # so that it works nicely with the renderers.
-        self.name = "substring"
-        # this may be None if the substring doesn't match
-        self.matches = matches
-
-    def __str__(self):
-        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
-        assert isinstance(self.value, str)
-        return f'substring("{self.value}", matches = {matches})'
-
-
-class Regex(String):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-        self.value = value
-
-        pat = self.value[len("/") : -len("/")]
-        flags = re.DOTALL
-        if value.endswith("/i"):
-            pat = self.value[len("/") : -len("/i")]
-            flags |= re.IGNORECASE
-        try:
-            self.re = re.compile(pat, flags)
-        except re.error as exc:
-            if value.endswith("/i"):
-                value = value[: -len("i")]
-            raise ValueError(
-                f"invalid regular expression: {value} it should use Python syntax, try it at https://pythex.org"
-            ) from exc
-
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.regex"] += 1
-
-        # mapping from string value to list of locations.
-        # will unique the locations later on.
-        matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
-
-        for feature, locations in ctx.items():
-            if not isinstance(feature, (String,)):
-                continue
-
-            if not isinstance(feature.value, str):
-                # this is a programming error: String should only contain str
-                raise ValueError("unexpected feature value type")
-
-            # `re.search` finds a match anywhere in the given string
-            # which implies leading and/or trailing whitespace.
-            # using this mode cleans is more convenient for rule authors,
-            # so that they don't have to prefix/suffix their terms like: /.*foo.*/.
-            if self.re.search(feature.value):
-                matches[feature.value].update(locations)
-                if short_circuit:
-                    # we found one matching string, thats sufficient to match.
-                    # don't collect other matching strings in this mode.
-                    break
-
-        if matches:
-            # collect all locations
-            locations = set()
-            for locs in matches.values():
-                locations.update(locs)
-
-            # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
-            # this is because `self` may match on many strings, so we can't stuff the matched value into it.
-            # instead, return a new instance that has a reference to both the regex and the matched values.
-            # see #262.
-            return Result(True, _MatchedRegex(self, dict(matches)), [], locations=locations)
-        else:
-            return Result(False, _MatchedRegex(self, {}), [])
-
-    def __str__(self):
-        assert isinstance(self.value, str)
-        return f"regex(string =~ {self.value})"
-
-
-class _MatchedRegex(Regex):
-    """
-    this represents specific match instances of a regular expression feature.
-    treat it the same as a `Regex` except it has the `matches` field that contains the complete strings that matched.
-
-    note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
-    """
-
-    def __init__(self, regex: Regex, matches: Dict[str, Set[Address]]):
-        """
-        args:
-          regex: the regex feature that matches.
-          matches: mapping from matching string to its locations.
-        """
-        super().__init__(str(regex.value), description=regex.description)
-        # we want this to collide with the name of `Regex` above,
-        # so that it works nicely with the renderers.
-        self.name = "regex"
-        # this may be None if the regex doesn't match
-        self.matches = matches
-
-    def __str__(self):
-        matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
-        assert isinstance(self.value, str)
-        return f"regex(string =~ {self.value}, matches = {matches})"
-
-
-class StringFactory:
-    def __new__(cls, value: str, description=None):
-        if value.startswith("/") and (value.endswith("/") or value.endswith("/i")):
-            return Regex(value, description=description)
-        return String(value, description=description)
-
-
-class Bytes(Feature):
-    def __init__(self, value: bytes, description=None):
-        super().__init__(value, description=description)
-        self.value = value
-
-    def evaluate(self, ctx, **kwargs):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.bytes"] += 1
-
-        assert isinstance(self.value, bytes)
-        for feature, locations in ctx.items():
-            if not isinstance(feature, (Bytes,)):
-                continue
-
-            assert isinstance(feature.value, bytes)
-            if feature.value.startswith(self.value):
-                return Result(True, self, [], locations=locations)
-
-        return Result(False, self, [])
-
-    def get_value_str(self):
-        assert isinstance(self.value, bytes)
-        return hex_string(bytes_to_str(self.value))
-
-
-# other candidates here: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types
-ARCH_I386 = "i386"
-ARCH_AMD64 = "amd64"
-# dotnet
-ARCH_ANY = "any"
-VALID_ARCH = (ARCH_I386, ARCH_AMD64, ARCH_ANY)
-
-
-class Arch(Feature):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-        self.name = "arch"
-
-
-OS_WINDOWS = "windows"
-OS_LINUX = "linux"
-OS_MACOS = "macos"
-# dotnet
-OS_ANY = "any"
-VALID_OS = {os.value for os in capa.features.extractors.elf.OS}
-VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS, OS_ANY})
-# internal only, not to be used in rules
-OS_AUTO = "auto"
-
-
-class OS(Feature):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-        self.name = "os"
-
-    def evaluate(self, ctx, **kwargs):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature." + self.name] += 1
-
-        for feature, locations in ctx.items():
-            if not isinstance(feature, (OS,)):
-                continue
-
-            assert isinstance(feature.value, str)
-            if OS_ANY in (self.value, feature.value) or self.value == feature.value:
-                return Result(True, self, [], locations=locations)
-
-        return Result(False, self, [])
-
-
-FORMAT_PE = "pe"
-FORMAT_ELF = "elf"
-FORMAT_DOTNET = "dotnet"
-VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET)
-# internal only, not to be used in rules
-FORMAT_AUTO = "auto"
-FORMAT_SC32 = "sc32"
-FORMAT_SC64 = "sc64"
-FORMAT_CAPE = "cape"
-FORMAT_FREEZE = "freeze"
-FORMAT_RESULT = "result"
-STATIC_FORMATS = {
-    FORMAT_SC32,
-    FORMAT_SC64,
-    FORMAT_PE,
-    FORMAT_ELF,
-    FORMAT_DOTNET,
-    FORMAT_FREEZE,
-    FORMAT_RESULT,
-}
-DYNAMIC_FORMATS = {
-    FORMAT_CAPE,
-    FORMAT_FREEZE,
-    FORMAT_RESULT,
-}
-FORMAT_UNKNOWN = "unknown"
-
-
-class Format(Feature):
-    def __init__(self, value: str, description=None):
-        super().__init__(value, description=description)
-        self.name = "format"
-
-
-def is_global_feature(feature):
-    """
-    is this a feature that is extracted at every scope?
-    today, these are OS and arch features.
-    """
-    return isinstance(feature, (OS, Arch))

capa/features/extractors/__init__.py

@@ -0,0 +1,294 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import abc
+
+from capa.helpers import oint
+
+
+class FeatureExtractor(object):
+    """
+    FeatureExtractor defines the interface for fetching features from a sample.
+
+    There may be multiple backends that support fetching features for capa.
+    For example, we use vivisect by default, but also want to support saving
+     and restoring features from a JSON file.
+    When we restore the features, we'd like to use exactly the same matching logic
+     to find matching rules.
+    Therefore, we can define a FeatureExtractor that provides features from the
+     serialized JSON file and do matching without a binary analysis pass.
+    Also, this provides a way to hook in an IDA backend.
+
+    This class is not instantiated directly; it is the base class for other implementations.
+    """
+
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self):
+        #
+        # note: a subclass should define ctor parameters for its own use.
+        #  for example, the Vivisect feature extract might require the vw and/or path.
+        # this base class doesn't know what to do with that info, though.
+        #
+        super(FeatureExtractor, self).__init__()
+
+    def block_offset(self, bb):
+        return oint(bb)
+
+    def function_offset(self, f):
+        return oint(f)
+
+    @abc.abstractmethod
+    def get_base_address(self):
+        """
+        fetch the preferred load address at which the sample was analyzed.
+
+        returns: int
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def extract_file_features(self):
+        """
+        extract file-scope features.
+
+        example::
+
+            extractor = VivisectFeatureExtractor(vw, path)
+            for feature, va in extractor.get_file_features():
+                print('0x%x: %s', va, feature)
+
+        yields:
+          Tuple[capa.features.Feature, int]: feature and its location
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def get_functions(self):
+        """
+        enumerate the functions and provide opaque values that will
+         subsequently be provided to `.extract_function_features()`, etc.
+
+        by "opaque value", we mean that this can be any object, as long as it
+         provides enough context to `.extract_function_features()`.
+
+        the opaque value should support casting to int (`__int__`) for the function start address.
+
+        yields:
+          any: the opaque function value.
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def extract_function_features(self, f):
+        """
+        extract function-scope features.
+        the arguments are opaque values previously provided by `.get_functions()`, etc.
+
+        example::
+
+            extractor = VivisectFeatureExtractor(vw, path)
+            for function in extractor.get_functions():
+                for feature, va in extractor.extract_function_features(function):
+                    print('0x%x: %s', va, feature)
+
+        args:
+          f [any]: an opaque value previously fetched from `.get_functions()`.
+
+        yields:
+          Tuple[capa.features.Feature, int]: feature and its location
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def get_basic_blocks(self, f):
+        """
+        enumerate the basic blocks in the given function and provide opaque values that will
+         subsequently be provided to `.extract_basic_block_features()`, etc.
+
+        by "opaque value", we mean that this can be any object, as long as it
+         provides enough context to `.extract_basic_block_features()`.
+
+        the opaque value should support casting to int (`__int__`) for the basic block start address.
+
+        yields:
+          any: the opaque basic block value.
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def extract_basic_block_features(self, f, bb):
+        """
+        extract basic block-scope features.
+        the arguments are opaque values previously provided by `.get_functions()`, etc.
+
+        example::
+
+            extractor = VivisectFeatureExtractor(vw, path)
+            for function in extractor.get_functions():
+                for bb in extractor.get_basic_blocks(function):
+                    for feature, va in extractor.extract_basic_block_features(function, bb):
+                        print('0x%x: %s', va, feature)
+
+        args:
+          f [any]: an opaque value previously fetched from `.get_functions()`.
+          bb [any]: an opaque value previously fetched from `.get_basic_blocks()`.
+
+        yields:
+          Tuple[capa.features.Feature, int]: feature and its location
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def get_instructions(self, f, bb):
+        """
+        enumerate the instructions in the given basic block and provide opaque values that will
+         subsequently be provided to `.extract_insn_features()`, etc.
+
+        by "opaque value", we mean that this can be any object, as long as it
+         provides enough context to `.extract_insn_features()`.
+
+        the opaque value should support casting to int (`__int__`) for the instruction address.
+
+        yields:
+          any: the opaque function value.
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def extract_insn_features(self, f, bb, insn):
+        """
+        extract instruction-scope features.
+        the arguments are opaque values previously provided by `.get_functions()`, etc.
+
+        example::
+
+            extractor = VivisectFeatureExtractor(vw, path)
+            for function in extractor.get_functions():
+                for bb in extractor.get_basic_blocks(function):
+                    for insn in extractor.get_instructions(function, bb):
+                        for feature, va in extractor.extract_insn_features(function, bb, insn):
+                            print('0x%x: %s', va, feature)
+
+        args:
+          f [any]: an opaque value previously fetched from `.get_functions()`.
+          bb [any]: an opaque value previously fetched from `.get_basic_blocks()`.
+          insn [any]: an opaque value previously fetched from `.get_instructions()`.
+
+        yields:
+          Tuple[capa.features.Feature, int]: feature and its location
+        """
+        raise NotImplemented
+
+
+class NullFeatureExtractor(FeatureExtractor):
+    """
+    An extractor that extracts some user-provided features.
+    The structure of the single parameter is demonstrated in the example below.
+
+    This is useful for testing, as we can provide expected values and see if matching works.
+    Also, this is how we represent features deserialized from a freeze file.
+
+    example::
+
+        extractor = NullFeatureExtractor({
+            'base address: 0x401000,
+            'file features': [
+                (0x402345, capa.features.Characteristic('embedded pe')),
+            ],
+            'functions': {
+                0x401000: {
+                    'features': [
+                        (0x401000, capa.features.Characteristic('nzxor')),
+                    ],
+                    'basic blocks': {
+                        0x401000: {
+                            'features': [
+                                (0x401000, capa.features.Characteristic('tight-loop')),
+                            ],
+                            'instructions': {
+                                0x401000: {
+                                    'features': [
+                                        (0x401000, capa.features.Characteristic('nzxor')),
+                                    ],
+                                },
+                                0x401002: ...
+                            }
+                        },
+                        0x401005: ...
+                    }
+                },
+                0x40200: ...
+            }
+        )
+    """
+
+    def __init__(self, features):
+        super(NullFeatureExtractor, self).__init__()
+        self.features = features
+
+    def get_base_address(self):
+        return self.features["base address"]
+
+    def extract_file_features(self):
+        for p in self.features.get("file features", []):
+            va, feature = p
+            yield feature, va
+
+    def get_functions(self):
+        for va in sorted(self.features["functions"].keys()):
+            yield va
+
+    def extract_function_features(self, f):
+        for p in self.features.get("functions", {}).get(f, {}).get("features", []):  # noqa: E127 line over-indented
+            va, feature = p
+            yield feature, va
+
+    def get_basic_blocks(self, f):
+        for va in sorted(
+            self.features.get("functions", {})  # noqa: E127 line over-indented
+            .get(f, {})
+            .get("basic blocks", {})
+            .keys()
+        ):
+            yield va
+
+    def extract_basic_block_features(self, f, bb):
+        for p in (
+            self.features.get("functions", {})  # noqa: E127 line over-indented
+            .get(f, {})
+            .get("basic blocks", {})
+            .get(bb, {})
+            .get("features", [])
+        ):
+            va, feature = p
+            yield feature, va
+
+    def get_instructions(self, f, bb):
+        for va in sorted(
+            self.features.get("functions", {})  # noqa: E127 line over-indented
+            .get(f, {})
+            .get("basic blocks", {})
+            .get(bb, {})
+            .get("instructions", {})
+            .keys()
+        ):
+            yield va
+
+    def extract_insn_features(self, f, bb, insn):
+        for p in (
+            self.features.get("functions", {})  # noqa: E127 line over-indented
+            .get(f, {})
+            .get("basic blocks", {})
+            .get(bb, {})
+            .get("instructions", {})
+            .get(insn, {})
+            .get("features", [])
+        ):
+            va, feature = p
+            yield feature, va

capa/features/extractors/base_extractor.py

@@ -1,470 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import abc
-import hashlib
-import dataclasses
-from typing import Any, Dict, Tuple, Union, Iterator
-from dataclasses import dataclass
-
-# TODO(williballenthin): use typing.TypeAlias directly when Python 3.9 is deprecated
-# https://github.com/mandiant/capa/issues/1699
-from typing_extensions import TypeAlias
-
-import capa.features.address
-from capa.features.common import Feature
-from capa.features.address import Address, ThreadAddress, ProcessAddress, DynamicCallAddress, AbsoluteVirtualAddress
-
-# feature extractors may reference functions, BBs, insns by opaque handle values.
-# you can use the `.address` property to get and render the address of the feature.
-#
-# these handles are only consumed by routines on
-# the feature extractor from which they were created.
-
-
-@dataclass
-class SampleHashes:
-    md5: str
-    sha1: str
-    sha256: str
-
-    @classmethod
-    def from_bytes(cls, buf: bytes) -> "SampleHashes":
-        md5 = hashlib.md5()
-        sha1 = hashlib.sha1()
-        sha256 = hashlib.sha256()
-        md5.update(buf)
-        sha1.update(buf)
-        sha256.update(buf)
-
-        return cls(md5=md5.hexdigest(), sha1=sha1.hexdigest(), sha256=sha256.hexdigest())
-
-
-@dataclass
-class FunctionHandle:
-    """reference to a function recognized by a feature extractor.
-
-    Attributes:
-        address: the address of the function.
-        inner: extractor-specific data.
-        ctx: a context object for the extractor.
-    """
-
-    address: Address
-    inner: Any
-    ctx: Dict[str, Any] = dataclasses.field(default_factory=dict)
-
-
-@dataclass
-class BBHandle:
-    """reference to a basic block recognized by a feature extractor.
-
-    Attributes:
-        address: the address of the basic block start address.
-        inner: extractor-specific data.
-    """
-
-    address: Address
-    inner: Any
-
-
-@dataclass
-class InsnHandle:
-    """reference to a instruction recognized by a feature extractor.
-
-    Attributes:
-        address: the address of the instruction address.
-        inner: extractor-specific data.
-    """
-
-    address: Address
-    inner: Any
-
-
-class StaticFeatureExtractor:
-    """
-    StaticFeatureExtractor defines the interface for fetching features from a
-    sample without running it; extractors that rely on the execution trace of
-    a sample must implement the other sibling class, DynamicFeatureExtracor.
-
-    There may be multiple backends that support fetching features for capa.
-    For example, we use vivisect by default, but also want to support saving
-     and restoring features from a JSON file.
-    When we restore the features, we'd like to use exactly the same matching logic
-     to find matching rules.
-    Therefore, we can define a StaticFeatureExtractor that provides features from the
-     serialized JSON file and do matching without a binary analysis pass.
-    Also, this provides a way to hook in an IDA backend.
-
-    This class is not instantiated directly; it is the base class for other implementations.
-    """
-
-    __metaclass__ = abc.ABCMeta
-
-    def __init__(self, hashes: SampleHashes):
-        #
-        # note: a subclass should define ctor parameters for its own use.
-        #  for example, the Vivisect feature extract might require the vw and/or path.
-        # this base class doesn't know what to do with that info, though.
-        #
-        super().__init__()
-        self._sample_hashes = hashes
-
-    @abc.abstractmethod
-    def get_base_address(self) -> Union[AbsoluteVirtualAddress, capa.features.address._NoAddress]:
-        """
-        fetch the preferred load address at which the sample was analyzed.
-
-        when the base address is `NO_ADDRESS`, then the loader has no concept of a preferred load address.
-        such as: shellcode, .NET modules, etc.
-        in these scenarios, RelativeVirtualAddresses aren't used.
-        """
-        raise NotImplementedError()
-
-    def get_sample_hashes(self) -> SampleHashes:
-        """
-        fetch the hashes for the sample contained within the extractor.
-        """
-        return self._sample_hashes
-
-    @abc.abstractmethod
-    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract features found at every scope ("global").
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for feature, va in extractor.get_global_features():
-                print('0x%x: %s', va, feature)
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract file-scope features.
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for feature, va in extractor.get_file_features():
-                print('0x%x: %s', va, feature)
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        """
-        enumerate the functions and provide opaque values that will
-         subsequently be provided to `.extract_function_features()`, etc.
-        """
-        raise NotImplementedError()
-
-    def is_library_function(self, addr: Address) -> bool:
-        """
-        is the given address a library function?
-        the backend may implement its own function matching algorithm, or none at all.
-        we accept an address here, rather than function object,
-         to handle addresses identified in instructions.
-
-        this information is used to:
-          - filter out matches in library functions (by default), and
-          - recognize when to fetch symbol names for called (non-API) functions
-
-        args:
-          addr (Address): the address of a function.
-
-        returns:
-          bool: True if the given address is the start of a library function.
-        """
-        return False
-
-    def get_function_name(self, addr: Address) -> str:
-        """
-        fetch any recognized name for the given address.
-        this is only guaranteed to return a value when the given function is a recognized library function.
-        we accept a VA here, rather than function object, to handle addresses identified in instructions.
-
-        args:
-          addr (Address): the address of a function.
-
-        returns:
-          str: the function name
-
-        raises:
-          KeyError: when the given function does not have a name.
-        """
-        raise KeyError(addr)
-
-    @abc.abstractmethod
-    def extract_function_features(self, f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract function-scope features.
-        the arguments are opaque values previously provided by `.get_functions()`, etc.
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for function in extractor.get_functions():
-                for feature, address in extractor.extract_function_features(function):
-                    print('0x%x: %s', address, feature)
-
-        args:
-          f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_basic_blocks(self, f: FunctionHandle) -> Iterator[BBHandle]:
-        """
-        enumerate the basic blocks in the given function and provide opaque values that will
-         subsequently be provided to `.extract_basic_block_features()`, etc.
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract basic block-scope features.
-        the arguments are opaque values previously provided by `.get_functions()`, etc.
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for function in extractor.get_functions():
-                for bb in extractor.get_basic_blocks(function):
-                    for feature, address in extractor.extract_basic_block_features(function, bb):
-                        print('0x%x: %s', address, feature)
-
-        args:
-          f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
-          bb [BBHandle]: an opaque value previously fetched from `.get_basic_blocks()`.
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_instructions(self, f: FunctionHandle, bb: BBHandle) -> Iterator[InsnHandle]:
-        """
-        enumerate the instructions in the given basic block and provide opaque values that will
-         subsequently be provided to `.extract_insn_features()`, etc.
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_insn_features(
-        self, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
-    ) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract instruction-scope features.
-        the arguments are opaque values previously provided by `.get_functions()`, etc.
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for function in extractor.get_functions():
-                for bb in extractor.get_basic_blocks(function):
-                    for insn in extractor.get_instructions(function, bb):
-                        for feature, address in extractor.extract_insn_features(function, bb, insn):
-                            print('0x%x: %s', address, feature)
-
-        args:
-          f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
-          bb [BBHandle]: an opaque value previously fetched from `.get_basic_blocks()`.
-          insn [InsnHandle]: an opaque value previously fetched from `.get_instructions()`.
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-
-@dataclass
-class ProcessHandle:
-    """
-    reference to a process extracted by the sandbox.
-
-    Attributes:
-        address: process's address (pid)
-        inner: sandbox-specific data
-    """
-
-    address: ProcessAddress
-    inner: Any
-
-
-@dataclass
-class ThreadHandle:
-    """
-    reference to a thread extracted by the sandbox.
-
-    Attributes:
-        address: thread's address (tid)
-        inner: sandbox-specific data
-    """
-
-    address: ThreadAddress
-    inner: Any
-
-
-@dataclass
-class CallHandle:
-    """
-    reference to an api call extracted by the sandbox.
-
-    Attributes:
-        address: call's address, such as event index or id
-        inner: sandbox-specific data
-    """
-
-    address: DynamicCallAddress
-    inner: Any
-
-
-class DynamicFeatureExtractor:
-    """
-    DynamicFeatureExtractor defines the interface for fetching features from a
-    sandbox' analysis of a sample; extractors that rely on statically analyzing
-    a sample must implement the sibling extractor, StaticFeatureExtractor.
-
-    Features are grouped mainly into threads that alongside their meta-features are also grouped into
-    processes (that also have their own features). Other scopes (such as function and file) may also apply
-    for a specific sandbox.
-
-    This class is not instantiated directly; it is the base class for other implementations.
-    """
-
-    __metaclass__ = abc.ABCMeta
-
-    def __init__(self, hashes: SampleHashes):
-        #
-        # note: a subclass should define ctor parameters for its own use.
-        #  for example, the Vivisect feature extract might require the vw and/or path.
-        # this base class doesn't know what to do with that info, though.
-        #
-        super().__init__()
-        self._sample_hashes = hashes
-
-    def get_sample_hashes(self) -> SampleHashes:
-        """
-        fetch the hashes for the sample contained within the extractor.
-        """
-        return self._sample_hashes
-
-    @abc.abstractmethod
-    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract features found at every scope ("global").
-
-        example::
-
-            extractor = CapeFeatureExtractor.from_report(json.loads(buf))
-            for feature, addr in extractor.get_global_features():
-                print(addr, feature)
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract file-scope features.
-
-        example::
-
-            extractor = CapeFeatureExtractor.from_report(json.loads(buf))
-            for feature, addr in extractor.get_file_features():
-                print(addr, feature)
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_processes(self) -> Iterator[ProcessHandle]:
-        """
-        Enumerate processes in the trace.
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
-        """
-        Yields all the features of a process. These include:
-        - file features of the process' image
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_process_name(self, ph: ProcessHandle) -> str:
-        """
-        Returns the human-readable name for the given process,
-        such as the filename.
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
-        """
-        Enumerate threads in the given process.
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
-        """
-        Yields all the features of a thread. These include:
-        - sequenced api traces
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
-        """
-        Enumerate calls in the given thread
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_call_features(
-        self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
-    ) -> Iterator[Tuple[Feature, Address]]:
-        """
-        Yields all features of a call. These include:
-        - api name
-        - bytes/strings/numbers extracted from arguments
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_call_name(self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> str:
-        """
-        Returns the human-readable name for the given call,
-        such as as rendered API log entry, like:
-
-            Foo(1, "two", b"\x00\x11") -> -1
-        """
-        raise NotImplementedError()
-
-
-FeatureExtractor: TypeAlias = Union[StaticFeatureExtractor, DynamicFeatureExtractor]

capa/features/extractors/binja/basicblock.py

@@ -1,184 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import string
-import struct
-from typing import Tuple, Iterator
-
-from binaryninja import Function, Settings
-from binaryninja import BasicBlock as BinjaBasicBlock
-from binaryninja import (
-    BinaryView,
-    SymbolType,
-    RegisterValueType,
-    VariableSourceType,
-    MediumLevelILSetVar,
-    MediumLevelILOperation,
-    MediumLevelILBasicBlock,
-    MediumLevelILInstruction,
-)
-
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address
-from capa.features.basicblock import BasicBlock
-from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
-
-use_const_outline: bool = False
-settings: Settings = Settings()
-if settings.contains("analysis.outlining.builtins") and settings.get_bool("analysis.outlining.builtins"):
-    use_const_outline = True
-
-
-def get_printable_len_ascii(s: bytes) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
-    count = 0
-    for c in s:
-        if c == 0:
-            return count
-        if c < 127 and chr(c) in string.printable:
-            count += 1
-    return count
-
-
-def get_printable_len_wide(s: bytes) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
-    if all(c == 0x00 for c in s[1::2]):
-        return get_printable_len_ascii(s[::2])
-    return 0
-
-
-def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
-    bv: BinaryView = f.view
-
-    if il.operation != MediumLevelILOperation.MLIL_CALL:
-        return 0
-
-    target = il.dest
-    if target.operation not in [MediumLevelILOperation.MLIL_CONST, MediumLevelILOperation.MLIL_CONST_PTR]:
-        return 0
-
-    addr = target.value.value
-    sym = bv.get_symbol_at(addr)
-    if not sym or sym.type != SymbolType.LibraryFunctionSymbol:
-        return 0
-
-    if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
-        return 0
-
-    if len(il.params) < 2:
-        return 0
-
-    dest = il.params[0]
-    if dest.operation in [MediumLevelILOperation.MLIL_ADDRESS_OF, MediumLevelILOperation.MLIL_VAR]:
-        var = dest.src
-    else:
-        return 0
-
-    if var.source_type != VariableSourceType.StackVariableSourceType:
-        return 0
-
-    src = il.params[1]
-    if src.value.type != RegisterValueType.ConstantDataAggregateValue:
-        return 0
-
-    s = f.get_constant_data(RegisterValueType.ConstantDataAggregateValue, src.value.value)
-    return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
-
-
-def get_printable_len(il: MediumLevelILSetVar) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
-    width = il.dest.type.width
-    value = il.src.value.value
-
-    if width == 1:
-        chars = struct.pack("<B", value & 0xFF)
-    elif width == 2:
-        chars = struct.pack("<H", value & 0xFFFF)
-    elif width == 4:
-        chars = struct.pack("<I", value & 0xFFFFFFFF)
-    elif width == 8:
-        chars = struct.pack("<Q", value & 0xFFFFFFFFFFFFFFFF)
-    else:
-        return 0
-
-    def is_printable_ascii(chars_: bytes):
-        return all(c < 127 and chr(c) in string.printable for c in chars_)
-
-    def is_printable_utf16le(chars_: bytes):
-        if all(c == 0x00 for c in chars_[1::2]):
-            return is_printable_ascii(chars_[::2])
-
-    if is_printable_ascii(chars):
-        return width
-
-    if is_printable_utf16le(chars):
-        return width // 2
-
-    return 0
-
-
-def is_mov_imm_to_stack(il: MediumLevelILInstruction) -> bool:
-    """verify instruction moves immediate onto stack"""
-    if il.operation != MediumLevelILOperation.MLIL_SET_VAR:
-        return False
-
-    if il.src.operation != MediumLevelILOperation.MLIL_CONST:
-        return False
-
-    if il.dest.source_type != VariableSourceType.StackVariableSourceType:
-        return False
-
-    return True
-
-
-def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
-    """check basic block for stackstring indicators
-
-    true if basic block contains enough moves of constant bytes to the stack
-    """
-    count = 0
-    for il in bb:
-        if use_const_outline:
-            count += get_stack_string_len(f, il)
-        else:
-            if is_mov_imm_to_stack(il):
-                count += get_printable_len(il)
-
-    if count > MIN_STACKSTRING_LEN:
-        return True
-    return False
-
-
-def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract stackstring indicators from basic block"""
-    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
-    if bb[1] is not None and bb_contains_stackstring(fh.inner, bb[1]):
-        yield Characteristic("stack string"), bbh.address
-
-
-def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract tight loop indicators from a basic block"""
-    bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
-    for edge in bb[0].outgoing_edges:
-        if edge.target.start == bb[0].start:
-            yield Characteristic("tight loop"), bbh.address
-
-
-def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract basic block features"""
-    for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, addr in bb_handler(fh, bbh):
-            yield feature, addr
-    yield BasicBlock(), bbh.address
-
-
-BASIC_BLOCK_HANDLERS = (
-    extract_bb_tight_loop,
-    extract_bb_stackstring,
-)

capa/features/extractors/binja/extractor.py

@@ -1,81 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import List, Tuple, Iterator
-
-import binaryninja as binja
-
-import capa.features.extractors.elf
-import capa.features.extractors.binja.file
-import capa.features.extractors.binja.insn
-import capa.features.extractors.binja.global_
-import capa.features.extractors.binja.function
-import capa.features.extractors.binja.basicblock
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import (
-    BBHandle,
-    InsnHandle,
-    SampleHashes,
-    FunctionHandle,
-    StaticFeatureExtractor,
-)
-
-
-class BinjaFeatureExtractor(StaticFeatureExtractor):
-    def __init__(self, bv: binja.BinaryView):
-        super().__init__(hashes=SampleHashes.from_bytes(bv.file.raw.read(0, len(bv.file.raw))))
-        self.bv = bv
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv))
-        self.global_features.extend(capa.features.extractors.binja.global_.extract_os(self.bv))
-        self.global_features.extend(capa.features.extractors.binja.global_.extract_arch(self.bv))
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(self.bv.start)
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.binja.file.extract_features(self.bv)
-
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        for f in self.bv.functions:
-            yield FunctionHandle(address=AbsoluteVirtualAddress(f.start), inner=f)
-
-    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.binja.function.extract_features(fh)
-
-    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
-        f: binja.Function = fh.inner
-        # Set up a MLIL basic block dict look up to associate the disassembly basic block with its MLIL basic block
-        mlil_lookup = {}
-        for mlil_bb in f.mlil.basic_blocks:
-            mlil_lookup[mlil_bb.source_block.start] = mlil_bb
-
-        for bb in f.basic_blocks:
-            mlil_bb = mlil_lookup.get(bb.start)
-
-            yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=(bb, mlil_bb))
-
-    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.binja.basicblock.extract_features(fh, bbh)
-
-    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
-        import capa.features.extractors.binja.helpers as binja_helpers
-
-        bb: Tuple[binja.BasicBlock, binja.MediumLevelILBasicBlock] = bbh.inner
-        addr = bb[0].start
-
-        for text, length in bb[0]:
-            insn = binja_helpers.DisassemblyInstruction(addr, length, text)
-            yield InsnHandle(address=AbsoluteVirtualAddress(addr), inner=insn)
-            addr += length
-
-    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
-        yield from capa.features.extractors.binja.insn.extract_features(fh, bbh, ih)

capa/features/extractors/binja/file.py

@@ -1,187 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import struct
-from typing import Tuple, Iterator
-
-from binaryninja import Segment, BinaryView, SymbolType, SymbolBinding
-
-import capa.features.extractors.common
-import capa.features.extractors.helpers
-import capa.features.extractors.strings
-from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
-from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.binja.helpers import read_c_string, unmangle_c_name
-
-
-def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[int, int]]:
-    """check segment for embedded PE
-
-    adapted for binja from:
-    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
-    """
-    mz_xor = [
-        (
-            capa.features.extractors.helpers.xor_static(b"MZ", i),
-            capa.features.extractors.helpers.xor_static(b"PE", i),
-            i,
-        )
-        for i in range(256)
-    ]
-
-    todo = []
-    # If this is the first segment of the binary, skip the first bytes. Otherwise, there will always be a matched
-    # PE at the start of the binaryview.
-    start = seg.start
-    if bv.view_type == "PE" and start == bv.start:
-        start += 1
-
-    for mzx, pex, i in mz_xor:
-        for off, _ in bv.find_all_data(start, seg.end, mzx):
-            todo.append((off, mzx, pex, i))
-
-    while len(todo):
-        off, mzx, pex, i = todo.pop()
-
-        # The MZ header has one field we will check e_lfanew is at 0x3c
-        e_lfanew = off + 0x3C
-
-        if seg.end < (e_lfanew + 4):
-            continue
-
-        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(bv.read(e_lfanew, 4), i))[0]
-
-        peoff = off + newoff
-        if seg.end < (peoff + 2):
-            continue
-
-        if bv.read(peoff, 2) == pex:
-            yield off, i
-
-
-def extract_file_embedded_pe(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract embedded PE features"""
-    for seg in bv.segments:
-        for ea, _ in check_segment_for_pe(bv, seg):
-            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
-
-
-def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract function exports"""
-    for sym in bv.get_symbols_of_type(SymbolType.FunctionSymbol):
-        if sym.binding in [SymbolBinding.GlobalBinding, SymbolBinding.WeakBinding]:
-            name = sym.short_name
-            yield Export(name), AbsoluteVirtualAddress(sym.address)
-            unmangled_name = unmangle_c_name(name)
-            if name != unmangled_name:
-                yield Export(unmangled_name), AbsoluteVirtualAddress(sym.address)
-
-    for sym in bv.get_symbols_of_type(SymbolType.DataSymbol):
-        if sym.binding not in [SymbolBinding.GlobalBinding]:
-            continue
-
-        name = sym.short_name
-        if not name.startswith("__forwarder_name"):
-            continue
-
-        # Due to https://github.com/Vector35/binaryninja-api/issues/4641, in binja version 3.5, the symbol's name
-        # does not contain the DLL name. As a workaround, we read the C string at the symbol's address, which contains
-        # both the DLL name and the function name.
-        # Once the above issue is closed in the next binjs stable release, we can update the code here to use the
-        # symbol name directly.
-        name = read_c_string(bv, sym.address, 1024)
-        forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(name)
-        yield Export(forwarded_name), AbsoluteVirtualAddress(sym.address)
-        yield Characteristic("forwarded export"), AbsoluteVirtualAddress(sym.address)
-
-
-def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract function imports
-
-    1. imports by ordinal:
-     - modulename.#ordinal
-
-    2. imports by name, results in two features to support importname-only
-       matching:
-     - modulename.importname
-     - importname
-    """
-    for sym in bv.get_symbols_of_type(SymbolType.ImportAddressSymbol):
-        lib_name = str(sym.namespace)
-        addr = AbsoluteVirtualAddress(sym.address)
-        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym.short_name, include_dll=True):
-            yield Import(name), addr
-
-        ordinal = sym.ordinal
-        if ordinal != 0 and (lib_name != ""):
-            ordinal_name = f"#{ordinal}"
-            for name in capa.features.extractors.helpers.generate_symbols(lib_name, ordinal_name, include_dll=True):
-                yield Import(name), addr
-
-
-def extract_file_section_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract section names"""
-    for name, section in bv.sections.items():
-        yield Section(name), AbsoluteVirtualAddress(section.start)
-
-
-def extract_file_strings(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract ASCII and UTF-16 LE strings"""
-    for s in bv.strings:
-        yield String(s.value), FileOffsetAddress(s.start)
-
-
-def extract_file_function_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract the names of statically-linked library functions.
-    """
-    for sym_name in bv.symbols:
-        for sym in bv.symbols[sym_name]:
-            if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
-                continue
-
-            name = sym.short_name
-            yield FunctionName(name), sym.address
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                yield FunctionName(name[1:]), sym.address
-
-
-def extract_file_format(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    view_type = bv.view_type
-    if view_type in ["PE", "COFF"]:
-        yield Format(FORMAT_PE), NO_ADDRESS
-    elif view_type == "ELF":
-        yield Format(FORMAT_ELF), NO_ADDRESS
-    elif view_type == "Raw":
-        # no file type to return when processing a binary file, but we want to continue processing
-        return
-    else:
-        raise NotImplementedError(f"unexpected file format: {view_type}")
-
-
-def extract_features(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    """extract file features"""
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler(bv):
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_strings,
-    extract_file_section_names,
-    extract_file_embedded_pe,
-    extract_file_function_names,
-    extract_file_format,
-)

capa/features/extractors/binja/find_binja_api.py

@@ -1,35 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import subprocess
-from pathlib import Path
-
-# When the script gets executed as a standalone executable (via PyInstaller), `import binaryninja` does not work because
-# we have excluded the binaryninja module in `pyinstaller.spec`. The trick here is to call the system Python and try
-# to find out the path of the binaryninja module that has been installed.
-# Note, including the binaryninja module in the `pyintaller.spec` would not work, since the binaryninja module tries to
-# find the binaryninja core e.g., `libbinaryninjacore.dylib`, using a relative path. And this does not work when the
-# binaryninja module is extracted by the PyInstaller.
-code = r"""
-from pathlib import Path
-from importlib import util
-spec = util.find_spec('binaryninja')
-if spec is not None:
-    if len(spec.submodule_search_locations) > 0:
-            path = Path(spec.submodule_search_locations[0])
-            # encode the path with utf8 then convert to hex, make sure it can be read and restored properly
-            print(str(path.parent).encode('utf8').hex())
-"""
-
-
-def find_binja_path() -> Path:
-    raw_output = subprocess.check_output(["python", "-c", code]).decode("ascii").strip()
-    return Path(bytes.fromhex(raw_output).decode("utf8"))
-
-
-if __name__ == "__main__":
-    print(find_binja_path())

capa/features/extractors/binja/function.py

@@ -1,104 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Tuple, Iterator
-
-from binaryninja import Function, BinaryView, SymbolType, RegisterValueType, LowLevelILOperation
-
-from capa.features.file import FunctionName
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors import loops
-from capa.features.extractors.base_extractor import FunctionHandle
-
-
-def extract_function_calls_to(fh: FunctionHandle):
-    """extract callers to a function"""
-    func: Function = fh.inner
-
-    for caller in func.caller_sites:
-        # Everything that is a code reference to the current function is considered a caller, which actually includes
-        # many other references that are NOT a caller. For example, an instruction `push function_start` will also be
-        # considered a caller to the function
-        llil = caller.llil
-        if (llil is None) or llil.operation not in [
-            LowLevelILOperation.LLIL_CALL,
-            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-            LowLevelILOperation.LLIL_JUMP,
-            LowLevelILOperation.LLIL_TAILCALL,
-        ]:
-            continue
-
-        if llil.dest.value.type not in [
-            RegisterValueType.ImportedAddressValue,
-            RegisterValueType.ConstantValue,
-            RegisterValueType.ConstantPointerValue,
-        ]:
-            continue
-
-        address = llil.dest.value.value
-        if address != func.start:
-            continue
-
-        yield Characteristic("calls to"), AbsoluteVirtualAddress(caller.address)
-
-
-def extract_function_loop(fh: FunctionHandle):
-    """extract loop indicators from a function"""
-    func: Function = fh.inner
-
-    edges = []
-
-    # construct control flow graph
-    for bb in func.basic_blocks:
-        for edge in bb.outgoing_edges:
-            edges.append((bb.start, edge.target.start))
-
-    if loops.has_loop(edges):
-        yield Characteristic("loop"), fh.address
-
-
-def extract_recursive_call(fh: FunctionHandle):
-    """extract recursive function call"""
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-    if bv is None:
-        return
-
-    for ref in bv.get_code_refs(func.start):
-        if ref.function == func:
-            yield Characteristic("recursive call"), fh.address
-
-
-def extract_function_name(fh: FunctionHandle):
-    """extract function names (e.g., symtab names)"""
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-    if bv is None:
-        return
-
-    for sym in bv.get_symbols(func.start):
-        if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
-            continue
-
-        name = sym.short_name
-        yield FunctionName(name), sym.address
-        if name.startswith("_"):
-            # some linkers may prefix linked routines with a `_` to avoid name collisions.
-            # extract features for both the mangled and un-mangled representations.
-            # e.g. `_fwrite` -> `fwrite`
-            # see: https://stackoverflow.com/a/2628384/87207
-            yield FunctionName(name[1:]), sym.address
-
-
-def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    for func_handler in FUNCTION_HANDLERS:
-        for feature, addr in func_handler(fh):
-            yield feature, addr
-
-
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call, extract_function_name)

capa/features/extractors/binja/global_.py

@@ -1,60 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-from typing import Tuple, Iterator
-
-from binaryninja import BinaryView
-
-from capa.features.common import OS, OS_MACOS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
-from capa.features.address import NO_ADDRESS, Address
-
-logger = logging.getLogger(__name__)
-
-
-def extract_os(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    name = bv.platform.name
-    if "-" in name:
-        name = name.split("-")[0]
-
-    if name == "windows":
-        yield OS(OS_WINDOWS), NO_ADDRESS
-
-    elif name == "macos":
-        yield OS(OS_MACOS), NO_ADDRESS
-
-    elif name in ["linux", "freebsd", "decree"]:
-        yield OS(name), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a further CLI argument to specify the OS,
-        # but i think this would be rarely used.
-        # rules that rely on OS conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess OS", name)
-        return
-
-
-def extract_arch(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
-    arch = bv.arch.name
-    if arch == "x86_64":
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    elif arch == "x86":
-        yield Arch(ARCH_I386), NO_ADDRESS
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", arch)
-        return

capa/features/extractors/binja/helpers.py

@@ -1,69 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import re
-from typing import List, Callable
-from dataclasses import dataclass
-
-from binaryninja import BinaryView, LowLevelILInstruction
-from binaryninja.architecture import InstructionTextToken
-
-
-@dataclass
-class DisassemblyInstruction:
-    address: int
-    length: int
-    text: List[InstructionTextToken]
-
-
-LLIL_VISITOR = Callable[[LowLevelILInstruction, LowLevelILInstruction, int], bool]
-
-
-def visit_llil_exprs(il: LowLevelILInstruction, func: LLIL_VISITOR):
-    # BN does not really support operand index at the disassembly level, so use the LLIL operand index as a substitute.
-    # Note, this is NOT always guaranteed to be the same as disassembly operand.
-    for i, op in enumerate(il.operands):
-        if isinstance(op, LowLevelILInstruction) and func(op, il, i):
-            visit_llil_exprs(op, func)
-
-
-def unmangle_c_name(name: str) -> str:
-    # https://learn.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170#FormatC
-    # Possible variations for BaseThreadInitThunk:
-    # @BaseThreadInitThunk@12
-    # _BaseThreadInitThunk
-    # _BaseThreadInitThunk@12
-    # It is also possible for a function to have a `Stub` appended to its name:
-    # _lstrlenWStub@4
-
-    # A small optimization to avoid running the regex too many times
-    # this still increases the unit test execution time from 170s to 200s, should be able to accelerate it
-    #
-    # TODO(xusheng): performance optimizations to improve test execution time
-    # https://github.com/mandiant/capa/issues/1610
-    if name[0] in ["@", "_"]:
-        match = re.match(r"^[@|_](.*?)(Stub)?(@\d+)?$", name)
-        if match:
-            return match.group(1)
-
-    return name
-
-
-def read_c_string(bv: BinaryView, offset: int, max_len: int) -> str:
-    s: List[str] = []
-    while len(s) < max_len:
-        try:
-            c = bv.read(offset + len(s), 1)[0]
-        except Exception:
-            break
-
-        if c == 0:
-            break
-
-        s.append(chr(c))
-
-    return "".join(s)

capa/features/extractors/binja/insn.py

@@ -1,586 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Any, List, Tuple, Iterator, Optional
-
-from binaryninja import Function
-from binaryninja import BasicBlock as BinjaBasicBlock
-from binaryninja import (
-    BinaryView,
-    ILRegister,
-    SymbolType,
-    BinaryReader,
-    RegisterValueType,
-    LowLevelILOperation,
-    LowLevelILInstruction,
-)
-
-import capa.features.extractors.helpers
-from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
-from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.binja.helpers import DisassemblyInstruction, visit_llil_exprs
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-# security cookie checks may perform non-zeroing XORs, these are expected within a certain
-# byte range within the first and returning basic blocks, this helps to reduce FP features
-SECURITY_COOKIE_BYTES_DELTA = 0x40
-
-
-# check if a function is a stub function to another function/symbol. The criteria is:
-# 1. The function must only have one basic block
-# 2. The function must only make one call/jump to another address
-# If the function being checked is a stub function, returns the target address. Otherwise, return None.
-def is_stub_function(bv: BinaryView, addr: int) -> Optional[int]:
-    funcs = bv.get_functions_at(addr)
-    for func in funcs:
-        if len(func.basic_blocks) != 1:
-            continue
-
-        call_count = 0
-        call_target = None
-        for il in func.llil.instructions:
-            if il.operation in [
-                LowLevelILOperation.LLIL_CALL,
-                LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-                LowLevelILOperation.LLIL_JUMP,
-                LowLevelILOperation.LLIL_TAILCALL,
-            ]:
-                call_count += 1
-                if il.dest.value.type in [
-                    RegisterValueType.ImportedAddressValue,
-                    RegisterValueType.ConstantValue,
-                    RegisterValueType.ConstantPointerValue,
-                ]:
-                    call_target = il.dest.value.value
-
-        if call_count == 1 and call_target is not None:
-            return call_target
-
-    return None
-
-
-def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction API features
-
-    example:
-       call dword [0x00473038]
-    """
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    for llil in func.get_llils_at(ih.address):
-        if llil.operation in [
-            LowLevelILOperation.LLIL_CALL,
-            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-            LowLevelILOperation.LLIL_JUMP,
-            LowLevelILOperation.LLIL_TAILCALL,
-        ]:
-            if llil.dest.value.type not in [
-                RegisterValueType.ImportedAddressValue,
-                RegisterValueType.ConstantValue,
-                RegisterValueType.ConstantPointerValue,
-            ]:
-                continue
-            address = llil.dest.value.value
-            candidate_addrs = [address]
-            stub_addr = is_stub_function(bv, address)
-            if stub_addr is not None:
-                candidate_addrs.append(stub_addr)
-
-            for address in candidate_addrs:
-                for sym in func.view.get_symbols(address):
-                    if sym is None or sym.type not in [
-                        SymbolType.ImportAddressSymbol,
-                        SymbolType.ImportedFunctionSymbol,
-                        SymbolType.FunctionSymbol,
-                    ]:
-                        continue
-
-                    sym_name = sym.short_name
-
-                    lib_name = ""
-                    import_lib = bv.lookup_imported_object_library(sym.address)
-                    if import_lib is not None:
-                        lib_name = import_lib[0].name
-                        if lib_name.endswith(".dll"):
-                            lib_name = lib_name[:-4]
-                        elif lib_name.endswith(".so"):
-                            lib_name = lib_name[:-3]
-
-                    for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name):
-                        yield API(name), ih.address
-
-                    if sym_name.startswith("_"):
-                        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name[1:]):
-                            yield API(name), ih.address
-
-
-def extract_insn_number_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction number features
-    example:
-        push    3136B0h         ; dwControlCode
-    """
-    func: Function = fh.inner
-
-    results: List[Tuple[Any[Number, OperandNumber], Address]] = []
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        if il.operation == LowLevelILOperation.LLIL_LOAD:
-            return False
-
-        if il.operation not in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-            return True
-
-        for op in parent.operands:
-            if isinstance(op, ILRegister) and op.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
-                return False
-            elif isinstance(op, LowLevelILInstruction) and op.operation == LowLevelILOperation.LLIL_REG:
-                if op.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
-                    return False
-
-        raw_value = il.value.value
-        if parent.operation == LowLevelILOperation.LLIL_SUB:
-            raw_value = -raw_value
-
-        results.append((Number(raw_value), ih.address))
-        results.append((OperandNumber(index, raw_value), ih.address))
-
-        return False
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse referenced byte sequences
-    example:
-        push    offset iid_004118d4_IShellLinkA ; riid
-    """
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    candidate_addrs = set()
-
-    llil = func.get_llil_at(ih.address)
-    if llil is None or llil.operation in [LowLevelILOperation.LLIL_CALL, LowLevelILOperation.LLIL_CALL_STACK_ADJUST]:
-        return
-
-    for ref in bv.get_code_refs_from(ih.address):
-        if ref == ih.address:
-            continue
-
-        if len(bv.get_functions_containing(ref)) > 0:
-            continue
-
-        candidate_addrs.add(ref)
-
-    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-            value = il.value.value
-            if value > 0:
-                candidate_addrs.add(value)
-            return False
-
-        return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    for addr in candidate_addrs:
-        extracted_bytes = bv.read(addr, MAX_BYTES_FEATURE_SIZE)
-        if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-            if bv.get_string_at(addr) is None:
-                # don't extract byte features for obvious strings
-                yield Bytes(extracted_bytes), ih.address
-
-
-def extract_insn_string_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction string features
-
-    example:
-        push offset aAcr     ; "ACR  > "
-    """
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    candidate_addrs = set()
-
-    # collect candidate address from code refs directly
-    for ref in bv.get_code_refs_from(ih.address):
-        if ref == ih.address:
-            continue
-
-        if len(bv.get_functions_containing(ref)) > 0:
-            continue
-
-        candidate_addrs.add(ref)
-
-    # collect candidate address by enumerating all integers, https://github.com/Vector35/binaryninja-api/issues/3966
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        if il.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-            value = il.value.value
-            if value > 0:
-                candidate_addrs.add(value)
-            return False
-
-        return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    # Now we have all the candidate address, check them for string or pointer to string
-    br = BinaryReader(bv)
-    for addr in candidate_addrs:
-        found = bv.get_string_at(addr)
-        if found:
-            yield String(found.value), ih.address
-
-        br.seek(addr)
-        pointer = None
-        if bv.arch.address_size == 4:
-            pointer = br.read32()
-        elif bv.arch.address_size == 8:
-            pointer = br.read64()
-
-        if pointer is not None:
-            found = bv.get_string_at(pointer)
-            if found:
-                yield String(found.value), ih.address
-
-
-def extract_insn_offset_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction structure offset features
-
-    example:
-        .text:0040112F cmp [esi+4], ebx
-    """
-    func: Function = fh.inner
-
-    results: List[Tuple[Any[Offset, OperandOffset], Address]] = []
-    address_size = func.view.arch.address_size * 8
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        #  The most common case, read/write dereference to something like `dword [eax+0x28]`
-        if il.operation in [LowLevelILOperation.LLIL_ADD, LowLevelILOperation.LLIL_SUB]:
-            left = il.left
-            right = il.right
-            # Exclude offsets based on stack/franme pointers
-            if left.operation == LowLevelILOperation.LLIL_REG and left.src.name in ["esp", "ebp", "rsp", "rbp", "sp"]:
-                return True
-
-            if right.operation != LowLevelILOperation.LLIL_CONST:
-                return True
-
-            raw_value = right.value.value
-            # If this is not a dereference, then this must be an add and the offset must be in the range \
-            # [0, MAX_STRUCTURE_SIZE]. For example,
-            # add eax, 0x10,
-            # lea ebx, [eax + 1]
-            if parent.operation not in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
-                if il.operation != LowLevelILOperation.LLIL_ADD or (not 0 < raw_value < MAX_STRUCTURE_SIZE):
-                    return False
-
-            if address_size > 0:
-                # BN also encodes the constant value as two's complement, we need to restore its original value
-                value = capa.features.extractors.helpers.twos_complement(raw_value, address_size)
-            else:
-                value = raw_value
-
-            results.append((Offset(value), ih.address))
-            results.append((OperandOffset(index, value), ih.address))
-            return False
-
-        # An edge case: for code like `push dword [esi]`, we need to generate a feature for offset 0x0
-        elif il.operation in [LowLevelILOperation.LLIL_LOAD, LowLevelILOperation.LLIL_STORE]:
-            if il.operands[0].operation == LowLevelILOperation.LLIL_REG:
-                results.append((Offset(0), ih.address))
-                results.append((OperandOffset(index, 0), ih.address))
-                return False
-
-        return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def is_nzxor_stack_cookie(f: Function, bb: BinjaBasicBlock, llil: LowLevelILInstruction) -> bool:
-    """check if nzxor exists within stack cookie delta"""
-    # TODO(xusheng): use LLIL SSA to do more accurate analysis
-    # https://github.com/mandiant/capa/issues/1609
-
-    reg_names = []
-    if llil.left.operation == LowLevelILOperation.LLIL_REG:
-        reg_names.append(llil.left.src.name)
-
-    if llil.right.operation == LowLevelILOperation.LLIL_REG:
-        reg_names.append(llil.right.src.name)
-
-    # stack cookie reg should be stack/frame pointer
-    if not any(reg in ["ebp", "esp", "rbp", "rsp", "sp"] for reg in reg_names):
-        return False
-
-    # expect security cookie init in first basic block within first bytes (instructions)
-    if len(bb.incoming_edges) == 0 and llil.address < (bb.start + SECURITY_COOKIE_BYTES_DELTA):
-        return True
-
-    # ... or within last bytes (instructions) before a return
-    if len(bb.outgoing_edges) == 0 and llil.address > (bb.end - SECURITY_COOKIE_BYTES_DELTA):
-        return True
-
-    return False
-
-
-def extract_insn_nzxor_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction non-zeroing XOR instruction
-    ignore expected non-zeroing XORs, e.g. security cookies
-    """
-    func: Function = fh.inner
-
-    results = []
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        # If the two operands of the xor instruction are the same, the LLIL will be translated to other instructions,
-        # e.g., <llil: eax = 0>, (LLIL_SET_REG). So we do not need to check whether the two operands are the same.
-        if il.operation == LowLevelILOperation.LLIL_XOR:
-            # Exclude cases related to the stack cookie
-            if is_nzxor_stack_cookie(fh.inner, bbh.inner[0], il):
-                return False
-            results.append((Characteristic("nzxor"), ih.address))
-            return False
-        else:
-            return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def extract_insn_mnemonic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction mnemonic features"""
-    insn: DisassemblyInstruction = ih.inner
-    yield Mnemonic(insn.text[0].text), ih.address
-
-
-def extract_insn_obfs_call_plus_5_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse call $+5 instruction from the given instruction.
-    """
-    insn: DisassemblyInstruction = ih.inner
-    if insn.text[0].text == "call" and insn.text[2].text == "$+5" and insn.length == 5:
-        yield Characteristic("call $+5"), ih.address
-
-
-def extract_insn_peb_access_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction peb access
-
-    fs:[0x30] on x86, gs:[0x60] on x64
-    """
-    func: Function = fh.inner
-
-    results = []
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILOperation, index: int) -> bool:
-        if il.operation != LowLevelILOperation.LLIL_LOAD:
-            return True
-
-        src = il.src
-        if src.operation != LowLevelILOperation.LLIL_ADD:
-            return True
-
-        left = src.left
-        right = src.right
-
-        if left.operation != LowLevelILOperation.LLIL_REG:
-            return True
-
-        reg = left.src.name
-
-        if right.operation != LowLevelILOperation.LLIL_CONST:
-            return True
-
-        value = right.value.value
-        if (reg, value) not in (("fsbase", 0x30), ("gsbase", 0x60)):
-            return True
-
-        results.append((Characteristic("peb access"), ih.address))
-        return False
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def extract_insn_segment_access_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction fs or gs access"""
-    func: Function = fh.inner
-
-    results = []
-
-    def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
-        if il.operation == LowLevelILOperation.LLIL_REG:
-            reg = il.src.name
-            if reg == "fsbase":
-                results.append((Characteristic("fs access"), ih.address))
-                return False
-            elif reg == "gsbase":
-                results.append((Characteristic("gs access"), ih.address))
-                return False
-            return False
-
-        return True
-
-    for llil in func.get_llils_at(ih.address):
-        visit_llil_exprs(llil, llil_checker)
-
-    yield from results
-
-
-def extract_insn_cross_section_cflow(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    if bv is None:
-        return
-
-    seg1 = bv.get_segment_at(ih.address)
-    sections1 = bv.get_sections_at(ih.address)
-    for ref in bv.get_code_refs_from(ih.address):
-        if len(bv.get_functions_at(ref)) == 0:
-            continue
-
-        seg2 = bv.get_segment_at(ref)
-        sections2 = bv.get_sections_at(ref)
-        if seg1 != seg2 or sections1 != sections2:
-            yield Characteristic("cross section flow"), ih.address
-
-
-def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract functions calls from features
-
-    most relevant at the function scope, however, its most efficient to extract at the instruction scope
-    """
-    func: Function = fh.inner
-    bv: BinaryView = func.view
-
-    if bv is None:
-        return
-
-    for il in func.get_llils_at(ih.address):
-        if il.operation not in [
-            LowLevelILOperation.LLIL_CALL,
-            LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-            LowLevelILOperation.LLIL_TAILCALL,
-        ]:
-            continue
-
-        dest = il.dest
-        if dest.operation == LowLevelILOperation.LLIL_CONST_PTR:
-            value = dest.value.value
-            yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
-        elif dest.operation == LowLevelILOperation.LLIL_CONST:
-            yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value)
-        elif dest.operation == LowLevelILOperation.LLIL_LOAD:
-            indirect_src = dest.src
-            if indirect_src.operation == LowLevelILOperation.LLIL_CONST_PTR:
-                value = indirect_src.value.value
-                yield Characteristic("calls from"), AbsoluteVirtualAddress(value)
-            elif indirect_src.operation == LowLevelILOperation.LLIL_CONST:
-                yield Characteristic("calls from"), AbsoluteVirtualAddress(indirect_src.value)
-        elif dest.operation == LowLevelILOperation.LLIL_REG:
-            if dest.value.type in [
-                RegisterValueType.ImportedAddressValue,
-                RegisterValueType.ConstantValue,
-                RegisterValueType.ConstantPointerValue,
-            ]:
-                yield Characteristic("calls from"), AbsoluteVirtualAddress(dest.value.value)
-
-
-def extract_function_indirect_call_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
-    does not include calls like => call ds:dword_ABD4974
-
-    most relevant at the function or basic block scope;
-    however, its most efficient to extract at the instruction scope
-    """
-    func: Function = fh.inner
-
-    llil = func.get_llil_at(ih.address)
-    if llil is None or llil.operation not in [
-        LowLevelILOperation.LLIL_CALL,
-        LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
-        LowLevelILOperation.LLIL_TAILCALL,
-    ]:
-        return
-
-    if llil.dest.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-        return
-
-    if llil.dest.operation == LowLevelILOperation.LLIL_LOAD:
-        src = llil.dest.src
-        if src.operation in [LowLevelILOperation.LLIL_CONST, LowLevelILOperation.LLIL_CONST_PTR]:
-            return
-
-    yield Characteristic("indirect call"), ih.address
-
-
-def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract instruction features"""
-    for inst_handler in INSTRUCTION_HANDLERS:
-        for feature, ea in inst_handler(f, bbh, insn):
-            yield feature, ea
-
-
-INSTRUCTION_HANDLERS = (
-    extract_insn_api_features,
-    extract_insn_number_features,
-    extract_insn_bytes_features,
-    extract_insn_string_features,
-    extract_insn_offset_features,
-    extract_insn_nzxor_characteristic_features,
-    extract_insn_mnemonic_features,
-    extract_insn_obfs_call_plus_5_characteristic_features,
-    extract_insn_peb_access_characteristic_features,
-    extract_insn_cross_section_cflow,
-    extract_insn_segment_access_features,
-    extract_function_calls_from,
-    extract_function_indirect_call_characteristic_features,
-)

capa/features/extractors/cape/call.py

@@ -1,62 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import logging
-from typing import Tuple, Iterator
-
-from capa.helpers import assert_never
-from capa.features.insn import API, Number
-from capa.features.common import String, Feature
-from capa.features.address import Address
-from capa.features.extractors.cape.models import Call
-from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
-
-logger = logging.getLogger(__name__)
-
-
-def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    this method extracts the given call's features (such as API name and arguments),
-    and returns them as API, Number, and String features.
-
-    args:
-      ph: process handle (for defining the extraction scope)
-      th: thread handle (for defining the extraction scope)
-      ch: call handle (for defining the extraction scope)
-
-    yields:
-      Feature, address; where Feature is either: API, Number, or String.
-    """
-    call: Call = ch.inner
-
-    # list similar to disassembly: arguments right-to-left, call
-    for arg in reversed(call.arguments):
-        value = arg.value
-        if isinstance(value, list) and len(value) == 0:
-            # unsure why CAPE captures arguments as empty lists?
-            continue
-
-        elif isinstance(value, str):
-            yield String(value), ch.address
-
-        elif isinstance(value, int):
-            yield Number(value), ch.address
-
-        else:
-            assert_never(value)
-
-    yield API(call.api), ch.address
-
-
-def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
-    for handler in CALL_HANDLERS:
-        for feature, addr in handler(ph, th, ch):
-            yield feature, addr
-
-
-CALL_HANDLERS = (extract_call_features,)

capa/features/extractors/cape/extractor.py

@@ -1,153 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import logging
-from typing import Dict, Tuple, Union, Iterator
-
-import capa.features.extractors.cape.call
-import capa.features.extractors.cape.file
-import capa.features.extractors.cape.thread
-import capa.features.extractors.cape.global_
-import capa.features.extractors.cape.process
-from capa.exceptions import EmptyReportError, UnsupportedFormatError
-from capa.features.common import Feature, Characteristic
-from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress, _NoAddress
-from capa.features.extractors.cape.models import Call, Static, Process, CapeReport
-from capa.features.extractors.base_extractor import (
-    CallHandle,
-    SampleHashes,
-    ThreadHandle,
-    ProcessHandle,
-    DynamicFeatureExtractor,
-)
-
-logger = logging.getLogger(__name__)
-
-TESTED_VERSIONS = {"2.2-CAPE", "2.4-CAPE"}
-
-
-class CapeExtractor(DynamicFeatureExtractor):
-    def __init__(self, report: CapeReport):
-        super().__init__(
-            hashes=SampleHashes(
-                md5=report.target.file.md5.lower(),
-                sha1=report.target.file.sha1.lower(),
-                sha256=report.target.file.sha256.lower(),
-            )
-        )
-        self.report: CapeReport = report
-
-        # pre-compute these because we'll yield them at *every* scope.
-        self.global_features = list(capa.features.extractors.cape.global_.extract_features(self.report))
-
-    def get_base_address(self) -> Union[AbsoluteVirtualAddress, _NoAddress, None]:
-        # value according to the PE header, the actual trace may use a different imagebase
-        assert self.report.static is not None and self.report.static.pe is not None
-        return AbsoluteVirtualAddress(self.report.static.pe.imagebase)
-
-    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
-        yield from self.global_features
-
-    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.cape.file.extract_features(self.report)
-
-    def get_processes(self) -> Iterator[ProcessHandle]:
-        yield from capa.features.extractors.cape.file.get_processes(self.report)
-
-    def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.cape.process.extract_features(ph)
-
-    def get_process_name(self, ph) -> str:
-        process: Process = ph.inner
-        return process.process_name
-
-    def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
-        yield from capa.features.extractors.cape.process.get_threads(ph)
-
-    def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
-        if False:
-            # force this routine to be a generator,
-            # but we don't actually have any elements to generate.
-            yield Characteristic("never"), NO_ADDRESS
-        return
-
-    def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
-        yield from capa.features.extractors.cape.thread.get_calls(ph, th)
-
-    def extract_call_features(
-        self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
-    ) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.cape.call.extract_features(ph, th, ch)
-
-    def get_call_name(self, ph, th, ch) -> str:
-        call: Call = ch.inner
-
-        parts = []
-        parts.append(call.api)
-        parts.append("(")
-        for argument in call.arguments:
-            parts.append(argument.name)
-            parts.append("=")
-
-            if argument.pretty_value:
-                parts.append(argument.pretty_value)
-            else:
-                if isinstance(argument.value, int):
-                    parts.append(hex(argument.value))
-                elif isinstance(argument.value, str):
-                    parts.append('"')
-                    parts.append(argument.value)
-                    parts.append('"')
-                elif isinstance(argument.value, list):
-                    pass
-                else:
-                    capa.helpers.assert_never(argument.value)
-
-            parts.append(", ")
-        if call.arguments:
-            # remove the trailing comma
-            parts.pop()
-        parts.append(")")
-        parts.append(" -> ")
-        if call.pretty_return:
-            parts.append(call.pretty_return)
-        else:
-            parts.append(hex(call.return_))
-
-        return "".join(parts)
-
-    @classmethod
-    def from_report(cls, report: Dict) -> "CapeExtractor":
-        cr = CapeReport.model_validate(report)
-
-        if cr.info.version not in TESTED_VERSIONS:
-            logger.warning("CAPE version '%s' not tested/supported yet", cr.info.version)
-
-        # TODO(mr-tz): support more file types
-        # https://github.com/mandiant/capa/issues/1933
-        if "PE" not in cr.target.file.type:
-            logger.error(
-                "capa currently only supports PE target files, this target file's type is: '%s'.\nPlease report this at: https://github.com/mandiant/capa/issues/1933",
-                cr.target.file.type,
-            )
-
-        # observed in 2.4-CAPE reports from capesandbox.com
-        if cr.static is None and cr.target.file.pe is not None:
-            cr.static = Static()
-            cr.static.pe = cr.target.file.pe
-
-        if cr.static is None:
-            raise UnsupportedFormatError("CAPE report missing static analysis")
-
-        if cr.static.pe is None:
-            raise UnsupportedFormatError("CAPE report missing PE analysis")
-
-        if len(cr.behavior.processes) == 0:
-            raise EmptyReportError("CAPE did not capture any processes")
-
-        return cls(cr)

capa/features/extractors/cape/file.py

@@ -1,132 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import logging
-from typing import Tuple, Iterator
-
-from capa.features.file import Export, Import, Section
-from capa.features.common import String, Feature
-from capa.features.address import NO_ADDRESS, Address, ProcessAddress, AbsoluteVirtualAddress
-from capa.features.extractors.helpers import generate_symbols
-from capa.features.extractors.cape.models import CapeReport
-from capa.features.extractors.base_extractor import ProcessHandle
-
-logger = logging.getLogger(__name__)
-
-
-def get_processes(report: CapeReport) -> Iterator[ProcessHandle]:
-    """
-    get all the created processes for a sample
-    """
-    seen_processes = {}
-    for process in report.behavior.processes:
-        addr = ProcessAddress(pid=process.process_id, ppid=process.parent_id)
-        yield ProcessHandle(address=addr, inner=process)
-
-        # check for pid and ppid reuse
-        if addr not in seen_processes:
-            seen_processes[addr] = [process]
-        else:
-            logger.warning(
-                "pid and ppid reuse detected between process %s and process%s: %s",
-                process,
-                "es" if len(seen_processes[addr]) > 1 else "",
-                seen_processes[addr],
-            )
-            seen_processes[addr].append(process)
-
-
-def extract_import_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract imported function names
-    """
-    assert report.static is not None and report.static.pe is not None
-    imports = report.static.pe.imports
-
-    if isinstance(imports, dict):
-        imports = list(imports.values())
-
-    assert isinstance(imports, list)
-
-    for library in imports:
-        for function in library.imports:
-            if not function.name:
-                continue
-
-            for name in generate_symbols(library.dll, function.name, include_dll=True):
-                yield Import(name), AbsoluteVirtualAddress(function.address)
-
-
-def extract_export_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    assert report.static is not None and report.static.pe is not None
-    for function in report.static.pe.exports:
-        yield Export(function.name), AbsoluteVirtualAddress(function.address)
-
-
-def extract_section_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    assert report.static is not None and report.static.pe is not None
-    for section in report.static.pe.sections:
-        yield Section(section.name), AbsoluteVirtualAddress(section.virtual_address)
-
-
-def extract_file_strings(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    if report.strings is not None:
-        for string in report.strings:
-            yield String(string), NO_ADDRESS
-
-
-def extract_used_regkeys(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    for regkey in report.behavior.summary.keys:
-        yield String(regkey), NO_ADDRESS
-
-
-def extract_used_files(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    for file in report.behavior.summary.files:
-        yield String(file), NO_ADDRESS
-
-
-def extract_used_mutexes(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    for mutex in report.behavior.summary.mutexes:
-        yield String(mutex), NO_ADDRESS
-
-
-def extract_used_commands(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    for cmd in report.behavior.summary.executed_commands:
-        yield String(cmd), NO_ADDRESS
-
-
-def extract_used_apis(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    for symbol in report.behavior.summary.resolved_apis:
-        yield String(symbol), NO_ADDRESS
-
-
-def extract_used_services(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    for svc in report.behavior.summary.created_services:
-        yield String(svc), NO_ADDRESS
-    for svc in report.behavior.summary.started_services:
-        yield String(svc), NO_ADDRESS
-
-
-def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    for handler in FILE_HANDLERS:
-        for feature, addr in handler(report):
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_import_names,
-    extract_export_names,
-    extract_section_names,
-    extract_file_strings,
-    extract_used_regkeys,
-    extract_used_files,
-    extract_used_mutexes,
-    extract_used_commands,
-    extract_used_apis,
-    extract_used_services,
-)

capa/features/extractors/cape/global_.py

@@ -1,93 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import logging
-from typing import Tuple, Iterator
-
-from capa.features.common import (
-    OS,
-    OS_ANY,
-    OS_LINUX,
-    ARCH_I386,
-    FORMAT_PE,
-    ARCH_AMD64,
-    FORMAT_ELF,
-    OS_WINDOWS,
-    Arch,
-    Format,
-    Feature,
-)
-from capa.features.address import NO_ADDRESS, Address
-from capa.features.extractors.cape.models import CapeReport
-
-logger = logging.getLogger(__name__)
-
-
-def extract_arch(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    if "Intel 80386" in report.target.file.type:
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif "x86-64" in report.target.file.type:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        logger.warning("unrecognized Architecture: %s", report.target.file.type)
-        raise ValueError(
-            f"unrecognized Architecture from the CAPE report; output of file command: {report.target.file.type}"
-        )
-
-
-def extract_format(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    if "PE" in report.target.file.type:
-        yield Format(FORMAT_PE), NO_ADDRESS
-    elif "ELF" in report.target.file.type:
-        yield Format(FORMAT_ELF), NO_ADDRESS
-    else:
-        logger.warning("unknown file format, file command output: %s", report.target.file.type)
-        raise ValueError(
-            "unrecognized file format from the CAPE report; output of file command: {report.target.file.type}"
-        )
-
-
-def extract_os(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    # this variable contains the output of the file command
-    file_output = report.target.file.type
-
-    if "windows" in file_output.lower():
-        yield OS(OS_WINDOWS), NO_ADDRESS
-    elif "elf" in file_output.lower():
-        # operating systems recognized by the file command: https://github.com/file/file/blob/master/src/readelf.c#L609
-        if "Linux" in file_output:
-            yield OS(OS_LINUX), NO_ADDRESS
-        elif "Hurd" in file_output:
-            yield OS("hurd"), NO_ADDRESS
-        elif "Solaris" in file_output:
-            yield OS("solaris"), NO_ADDRESS
-        elif "kFreeBSD" in file_output:
-            yield OS("freebsd"), NO_ADDRESS
-        elif "kNetBSD" in file_output:
-            yield OS("netbsd"), NO_ADDRESS
-        else:
-            # if the operating system information is missing from the cape report, it's likely a bug
-            logger.warning("unrecognized OS: %s", file_output)
-            raise ValueError("unrecognized OS from the CAPE report; output of file command: {file_output}")
-    else:
-        # the sample is shellcode
-        logger.debug("unsupported file format, file command output: %s", file_output)
-        yield OS(OS_ANY), NO_ADDRESS
-
-
-def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
-    for global_handler in GLOBAL_HANDLER:
-        for feature, addr in global_handler(report):
-            yield feature, addr
-
-
-GLOBAL_HANDLER = (
-    extract_format,
-    extract_os,
-    extract_arch,
-)

capa/features/extractors/cape/helpers.py

@@ -1,29 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from typing import Any, Dict, List
-
-from capa.features.extractors.base_extractor import ProcessHandle
-
-
-def find_process(processes: List[Dict[str, Any]], ph: ProcessHandle) -> Dict[str, Any]:
-    """
-    find a specific process identified by a process handler.
-
-    args:
-      processes: a list of processes extracted by CAPE
-      ph: handle of the sought process
-
-    return:
-      a CAPE-defined dictionary for the sought process' information
-    """
-
-    for process in processes:
-        if ph.address.ppid == process["parent_id"] and ph.address.pid == process["process_id"]:
-            return process
-    return {}

capa/features/extractors/cape/models.py

@@ -1,446 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import binascii
-from typing import Any, Dict, List, Union, Literal, Optional
-
-from pydantic import Field, BaseModel, ConfigDict
-from typing_extensions import Annotated, TypeAlias
-from pydantic.functional_validators import BeforeValidator
-
-
-def validate_hex_int(value):
-    if isinstance(value, str):
-        return int(value, 16) if value.startswith("0x") else int(value, 10)
-    else:
-        return value
-
-
-def validate_hex_bytes(value):
-    return binascii.unhexlify(value) if isinstance(value, str) else value
-
-
-HexInt = Annotated[int, BeforeValidator(validate_hex_int)]
-HexBytes = Annotated[bytes, BeforeValidator(validate_hex_bytes)]
-
-
-# a model that *cannot* have extra fields
-# if they do, pydantic raises an exception.
-# use this for models we rely upon and cannot change.
-#
-# for things that may be extended and we don't care,
-# use FlexibleModel.
-class ExactModel(BaseModel):
-    model_config = ConfigDict(extra="forbid")
-
-
-# a model that can have extra fields that we ignore.
-# use this if we don't want to raise an exception for extra
-# data fields that we didn't expect.
-class FlexibleModel(BaseModel):
-    pass
-
-
-# use this type to indicate that we won't model this data.
-# because its not relevant to our use in capa.
-#
-# while its nice to have full coverage of the data shape,
-# it can easily change and break our parsing.
-# so we really only want to describe what we'll use.
-Skip: TypeAlias = Optional[Any]
-
-
-# mark fields that we haven't seen yet and need to model.
-# pydantic should raise an error when encountering data
-# in a field with this type.
-# then we can update the model with the discovered shape.
-TODO: TypeAlias = None
-ListTODO: TypeAlias = List[None]
-DictTODO: TypeAlias = ExactModel
-
-EmptyDict: TypeAlias = BaseModel
-EmptyList: TypeAlias = List[Any]
-
-
-class Info(FlexibleModel):
-    version: str
-
-
-class ImportedSymbol(ExactModel):
-    address: HexInt
-    name: Optional[str] = None
-
-
-class ImportedDll(ExactModel):
-    dll: str
-    imports: List[ImportedSymbol]
-
-
-class DirectoryEntry(ExactModel):
-    name: str
-    virtual_address: HexInt
-    size: HexInt
-
-
-class Section(ExactModel):
-    name: str
-    raw_address: HexInt
-    virtual_address: HexInt
-    virtual_size: HexInt
-    size_of_data: HexInt
-    characteristics: str
-    characteristics_raw: HexInt
-    entropy: float
-
-
-class Resource(ExactModel):
-    name: str
-    language: Optional[str] = None
-    sublanguage: str
-    filetype: Optional[str]
-    offset: HexInt
-    size: HexInt
-    entropy: float
-
-
-class DigitalSigner(FlexibleModel):
-    md5_fingerprint: str
-    not_after: str
-    not_before: str
-    serial_number: str
-    sha1_fingerprint: str
-    sha256_fingerprint: str
-
-    issuer_commonName: Optional[str] = None
-    issuer_countryName: Optional[str] = None
-    issuer_localityName: Optional[str] = None
-    issuer_organizationName: Optional[str] = None
-    issuer_stateOrProvinceName: Optional[str] = None
-
-    subject_commonName: Optional[str] = None
-    subject_countryName: Optional[str] = None
-    subject_localityName: Optional[str] = None
-    subject_organizationName: Optional[str] = None
-    subject_stateOrProvinceName: Optional[str] = None
-
-    extensions_authorityInfoAccess_caIssuers: Optional[str] = None
-    extensions_authorityKeyIdentifier: Optional[str] = None
-    extensions_cRLDistributionPoints_0: Optional[str] = None
-    extensions_certificatePolicies_0: Optional[str] = None
-    extensions_subjectAltName_0: Optional[str] = None
-    extensions_subjectKeyIdentifier: Optional[str] = None
-
-
-class AuxSigner(ExactModel):
-    name: str
-    issued_to: str = Field(alias="Issued to")
-    issued_by: str = Field(alias="Issued by")
-    expires: str = Field(alias="Expires")
-    sha1_hash: str = Field(alias="SHA1 hash")
-
-
-class Signer(ExactModel):
-    aux_sha1: Optional[str] = None
-    aux_timestamp: Optional[str] = None
-    aux_valid: Optional[bool] = None
-    aux_error: Optional[bool] = None
-    aux_error_desc: Optional[str] = None
-    aux_signers: Optional[List[AuxSigner]] = None
-
-
-class Overlay(ExactModel):
-    offset: HexInt
-    size: HexInt
-
-
-class KV(ExactModel):
-    name: str
-    value: str
-
-
-class ExportedSymbol(ExactModel):
-    address: HexInt
-    name: str
-    ordinal: int
-
-
-class PE(ExactModel):
-    peid_signatures: TODO
-    imagebase: HexInt
-    entrypoint: HexInt
-    reported_checksum: HexInt
-    actual_checksum: HexInt
-    osversion: str
-    pdbpath: Optional[str] = None
-    timestamp: str
-
-    # List[ImportedDll], or Dict[basename(dll), ImportedDll]
-    imports: Union[List[ImportedDll], Dict[str, ImportedDll]]
-    imported_dll_count: Optional[int] = None
-    imphash: str
-
-    exported_dll_name: Optional[str] = None
-    exports: List[ExportedSymbol]
-
-    dirents: List[DirectoryEntry]
-    sections: List[Section]
-
-    ep_bytes: Optional[HexBytes] = None
-
-    overlay: Optional[Overlay] = None
-    resources: List[Resource]
-    versioninfo: List[KV]
-
-    # base64 encoded data
-    icon: Optional[str] = None
-    # MD5-like hash
-    icon_hash: Optional[str] = None
-    # MD5-like hash
-    icon_fuzzy: Optional[str] = None
-    # short hex string
-    icon_dhash: Optional[str] = None
-
-    digital_signers: List[DigitalSigner]
-    guest_signers: Signer
-
-
-# TODO(mr-tz): target.file.dotnet, target.file.extracted_files, target.file.extracted_files_tool,
-#  target.file.extracted_files_time
-# https://github.com/mandiant/capa/issues/1814
-class File(FlexibleModel):
-    type: str
-    cape_type_code: Optional[int] = None
-    cape_type: Optional[str] = None
-
-    pid: Optional[Union[int, Literal[""]]] = None
-    name: Union[List[str], str]
-    path: str
-    guest_paths: Union[List[str], str, None]
-    timestamp: Optional[str] = None
-
-    #
-    # hashes
-    #
-    crc32: str
-    md5: str
-    sha1: str
-    sha256: str
-    sha512: str
-    sha3_384: str
-    ssdeep: str
-    # unsure why this would ever be "False"
-    tlsh: Optional[Union[str, bool]] = None
-    rh_hash: Optional[str] = None
-
-    #
-    # other metadata, static analysis
-    #
-    size: int
-    pe: Optional[PE] = None
-    ep_bytes: Optional[HexBytes] = None
-    entrypoint: Optional[int] = None
-    data: Optional[str] = None
-    strings: Optional[List[str]] = None
-
-    #
-    # detections (skip)
-    #
-    yara: Skip = None
-    cape_yara: Skip = None
-    clamav: Skip = None
-    virustotal: Skip = None
-
-
-class ProcessFile(File):
-    #
-    # like a File, but also has dynamic analysis results
-    #
-    pid: Optional[int] = None
-    process_path: Optional[str] = None
-    process_name: Optional[str] = None
-    module_path: Optional[str] = None
-    virtual_address: Optional[HexInt] = None
-    target_pid: Optional[Union[int, str]] = None
-    target_path: Optional[str] = None
-    target_process: Optional[str] = None
-
-
-class Argument(ExactModel):
-    name: str
-    # unsure why empty list is provided here
-    value: Union[HexInt, int, str, EmptyList]
-    pretty_value: Optional[str] = None
-
-
-class Call(ExactModel):
-    timestamp: str
-    thread_id: int
-    category: str
-
-    api: str
-
-    arguments: List[Argument]
-    status: bool
-    return_: HexInt = Field(alias="return")
-    pretty_return: Optional[str] = None
-
-    repeated: int
-
-    # virtual addresses
-    caller: HexInt
-    parentcaller: HexInt
-
-    # index into calls array
-    id: int
-
-
-class Process(ExactModel):
-    process_id: int
-    process_name: str
-    parent_id: int
-    module_path: str
-    first_seen: str
-    calls: List[Call]
-    threads: List[int]
-    environ: Dict[str, str]
-
-
-class ProcessTree(ExactModel):
-    name: str
-    pid: int
-    parent_id: int
-    module_path: str
-    threads: List[int]
-    environ: Dict[str, str]
-    children: List["ProcessTree"]
-
-
-class Summary(ExactModel):
-    files: List[str]
-    read_files: List[str]
-    write_files: List[str]
-    delete_files: List[str]
-    keys: List[str]
-    read_keys: List[str]
-    write_keys: List[str]
-    delete_keys: List[str]
-    executed_commands: List[str]
-    resolved_apis: List[str]
-    mutexes: List[str]
-    created_services: List[str]
-    started_services: List[str]
-
-
-class EncryptedBuffer(ExactModel):
-    process_name: str
-    pid: int
-
-    api_call: str
-    buffer: str
-    buffer_size: Optional[int] = None
-    crypt_key: Optional[Union[HexInt, str]] = None
-
-
-class Behavior(ExactModel):
-    summary: Summary
-
-    # list of processes, of threads, of calls
-    processes: List[Process]
-    # tree of processes
-    processtree: List[ProcessTree]
-
-    anomaly: List[str]
-    encryptedbuffers: List[EncryptedBuffer]
-    # these are small objects that describe atomic events,
-    # like file move, registery access.
-    # we'll detect the same with our API call analyis.
-    enhanced: Skip = None
-
-
-class Target(ExactModel):
-    category: str
-    file: File
-    pe: Optional[PE] = None
-
-
-class Static(ExactModel):
-    pe: Optional[PE] = None
-    flare_capa: Skip = None
-
-
-class Cape(ExactModel):
-    payloads: List[ProcessFile]
-    configs: Skip = None
-
-
-# flexible because there may be more sorts of analysis
-# but we only care about the ones described here.
-class CapeReport(FlexibleModel):
-    # the input file, I think
-    target: Target
-    # info about the processing job, like machine and distributed metadata.
-    info: Info
-
-    #
-    # static analysis results
-    #
-    static: Optional[Static] = None
-    strings: Optional[List[str]] = None
-
-    #
-    # dynamic analysis results
-    #
-    # post-processed results: process tree, anomalies, etc
-    behavior: Behavior
-
-    # post-processed results: payloads and extracted configs
-    CAPE: Optional[Cape] = None
-    dropped: Optional[List[File]] = None
-    procdump: Optional[List[ProcessFile]] = None
-    procmemory: ListTODO
-
-    # =========================================================================
-    # information we won't use in capa
-    #
-
-    #
-    # NBIs and HBIs
-    # these are super interesting, but they don't enable use to detect behaviors.
-    # they take a lot of code to model and details to maintain.
-    #
-    # if we come up with a future use for this, go ahead and re-enable!
-    #
-    network: Skip = None
-    suricata: Skip = None
-    curtain: Skip = None
-    sysmon: Skip = None
-    url_analysis: Skip = None
-
-    # screenshot hash values
-    deduplicated_shots: Skip = None
-    # k-v pairs describing the time it took to run each stage.
-    statistics: Skip = None
-    # k-v pairs of ATT&CK ID to signature name or similar.
-    ttps: Skip = None
-    # debug log messages
-    debug: Skip = None
-
-    # various signature matches
-    # we could potentially extend capa to use this info one day,
-    # though it would be quite sandbox-specific,
-    # and more detection-oriented than capability detection.
-    signatures: Skip = None
-    malfamily_tag: Optional[str] = None
-    malscore: float
-    detections: Skip = None
-    detections2pid: Optional[Dict[int, List[str]]] = None
-    # AV detections for the sample.
-    virustotal: Skip = None
-
-    @classmethod
-    def from_buf(cls, buf: bytes) -> "CapeReport":
-        return cls.model_validate_json(buf)

capa/features/extractors/cape/process.py

@@ -1,48 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import logging
-from typing import List, Tuple, Iterator
-
-from capa.features.common import String, Feature
-from capa.features.address import Address, ThreadAddress
-from capa.features.extractors.cape.models import Process
-from capa.features.extractors.base_extractor import ThreadHandle, ProcessHandle
-
-logger = logging.getLogger(__name__)
-
-
-def get_threads(ph: ProcessHandle) -> Iterator[ThreadHandle]:
-    """
-    get the threads associated with a given process
-    """
-    process: Process = ph.inner
-    threads: List[int] = process.threads
-
-    for thread in threads:
-        address: ThreadAddress = ThreadAddress(process=ph.address, tid=thread)
-        yield ThreadHandle(address=address, inner={})
-
-
-def extract_environ_strings(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract strings from a process' provided environment variables.
-    """
-    process: Process = ph.inner
-
-    for value in (value for value in process.environ.values() if value):
-        yield String(value), ph.address
-
-
-def extract_features(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
-    for handler in PROCESS_HANDLERS:
-        for feature, addr in handler(ph):
-            yield feature, addr
-
-
-PROCESS_HANDLERS = (extract_environ_strings,)

capa/features/extractors/cape/thread.py

@@ -1,32 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import logging
-from typing import Iterator
-
-from capa.features.address import DynamicCallAddress
-from capa.features.extractors.helpers import generate_symbols
-from capa.features.extractors.cape.models import Process
-from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
-
-logger = logging.getLogger(__name__)
-
-
-def get_calls(ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
-    process: Process = ph.inner
-
-    tid = th.address.tid
-    for call_index, call in enumerate(process.calls):
-        if call.thread_id != tid:
-            continue
-
-        for symbol in generate_symbols("", call.api):
-            call.api = symbol
-
-            addr = DynamicCallAddress(thread=th.address, id=call_index)
-            yield CallHandle(address=addr, inner=call)

capa/features/extractors/common.py

@@ -1,142 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import io
-import re
-import logging
-import binascii
-import contextlib
-from typing import Tuple, Iterator
-
-import pefile
-
-import capa.features
-import capa.features.extractors.elf
-import capa.features.extractors.pefile
-import capa.features.extractors.strings
-from capa.features.common import (
-    OS,
-    OS_ANY,
-    OS_AUTO,
-    ARCH_ANY,
-    FORMAT_PE,
-    FORMAT_ELF,
-    OS_WINDOWS,
-    FORMAT_FREEZE,
-    FORMAT_RESULT,
-    Arch,
-    Format,
-    String,
-    Feature,
-)
-from capa.features.freeze import is_freeze
-from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress
-
-logger = logging.getLogger(__name__)
-
-# match strings for formats
-MATCH_PE = b"MZ"
-MATCH_ELF = b"\x7fELF"
-MATCH_RESULT = b'{"meta":'
-MATCH_JSON_OBJECT = b'{"'
-
-
-def extract_file_strings(buf: bytes, **kwargs) -> Iterator[Tuple[String, Address]]:
-    """
-    extract ASCII and UTF-16 LE strings from file
-    """
-    for s in capa.features.extractors.strings.extract_ascii_strings(buf):
-        yield String(s.s), FileOffsetAddress(s.offset)
-
-    for s in capa.features.extractors.strings.extract_unicode_strings(buf):
-        yield String(s.s), FileOffsetAddress(s.offset)
-
-
-def extract_format(buf: bytes) -> Iterator[Tuple[Feature, Address]]:
-    if buf.startswith(MATCH_PE):
-        yield Format(FORMAT_PE), NO_ADDRESS
-    elif buf.startswith(MATCH_ELF):
-        yield Format(FORMAT_ELF), NO_ADDRESS
-    elif is_freeze(buf):
-        yield Format(FORMAT_FREEZE), NO_ADDRESS
-    elif buf.startswith(MATCH_RESULT):
-        yield Format(FORMAT_RESULT), NO_ADDRESS
-    elif re.sub(rb"\s", b"", buf[:20]).startswith(MATCH_JSON_OBJECT):
-        # potential start of JSON object data without whitespace
-        # we don't know what it is exactly, but may support it (e.g. a dynamic CAPE sandbox report)
-        # skip verdict here and let subsequent code analyze this further
-        return
-    else:
-        # we likely end up here:
-        #  1. handling a file format (e.g. macho)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s", binascii.hexlify(buf[:4]).decode("ascii"))
-        return
-
-
-def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
-    if buf.startswith(MATCH_PE):
-        yield from capa.features.extractors.pefile.extract_file_arch(pe=pefile.PE(data=buf))
-
-    elif buf.startswith(MATCH_RESULT):
-        yield Arch(ARCH_ANY), NO_ADDRESS
-
-    elif buf.startswith(MATCH_ELF):
-        with contextlib.closing(io.BytesIO(buf)) as f:
-            arch = capa.features.extractors.elf.detect_elf_arch(f)
-
-        if arch not in capa.features.common.VALID_ARCH:
-            logger.debug("unsupported arch: %s", arch)
-            return
-
-        yield Arch(arch), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a further CLI argument to specify the arch,
-        # but i think this would be rarely used.
-        # rules that rely on arch conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess Arch", binascii.hexlify(buf[:4]).decode("ascii"))
-        return
-
-
-def extract_os(buf, os=OS_AUTO) -> Iterator[Tuple[Feature, Address]]:
-    if os != OS_AUTO:
-        yield OS(os), NO_ADDRESS
-
-    if buf.startswith(MATCH_PE):
-        yield OS(OS_WINDOWS), NO_ADDRESS
-    elif buf.startswith(MATCH_RESULT):
-        yield OS(OS_ANY), NO_ADDRESS
-    elif buf.startswith(MATCH_ELF):
-        with contextlib.closing(io.BytesIO(buf)) as f:
-            os = capa.features.extractors.elf.detect_elf_os(f)
-
-        if os not in capa.features.common.VALID_OS:
-            logger.debug("unsupported os: %s", os)
-            return
-
-        yield OS(os), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # rules that rely on OS conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess OS", binascii.hexlify(buf[:4]).decode("ascii"))
-        return

capa/features/extractors/dnfile/extractor.py

@@ -1,161 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-from typing import Dict, List, Tuple, Union, Iterator, Optional
-from pathlib import Path
-
-import dnfile
-from dncil.cil.opcode import OpCodes
-
-import capa.features.extractors
-import capa.features.extractors.dotnetfile
-import capa.features.extractors.dnfile.file
-import capa.features.extractors.dnfile.insn
-import capa.features.extractors.dnfile.function
-from capa.features.common import Feature
-from capa.features.address import NO_ADDRESS, Address, DNTokenAddress, DNTokenOffsetAddress
-from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
-from capa.features.extractors.base_extractor import (
-    BBHandle,
-    InsnHandle,
-    SampleHashes,
-    FunctionHandle,
-    StaticFeatureExtractor,
-)
-from capa.features.extractors.dnfile.helpers import (
-    get_dotnet_types,
-    get_dotnet_fields,
-    get_dotnet_managed_imports,
-    get_dotnet_managed_methods,
-    get_dotnet_unmanaged_imports,
-    get_dotnet_managed_method_bodies,
-)
-
-
-class DnFileFeatureExtractorCache:
-    def __init__(self, pe: dnfile.dnPE):
-        self.imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
-        self.native_imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
-        self.methods: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
-        self.fields: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
-        self.types: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
-
-        for import_ in get_dotnet_managed_imports(pe):
-            self.imports[import_.token] = import_
-        for native_import in get_dotnet_unmanaged_imports(pe):
-            self.native_imports[native_import.token] = native_import
-        for method in get_dotnet_managed_methods(pe):
-            self.methods[method.token] = method
-        for field in get_dotnet_fields(pe):
-            self.fields[field.token] = field
-        for type_ in get_dotnet_types(pe):
-            self.types[type_.token] = type_
-
-    def get_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.imports.get(token)
-
-    def get_native_import(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.native_imports.get(token)
-
-    def get_method(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.methods.get(token)
-
-    def get_field(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.fields.get(token)
-
-    def get_type(self, token: int) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-        return self.types.get(token)
-
-
-class DnfileFeatureExtractor(StaticFeatureExtractor):
-    def __init__(self, path: Path):
-        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
-        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
-
-        # pre-compute .NET token lookup tables; each .NET method has access to this cache for feature extraction
-        # most relevant at instruction scope
-        self.token_cache: DnFileFeatureExtractorCache = DnFileFeatureExtractorCache(self.pe)
-
-        # pre-compute these because we'll yield them at *every* scope.
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_format())
-        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_os(pe=self.pe))
-        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_arch(pe=self.pe))
-
-    def get_base_address(self):
-        return NO_ADDRESS
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.dnfile.file.extract_features(self.pe)
-
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        # create a method lookup table
-        methods: Dict[Address, FunctionHandle] = {}
-        for token, method in get_dotnet_managed_method_bodies(self.pe):
-            fh: FunctionHandle = FunctionHandle(
-                address=DNTokenAddress(token),
-                inner=method,
-                ctx={"pe": self.pe, "calls_from": set(), "calls_to": set(), "cache": self.token_cache},
-            )
-
-            # method tokens should be unique
-            assert fh.address not in methods.keys()
-            methods[fh.address] = fh
-
-        # calculate unique calls to/from each method
-        for fh in methods.values():
-            for insn in fh.inner.instructions:
-                if insn.opcode not in (
-                    OpCodes.Call,
-                    OpCodes.Callvirt,
-                    OpCodes.Jmp,
-                    OpCodes.Newobj,
-                ):
-                    continue
-
-                address: DNTokenAddress = DNTokenAddress(insn.operand.value)
-
-                # record call to destination method; note: we only consider MethodDef methods for destinations
-                dest: Optional[FunctionHandle] = methods.get(address)
-                if dest is not None:
-                    dest.ctx["calls_to"].add(fh.address)
-
-                # record call from source method; note: we record all unique calls from a MethodDef method, not just
-                # those calls to other MethodDef methods e.g. calls to imported MemberRef methods
-                fh.ctx["calls_from"].add(address)
-
-        yield from methods.values()
-
-    def extract_function_features(self, fh) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.dnfile.function.extract_features(fh)
-
-    def get_basic_blocks(self, f) -> Iterator[BBHandle]:
-        # each dotnet method is considered 1 basic block
-        yield BBHandle(
-            address=f.address,
-            inner=f.inner,
-        )
-
-    def extract_basic_block_features(self, fh, bbh):
-        # we don't support basic block features
-        yield from []
-
-    def get_instructions(self, fh, bbh):
-        for insn in bbh.inner.instructions:
-            yield InsnHandle(
-                address=DNTokenOffsetAddress(bbh.address, insn.offset - (fh.inner.offset + fh.inner.header_size)),
-                inner=insn,
-            )
-
-    def extract_insn_features(self, fh, bbh, ih) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.dnfile.insn.extract_features(fh, bbh, ih)

capa/features/extractors/dnfile/file.py

@@ -1,63 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-from typing import Tuple, Iterator
-
-import dnfile
-
-import capa.features.extractors.dotnetfile
-from capa.features.file import Import, FunctionName
-from capa.features.common import Class, Format, String, Feature, Namespace, Characteristic
-from capa.features.address import Address
-
-
-def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[Tuple[Import, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_import_names(pe=pe)
-
-
-def extract_file_format(pe: dnfile.dnPE) -> Iterator[Tuple[Format, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_format(pe=pe)
-
-
-def extract_file_function_names(pe: dnfile.dnPE) -> Iterator[Tuple[FunctionName, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_function_names(pe=pe)
-
-
-def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe)
-
-
-def extract_file_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[Tuple[Characteristic, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_mixed_mode_characteristic_features(pe=pe)
-
-
-def extract_file_namespace_features(pe: dnfile.dnPE) -> Iterator[Tuple[Namespace, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_namespace_features(pe=pe)
-
-
-def extract_file_class_features(pe: dnfile.dnPE) -> Iterator[Tuple[Class, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_class_features(pe=pe)
-
-
-def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for file_handler in FILE_HANDLERS:
-        for feature, address in file_handler(pe):
-            yield feature, address
-
-
-FILE_HANDLERS = (
-    extract_file_import_names,
-    extract_file_function_names,
-    extract_file_strings,
-    extract_file_format,
-    extract_file_mixed_mode_characteristic_features,
-    extract_file_namespace_features,
-    extract_file_class_features,
-)

capa/features/extractors/dnfile/function.py

@@ -1,50 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-import logging
-from typing import Tuple, Iterator
-
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address
-from capa.features.extractors.base_extractor import FunctionHandle
-
-logger = logging.getLogger(__name__)
-
-
-def extract_function_calls_to(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
-    """extract callers to a function"""
-    for dest in fh.ctx["calls_to"]:
-        yield Characteristic("calls to"), dest
-
-
-def extract_function_calls_from(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
-    """extract callers from a function"""
-    for src in fh.ctx["calls_from"]:
-        yield Characteristic("calls from"), src
-
-
-def extract_recursive_call(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
-    """extract recursive function call"""
-    if fh.address in fh.ctx["calls_to"]:
-        yield Characteristic("recursive call"), fh.address
-
-
-def extract_function_loop(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
-    """extract loop indicators from a function"""
-    raise NotImplementedError()
-
-
-def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    for func_handler in FUNCTION_HANDLERS:
-        for feature, addr in func_handler(fh):
-            yield feature, addr
-
-
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_calls_from, extract_recursive_call)

capa/features/extractors/dnfile/helpers.py

@@ -1,448 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-import logging
-from typing import Dict, Tuple, Union, Iterator, Optional
-
-import dnfile
-from dncil.cil.body import CilMethodBody
-from dncil.cil.error import MethodBodyFormatError
-from dncil.clr.token import Token, StringToken, InvalidToken
-from dncil.cil.body.reader import CilMethodBodyReaderBase
-
-from capa.features.common import FeatureAccess
-from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
-
-logger = logging.getLogger(__name__)
-
-
-class DnfileMethodBodyReader(CilMethodBodyReaderBase):
-    def __init__(self, pe: dnfile.dnPE, row: dnfile.mdtable.MethodDefRow):
-        self.pe: dnfile.dnPE = pe
-        self.offset: int = self.pe.get_offset_from_rva(row.Rva)
-
-    def read(self, n: int) -> bytes:
-        data: bytes = self.pe.get_data(self.pe.get_rva_from_offset(self.offset), n)
-        self.offset += n
-        return data
-
-    def tell(self) -> int:
-        return self.offset
-
-    def seek(self, offset: int) -> int:
-        self.offset = offset
-        return self.offset
-
-
-def resolve_dotnet_token(pe: dnfile.dnPE, token: Token) -> Union[dnfile.base.MDTableRow, InvalidToken, str]:
-    """map generic token to string or table row"""
-    assert pe.net is not None
-    assert pe.net.mdtables is not None
-
-    if isinstance(token, StringToken):
-        user_string: Optional[str] = read_dotnet_user_string(pe, token)
-        if user_string is None:
-            return InvalidToken(token.value)
-        return user_string
-
-    table: Optional[dnfile.base.ClrMetaDataTable] = pe.net.mdtables.tables.get(token.table)
-    if table is None:
-        # table index is not valid
-        return InvalidToken(token.value)
-
-    try:
-        return table.rows[token.rid - 1]
-    except IndexError:
-        # table index is valid but row index is not valid
-        return InvalidToken(token.value)
-
-
-def read_dotnet_method_body(pe: dnfile.dnPE, row: dnfile.mdtable.MethodDefRow) -> Optional[CilMethodBody]:
-    """read dotnet method body"""
-    try:
-        return CilMethodBody(DnfileMethodBodyReader(pe, row))
-    except MethodBodyFormatError as e:
-        logger.debug("failed to parse managed method body @ 0x%08x (%s)", row.Rva, e)
-        return None
-
-
-def read_dotnet_user_string(pe: dnfile.dnPE, token: StringToken) -> Optional[str]:
-    """read user string from #US stream"""
-    assert pe.net is not None
-
-    if pe.net.user_strings is None:
-        # stream may not exist (seen in obfuscated .NET)
-        logger.debug("#US stream does not exist for stream index 0x%08x", token.rid)
-        return None
-
-    try:
-        user_string: Optional[dnfile.stream.UserString] = pe.net.user_strings.get_us(token.rid)
-    except UnicodeDecodeError as e:
-        logger.debug("failed to decode #US stream index 0x%08x (%s)", token.rid, e)
-        return None
-
-    if user_string is None:
-        return None
-
-    return user_string.value
-
-
-def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[DnType]:
-    """get managed imports from MemberRef table
-
-    see https://www.ntcore.com/files/dotnetformat.htm
-
-    10 - MemberRef Table
-        Each row represents an imported method
-            Class (index into the TypeRef, ModuleRef, MethodDef, TypeSpec or TypeDef tables)
-            Name (index into String heap)
-    01 - TypeRef Table
-        Each row represents an imported class, its namespace and the assembly which contains it
-            TypeName (index into String heap)
-            TypeNamespace (index into String heap)
-    """
-    for rid, member_ref in iter_dotnet_table(pe, dnfile.mdtable.MemberRef.number):
-        assert isinstance(member_ref, dnfile.mdtable.MemberRefRow)
-
-        if not isinstance(member_ref.Class.row, dnfile.mdtable.TypeRefRow):
-            # only process class imports from TypeRef table
-            continue
-
-        token: int = calculate_dotnet_token_value(dnfile.mdtable.MemberRef.number, rid)
-        access: Optional[str]
-
-        # assume .NET imports starting with get_/set_ are used to access a property
-        if member_ref.Name.startswith("get_"):
-            access = FeatureAccess.READ
-        elif member_ref.Name.startswith("set_"):
-            access = FeatureAccess.WRITE
-        else:
-            access = None
-
-        member_ref_name: str = member_ref.Name
-        if member_ref_name.startswith(("get_", "set_")):
-            # remove get_/set_ from MemberRef name
-            member_ref_name = member_ref_name[4:]
-
-        typerefnamespace, typerefname = resolve_nested_typeref_name(
-            member_ref.Class.row_index, member_ref.Class.row, pe
-        )
-
-        yield DnType(
-            token,
-            typerefname,
-            namespace=typerefnamespace,
-            member=member_ref_name,
-            access=access,
-        )
-
-
-def get_dotnet_methoddef_property_accessors(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
-    """get MethodDef methods used to access properties
-
-    see https://www.ntcore.com/files/dotnetformat.htm
-
-    24 - MethodSemantics Table
-        Links Events and Properties to specific methods. For example one Event can be associated to more methods. A property uses this table to associate get/set methods.
-            Semantics (a 2-byte bitmask of type MethodSemanticsAttributes)
-            Method (index into the MethodDef table)
-            Association (index into the Event or Property table; more precisely, a HasSemantics coded index)
-    """
-    for rid, method_semantics in iter_dotnet_table(pe, dnfile.mdtable.MethodSemantics.number):
-        assert isinstance(method_semantics, dnfile.mdtable.MethodSemanticsRow)
-
-        if method_semantics.Association.row is None:
-            logger.debug("MethodSemantics[0x%X] Association row is None", rid)
-            continue
-
-        if isinstance(method_semantics.Association.row, dnfile.mdtable.EventRow):
-            # ignore events
-            logger.debug("MethodSemantics[0x%X] ignoring Event", rid)
-            continue
-
-        if method_semantics.Method.table is None:
-            logger.debug("MethodSemantics[0x%X] Method table is None", rid)
-            continue
-
-        token: int = calculate_dotnet_token_value(
-            method_semantics.Method.table.number, method_semantics.Method.row_index
-        )
-
-        if method_semantics.Semantics.msSetter:
-            yield token, FeatureAccess.WRITE
-        elif method_semantics.Semantics.msGetter:
-            yield token, FeatureAccess.READ
-
-
-def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
-    """get managed method names from TypeDef table
-
-    see https://www.ntcore.com/files/dotnetformat.htm
-
-    02 - TypeDef Table
-        Each row represents a class in the current assembly.
-            TypeName (index into String heap)
-            TypeNamespace (index into String heap)
-            MethodList (index into MethodDef table; it marks the first of a contiguous run of Methods owned by this Type)
-    """
-    nested_class_table = get_dotnet_nested_class_table_index(pe)
-
-    accessor_map: Dict[int, str] = {}
-    for methoddef, methoddef_access in get_dotnet_methoddef_property_accessors(pe):
-        accessor_map[methoddef] = methoddef_access
-
-    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
-        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
-
-        for idx, method in enumerate(typedef.MethodList):
-            if method.table is None:
-                logger.debug("TypeDef[0x%X] MethodList[0x%X] table is None", rid, idx)
-                continue
-            if method.row is None:
-                logger.debug("TypeDef[0x%X] MethodList[0x%X] row is None", rid, idx)
-                continue
-
-            token: int = calculate_dotnet_token_value(method.table.number, method.row_index)
-            access: Optional[str] = accessor_map.get(token)
-
-            method_name: str = method.row.Name
-            if method_name.startswith(("get_", "set_")):
-                # remove get_/set_
-                method_name = method_name[4:]
-
-            typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
-
-            yield DnType(token, typedefname, namespace=typedefnamespace, member=method_name, access=access)
-
-
-def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
-    """get fields from TypeDef table
-
-    see https://www.ntcore.com/files/dotnetformat.htm
-
-    02 - TypeDef Table
-        Each row represents a class in the current assembly.
-            TypeName (index into String heap)
-            TypeNamespace (index into String heap)
-            FieldList (index into Field table; it marks the first of a contiguous run of Fields owned by this Type)
-    """
-    nested_class_table = get_dotnet_nested_class_table_index(pe)
-
-    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
-        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
-
-        for idx, field in enumerate(typedef.FieldList):
-            if field.table is None:
-                logger.debug("TypeDef[0x%X] FieldList[0x%X] table is None", rid, idx)
-                continue
-            if field.row is None:
-                logger.debug("TypeDef[0x%X] FieldList[0x%X] row is None", rid, idx)
-                continue
-
-            typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
-
-            token: int = calculate_dotnet_token_value(field.table.number, field.row_index)
-            yield DnType(token, typedefname, namespace=typedefnamespace, member=field.row.Name)
-
-
-def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
-    """get managed methods from MethodDef table"""
-    for rid, method_def in iter_dotnet_table(pe, dnfile.mdtable.MethodDef.number):
-        assert isinstance(method_def, dnfile.mdtable.MethodDefRow)
-
-        if not method_def.ImplFlags.miIL or any((method_def.Flags.mdAbstract, method_def.Flags.mdPinvokeImpl)):
-            # skip methods that do not have a method body
-            continue
-
-        body: Optional[CilMethodBody] = read_dotnet_method_body(pe, method_def)
-        if body is None:
-            logger.debug("MethodDef[0x%X] method body is None", rid)
-            continue
-
-        token: int = calculate_dotnet_token_value(dnfile.mdtable.MethodDef.number, rid)
-        yield token, body
-
-
-def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[DnUnmanagedMethod]:
-    """get unmanaged imports from ImplMap table
-
-    see https://www.ntcore.com/files/dotnetformat.htm
-
-    28 - ImplMap Table
-        ImplMap table holds information about unmanaged methods that can be reached from managed code, using PInvoke dispatch
-            MemberForwarded (index into the Field or MethodDef table; more precisely, a MemberForwarded coded index)
-            ImportName (index into the String heap)
-            ImportScope (index into the ModuleRef table)
-    """
-    for rid, impl_map in iter_dotnet_table(pe, dnfile.mdtable.ImplMap.number):
-        assert isinstance(impl_map, dnfile.mdtable.ImplMapRow)
-
-        module: str
-        if impl_map.ImportScope.row is None:
-            logger.debug("ImplMap[0x%X] ImportScope row is None", rid)
-            module = ""
-        else:
-            module = impl_map.ImportScope.row.Name
-        method: str = impl_map.ImportName
-
-        member_forward_table: int
-        if impl_map.MemberForwarded.table is None:
-            logger.debug("ImplMap[0x%X] MemberForwarded table is None", rid)
-            continue
-        else:
-            member_forward_table = impl_map.MemberForwarded.table.number
-        member_forward_row: int = impl_map.MemberForwarded.row_index
-
-        # ECMA says "Each row of the ImplMap table associates a row in the MethodDef table (MemberForwarded) with the
-        # name of a routine (ImportName) in some unmanaged DLL (ImportScope)"; so we calculate and map the MemberForwarded
-        # MethodDef table token to help us later record native import method calls made from CIL
-        token: int = calculate_dotnet_token_value(member_forward_table, member_forward_row)
-
-        # like Kernel32.dll
-        if module and "." in module:
-            module = module.split(".")[0]
-
-        # like kernel32.CreateFileA
-        yield DnUnmanagedMethod(token, module, method)
-
-
-def get_dotnet_table_row(pe: dnfile.dnPE, table_index: int, row_index: int) -> Optional[dnfile.base.MDTableRow]:
-    assert pe.net is not None
-    assert pe.net.mdtables is not None
-
-    if row_index - 1 <= 0:
-        return None
-
-    try:
-        table = pe.net.mdtables.tables.get(table_index, [])
-        return table[row_index - 1]
-    except IndexError:
-        return None
-
-
-def resolve_nested_typedef_name(
-    nested_class_table: dict, index: int, typedef: dnfile.mdtable.TypeDefRow, pe: dnfile.dnPE
-) -> Tuple[str, Tuple[str, ...]]:
-    """Resolves all nested TypeDef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
-
-    if index in nested_class_table:
-        typedef_name = []
-        name = typedef.TypeName
-
-        # Append the current typedef name
-        typedef_name.append(name)
-
-        while nested_class_table[index] in nested_class_table:
-            # Iterate through the typedef table to resolve the nested name
-            table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeDef.number, nested_class_table[index])
-            if table_row is None:
-                return typedef.TypeNamespace, tuple(typedef_name[::-1])
-
-            name = table_row.TypeName
-            typedef_name.append(name)
-            index = nested_class_table[index]
-
-        # Document the root enclosing details
-        table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeDef.number, nested_class_table[index])
-        if table_row is None:
-            return typedef.TypeNamespace, tuple(typedef_name[::-1])
-
-        enclosing_name = table_row.TypeName
-        typedef_name.append(enclosing_name)
-
-        return table_row.TypeNamespace, tuple(typedef_name[::-1])
-
-    else:
-        return typedef.TypeNamespace, (typedef.TypeName,)
-
-
-def resolve_nested_typeref_name(
-    index: int, typeref: dnfile.mdtable.TypeRefRow, pe: dnfile.dnPE
-) -> Tuple[str, Tuple[str, ...]]:
-    """Resolves all nested TypeRef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
-    # If the ResolutionScope decodes to a typeRef type then it is nested
-    if isinstance(typeref.ResolutionScope.table, dnfile.mdtable.TypeRef):
-        typeref_name = []
-        name = typeref.TypeName
-        # Not appending the current typeref name to avoid potential duplicate
-
-        # Validate index
-        table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeRef.number, index)
-        if table_row is None:
-            return typeref.TypeNamespace, (typeref.TypeName,)
-
-        while isinstance(table_row.ResolutionScope.table, dnfile.mdtable.TypeRef):
-            # Iterate through the typeref table to resolve the nested name
-            typeref_name.append(name)
-            name = table_row.TypeName
-            table_row = get_dotnet_table_row(pe, dnfile.mdtable.TypeRef.number, table_row.ResolutionScope.row_index)
-            if table_row is None:
-                return typeref.TypeNamespace, tuple(typeref_name[::-1])
-
-        # Document the root enclosing details
-        typeref_name.append(table_row.TypeName)
-
-        return table_row.TypeNamespace, tuple(typeref_name[::-1])
-
-    else:
-        return typeref.TypeNamespace, (typeref.TypeName,)
-
-
-def get_dotnet_nested_class_table_index(pe: dnfile.dnPE) -> Dict[int, int]:
-    """Build index for EnclosingClass based off the NestedClass row index in the nestedclass table"""
-    nested_class_table = {}
-
-    # Used to find nested classes in typedef
-    for _, nestedclass in iter_dotnet_table(pe, dnfile.mdtable.NestedClass.number):
-        assert isinstance(nestedclass, dnfile.mdtable.NestedClassRow)
-        nested_class_table[nestedclass.NestedClass.row_index] = nestedclass.EnclosingClass.row_index
-
-    return nested_class_table
-
-
-def get_dotnet_types(pe: dnfile.dnPE) -> Iterator[DnType]:
-    """get .NET types from TypeDef and TypeRef tables"""
-    nested_class_table = get_dotnet_nested_class_table_index(pe)
-
-    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
-        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
-
-        typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
-
-        typedef_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
-        yield DnType(typedef_token, typedefname, namespace=typedefnamespace)
-
-    for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
-        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
-
-        typerefnamespace, typerefname = resolve_nested_typeref_name(typeref.ResolutionScope.row_index, typeref, pe)
-
-        typeref_token: int = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
-        yield DnType(typeref_token, typerefname, namespace=typerefnamespace)
-
-
-def calculate_dotnet_token_value(table: int, rid: int) -> int:
-    return ((table & 0xFF) << Token.TABLE_SHIFT) | (rid & Token.RID_MASK)
-
-
-def is_dotnet_mixed_mode(pe: dnfile.dnPE) -> bool:
-    assert pe.net is not None
-    assert pe.net.Flags is not None
-
-    return not bool(pe.net.Flags.CLR_ILONLY)
-
-
-def iter_dotnet_table(pe: dnfile.dnPE, table_index: int) -> Iterator[Tuple[int, dnfile.base.MDTableRow]]:
-    assert pe.net is not None
-    assert pe.net.mdtables is not None
-
-    for rid, row in enumerate(pe.net.mdtables.tables.get(table_index, [])):
-        # .NET tables are 1-indexed
-        yield rid + 1, row

capa/features/extractors/dnfile/insn.py

@@ -1,227 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-import logging
-from typing import TYPE_CHECKING, Tuple, Union, Iterator, Optional
-
-if TYPE_CHECKING:
-    from capa.features.extractors.dnfile.extractor import DnFileFeatureExtractorCache
-
-import dnfile
-from dncil.clr.token import Token, StringToken, InvalidToken
-from dncil.cil.opcode import OpCodes
-
-import capa.features.extractors.helpers
-from capa.features.insn import API, Number, Property
-from capa.features.common import Class, String, Feature, Namespace, FeatureAccess, Characteristic
-from capa.features.address import Address
-from capa.features.extractors.dnfile.types import DnType, DnUnmanagedMethod
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-from capa.features.extractors.dnfile.helpers import (
-    resolve_dotnet_token,
-    read_dotnet_user_string,
-    calculate_dotnet_token_value,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def get_callee(
-    pe: dnfile.dnPE, cache: DnFileFeatureExtractorCache, token: Token
-) -> Optional[Union[DnType, DnUnmanagedMethod]]:
-    """map .NET token to un/managed (generic) method"""
-    token_: int
-    if token.table == dnfile.mdtable.MethodSpec.number:
-        # map MethodSpec to MethodDef or MemberRef
-        row: Union[dnfile.base.MDTableRow, InvalidToken, str] = resolve_dotnet_token(pe, token)
-        assert isinstance(row, dnfile.mdtable.MethodSpecRow)
-
-        if row.Method.table is None:
-            logger.debug("MethodSpec[0x%X] Method table is None", token.rid)
-            return None
-
-        token_ = calculate_dotnet_token_value(row.Method.table.number, row.Method.row_index)
-    else:
-        token_ = token.value
-
-    callee: Optional[Union[DnType, DnUnmanagedMethod]] = cache.get_import(token_)
-    if callee is None:
-        # we must check unmanaged imports before managed methods because we map forwarded managed methods
-        # to their unmanaged imports; we prefer a forwarded managed method be mapped to its unmanaged import for analysis
-        callee = cache.get_native_import(token_)
-        if callee is None:
-            callee = cache.get_method(token_)
-    return callee
-
-
-def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction API features"""
-    if ih.inner.opcode not in (
-        OpCodes.Call,
-        OpCodes.Callvirt,
-        OpCodes.Jmp,
-        OpCodes.Newobj,
-    ):
-        return
-
-    callee: Optional[Union[DnType, DnUnmanagedMethod]] = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
-    if isinstance(callee, DnType):
-        # ignore methods used to access properties
-        if callee.access is None:
-            # like System.IO.File::Delete
-            yield API(str(callee)), ih.address
-    elif isinstance(callee, DnUnmanagedMethod):
-        # like kernel32.CreateFileA
-        for name in capa.features.extractors.helpers.generate_symbols(callee.module, callee.method):
-            yield API(name), ih.address
-
-
-def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction property features"""
-    name: Optional[str] = None
-    access: Optional[str] = None
-
-    if ih.inner.opcode in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
-        # property access via MethodDef or MemberRef
-        callee: Optional[Union[DnType, DnUnmanagedMethod]] = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
-        if isinstance(callee, DnType):
-            if callee.access is not None:
-                name = str(callee)
-                access = callee.access
-
-    elif ih.inner.opcode in (OpCodes.Ldfld, OpCodes.Ldflda, OpCodes.Ldsfld, OpCodes.Ldsflda):
-        # property read via Field
-        read_field: Optional[Union[DnType, DnUnmanagedMethod]] = fh.ctx["cache"].get_field(ih.inner.operand.value)
-        if read_field is not None:
-            name = str(read_field)
-            access = FeatureAccess.READ
-
-    elif ih.inner.opcode in (OpCodes.Stfld, OpCodes.Stsfld):
-        # property write via Field
-        write_field: Optional[Union[DnType, DnUnmanagedMethod]] = fh.ctx["cache"].get_field(ih.inner.operand.value)
-        if write_field is not None:
-            name = str(write_field)
-            access = FeatureAccess.WRITE
-
-    if name is not None:
-        if access is not None:
-            yield Property(name, access=access), ih.address
-        yield Property(name), ih.address
-
-
-def extract_insn_namespace_class_features(
-    fh: FunctionHandle, bh, ih: InsnHandle
-) -> Iterator[Tuple[Union[Namespace, Class], Address]]:
-    """parse instruction namespace and class features"""
-    type_: Optional[Union[DnType, DnUnmanagedMethod]] = None
-
-    if ih.inner.opcode in (
-        OpCodes.Call,
-        OpCodes.Callvirt,
-        OpCodes.Jmp,
-        OpCodes.Ldvirtftn,
-        OpCodes.Ldftn,
-        OpCodes.Newobj,
-    ):
-        # method call - includes managed methods (MethodDef, TypeRef) and properties (MethodSemantics, TypeRef)
-        type_ = get_callee(fh.ctx["pe"], fh.ctx["cache"], ih.inner.operand)
-
-    elif ih.inner.opcode in (
-        OpCodes.Ldfld,
-        OpCodes.Ldflda,
-        OpCodes.Ldsfld,
-        OpCodes.Ldsflda,
-        OpCodes.Stfld,
-        OpCodes.Stsfld,
-    ):
-        # field access
-        type_ = fh.ctx["cache"].get_field(ih.inner.operand.value)
-
-    # ECMA 335 VI.C.4.10
-    elif ih.inner.opcode in (
-        OpCodes.Initobj,
-        OpCodes.Box,
-        OpCodes.Castclass,
-        OpCodes.Cpobj,
-        OpCodes.Isinst,
-        OpCodes.Ldelem,
-        OpCodes.Ldelema,
-        OpCodes.Ldobj,
-        OpCodes.Mkrefany,
-        OpCodes.Newarr,
-        OpCodes.Refanyval,
-        OpCodes.Sizeof,
-        OpCodes.Stobj,
-        OpCodes.Unbox,
-        OpCodes.Constrained,
-        OpCodes.Stelem,
-        OpCodes.Unbox_Any,
-    ):
-        # type access
-        type_ = fh.ctx["cache"].get_type(ih.inner.operand.value)
-
-    if isinstance(type_, DnType):
-        yield Class(DnType.format_name(type_.class_, namespace=type_.namespace)), ih.address
-        if type_.namespace:
-            yield Namespace(type_.namespace), ih.address
-
-
-def extract_insn_number_features(fh, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction number features"""
-    if ih.inner.is_ldc():
-        yield Number(ih.inner.get_ldc()), ih.address
-
-
-def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction string features"""
-    if not ih.inner.is_ldstr():
-        return
-
-    if not isinstance(ih.inner.operand, StringToken):
-        return
-
-    user_string: Optional[str] = read_dotnet_user_string(fh.ctx["pe"], ih.inner.operand)
-    if user_string is None:
-        return
-
-    if len(user_string) >= 4:
-        yield String(user_string), ih.address
-
-
-def extract_unmanaged_call_characteristic_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Characteristic, Address]]:
-    if ih.inner.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
-        return
-
-    row: Union[str, InvalidToken, dnfile.base.MDTableRow] = resolve_dotnet_token(fh.ctx["pe"], ih.inner.operand)
-    if not isinstance(row, dnfile.mdtable.MethodDefRow):
-        return
-
-    if any((row.Flags.mdPinvokeImpl, row.ImplFlags.miUnmanaged, row.ImplFlags.miNative)):
-        yield Characteristic("unmanaged call"), ih.address
-
-
-def extract_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract instruction features"""
-    for inst_handler in INSTRUCTION_HANDLERS:
-        for feature, addr in inst_handler(fh, bbh, ih):
-            assert isinstance(addr, Address)
-            yield feature, addr
-
-
-INSTRUCTION_HANDLERS = (
-    extract_insn_api_features,
-    extract_insn_property_features,
-    extract_insn_number_features,
-    extract_insn_string_features,
-    extract_insn_namespace_class_features,
-    extract_unmanaged_call_characteristic_features,
-)

capa/features/extractors/dnfile/types.py

@@ -1,80 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from typing import Tuple, Optional
-
-
-class DnType:
-    def __init__(
-        self, token: int, class_: Tuple[str, ...], namespace: str = "", member: str = "", access: Optional[str] = None
-    ):
-        self.token: int = token
-        self.access: Optional[str] = access
-        self.namespace: str = namespace
-        self.class_: Tuple[str, ...] = class_
-
-        if member == ".ctor":
-            member = "ctor"
-        if member == ".cctor":
-            member = "cctor"
-
-        self.member: str = member
-
-    def __hash__(self):
-        return hash((self.token, self.access, self.namespace, self.class_, self.member))
-
-    def __eq__(self, other):
-        return (
-            self.token == other.token
-            and self.access == other.access
-            and self.namespace == other.namespace
-            and self.class_ == other.class_
-            and self.member == other.member
-        )
-
-    def __str__(self):
-        return DnType.format_name(self.class_, namespace=self.namespace, member=self.member)
-
-    def __repr__(self):
-        return str(self)
-
-    @staticmethod
-    def format_name(class_: Tuple[str, ...], namespace: str = "", member: str = ""):
-        if len(class_) > 1:
-            class_str = "/".join(class_)  # Concat items in tuple, separated by a "/"
-        else:
-            class_str = "".join(class_)  # Convert tuple to str
-        # like File::OpenRead
-        name: str = f"{class_str}::{member}" if member else class_str
-        if namespace:
-            # like System.IO.File::OpenRead
-            name = f"{namespace}.{name}"
-        return name
-
-
-class DnUnmanagedMethod:
-    def __init__(self, token: int, module: str, method: str):
-        self.token: int = token
-        self.module: str = module
-        self.method: str = method
-
-    def __hash__(self):
-        return hash((self.token, self.module, self.method))
-
-    def __eq__(self, other):
-        return self.token == other.token and self.module == other.module and self.method == other.method
-
-    def __str__(self):
-        return DnUnmanagedMethod.format_name(self.module, self.method)
-
-    def __repr__(self):
-        return str(self)
-
-    @staticmethod
-    def format_name(module, method):
-        return f"{module}.{method}"

capa/features/extractors/dotnetfile.py

@@ -1,248 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-from typing import Tuple, Iterator
-from pathlib import Path
-
-import dnfile
-import pefile
-
-import capa.features.extractors.helpers
-from capa.features.file import Import, FunctionName
-from capa.features.common import (
-    OS,
-    OS_ANY,
-    ARCH_ANY,
-    ARCH_I386,
-    FORMAT_PE,
-    ARCH_AMD64,
-    FORMAT_DOTNET,
-    Arch,
-    Class,
-    Format,
-    String,
-    Feature,
-    Namespace,
-    Characteristic,
-)
-from capa.features.address import NO_ADDRESS, Address, DNTokenAddress
-from capa.features.extractors.dnfile.types import DnType
-from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
-from capa.features.extractors.dnfile.helpers import (
-    iter_dotnet_table,
-    is_dotnet_mixed_mode,
-    get_dotnet_managed_imports,
-    get_dotnet_managed_methods,
-    resolve_nested_typedef_name,
-    resolve_nested_typeref_name,
-    calculate_dotnet_token_value,
-    get_dotnet_unmanaged_imports,
-    get_dotnet_nested_class_table_index,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_format(**kwargs) -> Iterator[Tuple[Format, Address]]:
-    yield Format(FORMAT_PE), NO_ADDRESS
-    yield Format(FORMAT_DOTNET), NO_ADDRESS
-
-
-def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Import, Address]]:
-    for method in get_dotnet_managed_imports(pe):
-        # like System.IO.File::OpenRead
-        yield Import(str(method)), DNTokenAddress(method.token)
-
-    for imp in get_dotnet_unmanaged_imports(pe):
-        # like kernel32.CreateFileA
-        for name in capa.features.extractors.helpers.generate_symbols(imp.module, imp.method, include_dll=True):
-            yield Import(name), DNTokenAddress(imp.token)
-
-
-def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[FunctionName, Address]]:
-    for method in get_dotnet_managed_methods(pe):
-        yield FunctionName(str(method)), DNTokenAddress(method.token)
-
-
-def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Namespace, Address]]:
-    """emit namespace features from TypeRef and TypeDef tables"""
-
-    # namespaces may be referenced multiple times, so we need to filter
-    namespaces = set()
-
-    for _, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
-        # emit internal .NET namespaces
-        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
-        namespaces.add(typedef.TypeNamespace)
-
-    for _, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
-        # emit external .NET namespaces
-        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
-        namespaces.add(typeref.TypeNamespace)
-
-    # namespaces may be empty, discard
-    namespaces.discard("")
-
-    for namespace in namespaces:
-        # namespace do not have an associated token, so we yield 0x0
-        yield Namespace(namespace), NO_ADDRESS
-
-
-def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Class, Address]]:
-    """emit class features from TypeRef and TypeDef tables"""
-    nested_class_table = get_dotnet_nested_class_table_index(pe)
-
-    for rid, typedef in iter_dotnet_table(pe, dnfile.mdtable.TypeDef.number):
-        # emit internal .NET classes
-        assert isinstance(typedef, dnfile.mdtable.TypeDefRow)
-
-        typedefnamespace, typedefname = resolve_nested_typedef_name(nested_class_table, rid, typedef, pe)
-
-        token = calculate_dotnet_token_value(dnfile.mdtable.TypeDef.number, rid)
-        yield Class(DnType.format_name(typedefname, namespace=typedefnamespace)), DNTokenAddress(token)
-
-    for rid, typeref in iter_dotnet_table(pe, dnfile.mdtable.TypeRef.number):
-        # emit external .NET classes
-        assert isinstance(typeref, dnfile.mdtable.TypeRefRow)
-
-        typerefnamespace, typerefname = resolve_nested_typeref_name(typeref.ResolutionScope.row_index, typeref, pe)
-
-        token = calculate_dotnet_token_value(dnfile.mdtable.TypeRef.number, rid)
-        yield Class(DnType.format_name(typerefname, namespace=typerefnamespace)), DNTokenAddress(token)
-
-
-def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
-    yield OS(OS_ANY), NO_ADDRESS
-
-
-def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Arch, Address]]:
-    # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
-    # .NET 4.5 added option: any CPU, 32-bit preferred
-    assert pe.net is not None
-    assert pe.net.Flags is not None
-
-    if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        yield Arch(ARCH_ANY), NO_ADDRESS
-
-
-def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[String, Address]]:
-    yield from capa.features.extractors.common.extract_file_strings(pe.__data__)
-
-
-def extract_file_mixed_mode_characteristic_features(
-    pe: dnfile.dnPE, **kwargs
-) -> Iterator[Tuple[Characteristic, Address]]:
-    if is_dotnet_mixed_mode(pe):
-        yield Characteristic("mixed mode"), NO_ADDRESS
-
-
-def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler(pe=pe):  # type: ignore
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_file_import_names,
-    extract_file_function_names,
-    extract_file_strings,
-    extract_file_format,
-    extract_file_mixed_mode_characteristic_features,
-    extract_file_namespace_features,
-    extract_file_class_features,
-)
-
-
-def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for handler in GLOBAL_HANDLERS:
-        for feature, va in handler(pe=pe):  # type: ignore
-            yield feature, va
-
-
-GLOBAL_HANDLERS = (
-    extract_file_os,
-    extract_file_arch,
-)
-
-
-class DotnetFileFeatureExtractor(StaticFeatureExtractor):
-    def __init__(self, path: Path):
-        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
-        self.path: Path = path
-        self.pe: dnfile.dnPE = dnfile.dnPE(str(path))
-
-    def get_base_address(self):
-        return NO_ADDRESS
-
-    def get_entry_point(self) -> int:
-        # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
-        #  True: native EP: Token
-        #  False: managed EP: RVA
-        assert self.pe.net is not None
-        assert self.pe.net.struct is not None
-
-        return self.pe.net.struct.EntryPointTokenOrRva
-
-    def extract_global_features(self):
-        yield from extract_global_features(self.pe)
-
-    def extract_file_features(self):
-        yield from extract_file_features(self.pe)
-
-    def is_dotnet_file(self) -> bool:
-        return bool(self.pe.net)
-
-    def is_mixed_mode(self) -> bool:
-        return is_dotnet_mixed_mode(self.pe)
-
-    def get_runtime_version(self) -> Tuple[int, int]:
-        assert self.pe.net is not None
-        assert self.pe.net.struct is not None
-        assert self.pe.net.struct.MajorRuntimeVersion is not None
-        assert self.pe.net.struct.MinorRuntimeVersion is not None
-
-        return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
-
-    def get_meta_version_string(self) -> str:
-        assert self.pe.net is not None
-        assert self.pe.net.metadata is not None
-        assert self.pe.net.metadata.struct is not None
-        assert self.pe.net.metadata.struct.Version is not None
-
-        vbuf = self.pe.net.metadata.struct.Version
-        assert isinstance(vbuf, bytes)
-
-        return vbuf.rstrip(b"\x00").decode("utf-8")
-
-    def get_functions(self):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def extract_function_features(self, f):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def get_basic_blocks(self, f):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def extract_basic_block_features(self, f, bb):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def get_instructions(self, f, bb):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def extract_insn_features(self, f, bb, insn):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def is_library_function(self, va):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def get_function_name(self, va):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")

capa/features/extractors/elffile.py

@@ -1,203 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import io
-import logging
-from typing import Tuple, Iterator
-from pathlib import Path
-
-from elftools.elf.elffile import ELFFile, SymbolTableSection
-from elftools.elf.relocation import RelocationSection
-
-import capa.features.extractors.common
-from capa.features.file import Export, Import, Section
-from capa.features.common import OS, FORMAT_ELF, Arch, Format, Feature
-from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_export_names(elf: ELFFile, **kwargs):
-    for section in elf.iter_sections():
-        if not isinstance(section, SymbolTableSection):
-            continue
-
-        if section["sh_entsize"] == 0:
-            logger.debug("Symbol table '%s' has a sh_entsize of zero!", section.name)
-            continue
-
-        logger.debug("Symbol table '%s' contains %s entries:", section.name, section.num_symbols())
-
-        for symbol in section.iter_symbols():
-            # The following conditions are based on the following article
-            # http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
-            if not symbol.name:
-                continue
-            if symbol.entry.st_info.type not in ["STT_FUNC", "STT_OBJECT", "STT_IFUNC"]:
-                continue
-            if symbol.entry.st_value == 0:
-                continue
-            if symbol.entry.st_shndx == "SHN_UNDEF":
-                continue
-
-            yield Export(symbol.name), AbsoluteVirtualAddress(symbol.entry.st_value)
-
-
-def extract_file_import_names(elf: ELFFile, **kwargs):
-    # Create a dictionary to store symbol names by their index
-    symbol_names = {}
-
-    # Extract symbol names and store them in the dictionary
-    for section in elf.iter_sections():
-        if not isinstance(section, SymbolTableSection):
-            continue
-
-        for _, symbol in enumerate(section.iter_symbols()):
-            # The following conditions are based on the following article
-            # http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
-            if not symbol.name:
-                continue
-            if symbol.entry.st_info.type not in ["STT_FUNC", "STT_OBJECT", "STT_IFUNC"]:
-                continue
-            if symbol.entry.st_value != 0:
-                continue
-            if symbol.entry.st_shndx != "SHN_UNDEF":
-                continue
-            if symbol.entry.st_name == 0:
-                continue
-
-            symbol_names[_] = symbol.name
-
-    for section in elf.iter_sections():
-        if not isinstance(section, RelocationSection):
-            continue
-
-        if section["sh_entsize"] == 0:
-            logger.debug("Symbol table '%s' has a sh_entsize of zero!", section.name)
-            continue
-
-        logger.debug("Symbol table '%s' contains %s entries:", section.name, section.num_relocations())
-
-        for relocation in section.iter_relocations():
-            # Extract the symbol name from the symbol table using the symbol index in the relocation
-            if relocation["r_info_sym"] not in symbol_names:
-                continue
-            yield Import(symbol_names[relocation["r_info_sym"]]), FileOffsetAddress(relocation["r_offset"])
-
-
-def extract_file_section_names(elf: ELFFile, **kwargs):
-    for section in elf.iter_sections():
-        if section.name:
-            yield Section(section.name), AbsoluteVirtualAddress(section.header.sh_addr)
-        elif section.is_null():
-            yield Section("NULL"), AbsoluteVirtualAddress(section.header.sh_addr)
-
-
-def extract_file_strings(buf, **kwargs):
-    yield from capa.features.extractors.common.extract_file_strings(buf)
-
-
-def extract_file_os(elf: ELFFile, buf, **kwargs):
-    # our current approach does not always get an OS value, e.g. for packed samples
-    # for file limitation purposes, we're more lax here
-    try:
-        os_tuple = next(capa.features.extractors.common.extract_os(buf))
-        yield os_tuple
-    except StopIteration:
-        yield OS("unknown"), NO_ADDRESS
-
-
-def extract_file_format(**kwargs):
-    yield Format(FORMAT_ELF), NO_ADDRESS
-
-
-def extract_file_arch(elf: ELFFile, **kwargs):
-    arch = elf.get_machine_arch()
-    if arch == "x86":
-        yield Arch("i386"), NO_ADDRESS
-    elif arch == "x64":
-        yield Arch("amd64"), NO_ADDRESS
-    else:
-        logger.warning("unsupported architecture: %s", arch)
-
-
-def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler(elf=elf, buf=buf):  # type: ignore
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_section_names,
-    extract_file_strings,
-    # no library matching
-    extract_file_format,
-)
-
-
-def extract_global_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
-    for global_handler in GLOBAL_HANDLERS:
-        for feature, addr in global_handler(elf=elf, buf=buf):  # type: ignore
-            yield feature, addr
-
-
-GLOBAL_HANDLERS = (
-    extract_file_os,
-    extract_file_arch,
-)
-
-
-class ElfFeatureExtractor(StaticFeatureExtractor):
-    def __init__(self, path: Path):
-        super().__init__(SampleHashes.from_bytes(path.read_bytes()))
-        self.path: Path = path
-        self.elf = ELFFile(io.BytesIO(path.read_bytes()))
-
-    def get_base_address(self):
-        # virtual address of the first segment with type LOAD
-        for segment in self.elf.iter_segments():
-            if segment.header.p_type == "PT_LOAD":
-                return AbsoluteVirtualAddress(segment.header.p_vaddr)
-
-    def extract_global_features(self):
-        buf = self.path.read_bytes()
-
-        for feature, addr in extract_global_features(self.elf, buf):
-            yield feature, addr
-
-    def extract_file_features(self):
-        buf = self.path.read_bytes()
-
-        for feature, addr in extract_file_features(self.elf, buf):
-            yield feature, addr
-
-    def get_functions(self):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def extract_function_features(self, f):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def get_basic_blocks(self, f):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def extract_basic_block_features(self, f, bb):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def get_instructions(self, f, bb):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def extract_insn_features(self, f, bb, insn):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def is_library_function(self, addr):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def get_function_name(self, addr):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")

capa/features/extractors/ghidra/basicblock.py

@@ -1,152 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import string
-import struct
-from typing import Tuple, Iterator
-
-import ghidra
-from ghidra.program.model.lang import OperandType
-
-import capa.features.extractors.ghidra.helpers
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address
-from capa.features.basicblock import BasicBlock
-from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
-
-
-def get_printable_len(op: ghidra.program.model.scalar.Scalar) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
-    op_bit_len = op.bitLength()
-    op_byte_len = op_bit_len // 8
-    op_val = op.getValue()
-
-    if op_bit_len == 8:
-        chars = struct.pack("<B", op_val & 0xFF)
-    elif op_bit_len == 16:
-        chars = struct.pack("<H", op_val & 0xFFFF)
-    elif op_bit_len == 32:
-        chars = struct.pack("<I", op_val & 0xFFFFFFFF)
-    elif op_bit_len == 64:
-        chars = struct.pack("<Q", op_val & 0xFFFFFFFFFFFFFFFF)
-    else:
-        raise ValueError(f"Unhandled operand data type 0x{op_bit_len:x}.")
-
-    def is_printable_ascii(chars_: bytes):
-        return all(c < 127 and chr(c) in string.printable for c in chars_)
-
-    def is_printable_utf16le(chars_: bytes):
-        if all(c == 0x00 for c in chars_[1::2]):
-            return is_printable_ascii(chars_[::2])
-
-    if is_printable_ascii(chars):
-        return op_byte_len
-
-    if is_printable_utf16le(chars):
-        return op_byte_len
-
-    return 0
-
-
-def is_mov_imm_to_stack(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    """verify instruction moves immediate onto stack"""
-
-    # Ghidra will Bitwise OR the OperandTypes to assign multiple
-    # i.e., the first operand is a stackvar (dynamically allocated),
-    # and the second is a scalar value (single int/char/float/etc.)
-    mov_its_ops = [(OperandType.ADDRESS | OperandType.DYNAMIC), OperandType.SCALAR]
-    found = False
-
-    # MOV dword ptr [EBP + local_*], 0x65
-    if insn.getMnemonicString().startswith("MOV"):
-        found = all(insn.getOperandType(i) == mov_its_ops[i] for i in range(2))
-
-    return found
-
-
-def bb_contains_stackstring(bb: ghidra.program.model.block.CodeBlock) -> bool:
-    """check basic block for stackstring indicators
-
-    true if basic block contains enough moves of constant bytes to the stack
-    """
-    count = 0
-    for insn in currentProgram().getListing().getInstructions(bb, True):  # type: ignore [name-defined] # noqa: F821
-        if is_mov_imm_to_stack(insn):
-            count += get_printable_len(insn.getScalar(1))
-        if count > MIN_STACKSTRING_LEN:
-            return True
-    return False
-
-
-def _bb_has_tight_loop(bb: ghidra.program.model.block.CodeBlock):
-    """
-    parse tight loops, true if last instruction in basic block branches to bb start
-    """
-    # Reverse Ordered, first InstructionDB
-    last_insn = currentProgram().getListing().getInstructions(bb, False).next()  # type: ignore [name-defined] # noqa: F821
-
-    if last_insn.getFlowType().isJump():
-        return last_insn.getAddress(0) == bb.getMinAddress()
-
-    return False
-
-
-def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract stackstring indicators from basic block"""
-    bb: ghidra.program.model.block.CodeBlock = bbh.inner
-
-    if bb_contains_stackstring(bb):
-        yield Characteristic("stack string"), bbh.address
-
-
-def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """check basic block for tight loop indicators"""
-    bb: ghidra.program.model.block.CodeBlock = bbh.inner
-
-    if _bb_has_tight_loop(bb):
-        yield Characteristic("tight loop"), bbh.address
-
-
-BASIC_BLOCK_HANDLERS = (
-    extract_bb_tight_loop,
-    extract_bb_stackstring,
-)
-
-
-def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract features from the given basic block.
-
-    args:
-        bb: the basic block to process.
-
-    yields:
-      Tuple[Feature, int]: the features and their location found in this basic block.
-    """
-    yield BasicBlock(), bbh.address
-    for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, addr in bb_handler(fh, bbh):
-            yield feature, addr
-
-
-def main():
-    features = []
-    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
-
-    for fh in GhidraFeatureExtractor().get_functions():
-        for bbh in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
-            features.extend(list(extract_features(fh, bbh)))
-
-    import pprint
-
-    pprint.pprint(features)  # noqa: T203
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ghidra/extractor.py

@@ -1,93 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import List, Tuple, Iterator
-
-import capa.features.extractors.ghidra.file
-import capa.features.extractors.ghidra.insn
-import capa.features.extractors.ghidra.global_
-import capa.features.extractors.ghidra.function
-import capa.features.extractors.ghidra.basicblock
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import (
-    BBHandle,
-    InsnHandle,
-    SampleHashes,
-    FunctionHandle,
-    StaticFeatureExtractor,
-)
-
-
-class GhidraFeatureExtractor(StaticFeatureExtractor):
-    def __init__(self):
-        import capa.features.extractors.ghidra.helpers as ghidra_helpers
-
-        super().__init__(
-            SampleHashes(
-                md5=capa.ghidra.helpers.get_file_md5(),
-                # ghidra doesn't expose this hash.
-                # https://ghidra.re/ghidra_docs/api/ghidra/program/model/listing/Program.html
-                #
-                # the hashes are stored in the database, not computed on the fly,
-                # so its probably not trivial to add SHA1.
-                sha1="",
-                sha256=capa.ghidra.helpers.get_file_sha256(),
-            )
-        )
-
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.ghidra.file.extract_file_format())
-        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_os())
-        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_arch())
-        self.imports = ghidra_helpers.get_file_imports()
-        self.externs = ghidra_helpers.get_file_externs()
-        self.fakes = ghidra_helpers.map_fake_import_addrs()
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(currentProgram().getImageBase().getOffset())  # type: ignore [name-defined] # noqa: F821
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.ghidra.file.extract_features()
-
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        import capa.features.extractors.ghidra.helpers as ghidra_helpers
-
-        for fhandle in ghidra_helpers.get_function_symbols():
-            fh: FunctionHandle = FunctionHandle(
-                address=AbsoluteVirtualAddress(fhandle.getEntryPoint().getOffset()),
-                inner=fhandle,
-                ctx={"imports_cache": self.imports, "externs_cache": self.externs, "fakes_cache": self.fakes},
-            )
-            yield fh
-
-    @staticmethod
-    def get_function(addr: int) -> FunctionHandle:
-        func = getFunctionContaining(toAddr(addr))  # type: ignore [name-defined] # noqa: F821
-        return FunctionHandle(address=AbsoluteVirtualAddress(func.getEntryPoint().getOffset()), inner=func)
-
-    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.ghidra.function.extract_features(fh)
-
-    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
-        import capa.features.extractors.ghidra.helpers as ghidra_helpers
-
-        yield from ghidra_helpers.get_function_blocks(fh)
-
-    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.ghidra.basicblock.extract_features(fh, bbh)
-
-    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
-        import capa.features.extractors.ghidra.helpers as ghidra_helpers
-
-        yield from ghidra_helpers.get_insn_in_range(bbh)
-
-    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
-        yield from capa.features.extractors.ghidra.insn.extract_features(fh, bbh, ih)

capa/features/extractors/ghidra/file.py

@@ -1,204 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import re
-import struct
-from typing import List, Tuple, Iterator
-
-from ghidra.program.model.symbol import SourceType, SymbolType
-
-import capa.features.extractors.common
-import capa.features.extractors.helpers
-import capa.features.extractors.strings
-import capa.features.extractors.ghidra.helpers
-from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
-from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
-
-MAX_OFFSET_PE_AFTER_MZ = 0x200
-
-
-def find_embedded_pe(block_bytez: bytes, mz_xor: List[Tuple[bytes, bytes, int]]) -> Iterator[Tuple[int, int]]:
-    """check segment for embedded PE
-
-    adapted for Ghidra from:
-    https://github.com/vivisect/vivisect/blob/91e8419a861f4977https://github.com/vivisect/vivisect/blob/91e8419a861f49779f18316f155311967e696836/PE/carve.py#L259f18316f155311967e696836/PE/carve.py#L25
-    """
-    todo = []
-
-    for mzx, pex, i in mz_xor:
-        for match in re.finditer(re.escape(mzx), block_bytez):
-            todo.append((match.start(), mzx, pex, i))
-
-    seg_max = len(block_bytez)  # noqa: F821
-    while len(todo):
-        off, mzx, pex, i = todo.pop()
-
-        # MZ header has one field we will check e_lfanew is at 0x3c
-        e_lfanew = off + 0x3C
-
-        if seg_max < e_lfanew + 4:
-            continue
-
-        e_lfanew_bytes = block_bytez[e_lfanew : e_lfanew + 4]
-        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(e_lfanew_bytes, i))[0]
-
-        # assume XOR'd "PE" bytes exist within threshold
-        if newoff > MAX_OFFSET_PE_AFTER_MZ:
-            continue
-
-        peoff = off + newoff
-        if seg_max < peoff + 2:
-            continue
-
-        pe_bytes = block_bytez[peoff : peoff + 2]
-        if pe_bytes == pex:
-            yield off, i
-
-
-def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
-    """extract embedded PE features"""
-
-    # pre-compute XOR pairs
-    mz_xor: List[Tuple[bytes, bytes, int]] = [
-        (
-            capa.features.extractors.helpers.xor_static(b"MZ", i),
-            capa.features.extractors.helpers.xor_static(b"PE", i),
-            i,
-        )
-        for i in range(256)
-    ]
-
-    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
-        if not all((block.isLoaded(), block.isInitialized(), "Headers" not in block.getName())):
-            continue
-
-        for off, _ in find_embedded_pe(capa.features.extractors.ghidra.helpers.get_block_bytes(block), mz_xor):
-            # add offset back to block start
-            ea: int = block.getStart().add(off).getOffset()
-
-            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
-
-
-def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract function exports"""
-    st = currentProgram().getSymbolTable()  # type: ignore [name-defined] # noqa: F821
-    for addr in st.getExternalEntryPointIterator():
-        yield Export(st.getPrimarySymbol(addr).getName()), AbsoluteVirtualAddress(addr.getOffset())
-
-
-def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract function imports
-
-    1. imports by ordinal:
-     - modulename.#ordinal
-
-    2. imports by name, results in two features to support importname-only
-       matching:
-     - modulename.importname
-     - importname
-    """
-
-    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
-        for r in f.getSymbol().getReferences():
-            if r.getReferenceType().isData():
-                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
-
-        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_*
-        if "Ordinal_" in fstr[1]:
-            fstr[1] = f"#{fstr[1].split('_')[1]}"
-
-        for name in capa.features.extractors.helpers.generate_symbols(fstr[0][:-4], fstr[1], include_dll=True):
-            yield Import(name), AbsoluteVirtualAddress(addr)
-
-
-def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract section names"""
-
-    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
-        yield Section(block.getName()), AbsoluteVirtualAddress(block.getStart().getOffset())
-
-
-def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
-    """extract ASCII and UTF-16 LE strings"""
-
-    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
-        if not block.isInitialized():
-            continue
-
-        p_bytes = capa.features.extractors.ghidra.helpers.get_block_bytes(block)
-
-        for s in capa.features.extractors.strings.extract_ascii_strings(p_bytes):
-            offset = block.getStart().getOffset() + s.offset
-            yield String(s.s), FileOffsetAddress(offset)
-
-        for s in capa.features.extractors.strings.extract_unicode_strings(p_bytes):
-            offset = block.getStart().getOffset() + s.offset
-            yield String(s.s), FileOffsetAddress(offset)
-
-
-def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract the names of statically-linked library functions.
-    """
-
-    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
-        # .isExternal() misses more than this config for the function symbols
-        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
-            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
-            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
-                name = name[13:]
-            addr = AbsoluteVirtualAddress(sym.getAddress().getOffset())
-            yield FunctionName(name), addr
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                yield FunctionName(name[1:]), addr
-
-
-def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
-    ef = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
-    if "PE" in ef:
-        yield Format(FORMAT_PE), NO_ADDRESS
-    elif "ELF" in ef:
-        yield Format(FORMAT_ELF), NO_ADDRESS
-    elif "Raw" in ef:
-        # no file type to return when processing a binary file, but we want to continue processing
-        return
-    else:
-        raise NotImplementedError(f"unexpected file format: {ef}")
-
-
-def extract_features() -> Iterator[Tuple[Feature, Address]]:
-    """extract file features"""
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler():
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_file_embedded_pe,
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_section_names,
-    extract_file_strings,
-    extract_file_function_names,
-    extract_file_format,
-)
-
-
-def main():
-    """ """
-    import pprint
-
-    pprint.pprint(list(extract_features()))  # noqa: T203
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ghidra/function.py

@@ -1,73 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Tuple, Iterator
-
-import ghidra
-from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
-
-import capa.features.extractors.ghidra.helpers
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors import loops
-from capa.features.extractors.base_extractor import FunctionHandle
-
-
-def extract_function_calls_to(fh: FunctionHandle):
-    """extract callers to a function"""
-    f: ghidra.program.database.function.FunctionDB = fh.inner
-    for ref in f.getSymbol().getReferences():
-        if ref.getReferenceType().isCall():
-            yield Characteristic("calls to"), AbsoluteVirtualAddress(ref.getFromAddress().getOffset())
-
-
-def extract_function_loop(fh: FunctionHandle):
-    f: ghidra.program.database.function.FunctionDB = fh.inner
-
-    edges = []
-    for block in SimpleBlockIterator(BasicBlockModel(currentProgram()), f.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
-        dests = block.getDestinations(monitor())  # type: ignore [name-defined] # noqa: F821
-        s_addrs = block.getStartAddresses()
-
-        while dests.hasNext():  # For loop throws Python TypeError
-            for addr in s_addrs:
-                edges.append((addr.getOffset(), dests.next().getDestinationAddress().getOffset()))
-
-    if loops.has_loop(edges):
-        yield Characteristic("loop"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
-
-
-def extract_recursive_call(fh: FunctionHandle):
-    f: ghidra.program.database.function.FunctionDB = fh.inner
-
-    for func in f.getCalledFunctions(monitor()):  # type: ignore [name-defined] # noqa: F821
-        if func.getEntryPoint().getOffset() == f.getEntryPoint().getOffset():
-            yield Characteristic("recursive call"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
-
-
-def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    for func_handler in FUNCTION_HANDLERS:
-        for feature, addr in func_handler(fh):
-            yield feature, addr
-
-
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
-
-
-def main():
-    """ """
-    features = []
-    for fhandle in capa.features.extractors.ghidra.helpers.get_function_symbols():
-        features.extend(list(extract_features(fhandle)))
-
-    import pprint
-
-    pprint.pprint(features)  # noqa: T203
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/ghidra/global_.py

@@ -1,67 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-import contextlib
-from typing import Tuple, Iterator
-
-import capa.ghidra.helpers
-import capa.features.extractors.elf
-import capa.features.extractors.ghidra.helpers
-from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
-from capa.features.address import NO_ADDRESS, Address
-
-logger = logging.getLogger(__name__)
-
-
-def extract_os() -> Iterator[Tuple[Feature, Address]]:
-    format_name: str = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
-
-    if "PE" in format_name:
-        yield OS(OS_WINDOWS), NO_ADDRESS
-
-    elif "ELF" in format_name:
-        with contextlib.closing(capa.ghidra.helpers.GHIDRAIO()) as f:
-            os = capa.features.extractors.elf.detect_elf_os(f)
-
-        yield OS(os), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a further CLI argument to specify the OS,
-        # but i think this would be rarely used.
-        # rules that rely on OS conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess OS", format_name)
-        return
-
-
-def extract_arch() -> Iterator[Tuple[Feature, Address]]:
-    lang_id = currentProgram().getMetadata().get("Language ID")  # type: ignore [name-defined] # noqa: F821
-
-    if "x86" in lang_id and "64" in lang_id:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-
-    elif "x86" in lang_id and "32" in lang_id:
-        yield Arch(ARCH_I386), NO_ADDRESS
-
-    elif "x86" not in lang_id:
-        logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel")
-        return
-
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", lang_id)
-        return

capa/features/extractors/ghidra/helpers.py

@@ -1,301 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Dict, List, Iterator
-
-import ghidra
-import java.lang
-from ghidra.program.model.lang import OperandType
-from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
-from ghidra.program.model.symbol import SourceType, SymbolType
-from ghidra.program.model.address import AddressSpace
-
-import capa.features.extractors.helpers
-from capa.features.common import THUNK_CHAIN_DEPTH_DELTA
-from capa.features.address import AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-
-def ints_to_bytes(bytez: List[int]) -> bytes:
-    """convert Java signed ints to Python bytes
-
-    args:
-        bytez: list of Java signed ints
-    """
-    return bytes([b & 0xFF for b in bytez])
-
-
-def find_byte_sequence(addr: ghidra.program.model.address.Address, seq: bytes) -> Iterator[int]:
-    """yield all ea of a given byte sequence
-
-    args:
-        addr: start address
-        seq: bytes to search e.g. b"\x01\x03"
-    """
-    seqstr = "".join([f"\\x{b:02x}" for b in seq])
-    eas = findBytes(addr, seqstr, java.lang.Integer.MAX_VALUE, 1)  # type: ignore [name-defined] # noqa: F821
-
-    yield from eas
-
-
-def get_bytes(addr: ghidra.program.model.address.Address, length: int) -> bytes:
-    """yield length bytes at addr
-
-    args:
-        addr: Address to begin pull from
-        length: length of bytes to pull
-    """
-    try:
-        return ints_to_bytes(getBytes(addr, length))  # type: ignore [name-defined] # noqa: F821
-    except RuntimeError:
-        return b""
-
-
-def get_block_bytes(block: ghidra.program.model.mem.MemoryBlock) -> bytes:
-    """yield all bytes in a given block
-
-    args:
-        block: MemoryBlock to pull from
-    """
-    return get_bytes(block.getStart(), block.getSize())
-
-
-def get_function_symbols():
-    """yield all non-external function symbols"""
-    yield from currentProgram().getFunctionManager().getFunctionsNoStubs(True)  # type: ignore [name-defined] # noqa: F821
-
-
-def get_function_blocks(fh: FunctionHandle) -> Iterator[BBHandle]:
-    """yield BBHandle for each bb in a given function"""
-
-    func: ghidra.program.database.function.FunctionDB = fh.inner
-    for bb in SimpleBlockIterator(BasicBlockModel(currentProgram()), func.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
-        yield BBHandle(address=AbsoluteVirtualAddress(bb.getMinAddress().getOffset()), inner=bb)
-
-
-def get_insn_in_range(bbh: BBHandle) -> Iterator[InsnHandle]:
-    """yield InshHandle for each insn in a given basicblock"""
-    for insn in currentProgram().getListing().getInstructions(bbh.inner, True):  # type: ignore [name-defined] # noqa: F821
-        yield InsnHandle(address=AbsoluteVirtualAddress(insn.getAddress().getOffset()), inner=insn)
-
-
-def get_file_imports() -> Dict[int, List[str]]:
-    """get all import names & addrs"""
-
-    import_dict: Dict[int, List[str]] = {}
-
-    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
-        for r in f.getSymbol().getReferences():
-            if r.getReferenceType().isData():
-                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
-
-        ex_loc = f.getExternalLocation().getAddress()  # map external locations as well (offset into module files)
-
-        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_* / <EXTERNAL>::import
-        if "Ordinal_" in fstr[1]:
-            fstr[1] = f"#{fstr[1].split('_')[1]}"
-
-        # <EXTERNAL> mostly shows up in ELF files, otherwise, strip '.dll' w/ [:-4]
-        fstr[0] = "*" if "<EXTERNAL>" in fstr[0] else fstr[0][:-4]
-
-        for name in capa.features.extractors.helpers.generate_symbols(fstr[0], fstr[1]):
-            import_dict.setdefault(addr, []).append(name)
-            if ex_loc:
-                import_dict.setdefault(ex_loc.getOffset(), []).append(name)
-
-    return import_dict
-
-
-def get_file_externs() -> Dict[int, List[str]]:
-    """
-    Gets function names & addresses of statically-linked library functions
-
-    Ghidra's external namespace is mostly reserved for dynamically-linked
-    imports. Statically-linked functions are part of the global namespace.
-    Filtering on the type, source, and namespace of the symbols yield more
-    statically-linked library functions.
-
-    Example: (PMA Lab 16-01.exe_) 7faafc7e4a5c736ebfee6abbbc812d80:0x407490
-    - __aulldiv
-    - Note: See Symbol Table labels
-    """
-
-    extern_dict: Dict[int, List[str]] = {}
-
-    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
-        # .isExternal() misses more than this config for the function symbols
-        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
-            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
-            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
-                name = name[13:]
-            extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name)
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name[1:])
-
-    return extern_dict
-
-
-def map_fake_import_addrs() -> Dict[int, List[int]]:
-    """
-    Map ghidra's fake import entrypoints to their
-    real addresses
-
-    Helps as many Ghidra Scripting API calls end up returning
-    these external (fake) addresses.
-
-    Undocumented but intended Ghidra behavior:
-     - Import entryPoint fields are stored in the 'EXTERNAL:' AddressSpace.
-       'getEntryPoint()' returns the entryPoint field, which is an offset
-       from the beginning of the assigned AddressSpace. In the case of externals,
-       they start from 1 and increment.
-    https://github.com/NationalSecurityAgency/ghidra/blob/26d4bd9104809747c21f2528cab8aba9aef9acd5/Ghidra/Features/Base/src/test.slow/java/ghidra/program/database/function/ExternalFunctionDBTest.java#L90
-
-    Example: (mimikatz.exe_) 5f66b82558ca92e54e77f216ef4c066c:0x473090
-    - 0x473090 -> PTR_CreateServiceW_00473090
-    - 'EXTERNAL:00000025' -> External Address (ghidra.program.model.address.SpecialAddress)
-    """
-    fake_dict: Dict[int, List[int]] = {}
-
-    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
-        for r in f.getSymbol().getReferences():
-            if r.getReferenceType().isData():
-                fake_dict.setdefault(f.getEntryPoint().getOffset(), []).append(r.getFromAddress().getOffset())
-
-    return fake_dict
-
-
-def check_addr_for_api(
-    addr: ghidra.program.model.address.Address,
-    fakes: Dict[int, List[int]],
-    imports: Dict[int, List[str]],
-    externs: Dict[int, List[str]],
-) -> bool:
-    offset = addr.getOffset()
-
-    fake = fakes.get(offset)
-    if fake:
-        return True
-
-    imp = imports.get(offset)
-    if imp:
-        return True
-
-    extern = externs.get(offset)
-    if extern:
-        return True
-
-    return False
-
-
-def is_call_or_jmp(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    return any(mnem in insn.getMnemonicString() for mnem in ["CALL", "J"])  # JMP, JNE, JNZ, etc
-
-
-def is_sp_modified(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    for i in range(insn.getNumOperands()):
-        if insn.getOperandType(i) == OperandType.REGISTER:
-            return "SP" in insn.getRegister(i).getName() and insn.getOperandRefType(i).isWrite()
-    return False
-
-
-def is_stack_referenced(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    """generic catch-all for stack references"""
-    for i in range(insn.getNumOperands()):
-        if insn.getOperandType(i) == OperandType.REGISTER:
-            if "BP" in insn.getRegister(i).getName():
-                return True
-            else:
-                continue
-
-    return any(ref.isStackReference() for ref in insn.getReferencesFrom())
-
-
-def is_zxor(insn: ghidra.program.database.code.InstructionDB) -> bool:
-    # assume XOR insn
-    # XOR's against the same operand zero out
-    ops = []
-    operands = []
-    for i in range(insn.getNumOperands()):
-        ops.append(insn.getOpObjects(i))
-
-    # Operands stored in a 2D array
-    for j in range(len(ops)):
-        for k in range(len(ops[j])):
-            operands.append(ops[j][k])
-
-    return all(n == operands[0] for n in operands)
-
-
-def handle_thunk(addr: ghidra.program.model.address.Address):
-    """Follow thunk chains down to a reasonable depth"""
-    ref = addr
-    for _ in range(THUNK_CHAIN_DEPTH_DELTA):
-        thunk_jmp = getInstructionAt(ref)  # type: ignore [name-defined] # noqa: F821
-        if thunk_jmp and is_call_or_jmp(thunk_jmp):
-            if OperandType.isAddress(thunk_jmp.getOperandType(0)):
-                ref = thunk_jmp.getAddress(0)
-        else:
-            thunk_dat = getDataContaining(ref)  # type: ignore [name-defined] # noqa: F821
-            if thunk_dat and thunk_dat.isDefined() and thunk_dat.isPointer():
-                ref = thunk_dat.getValue()
-                break  # end of thunk chain reached
-    return ref
-
-
-def dereference_ptr(insn: ghidra.program.database.code.InstructionDB):
-    addr_code = OperandType.ADDRESS | OperandType.CODE
-    to_deref = insn.getAddress(0)
-    dat = getDataContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
-
-    if insn.getOperandType(0) == addr_code:
-        thfunc = getFunctionContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
-        if thfunc and thfunc.isThunk():
-            return handle_thunk(to_deref)
-        else:
-            # if it doesn't poin to a thunk, it's usually a jmp to a label
-            return to_deref
-    if not dat:
-        return to_deref
-    if dat.isDefined() and dat.isPointer():
-        addr = dat.getValue()
-        # now we need to check the addr space to see if it is truly resolvable
-        # ghidra sometimes likes to hand us direct RAM addrs, which typically point
-        # to api calls that we can't actually resolve as such
-        if addr.getAddressSpace().getType() == AddressSpace.TYPE_RAM:
-            return to_deref
-        else:
-            return addr
-    else:
-        return to_deref
-
-
-def find_data_references_from_insn(insn, max_depth: int = 10):
-    """yield data references from given instruction"""
-    for reference in insn.getReferencesFrom():
-        if not reference.getReferenceType().isData():
-            # only care about data references
-            continue
-
-        to_addr = reference.getToAddress()
-
-        for _ in range(max_depth - 1):
-            data = getDataAt(to_addr)  # type: ignore [name-defined] # noqa: F821
-            if data and data.isPointer():
-                ptr_value = data.getValue()
-
-                if ptr_value is None:
-                    break
-
-                to_addr = ptr_value
-            else:
-                break
-
-        yield to_addr

capa/features/extractors/ghidra/insn.py

@@ -1,503 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Any, Dict, Tuple, Iterator
-
-import ghidra
-from ghidra.program.model.lang import OperandType
-from ghidra.program.model.block import SimpleBlockModel
-
-import capa.features.extractors.helpers
-import capa.features.extractors.ghidra.helpers
-from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
-from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-# security cookie checks may perform non-zeroing XORs, these are expected within a certain
-# byte range within the first and returning basic blocks, this helps to reduce FP features
-SECURITY_COOKIE_BYTES_DELTA = 0x40
-
-
-OPERAND_TYPE_DYNAMIC_ADDRESS = OperandType.DYNAMIC | OperandType.ADDRESS
-
-
-def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
-    """Populate the import cache for this context"""
-    if "imports_cache" not in ctx:
-        ctx["imports_cache"] = capa.features.extractors.ghidra.helpers.get_file_imports()
-    return ctx["imports_cache"]
-
-
-def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
-    """Populate the externs cache for this context"""
-    if "externs_cache" not in ctx:
-        ctx["externs_cache"] = capa.features.extractors.ghidra.helpers.get_file_externs()
-    return ctx["externs_cache"]
-
-
-def get_fakes(ctx: Dict[str, Any]) -> Dict[int, Any]:
-    """Populate the fake import addrs cache for this context"""
-    if "fakes_cache" not in ctx:
-        ctx["fakes_cache"] = capa.features.extractors.ghidra.helpers.map_fake_import_addrs()
-    return ctx["fakes_cache"]
-
-
-def check_for_api_call(
-    insn, externs: Dict[int, Any], fakes: Dict[int, Any], imports: Dict[int, Any], imp_or_ex: bool
-) -> Iterator[Any]:
-    """check instruction for API call
-
-    params:
-        externs - external library functions cache
-        fakes - mapped fake import addresses cache
-        imports - imported functions cache
-        imp_or_ex - flag to check imports or externs
-
-    yields:
-        matched api calls
-    """
-    info = ()
-    funcs = imports if imp_or_ex else externs
-
-    # assume only CALLs or JMPs are passed
-    ref_type = insn.getOperandType(0)
-    addr_data = OperandType.ADDRESS | OperandType.DATA  # needs dereferencing
-    addr_code = OperandType.ADDRESS | OperandType.CODE  # needs dereferencing
-
-    if OperandType.isRegister(ref_type):
-        if OperandType.isAddress(ref_type):
-            # If it's an address in a register, check the mapped fake addrs
-            # since they're dereferenced to their fake addrs
-            op_ref = insn.getAddress(0).getOffset()
-            ref = fakes.get(op_ref)  # obtain the real addr
-            if not ref:
-                return
-        else:
-            return
-    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
-        # we must dereference and check if the addr is a pointer to an api function
-        addr_ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
-        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
-            return
-        ref = addr_ref.getOffset()
-    elif ref_type == OPERAND_TYPE_DYNAMIC_ADDRESS or ref_type == OperandType.DYNAMIC:
-        return  # cannot resolve dynamics statically
-    else:
-        # pure address does not need to get dereferenced/ handled
-        addr_ref = insn.getAddress(0)
-        if not addr_ref:
-            # If it returned null, it was an indirect
-            # that had no address reference.
-            # This check is faster than checking for (indirect and not address)
-            return
-        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
-            return
-        ref = addr_ref.getOffset()
-
-    if isinstance(ref, list):  # ref from REG | ADDR
-        for r in ref:
-            info = funcs.get(r)  # type: ignore
-            if info:
-                yield info
-    else:
-        info = funcs.get(ref)  # type: ignore
-        if info:
-            yield info
-
-
-def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
-        return
-
-    externs = get_externs(fh.ctx)
-    fakes = get_fakes(fh.ctx)
-    imports = get_imports(fh.ctx)
-
-    # check calls to imported functions
-    for api in check_for_api_call(insn, externs, fakes, imports, True):
-        for imp in api:
-            yield API(imp), ih.address
-
-    # check calls to extern functions
-    for api in check_for_api_call(insn, externs, fakes, imports, False):
-        for ext in api:
-            yield API(ext), ih.address
-
-
-def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction number features
-    example:
-        push    3136B0h         ; dwControlCode
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if insn.getMnemonicString().startswith("RET"):
-        # skip things like:
-        #   .text:0042250E retn 8
-        return
-
-    if capa.features.extractors.ghidra.helpers.is_sp_modified(insn):
-        # skip things like:
-        #   .text:00401145 add esp, 0Ch
-        return
-
-    for i in range(insn.getNumOperands()):
-        # Exceptions for LEA insn:
-        # invalid operand encoding, considered numbers instead of offsets
-        # see: mimikatz.exe_:0x4018C0
-        if insn.getOperandType(i) == OperandType.DYNAMIC and insn.getMnemonicString().startswith("LEA"):
-            # Additional check, avoid yielding "wide" values (ex. mimikatz.exe:0x471EE6 LEA EBX, [ECX + EAX*0x4])
-            op_objs = insn.getOpObjects(i)
-            if len(op_objs) == 3:  # ECX, EAX, 0x4
-                continue
-
-            if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
-                const = op_objs[-1].getUnsignedValue()
-                addr = ih.address
-
-                yield Number(const), addr
-                yield OperandNumber(i, const), addr
-        elif not OperandType.isScalar(insn.getOperandType(i)):
-            # skip things like:
-            #   references, void types
-            continue
-        else:
-            const = insn.getScalar(i).getUnsignedValue()
-            addr = ih.address
-
-            yield Number(const), addr
-            yield OperandNumber(i, const), addr
-
-            if insn.getMnemonicString().startswith("ADD") and 0 < const < MAX_STRUCTURE_SIZE:
-                # for pattern like:
-                #
-                #     add eax, 0x10
-                #
-                # assume 0x10 is also an offset (imagine eax is a pointer).
-                yield Offset(const), addr
-                yield OperandOffset(i, const), addr
-
-
-def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction structure offset features
-
-    example:
-        .text:0040112F cmp [esi+4], ebx
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if insn.getMnemonicString().startswith("LEA"):
-        return
-
-    if capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
-        # ignore stack references
-        return
-
-    # Ghidra stores operands in 2D arrays if they contain offsets
-    for i in range(insn.getNumOperands()):
-        if insn.getOperandType(i) == OperandType.DYNAMIC:  # e.g. [esi + 4]
-            # manual extraction, since the default api calls only work on the 1st dimension of the array
-            op_objs = insn.getOpObjects(i)
-            if not op_objs:
-                continue
-
-            if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
-                op_off = op_objs[-1].getValue()
-            else:
-                op_off = 0
-
-            yield Offset(op_off), ih.address
-            yield OperandOffset(i, op_off), ih.address
-
-
-def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse referenced byte sequences
-
-    example:
-        push    offset iid_004118d4_IShellLinkA ; riid
-    """
-    for addr in capa.features.extractors.ghidra.helpers.find_data_references_from_insn(ih.inner):
-        data = getDataAt(addr)  # type: ignore [name-defined] # noqa: F821
-        if data and not data.hasStringValue():
-            extracted_bytes = capa.features.extractors.ghidra.helpers.get_bytes(addr, MAX_BYTES_FEATURE_SIZE)
-            if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-                yield Bytes(extracted_bytes), ih.address
-
-
-def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction string features
-
-    example:
-        push offset aAcr     ; "ACR  > "
-    """
-    for addr in capa.features.extractors.ghidra.helpers.find_data_references_from_insn(ih.inner):
-        data = getDataAt(addr)  # type: ignore [name-defined] # noqa: F821
-        if data and data.hasStringValue():
-            yield String(data.getValue()), ih.address
-
-
-def extract_insn_mnemonic_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction mnemonic features"""
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    yield Mnemonic(insn.getMnemonicString().lower()), ih.address
-
-
-def extract_insn_obfs_call_plus_5_characteristic_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse call $+5 instruction from the given instruction.
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
-        return
-
-    code_ref = OperandType.ADDRESS | OperandType.CODE
-    ref = insn.getAddress()
-    for i in range(insn.getNumOperands()):
-        if insn.getOperandType(i) == code_ref:
-            ref = insn.getAddress(i)
-
-    if insn.getAddress().add(5) == ref:
-        yield Characteristic("call $+5"), ih.address
-
-
-def extract_insn_segment_access_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction fs or gs access"""
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    insn_str = insn.toString()
-
-    if "FS:" in insn_str:
-        yield Characteristic("fs access"), ih.address
-
-    if "GS:" in insn_str:
-        yield Characteristic("gs access"), ih.address
-
-
-def extract_insn_peb_access_characteristic_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction peb access
-
-    fs:[0x30] on x86, gs:[0x60] on x64
-
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    insn_str = insn.toString()
-    if insn_str.startswith(("PUSH", "MOV")):
-        if "FS:[0x30]" in insn_str or "GS:[0x60]" in insn_str:
-            yield Characteristic("peb access"), ih.address
-
-
-def extract_insn_cross_section_cflow(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
-        return
-
-    externs = get_externs(fh.ctx)
-    fakes = get_fakes(fh.ctx)
-    imports = get_imports(fh.ctx)
-
-    # OperandType to dereference
-    addr_data = OperandType.ADDRESS | OperandType.DATA
-    addr_code = OperandType.ADDRESS | OperandType.CODE
-
-    ref_type = insn.getOperandType(0)
-
-    # both OperandType flags must be present
-    # bail on REGISTER alone
-    if OperandType.isRegister(ref_type):
-        if OperandType.isAddress(ref_type):
-            ref = insn.getAddress(0)  # Ghidra dereferences REG | ADDR
-            if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
-                return
-        else:
-            return
-    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
-        # we must dereference and check if the addr is a pointer to an api function
-        ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
-        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
-            return
-    elif ref_type == OPERAND_TYPE_DYNAMIC_ADDRESS or ref_type == OperandType.DYNAMIC:
-        return  # cannot resolve dynamics statically
-    else:
-        # pure address does not need to get dereferenced/ handled
-        ref = insn.getAddress(0)
-        if not ref:
-            # If it returned null, it was an indirect
-            # that had no address reference.
-            # This check is faster than checking for (indirect and not address)
-            return
-        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
-            return
-
-    this_mem_block = getMemoryBlock(insn.getAddress())  # type: ignore [name-defined] # noqa: F821
-    ref_block = getMemoryBlock(ref)  # type: ignore [name-defined] # noqa: F821
-    if ref_block != this_mem_block:
-        yield Characteristic("cross section flow"), ih.address
-
-
-def extract_function_calls_from(
-    fh: FunctionHandle,
-    bb: BBHandle,
-    ih: InsnHandle,
-) -> Iterator[Tuple[Feature, Address]]:
-    """extract functions calls from features
-
-    most relevant at the function scope, however, its most efficient to extract at the instruction scope
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if insn.getMnemonicString().startswith("CALL"):
-        # This method of "dereferencing" addresses/ pointers
-        # is not as robust as methods in other functions,
-        # but works just fine for this one
-        reference = 0
-        for ref in insn.getReferencesFrom():
-            addr = ref.getToAddress()
-
-            # avoid returning fake addrs
-            if not addr.isExternalAddress():
-                reference = addr.getOffset()
-
-            # if a reference is < 0, then ghidra pulled an offset from a DYNAMIC | ADDR (usually a stackvar)
-            # these cannot be resolved to actual addrs
-            if reference > 0:
-                yield Characteristic("calls from"), AbsoluteVirtualAddress(reference)
-
-
-def extract_function_indirect_call_characteristic_features(
-    fh: FunctionHandle,
-    bb: BBHandle,
-    ih: InsnHandle,
-) -> Iterator[Tuple[Feature, Address]]:
-    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
-    does not include calls like => call ds:dword_ABD4974
-
-    most relevant at the function or basic block scope;
-    however, its most efficient to extract at the instruction scope
-    """
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if insn.getMnemonicString().startswith("CALL"):
-        if OperandType.isRegister(insn.getOperandType(0)):
-            yield Characteristic("indirect call"), ih.address
-        if OperandType.isIndirect(insn.getOperandType(0)):
-            yield Characteristic("indirect call"), ih.address
-
-
-def check_nzxor_security_cookie_delta(
-    fh: ghidra.program.database.function.FunctionDB, insn: ghidra.program.database.code.InstructionDB
-):
-    """Get the function containing the insn
-    Get the last block of the function that contains the insn
-
-    Check the bb containing the insn
-    Check the last bb of the function containing the insn
-    """
-
-    model = SimpleBlockModel(currentProgram())  # type: ignore [name-defined] # noqa: F821
-    insn_addr = insn.getAddress()
-    func_asv = fh.getBody()
-    first_addr = func_asv.getMinAddress()
-    last_addr = func_asv.getMaxAddress()
-
-    if model.getFirstCodeBlockContaining(
-        first_addr, monitor()  # type: ignore [name-defined] # noqa: F821
-    ) == model.getFirstCodeBlockContaining(
-        last_addr, monitor()  # type: ignore [name-defined] # noqa: F821
-    ):
-        if insn_addr < first_addr.add(SECURITY_COOKIE_BYTES_DELTA):
-            return True
-        else:
-            return insn_addr > last_addr.add(SECURITY_COOKIE_BYTES_DELTA * -1)
-    else:
-        return False
-
-
-def extract_insn_nzxor_characteristic_features(
-    fh: FunctionHandle,
-    bb: BBHandle,
-    ih: InsnHandle,
-) -> Iterator[Tuple[Feature, Address]]:
-    f: ghidra.program.database.function.FunctionDB = fh.inner
-    insn: ghidra.program.database.code.InstructionDB = ih.inner
-
-    if "XOR" not in insn.getMnemonicString():
-        return
-    if capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
-        return
-    if capa.features.extractors.ghidra.helpers.is_zxor(insn):
-        return
-    if check_nzxor_security_cookie_delta(f, insn):
-        return
-    yield Characteristic("nzxor"), ih.address
-
-
-def extract_features(
-    fh: FunctionHandle,
-    bb: BBHandle,
-    insn: InsnHandle,
-) -> Iterator[Tuple[Feature, Address]]:
-    for insn_handler in INSTRUCTION_HANDLERS:
-        for feature, addr in insn_handler(fh, bb, insn):
-            yield feature, addr
-
-
-INSTRUCTION_HANDLERS = (
-    extract_insn_api_features,
-    extract_insn_number_features,
-    extract_insn_bytes_features,
-    extract_insn_string_features,
-    extract_insn_offset_features,
-    extract_insn_nzxor_characteristic_features,
-    extract_insn_mnemonic_features,
-    extract_insn_obfs_call_plus_5_characteristic_features,
-    extract_insn_peb_access_characteristic_features,
-    extract_insn_cross_section_cflow,
-    extract_insn_segment_access_features,
-    extract_function_calls_from,
-    extract_function_indirect_call_characteristic_features,
-)
-
-
-def main():
-    """ """
-    features = []
-    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
-
-    for fh in GhidraFeatureExtractor().get_functions():
-        for bb in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
-            for insn in capa.features.extractors.ghidra.helpers.get_insn_in_range(bb):
-                features.extend(list(extract_features(fh, bb, insn)))
-
-    import pprint
-
-    pprint.pprint(features)  # noqa: T203
-
-
-if __name__ == "__main__":
-    main()

capa/features/extractors/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,18 +6,23 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-import struct
+import sys
 import builtins
-from typing import Tuple, Iterator
+
+from capa.features.file import Import
+from capa.features.insn import API
 
 MIN_STACKSTRING_LEN = 8
 
 
-def xor_static(data: bytes, i: int) -> bytes:
-    return bytes(c ^ i for c in data)
+def xor_static(data, i):
+    if sys.version_info >= (3, 0):
+        return bytes(c ^ i for c in data)
+    else:
+        return "".join(chr(ord(c) ^ i) for c in data)
 
 
-def is_aw_function(symbol: str) -> bool:
+def is_aw_function(symbol):
     """
     is the given function name an A/W function?
     these are variants of functions that, on Windows, accept either a narrow or wide string.
@@ -29,10 +34,11 @@ def is_aw_function(symbol: str) -> bool:
     if symbol[-1] not in ("A", "W"):
         return False
 
-    return True
+    # second to last character should be lowercase letter
+    return "a" <= symbol[-2] <= "z" or "0" <= symbol[-2] <= "9"
 
 
-def is_ordinal(symbol: str) -> bool:
+def is_ordinal(symbol):
     """
     is the given symbol an ordinal that is prefixed by "#"?
     """
@@ -41,69 +47,37 @@ def is_ordinal(symbol: str) -> bool:
     return False
 
 
-def generate_symbols(dll: str, symbol: str, include_dll=False) -> Iterator[str]:
+def generate_symbols(dll, symbol):
     """
     for a given dll and symbol name, generate variants.
     we over-generate features to make matching easier.
     these include:
-      - CreateFileA
-      - CreateFile
-      - ws2_32.#1
-
-    note that since capa v7 only `import` features and APIs called via ordinal include DLL names:
       - kernel32.CreateFileA
       - kernel32.CreateFile
-      - ws2_32.#1
-
-    for `api` features dll names are good for documentation but not used during matching
+      - CreateFileA
+      - CreateFile
     """
-    # normalize dll name
-    dll = dll.lower()
-
-    # trim extensions observed in dynamic traces
-    dll = dll[0:-4] if dll.endswith(".dll") else dll
-    dll = dll[0:-4] if dll.endswith(".drv") else dll
-
-    if include_dll or is_ordinal(symbol):
-        # ws2_32.#1
-        # kernel32.CreateFileA
-        yield f"{dll}.{symbol}"
+    # kernel32.CreateFileA
+    yield "%s.%s" % (dll, symbol)
 
     if not is_ordinal(symbol):
         # CreateFileA
         yield symbol
 
-        if is_aw_function(symbol):
-            if include_dll:
-                # kernel32.CreateFile
-                yield f"{dll}.{symbol[:-1]}"
+    if is_aw_function(symbol):
+        # kernel32.CreateFile
+        yield "%s.%s" % (dll, symbol[:-1])
 
+        if not is_ordinal(symbol):
             # CreateFile
             yield symbol[:-1]
 
 
-def reformat_forwarded_export_name(forwarded_name: str) -> str:
-    """
-    a forwarded export has a DLL name/path and symbol name.
-    we want the former to be lowercase, and the latter to be verbatim.
-    """
-
-    # use rpartition so we can split on separator between dll and name.
-    # the dll name can be a full path, like in the case of
-    # ef64d6d7c34250af8e21a10feb931c9b
-    # which i assume means the path can have embedded periods.
-    # so we don't want the first period, we want the last.
-    forwarded_dll, _, forwarded_symbol = forwarded_name.rpartition(".")
-    forwarded_dll = forwarded_dll.lower()
-
-    return f"{forwarded_dll}.{forwarded_symbol}"
-
-
-def all_zeros(bytez: bytes) -> bool:
+def all_zeros(bytez):
     return all(b == 0 for b in builtins.bytes(bytez))
 
 
-def twos_complement(val: int, bits: int) -> int:
+def twos_complement(val, bits):
     """
     compute the 2's complement of int value val
 
@@ -116,48 +90,3 @@ def twos_complement(val: int, bits: int) -> int:
     else:
         # return positive value as is
         return val
-
-
-def carve_pe(pbytes: bytes, offset: int = 0) -> Iterator[Tuple[int, int]]:
-    """
-    Generate (offset, key) tuples of embedded PEs
-
-    Based on the version from vivisect:
-      https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
-    And its IDA adaptation:
-      capa/features/extractors/ida/file.py
-    """
-    mz_xor = [
-        (
-            xor_static(b"MZ", key),
-            xor_static(b"PE", key),
-            key,
-        )
-        for key in range(256)
-    ]
-
-    pblen = len(pbytes)
-    todo = [(pbytes.find(mzx, offset), mzx, pex, key) for mzx, pex, key in mz_xor]
-    todo = [(off, mzx, pex, key) for (off, mzx, pex, key) in todo if off != -1]
-
-    while len(todo):
-        off, mzx, pex, key = todo.pop()
-
-        # The MZ header has one field we will check
-        # e_lfanew is at 0x3c
-        e_lfanew = off + 0x3C
-        if pblen < (e_lfanew + 4):
-            continue
-
-        newoff = struct.unpack("<I", xor_static(pbytes[e_lfanew : e_lfanew + 4], key))[0]
-
-        nextres = pbytes.find(mzx, off + 1)
-        if nextres != -1:
-            todo.append((nextres, mzx, pex, key))
-
-        peoff = off + newoff
-        if pblen < (peoff + 2):
-            continue
-
-        if pbytes[peoff : peoff + 2] == pex:
-            yield (off, key)

capa/features/extractors/ida/__init__.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import sys
+import types
+
+import idaapi
+
+import capa.features.extractors.ida.file
+import capa.features.extractors.ida.insn
+import capa.features.extractors.ida.function
+import capa.features.extractors.ida.basicblock
+from capa.features.extractors import FeatureExtractor
+
+
+def get_ea(self):
+    """ """
+    if isinstance(self, (idaapi.BasicBlock, idaapi.func_t)):
+        return self.start_ea
+    if isinstance(self, idaapi.insn_t):
+        return self.ea
+    raise TypeError
+
+
+def add_ea_int_cast(o):
+    """
+    dynamically add a cast-to-int (`__int__`) method to the given object
+    that returns the value of the `.ea` property.
+    this bit of skullduggery lets use cast viv-utils objects as ints.
+    the correct way of doing this is to update viv-utils (or subclass the objects here).
+    """
+    if sys.version_info[0] >= 3:
+        setattr(o, "__int__", types.MethodType(get_ea, o))
+    else:
+        setattr(o, "__int__", types.MethodType(get_ea, o, type(o)))
+    return o
+
+
+class IdaFeatureExtractor(FeatureExtractor):
+    def __init__(self):
+        super(IdaFeatureExtractor, self).__init__()
+
+    def get_base_address(self):
+        return idaapi.get_imagebase()
+
+    def extract_file_features(self):
+        for (feature, ea) in capa.features.extractors.ida.file.extract_features():
+            yield feature, ea
+
+    def get_functions(self):
+        import capa.features.extractors.ida.helpers as ida_helpers
+
+        # data structure shared across functions yielded here.
+        # useful for caching analysis relevant across a single workspace.
+        ctx = {}
+
+        # ignore library functions and thunk functions as identified by IDA
+        for f in ida_helpers.get_functions(skip_thunks=True, skip_libs=True):
+            setattr(f, "ctx", ctx)
+            yield add_ea_int_cast(f)
+
+    @staticmethod
+    def get_function(ea):
+        f = idaapi.get_func(ea)
+        setattr(f, "ctx", {})
+        return add_ea_int_cast(f)
+
+    def extract_function_features(self, f):
+        for (feature, ea) in capa.features.extractors.ida.function.extract_features(f):
+            yield feature, ea
+
+    def get_basic_blocks(self, f):
+        for bb in capa.features.extractors.ida.helpers.get_function_blocks(f):
+            yield add_ea_int_cast(bb)
+
+    def extract_basic_block_features(self, f, bb):
+        for (feature, ea) in capa.features.extractors.ida.basicblock.extract_features(f, bb):
+            yield feature, ea
+
+    def get_instructions(self, f, bb):
+        import capa.features.extractors.ida.helpers as ida_helpers
+
+        for insn in ida_helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
+            yield add_ea_int_cast(insn)
+
+    def extract_insn_features(self, f, bb, insn):
+        for (feature, ea) in capa.features.extractors.ida.insn.extract_features(f, bb, insn):
+            yield feature, ea

capa/features/extractors/ida/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,23 +6,25 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+import sys
 import string
 import struct
-from typing import Tuple, Iterator
 
 import idaapi
 
 import capa.features.extractors.ida.helpers
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address
+from capa.features import Characteristic
 from capa.features.basicblock import BasicBlock
 from capa.features.extractors.ida import helpers
 from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
 
 
-def get_printable_len(op: idaapi.op_t) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
+def get_printable_len(op):
+    """Return string length if all operand bytes are ascii or utf16-le printable
+
+    args:
+        op (IDA op_t)
+    """
     op_val = capa.features.extractors.ida.helpers.mask_op_val(op)
 
     if op.dtype == idaapi.dt_byte:
@@ -34,14 +36,21 @@ def get_printable_len(op: idaapi.op_t) -> int:
     elif op.dtype == idaapi.dt_qword:
         chars = struct.pack("<Q", op_val)
     else:
-        raise ValueError(f"Unhandled operand data type 0x{op.dtype:x}.")
+        raise ValueError("Unhandled operand data type 0x%x." % op.dtype)
 
-    def is_printable_ascii(chars_: bytes):
-        return all(c < 127 and chr(c) in string.printable for c in chars_)
+    def is_printable_ascii(chars):
+        if sys.version_info[0] >= 3:
+            return all(c < 127 and chr(c) in string.printable for c in chars)
+        else:
+            return all(ord(c) < 127 and c in string.printable for c in chars)
 
-    def is_printable_utf16le(chars_: bytes):
-        if all(c == 0x00 for c in chars_[1::2]):
-            return is_printable_ascii(chars_[::2])
+    def is_printable_utf16le(chars):
+        if sys.version_info[0] >= 3:
+            if all(c == 0x00 for c in chars[1::2]):
+                return is_printable_ascii(chars[::2])
+        else:
+            if all(c == "\x00" for c in chars[1::2]):
+                return is_printable_ascii(chars[::2])
 
     if is_printable_ascii(chars):
         return idaapi.get_dtype_size(op.dtype)
@@ -52,8 +61,12 @@ def get_printable_len(op: idaapi.op_t) -> int:
     return 0
 
 
-def is_mov_imm_to_stack(insn: idaapi.insn_t) -> bool:
-    """verify instruction moves immediate onto stack"""
+def is_mov_imm_to_stack(insn):
+    """verify instruction moves immediate onto stack
+
+    args:
+        insn (IDA insn_t)
+    """
     if insn.Op2.type != idaapi.o_imm:
         return False
 
@@ -66,10 +79,14 @@ def is_mov_imm_to_stack(insn: idaapi.insn_t) -> bool:
     return True
 
 
-def bb_contains_stackstring(f: idaapi.func_t, bb: idaapi.BasicBlock) -> bool:
+def bb_contains_stackstring(f, bb):
     """check basic block for stackstring indicators
 
     true if basic block contains enough moves of constant bytes to the stack
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
     """
     count = 0
     for insn in capa.features.extractors.ida.helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
@@ -80,27 +97,57 @@ def bb_contains_stackstring(f: idaapi.func_t, bb: idaapi.BasicBlock) -> bool:
     return False
 
 
-def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract stackstring indicators from basic block"""
-    if bb_contains_stackstring(fh.inner, bbh.inner):
-        yield Characteristic("stack string"), bbh.address
+def extract_bb_stackstring(f, bb):
+    """extract stackstring indicators from basic block
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+    """
+    if bb_contains_stackstring(f, bb):
+        yield Characteristic("stack string"), bb.start_ea
 
 
-def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract tight loop indicators from a basic block"""
-    if capa.features.extractors.ida.helpers.is_basic_block_tight_loop(bbh.inner):
-        yield Characteristic("tight loop"), bbh.address
+def extract_bb_tight_loop(f, bb):
+    """extract tight loop indicators from a basic block
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+    """
+    if capa.features.extractors.ida.helpers.is_basic_block_tight_loop(bb):
+        yield Characteristic("tight loop"), bb.start_ea
 
 
-def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract basic block features"""
+def extract_features(f, bb):
+    """extract basic block features
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+    """
     for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, addr in bb_handler(fh, bbh):
-            yield feature, addr
-    yield BasicBlock(), bbh.address
+        for (feature, ea) in bb_handler(f, bb):
+            yield feature, ea
+    yield BasicBlock(), bb.start_ea
 
 
 BASIC_BLOCK_HANDLERS = (
     extract_bb_tight_loop,
     extract_bb_stackstring,
 )
+
+
+def main():
+    features = []
+    for f in helpers.get_functions(skip_thunks=True, skip_libs=True):
+        for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
+            features.extend(list(extract_features(f, bb)))
+
+    import pprint
+
+    pprint.pprint(features)
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ida/extractor.py

@@ -1,82 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import List, Tuple, Iterator
-
-import idaapi
-import ida_nalt
-
-import capa.ida.helpers
-import capa.features.extractors.elf
-import capa.features.extractors.ida.file
-import capa.features.extractors.ida.insn
-import capa.features.extractors.ida.global_
-import capa.features.extractors.ida.function
-import capa.features.extractors.ida.basicblock
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import (
-    BBHandle,
-    InsnHandle,
-    SampleHashes,
-    FunctionHandle,
-    StaticFeatureExtractor,
-)
-
-
-class IdaFeatureExtractor(StaticFeatureExtractor):
-    def __init__(self):
-        super().__init__(
-            hashes=SampleHashes(
-                md5=ida_nalt.retrieve_input_file_md5(), sha1="(unknown)", sha256=ida_nalt.retrieve_input_file_sha256()
-            )
-        )
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.ida.file.extract_file_format())
-        self.global_features.extend(capa.features.extractors.ida.global_.extract_os())
-        self.global_features.extend(capa.features.extractors.ida.global_.extract_arch())
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(idaapi.get_imagebase())
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.ida.file.extract_features()
-
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        import capa.features.extractors.ida.helpers as ida_helpers
-
-        # ignore library functions and thunk functions as identified by IDA
-        yield from ida_helpers.get_functions(skip_thunks=True, skip_libs=True)
-
-    @staticmethod
-    def get_function(ea: int) -> FunctionHandle:
-        f = idaapi.get_func(ea)
-        return FunctionHandle(address=AbsoluteVirtualAddress(f.start_ea), inner=f)
-
-    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.ida.function.extract_features(fh)
-
-    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
-        import capa.features.extractors.ida.helpers as ida_helpers
-
-        for bb in ida_helpers.get_function_blocks(fh.inner):
-            yield BBHandle(address=AbsoluteVirtualAddress(bb.start_ea), inner=bb)
-
-    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.ida.basicblock.extract_features(fh, bbh)
-
-    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
-        import capa.features.extractors.ida.helpers as ida_helpers
-
-        for insn in ida_helpers.get_instructions_in_range(bbh.inner.start_ea, bbh.inner.end_ea):
-            yield InsnHandle(address=AbsoluteVirtualAddress(insn.ea), inner=insn)
-
-    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
-        yield from capa.features.extractors.ida.insn.extract_features(fh, bbh, ih)

capa/features/extractors/ida/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,29 +7,26 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import struct
-from typing import Tuple, Iterator
 
 import idc
 import idaapi
 import idautils
-import ida_entry
 
-import capa.features.extractors.common
 import capa.features.extractors.helpers
 import capa.features.extractors.strings
 import capa.features.extractors.ida.helpers
-from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
-from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
-
-MAX_OFFSET_PE_AFTER_MZ = 0x200
+from capa.features import String, Characteristic
+from capa.features.file import Export, Import, Section
 
 
-def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
+def check_segment_for_pe(seg):
     """check segment for embedded PE
 
     adapted for IDA from:
-    https://github.com/vivisect/vivisect/blob/91e8419a861f49779f18316f155311967e696836/PE/carve.py#L25
+    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
+
+    args:
+        seg (IDA segment_t)
     """
     seg_max = seg.end_ea
     mz_xor = [
@@ -42,15 +39,14 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
     ]
 
     todo = []
-    for mzx, pex, i in mz_xor:
-        # find all segment offsets containing XOR'd "MZ" bytes
+    for (mzx, pex, i) in mz_xor:
         for off in capa.features.extractors.ida.helpers.find_byte_sequence(seg.start_ea, seg.end_ea, mzx):
             todo.append((off, mzx, pex, i))
 
     while len(todo):
         off, mzx, pex, i = todo.pop()
 
-        # MZ header has one field we will check e_lfanew is at 0x3c
+        # The MZ header has one field we will check e_lfanew is at 0x3c
         e_lfanew = off + 0x3C
 
         if seg_max < (e_lfanew + 4):
@@ -58,19 +54,18 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
 
         newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(idc.get_bytes(e_lfanew, 4), i))[0]
 
-        # assume XOR'd "PE" bytes exist within threshold
-        if newoff > MAX_OFFSET_PE_AFTER_MZ:
-            continue
-
         peoff = off + newoff
         if seg_max < (peoff + 2):
             continue
 
         if idc.get_bytes(peoff, 2) == pex:
-            yield off, i
+            yield (off, i)
+
+        for nextres in capa.features.extractors.ida.helpers.find_byte_sequence(off + 1, seg.end_ea, mzx):
+            todo.append((nextres, mzx, pex, i))
 
 
-def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
+def extract_file_embedded_pe():
     """extract embedded PE features
 
     IDA must load resource sections for this to be complete
@@ -78,23 +73,17 @@ def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
         - Check 'Load resource sections' when opening binary in IDA manually
     """
     for seg in capa.features.extractors.ida.helpers.get_segments(skip_header_segments=True):
-        for ea, _ in check_segment_for_pe(seg):
-            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
+        for (ea, _) in check_segment_for_pe(seg):
+            yield Characteristic("embedded pe"), ea
 
 
-def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract function exports"""
-    for _, ordinal, ea, name in idautils.Entries():
-        forwarded_name = ida_entry.get_entry_forwarder(ordinal)
-        if forwarded_name is None:
-            yield Export(name), AbsoluteVirtualAddress(ea)
-        else:
-            forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
-            yield Export(forwarded_name), AbsoluteVirtualAddress(ea)
-            yield Characteristic("forwarded export"), AbsoluteVirtualAddress(ea)
+def extract_file_export_names():
+    """ extract function exports """
+    for (_, _, ea, name) in idautils.Entries():
+        yield Export(name), ea
 
 
-def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
+def extract_file_import_names():
     """extract function imports
 
     1. imports by ordinal:
@@ -105,32 +94,28 @@ def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
      - modulename.importname
      - importname
     """
-    for ea, info in capa.features.extractors.ida.helpers.get_file_imports().items():
-        addr = AbsoluteVirtualAddress(ea)
+    for (ea, info) in capa.features.extractors.ida.helpers.get_file_imports().items():
         if info[1] and info[2]:
             # e.g. in mimikatz: ('cabinet', 'FCIAddFile', 11L)
             # extract by name here and by ordinal below
-            for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1], include_dll=True):
-                yield Import(name), addr
+            for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1]):
+                yield Import(name), ea
             dll = info[0]
-            symbol = f"#{info[2]}"
+            symbol = "#%d" % (info[2])
         elif info[1]:
             dll = info[0]
             symbol = info[1]
         elif info[2]:
             dll = info[0]
-            symbol = f"#{info[2]}"
+            symbol = "#%d" % (info[2])
         else:
             continue
 
-        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol, include_dll=True):
-            yield Import(name), addr
-
-    for ea, info in capa.features.extractors.ida.helpers.get_file_externs().items():
-        yield Import(info[1]), AbsoluteVirtualAddress(ea)
+        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+            yield Import(name), ea
 
 
-def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
+def extract_file_section_names():
     """extract section names
 
     IDA must load resource sections for this to be complete
@@ -138,10 +123,10 @@ def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
         - Check 'Load resource sections' when opening binary in IDA manually
     """
     for seg in capa.features.extractors.ida.helpers.get_segments(skip_header_segments=True):
-        yield Section(idaapi.get_segm_name(seg)), AbsoluteVirtualAddress(seg.start_ea)
+        yield Section(idaapi.get_segm_name(seg)), seg.start_ea
 
 
-def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
+def extract_file_strings():
     """extract ASCII and UTF-16 LE strings
 
     IDA must load resource sections for this to be complete
@@ -151,50 +136,18 @@ def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
     for seg in capa.features.extractors.ida.helpers.get_segments():
         seg_buff = capa.features.extractors.ida.helpers.get_segment_buffer(seg)
 
-        # differing to common string extractor factor in segment offset here
         for s in capa.features.extractors.strings.extract_ascii_strings(seg_buff):
-            yield String(s.s), FileOffsetAddress(seg.start_ea + s.offset)
+            yield String(s.s), (seg.start_ea + s.offset)
 
         for s in capa.features.extractors.strings.extract_unicode_strings(seg_buff):
-            yield String(s.s), FileOffsetAddress(seg.start_ea + s.offset)
+            yield String(s.s), (seg.start_ea + s.offset)
 
 
-def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract the names of statically-linked library functions.
-    """
-    for ea in idautils.Functions():
-        addr = AbsoluteVirtualAddress(ea)
-        if idaapi.get_func(ea).flags & idaapi.FUNC_LIB:
-            name = idaapi.get_name(ea)
-            yield FunctionName(name), addr
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                yield FunctionName(name[1:]), addr
-
-
-def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
-    file_info = idaapi.get_inf_structure()
-
-    if file_info.filetype in (idaapi.f_PE, idaapi.f_COFF):
-        yield Format(FORMAT_PE), NO_ADDRESS
-    elif file_info.filetype == idaapi.f_ELF:
-        yield Format(FORMAT_ELF), NO_ADDRESS
-    elif file_info.filetype == idaapi.f_BIN:
-        # no file type to return when processing a binary file, but we want to continue processing
-        return
-    else:
-        raise NotImplementedError(f"unexpected file format: {file_info.filetype}")
-
-
-def extract_features() -> Iterator[Tuple[Feature, Address]]:
-    """extract file features"""
+def extract_features():
+    """ extract file features """
     for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler():
-            yield feature, addr
+        for feature, va in file_handler():
+            yield feature, va
 
 
 FILE_HANDLERS = (
@@ -203,6 +156,15 @@ FILE_HANDLERS = (
     extract_file_strings,
     extract_file_section_names,
     extract_file_embedded_pe,
-    extract_file_function_names,
-    extract_file_format,
 )
+
+
+def main():
+    """ """
+    import pprint
+
+    pprint.pprint(list(extract_features()))
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ida/function.py

@@ -1,31 +1,35 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Tuple, Iterator
 
 import idaapi
 import idautils
 
 import capa.features.extractors.ida.helpers
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features import Characteristic
 from capa.features.extractors import loops
-from capa.features.extractors.base_extractor import FunctionHandle
 
 
-def extract_function_calls_to(fh: FunctionHandle):
-    """extract callers to a function"""
-    for ea in idautils.CodeRefsTo(fh.inner.start_ea, True):
-        yield Characteristic("calls to"), AbsoluteVirtualAddress(ea)
+def extract_function_calls_to(f):
+    """extract callers to a function
+
+    args:
+        f (IDA func_t)
+    """
+    for ea in idautils.CodeRefsTo(f.start_ea, True):
+        yield Characteristic("calls to"), ea
 
 
-def extract_function_loop(fh: FunctionHandle):
-    """extract loop indicators from a function"""
-    f: idaapi.func_t = fh.inner
+def extract_function_loop(f):
+    """extract loop indicators from a function
+
+    args:
+        f (IDA func_t)
+    """
     edges = []
 
     # construct control flow graph
@@ -34,19 +38,43 @@ def extract_function_loop(fh: FunctionHandle):
             edges.append((bb.start_ea, succ.start_ea))
 
     if loops.has_loop(edges):
-        yield Characteristic("loop"), fh.address
+        yield Characteristic("loop"), f.start_ea
 
 
-def extract_recursive_call(fh: FunctionHandle):
-    """extract recursive function call"""
-    if capa.features.extractors.ida.helpers.is_function_recursive(fh.inner):
-        yield Characteristic("recursive call"), fh.address
+def extract_recursive_call(f):
+    """extract recursive function call
+
+    args:
+        f (IDA func_t)
+    """
+    if capa.features.extractors.ida.helpers.is_function_recursive(f):
+        yield Characteristic("recursive call"), f.start_ea
 
 
-def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_features(f):
+    """extract function features
+
+    arg:
+        f (IDA func_t)
+    """
     for func_handler in FUNCTION_HANDLERS:
-        for feature, addr in func_handler(fh):
-            yield feature, addr
+        for (feature, ea) in func_handler(f):
+            yield feature, ea
 
 
 FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
+
+
+def main():
+    """ """
+    features = []
+    for f in capa.features.extractors.ida.get_functions(skip_thunks=True, skip_libs=True):
+        features.extend(list(extract_features(f)))
+
+    import pprint
+
+    pprint.pprint(features)
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ida/global_.py

@@ -1,65 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-import contextlib
-from typing import Tuple, Iterator
-
-import idaapi
-import ida_loader
-
-import capa.ida.helpers
-import capa.features.extractors.elf
-from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
-from capa.features.address import NO_ADDRESS, Address
-
-logger = logging.getLogger(__name__)
-
-
-def extract_os() -> Iterator[Tuple[Feature, Address]]:
-    format_name: str = ida_loader.get_file_type_name()
-
-    if "PE" in format_name:
-        yield OS(OS_WINDOWS), NO_ADDRESS
-
-    elif "ELF" in format_name:
-        with contextlib.closing(capa.ida.helpers.IDAIO()) as f:
-            os = capa.features.extractors.elf.detect_elf_os(f)
-
-        yield OS(os), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a further CLI argument to specify the OS,
-        # but i think this would be rarely used.
-        # rules that rely on OS conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess OS", format_name)
-        return
-
-
-def extract_arch() -> Iterator[Tuple[Feature, Address]]:
-    info: idaapi.idainfo = idaapi.get_inf_structure()
-    if info.procname == "metapc" and info.is_64bit():
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    elif info.procname == "metapc" and info.is_32bit():
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif info.procname == "metapc":
-        logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel")
-        return
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", info.procname)
-        return

capa/features/extractors/ida/helpers.py

@@ -1,24 +1,21 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import functools
-from typing import Any, Dict, Tuple, Iterator, Optional
+
+import sys
+import string
 
 import idc
 import idaapi
 import idautils
 import ida_bytes
-import ida_segment
-
-from capa.features.address import AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FunctionHandle
 
 
-def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
+def find_byte_sequence(start, end, seq):
     """yield all ea of a given byte sequence
 
     args:
@@ -26,33 +23,36 @@ def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
         end: max virtual address
         seq: bytes to search e.g. b"\x01\x03"
     """
-    seqstr = " ".join([f"{b:02x}" for b in seq])
+    if sys.version_info[0] >= 3:
+        seq = " ".join(["%02x" % b for b in seq])
+    else:
+        seq = " ".join(["%02x" % ord(b) for b in seq])
+
     while True:
-        # TODO(mike-hunhoff): find_binary is deprecated. Please use ida_bytes.bin_search() instead.
-        # https://github.com/mandiant/capa/issues/1606
-        ea = idaapi.find_binary(start, end, seqstr, 0, idaapi.SEARCH_DOWN)
+        ea = idaapi.find_binary(start, end, seq, 0, idaapi.SEARCH_DOWN)
         if ea == idaapi.BADADDR:
             break
         start = ea + 1
         yield ea
 
 
-def get_functions(
-    start: Optional[int] = None, end: Optional[int] = None, skip_thunks: bool = False, skip_libs: bool = False
-) -> Iterator[FunctionHandle]:
+def get_functions(start=None, end=None, skip_thunks=False, skip_libs=False):
     """get functions, range optional
 
     args:
         start: min virtual address
         end: max virtual address
+
+    ret:
+        yield func_t*
     """
     for ea in idautils.Functions(start=start, end=end):
         f = idaapi.get_func(ea)
         if not (skip_thunks and (f.flags & idaapi.FUNC_THUNK) or skip_libs and (f.flags & idaapi.FUNC_LIB)):
-            yield FunctionHandle(address=AbsoluteVirtualAddress(ea), inner=f)
+            yield f
 
 
-def get_segments(skip_header_segments=False) -> Iterator[idaapi.segment_t]:
+def get_segments(skip_header_segments=False):
     """get list of segments (sections) in the binary image
 
     args:
@@ -64,7 +64,7 @@ def get_segments(skip_header_segments=False) -> Iterator[idaapi.segment_t]:
             yield seg
 
 
-def get_segment_buffer(seg: idaapi.segment_t) -> bytes:
+def get_segment_buffer(seg):
     """return bytes stored in a given segment
 
     decrease buffer size until IDA is able to read bytes from the segment
@@ -82,22 +82,9 @@ def get_segment_buffer(seg: idaapi.segment_t) -> bytes:
     return buff if buff else b""
 
 
-def inspect_import(imports, library, ea, function, ordinal):
-    if function and function.startswith("__imp_"):
-        # handle mangled PE imports
-        function = function[len("__imp_") :]
-
-    if function and "@@" in function:
-        # handle mangled ELF imports, like "fopen@@glibc_2.2.5"
-        function, _, _ = function.partition("@@")
-
-    imports[ea] = (library.lower(), function, ordinal)
-    return True
-
-
-def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
-    """get file imports"""
-    imports: Dict[int, Tuple[str, str, int]] = {}
+def get_file_imports():
+    """ get file imports """
+    imports = {}
 
     for idx in range(idaapi.get_import_module_qty()):
         library = idaapi.get_import_module_name(idx)
@@ -105,38 +92,26 @@ def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
         if not library:
             continue
 
-        # IDA uses section names for the library of ELF imports, like ".dynsym".
-        # These are not useful to us, we may need to expand this list over time
-        # TODO(williballenthin): find all section names used by IDA
-        # https://github.com/mandiant/capa/issues/1419
-        if library == ".dynsym":
-            library = ""
+        def inspect_import(ea, function, ordinal):
+            if function and function.startswith("__imp_"):
+                # handle mangled names starting
+                function = function[len("__imp_") :]
+            imports[ea] = (library.lower(), function, ordinal)
+            return True
 
-        cb = functools.partial(inspect_import, imports, library)
-        idaapi.enum_import_names(idx, cb)
+        idaapi.enum_import_names(idx, inspect_import)
 
     return imports
 
 
-def get_file_externs() -> Dict[int, Tuple[str, str, int]]:
-    externs = {}
-
-    for seg in get_segments(skip_header_segments=True):
-        if seg.type != ida_segment.SEG_XTRN:
-            continue
-
-        for ea in idautils.Functions(seg.start_ea, seg.end_ea):
-            externs[ea] = ("", idaapi.get_func_name(ea), -1)
-
-    return externs
-
-
-def get_instructions_in_range(start: int, end: int) -> Iterator[idaapi.insn_t]:
+def get_instructions_in_range(start, end):
     """yield instructions in range
 
     args:
         start: virtual address (inclusive)
         end: virtual address (exclusive)
+    yield:
+        (insn_t*)
     """
     for head in idautils.Heads(start, end):
         insn = idautils.DecodeInstruction(head)
@@ -144,8 +119,8 @@ def get_instructions_in_range(start: int, end: int) -> Iterator[idaapi.insn_t]:
             yield insn
 
 
-def is_operand_equal(op1: idaapi.op_t, op2: idaapi.op_t) -> bool:
-    """compare two IDA op_t"""
+def is_operand_equal(op1, op2):
+    """ compare two IDA op_t """
     if op1.flags != op2.flags:
         return False
 
@@ -170,8 +145,8 @@ def is_operand_equal(op1: idaapi.op_t, op2: idaapi.op_t) -> bool:
     return True
 
 
-def is_basic_block_equal(bb1: idaapi.BasicBlock, bb2: idaapi.BasicBlock) -> bool:
-    """compare two IDA BasicBlock"""
+def is_basic_block_equal(bb1, bb2):
+    """ compare two IDA BasicBlock """
     if bb1.start_ea != bb2.start_ea:
         return False
 
@@ -184,12 +159,12 @@ def is_basic_block_equal(bb1: idaapi.BasicBlock, bb2: idaapi.BasicBlock) -> bool
     return True
 
 
-def basic_block_size(bb: idaapi.BasicBlock) -> int:
-    """calculate size of basic block"""
+def basic_block_size(bb):
+    """ calculate size of basic block """
     return bb.end_ea - bb.start_ea
 
 
-def read_bytes_at(ea: int, count: int) -> bytes:
+def read_bytes_at(ea, count):
     """ """
     # check if byte has a value, see get_wide_byte doc
     if not idc.is_loaded(ea):
@@ -202,10 +177,10 @@ def read_bytes_at(ea: int, count: int) -> bytes:
         return idc.get_bytes(ea, count)
 
 
-def find_string_at(ea: int, min_: int = 4) -> str:
-    """check if ASCII string exists at a given virtual address"""
+def find_string_at(ea, min=4):
+    """ check if ASCII string exists at a given virtual address """
     found = idaapi.get_strlit_contents(ea, -1, idaapi.STRTYPE_C)
-    if found and len(found) >= min_:
+    if found and len(found) > min:
         try:
             found = found.decode("ascii")
             # hacky check for IDA bug; get_strlit_contents also reads Unicode as
@@ -219,7 +194,7 @@ def find_string_at(ea: int, min_: int = 4) -> str:
     return ""
 
 
-def get_op_phrase_info(op: idaapi.op_t) -> Dict:
+def get_op_phrase_info(op):
     """parse phrase features from operand
 
     Pretty much dup of sark's implementation:
@@ -229,8 +204,7 @@ def get_op_phrase_info(op: idaapi.op_t) -> Dict:
         return {}
 
     scale = 1 << ((op.specflag2 & 0xC0) >> 6)
-    # IDA ea_t may be 32- or 64-bit; we assume displacement can only be 32-bit
-    offset = op.addr & 0xFFFFFFFF
+    offset = op.addr
 
     if op.specflag1 == 0:
         index = None
@@ -257,45 +231,47 @@ def get_op_phrase_info(op: idaapi.op_t) -> Dict:
     return {"base": base, "index": index, "scale": scale, "offset": offset}
 
 
-def is_op_write(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
-    """Check if an operand is written to (destination operand)"""
+def is_op_write(insn, op):
+    """ Check if an operand is written to (destination operand) """
     return idaapi.has_cf_chg(insn.get_canon_feature(), op.n)
 
 
-def is_op_read(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
-    """Check if an operand is read from (source operand)"""
+def is_op_read(insn, op):
+    """ Check if an operand is read from (source operand) """
     return idaapi.has_cf_use(insn.get_canon_feature(), op.n)
 
 
-def is_op_offset(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
-    """Check is an operand has been marked as an offset (by auto-analysis or manually)"""
+def is_op_offset(insn, op):
+    """ Check is an operand has been marked as an offset (by auto-analysis or manually) """
     flags = idaapi.get_flags(insn.ea)
     return ida_bytes.is_off(flags, op.n)
 
 
-def is_sp_modified(insn: idaapi.insn_t) -> bool:
-    """determine if instruction modifies SP, ESP, RSP"""
-    return any(
-        op.reg == idautils.procregs.sp.reg and is_op_write(insn, op)
-        for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,))
-    )
+def is_sp_modified(insn):
+    """ determine if instruction modifies SP, ESP, RSP """
+    for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
+        if op.reg == idautils.procregs.sp.reg and is_op_write(insn, op):
+            # register is stack and written
+            return True
+    return False
 
 
-def is_bp_modified(insn: idaapi.insn_t) -> bool:
-    """check if instruction modifies BP, EBP, RBP"""
-    return any(
-        op.reg == idautils.procregs.bp.reg and is_op_write(insn, op)
-        for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,))
-    )
+def is_bp_modified(insn):
+    """ check if instruction modifies BP, EBP, RBP """
+    for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
+        if op.reg == idautils.procregs.bp.reg and is_op_write(insn, op):
+            # register is base and written
+            return True
+    return False
 
 
-def is_frame_register(reg: int) -> bool:
-    """check if register is sp or bp"""
+def is_frame_register(reg):
+    """ check if register is sp or bp """
     return reg in (idautils.procregs.sp.reg, idautils.procregs.bp.reg)
 
 
-def get_insn_ops(insn: idaapi.insn_t, target_ops: Optional[Tuple[Any]] = None) -> idaapi.op_t:
-    """yield op_t for instruction, filter on type if specified"""
+def get_insn_ops(insn, target_ops=()):
+    """ yield op_t for instruction, filter on type if specified """
     for op in insn.ops:
         if op.type == idaapi.o_void:
             # avoid looping all 6 ops if only subset exists
@@ -305,12 +281,12 @@ def get_insn_ops(insn: idaapi.insn_t, target_ops: Optional[Tuple[Any]] = None) -
         yield op
 
 
-def is_op_stack_var(ea: int, index: int) -> bool:
-    """check if operand is a stack variable"""
+def is_op_stack_var(ea, index):
+    """ check if operand is a stack variable """
     return idaapi.is_stkvar(idaapi.get_flags(ea), index)
 
 
-def mask_op_val(op: idaapi.op_t) -> int:
+def mask_op_val(op):
     """mask value by data type
 
     necessary due to a bug in AMD64
@@ -330,15 +306,26 @@ def mask_op_val(op: idaapi.op_t) -> int:
     return masks.get(op.dtype, op.value) & op.value
 
 
-def is_function_recursive(f: idaapi.func_t) -> bool:
-    """check if function is recursive"""
-    return any(f.contains(ref) for ref in idautils.CodeRefsTo(f.start_ea, True))
+def is_function_recursive(f):
+    """check if function is recursive
+
+    args:
+        f (IDA func_t)
+    """
+    for ref in idautils.CodeRefsTo(f.start_ea, True):
+        if f.contains(ref):
+            return True
+    return False
 
 
-def is_basic_block_tight_loop(bb: idaapi.BasicBlock) -> bool:
+def is_basic_block_tight_loop(bb):
     """check basic block loops to self
 
     true if last instruction in basic block branches to basic block start
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
     """
     bb_end = idc.prev_head(bb.end_ea)
     if bb.start_ea < bb_end:
@@ -348,8 +335,8 @@ def is_basic_block_tight_loop(bb: idaapi.BasicBlock) -> bool:
     return False
 
 
-def find_data_reference_from_insn(insn: idaapi.insn_t, max_depth: int = 10) -> int:
-    """search for data reference from instruction, return address of instruction if no reference exists"""
+def find_data_reference_from_insn(insn, max_depth=10):
+    """ search for data reference from instruction, return address of instruction if no reference exists """
     depth = 0
     ea = insn.ea
 
@@ -364,10 +351,6 @@ def find_data_reference_from_insn(insn: idaapi.insn_t, max_depth: int = 10) -> i
             # break if circular reference
             break
 
-        if not idaapi.is_mapped(data_refs[0]):
-            # break if address is not mapped
-            break
-
         depth += 1
         if depth > max_depth:
             # break if max depth
@@ -378,17 +361,19 @@ def find_data_reference_from_insn(insn: idaapi.insn_t, max_depth: int = 10) -> i
     return ea
 
 
-def get_function_blocks(f: idaapi.func_t) -> Iterator[idaapi.BasicBlock]:
-    """yield basic blocks contained in specified function"""
+def get_function_blocks(f):
+    """yield basic blocks contained in specified function
+
+    args:
+        f (IDA func_t)
+    yield:
+        block (IDA BasicBlock)
+    """
     # leverage idaapi.FC_NOEXT flag to ignore useless external blocks referenced by the function
-    yield from idaapi.FlowChart(f, flags=(idaapi.FC_PREDS | idaapi.FC_NOEXT))
+    for block in idaapi.FlowChart(f, flags=(idaapi.FC_PREDS | idaapi.FC_NOEXT)):
+        yield block
 
 
-def is_basic_block_return(bb: idaapi.BasicBlock) -> bool:
-    """check if basic block is return block"""
+def is_basic_block_return(bb):
+    """ check if basic block is return block """
     return bb.type == idaapi.fcb_ret
-
-
-def has_sib(oper: idaapi.op_t) -> bool:
-    # via: https://reverseengineering.stackexchange.com/a/14300
-    return oper.specflag1 == 1

capa/features/extractors/ida/insn.py

@@ -1,11 +1,10 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Any, Dict, Tuple, Iterator
 
 import idc
 import idaapi
@@ -13,30 +12,51 @@ import idautils
 
 import capa.features.extractors.helpers
 import capa.features.extractors.ida.helpers
-from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
-from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+from capa.features import (
+    ARCH_X32,
+    ARCH_X64,
+    MAX_BYTES_FEATURE_SIZE,
+    THUNK_CHAIN_DEPTH_DELTA,
+    Bytes,
+    String,
+    Characteristic,
+)
+from capa.features.insn import API, Number, Offset, Mnemonic
 
 # security cookie checks may perform non-zeroing XORs, these are expected within a certain
 # byte range within the first and returning basic blocks, this helps to reduce FP features
 SECURITY_COOKIE_BYTES_DELTA = 0x40
 
 
-def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
+def get_arch(ctx):
+    """
+    fetch the ARCH_* constant for the currently open workspace.
+
+    via Tamir Bahar/@tmr232
+    https://reverseengineering.stackexchange.com/a/11398/17194
+    """
+    if "arch" not in ctx:
+        info = idaapi.get_inf_structure()
+        if info.is_64bit():
+            ctx["arch"] = ARCH_X64
+        elif info.is_32bit():
+            ctx["arch"] = ARCH_X32
+        else:
+            raise ValueError("unexpected architecture")
+    return ctx["arch"]
+
+
+def get_imports(ctx):
     if "imports_cache" not in ctx:
         ctx["imports_cache"] = capa.features.extractors.ida.helpers.get_file_imports()
     return ctx["imports_cache"]
 
 
-def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
-    if "externs_cache" not in ctx:
-        ctx["externs_cache"] = capa.features.extractors.ida.helpers.get_file_externs()
-    return ctx["externs_cache"]
+def check_for_api_call(ctx, insn):
+    """ check instruction for API call """
+    if not insn.get_canon_mnem() in ("call", "jmp"):
+        return
 
-
-def check_for_api_call(insn: idaapi.insn_t, funcs: Dict[int, Any]) -> Iterator[Any]:
-    """check instruction for API call"""
     info = ()
     ref = insn.ea
 
@@ -52,7 +72,7 @@ def check_for_api_call(insn: idaapi.insn_t, funcs: Dict[int, Any]) -> Iterator[A
             except IndexError:
                 break
 
-        info = funcs.get(ref, ())
+        info = get_imports(ctx).get(ref, ())
         if info:
             break
 
@@ -61,64 +81,37 @@ def check_for_api_call(insn: idaapi.insn_t, funcs: Dict[int, Any]) -> Iterator[A
             break
 
     if info:
-        yield info
+        yield "%s.%s" % (info[0], info[1])
 
 
-def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction API features
+def extract_insn_api_features(f, bb, insn):
+    """parse instruction API features
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
 
     example:
-       call dword [0x00473038]
+        call dword [0x00473038]
     """
-    insn: idaapi.insn_t = ih.inner
-
-    if insn.get_canon_mnem() not in ("call", "jmp"):
-        return
-
-    # check calls to imported functions
-    for api in check_for_api_call(insn, get_imports(fh.ctx)):
-        # tuple (<module>, <function>, <ordinal>)
-        for name in capa.features.extractors.helpers.generate_symbols(api[0], api[1]):
-            yield API(name), ih.address
-
-    # check calls to extern functions
-    for api in check_for_api_call(insn, get_externs(fh.ctx)):
-        # tuple (<module>, <function>, <ordinal>)
-        yield API(api[1]), ih.address
-
-    # extract IDA/FLIRT recognized API functions
-    targets = tuple(idautils.CodeRefsFrom(insn.ea, False))
-    if not targets:
-        return
-
-    target = targets[0]
-    target_func = idaapi.get_func(target)
-    if not target_func or target_func.start_ea != target:
-        # not a function (start)
-        return
-
-    if target_func.flags & idaapi.FUNC_LIB:
-        name = idaapi.get_name(target_func.start_ea)
-        yield API(name), ih.address
-        if name.startswith("_"):
-            # some linkers may prefix linked routines with a `_` to avoid name collisions.
-            # extract features for both the mangled and un-mangled representations.
-            # e.g. `_fwrite` -> `fwrite`
-            # see: https://stackoverflow.com/a/2628384/87207
-            yield API(name[1:]), ih.address
+    for api in check_for_api_call(f.ctx, insn):
+        dll, _, symbol = api.rpartition(".")
+        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+            yield API(name), insn.ea
 
 
-def extract_insn_number_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction number features
+def extract_insn_number_features(f, bb, insn):
+    """parse instruction number features
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
+
     example:
         push    3136B0h         ; dwControlCode
     """
-    insn: idaapi.insn_t = ih.inner
-
     if idaapi.is_ret_insn(insn):
         # skip things like:
         #   .text:0042250E retn 8
@@ -129,11 +122,7 @@ def extract_insn_number_features(
         #   .text:00401145 add esp, 0Ch
         return
 
-    for i, op in enumerate(insn.ops):
-        if op.type == idaapi.o_void:
-            break
-        if op.type not in (idaapi.o_imm, idaapi.o_mem):
-            continue
+    for op in capa.features.extractors.ida.helpers.get_insn_ops(insn, target_ops=(idaapi.o_imm, idaapi.o_mem)):
         # skip things like:
         #   .text:00401100 shr eax, offset loc_C
         if capa.features.extractors.ida.helpers.is_op_offset(insn, op):
@@ -144,27 +133,21 @@ def extract_insn_number_features(
         else:
             const = op.addr
 
-        yield Number(const), ih.address
-        yield OperandNumber(i, const), ih.address
-
-        if insn.itype == idaapi.NN_add and 0 < const < MAX_STRUCTURE_SIZE and op.type == idaapi.o_imm:
-            # for pattern like:
-            #
-            #     add eax, 0x10
-            #
-            # assume 0x10 is also an offset (imagine eax is a pointer).
-            yield Offset(const), ih.address
-            yield OperandOffset(i, const), ih.address
+        yield Number(const), insn.ea
+        yield Number(const, arch=get_arch(f.ctx)), insn.ea
 
 
-def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse referenced byte sequences
+def extract_insn_bytes_features(f, bb, insn):
+    """parse referenced byte sequences
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
+
     example:
         push    offset iid_004118d4_IShellLinkA ; riid
     """
-    insn: idaapi.insn_t = ih.inner
-
     if idaapi.is_call_insn(insn):
         return
 
@@ -172,54 +155,43 @@ def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
     if ref != insn.ea:
         extracted_bytes = capa.features.extractors.ida.helpers.read_bytes_at(ref, MAX_BYTES_FEATURE_SIZE)
         if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-            if not capa.features.extractors.ida.helpers.find_string_at(ref):
-                # don't extract byte features for obvious strings
-                yield Bytes(extracted_bytes), ih.address
+            yield Bytes(extracted_bytes), insn.ea
 
 
-def extract_insn_string_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction string features
+def extract_insn_string_features(f, bb, insn):
+    """parse instruction string features
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
 
     example:
         push offset aAcr     ; "ACR  > "
     """
-    insn: idaapi.insn_t = ih.inner
-
     ref = capa.features.extractors.ida.helpers.find_data_reference_from_insn(insn)
     if ref != insn.ea:
         found = capa.features.extractors.ida.helpers.find_string_at(ref)
         if found:
-            yield String(found), ih.address
+            yield String(found), insn.ea
 
 
-def extract_insn_offset_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction structure offset features
+def extract_insn_offset_features(f, bb, insn):
+    """parse instruction structure offset features
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
 
     example:
         .text:0040112F cmp [esi+4], ebx
     """
-    insn: idaapi.insn_t = ih.inner
-
-    for i, op in enumerate(insn.ops):
-        if op.type == idaapi.o_void:
-            break
-        if op.type not in (idaapi.o_phrase, idaapi.o_displ):
-            continue
+    for op in capa.features.extractors.ida.helpers.get_insn_ops(insn, target_ops=(idaapi.o_phrase, idaapi.o_displ)):
         if capa.features.extractors.ida.helpers.is_op_stack_var(insn.ea, op.n):
             continue
-
         p_info = capa.features.extractors.ida.helpers.get_op_phrase_info(op)
-
-        op_off = p_info.get("offset")
-        if op_off is None:
-            continue
-
+        op_off = p_info.get("offset", 0)
         if idaapi.is_mapped(op_off):
             # Ignore:
             #   mov esi, dword_1005B148[esi]
@@ -230,32 +202,12 @@ def extract_insn_offset_features(
         # https://stackoverflow.com/questions/31853189/x86-64-assembly-why-displacement-not-64-bits
         op_off = capa.features.extractors.helpers.twos_complement(op_off, 32)
 
-        yield Offset(op_off), ih.address
-        yield OperandOffset(i, op_off), ih.address
-
-        if (
-            insn.itype == idaapi.NN_lea
-            and i == 1
-            # o_displ is used for both:
-            #   [eax+1]
-            #   [eax+ebx+2]
-            and op.type == idaapi.o_displ
-            # but the SIB is only present for [eax+ebx+2]
-            # which we don't want
-            and not capa.features.extractors.ida.helpers.has_sib(op)
-        ):
-            # for pattern like:
-            #
-            #     lea eax, [ebx + 1]
-            #
-            # assume 1 is also an offset (imagine ebx is a zero register).
-            yield Number(op_off), ih.address
-            yield OperandNumber(i, op_off), ih.address
+        yield Offset(op_off), insn.ea
+        yield Offset(op_off, arch=get_arch(f.ctx)), insn.ea
 
 
-def contains_stack_cookie_keywords(s: str) -> bool:
-    """
-    check if string contains stack cookie keywords
+def contains_stack_cookie_keywords(s):
+    """check if string contains stack cookie keywords
 
     Examples:
         xor     ecx, ebp ; StackCookie
@@ -269,7 +221,7 @@ def contains_stack_cookie_keywords(s: str) -> bool:
     return any(keyword in s for keyword in ("stack", "security"))
 
 
-def bb_stack_cookie_registers(bb: idaapi.BasicBlock) -> Iterator[int]:
+def bb_stack_cookie_registers(bb):
     """scan basic block for stack cookie operations
 
     yield registers ids that may have been used for stack cookie operations
@@ -303,8 +255,8 @@ def bb_stack_cookie_registers(bb: idaapi.BasicBlock) -> Iterator[int]:
                     yield op.reg
 
 
-def is_nzxor_stack_cookie_delta(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.insn_t) -> bool:
-    """check if nzxor exists within stack cookie delta"""
+def is_nzxor_stack_cookie_delta(f, bb, insn):
+    """ check if nzxor exists within stack cookie delta """
     # security cookie check should use SP or BP
     if not capa.features.extractors.ida.helpers.is_frame_register(insn.Op2.reg):
         return False
@@ -326,8 +278,8 @@ def is_nzxor_stack_cookie_delta(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: i
     return False
 
 
-def is_nzxor_stack_cookie(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.insn_t) -> bool:
-    """check if nzxor is related to stack cookie"""
+def is_nzxor_stack_cookie(f, bb, insn):
+    """ check if nzxor is related to stack cookie """
     if contains_stack_cookie_keywords(idaapi.get_cmt(insn.ea, False)):
         # Example:
         #   xor     ecx, ebp        ; StackCookie
@@ -343,49 +295,37 @@ def is_nzxor_stack_cookie(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.
     return False
 
 
-def extract_insn_nzxor_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction non-zeroing XOR instruction
-    ignore expected non-zeroing XORs, e.g. security cookies
-    """
-    insn: idaapi.insn_t = ih.inner
+def extract_insn_nzxor_characteristic_features(f, bb, insn):
+    """parse instruction non-zeroing XOR instruction
 
+    ignore expected non-zeroing XORs, e.g. security cookies
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
+    """
     if insn.itype not in (idaapi.NN_xor, idaapi.NN_xorpd, idaapi.NN_xorps, idaapi.NN_pxor):
         return
     if capa.features.extractors.ida.helpers.is_operand_equal(insn.Op1, insn.Op2):
         return
-    if is_nzxor_stack_cookie(fh.inner, bbh.inner, insn):
+    if is_nzxor_stack_cookie(f, bb, insn):
         return
-    yield Characteristic("nzxor"), ih.address
+    yield Characteristic("nzxor"), insn.ea
 
 
-def extract_insn_mnemonic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction mnemonic features"""
-    yield Mnemonic(idc.print_insn_mnem(ih.inner.ea)), ih.address
+def extract_insn_mnemonic_features(f, bb, insn):
+    """parse instruction mnemonic features
 
-
-def extract_insn_obfs_call_plus_5_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
     """
-    parse call $+5 instruction from the given instruction.
-    """
-    insn: idaapi.insn_t = ih.inner
-
-    if not idaapi.is_call_insn(insn):
-        return
-
-    if insn.ea + 5 == idc.get_operand_value(insn.ea, 0):
-        yield Characteristic("call $+5"), ih.address
+    yield Mnemonic(insn.get_canon_mnem()), insn.ea
 
 
-def extract_insn_peb_access_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_peb_access_characteristic_features(f, bb, insn):
     """parse instruction peb access
 
     fs:[0x30] on x86, gs:[0x60] on x64
@@ -393,61 +333,51 @@ def extract_insn_peb_access_characteristic_features(
     TODO:
         IDA should be able to do this..
     """
-    insn: idaapi.insn_t = ih.inner
-
     if insn.itype not in (idaapi.NN_push, idaapi.NN_mov):
         return
 
-    if all(op.type != idaapi.o_mem for op in insn.ops):
+    if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
         # try to optimize for only memory references
         return
 
     disasm = idc.GetDisasm(insn.ea)
 
     if " fs:30h" in disasm or " gs:60h" in disasm:
-        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
-        # scanning the disassembly text is a hack.
-        # https://github.com/mandiant/capa/issues/1605
-        yield Characteristic("peb access"), ih.address
+        # TODO: replace above with proper IDA
+        yield Characteristic("peb access"), insn.ea
 
 
-def extract_insn_segment_access_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_segment_access_features(f, bb, insn):
     """parse instruction fs or gs access
 
     TODO:
         IDA should be able to do this...
     """
-    insn: idaapi.insn_t = ih.inner
-
-    if all(op.type != idaapi.o_mem for op in insn.ops):
+    if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
         # try to optimize for only memory references
         return
 
     disasm = idc.GetDisasm(insn.ea)
 
     if " fs:" in disasm:
-        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
-        # scanning the disassembly text is a hack.
-        # https://github.com/mandiant/capa/issues/1605
-        yield Characteristic("fs access"), ih.address
+        # TODO: replace above with proper IDA
+        yield Characteristic("fs access"), insn.ea
 
     if " gs:" in disasm:
-        # TODO(mike-hunhoff): use proper IDA API for fetching segment access
-        # scanning the disassembly text is a hack.
-        # https://github.com/mandiant/capa/issues/1605
-        yield Characteristic("gs access"), ih.address
+        # TODO: replace above with proper IDA
+        yield Characteristic("gs access"), insn.ea
 
 
-def extract_insn_cross_section_cflow(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
-    insn: idaapi.insn_t = ih.inner
+def extract_insn_cross_section_cflow(f, bb, insn):
+    """inspect the instruction for a CALL or JMP that crosses section boundaries
 
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
+    """
     for ref in idautils.CodeRefsFrom(insn.ea, False):
-        if ref in get_imports(fh.ctx):
+        if ref in get_imports(f.ctx).keys():
             # ignore API calls
             continue
         if not idaapi.getseg(ref):
@@ -455,40 +385,50 @@ def extract_insn_cross_section_cflow(
             continue
         if idaapi.getseg(ref) == idaapi.getseg(insn.ea):
             continue
-        yield Characteristic("cross section flow"), ih.address
+        yield Characteristic("cross section flow"), insn.ea
 
 
-def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_function_calls_from(f, bb, insn):
     """extract functions calls from features
 
     most relevant at the function scope, however, its most efficient to extract at the instruction scope
-    """
-    insn: idaapi.insn_t = ih.inner
 
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
+    """
     if idaapi.is_call_insn(insn):
         for ref in idautils.CodeRefsFrom(insn.ea, False):
-            yield Characteristic("calls from"), AbsoluteVirtualAddress(ref)
+            yield Characteristic("calls from"), ref
 
 
-def extract_function_indirect_call_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
+def extract_function_indirect_call_characteristic_features(f, bb, insn):
     """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
     does not include calls like => call ds:dword_ABD4974
 
     most relevant at the function or basic block scope;
     however, its most efficient to extract at the instruction scope
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
     """
-    insn: idaapi.insn_t = ih.inner
-
     if idaapi.is_call_insn(insn) and idc.get_operand_type(insn.ea, 0) in (idc.o_reg, idc.o_phrase, idc.o_displ):
-        yield Characteristic("indirect call"), ih.address
+        yield Characteristic("indirect call"), insn.ea
 
 
-def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract instruction features"""
+def extract_features(f, bb, insn):
+    """extract instruction features
+
+    args:
+        f (IDA func_t)
+        bb (IDA BasicBlock)
+        insn (IDA insn_t)
+    """
     for inst_handler in INSTRUCTION_HANDLERS:
-        for feature, ea in inst_handler(f, bbh, insn):
+        for (feature, ea) in inst_handler(f, bb, insn):
             yield feature, ea
 
 
@@ -500,10 +440,26 @@ INSTRUCTION_HANDLERS = (
     extract_insn_offset_features,
     extract_insn_nzxor_characteristic_features,
     extract_insn_mnemonic_features,
-    extract_insn_obfs_call_plus_5_characteristic_features,
     extract_insn_peb_access_characteristic_features,
     extract_insn_cross_section_cflow,
     extract_insn_segment_access_features,
     extract_function_calls_from,
     extract_function_indirect_call_characteristic_features,
 )
+
+
+def main():
+    """ """
+    features = []
+    for f in capa.features.extractors.ida.helpers.get_functions(skip_thunks=True, skip_libs=True):
+        for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
+            for insn in capa.features.extractors.ida.helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
+                features.extend(list(extract_features(f, bb, insn)))
+
+    import pprint
+
+    pprint.pprint(features)
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/loops.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,7 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-import networkx
+from networkx import nx
 from networkx.algorithms.components import strongly_connected_components
 
 
@@ -20,6 +20,6 @@ def has_loop(edges, threshold=2):
     returns:
         bool
     """
-    g = networkx.DiGraph()
+    g = nx.DiGraph()
     g.add_edges_from(edges)
     return any(len(comp) >= threshold for comp in strongly_connected_components(g))

capa/features/extractors/miasm/__init__.py

@@ -0,0 +1,107 @@
+# Copyright (C) 2020 FireEye, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: https://github.com/fireeye/capa/blob/master/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import miasm.analysis.binary
+import miasm.analysis.machine
+from miasm.core.locationdb import LocationDB
+
+import capa.features.extractors.miasm.file
+import capa.features.extractors.miasm.insn
+import capa.features.extractors.miasm.function
+import capa.features.extractors.miasm.basicblock
+from capa.features.extractors import FeatureExtractor
+
+
+class MiasmFeatureExtractor(FeatureExtractor):
+    def __init__(self, buf):
+        super(MiasmFeatureExtractor, self).__init__()
+        self.buf = buf
+        self.loc_db = LocationDB()
+        self.container = miasm.analysis.binary.Container.from_string(buf, self.loc_db)
+        self.pe = self.container.executable
+        self.machine = miasm.analysis.machine.Machine(self.container.arch)
+        self.cfg = self._build_cfg()
+
+    def get_base_address(self):
+        return self.container.entry_point
+
+    def extract_file_features(self):
+        for feature, va in capa.features.extractors.miasm.file.extract_file_features(self):
+            yield feature, va
+
+    # TODO: Improve this function (it just considers all loc_keys target of calls a function), port to miasm
+    def get_functions(self):
+        """
+        returns all loc_keys which are the argument of any call function
+        """
+        functions = set()
+
+        for block in self.cfg.blocks:
+            for line in block.lines:
+                if line.is_subcall() and line.args[0].is_loc():
+                    loc_key = line.args[0].loc_key
+                    if loc_key not in functions:
+                        functions.add(loc_key)
+                        yield loc_key
+
+    def extract_function_features(self, loc_key):
+        for feature, va in capa.features.extractors.miasm.function.extract_features(self, loc_key):
+            yield feature, va
+
+    def block_offset(self, bb):
+        return bb.lines[0].offset
+
+    def function_offset(self, f):
+        return self.cfg.loc_key_to_block(f).lines[0].offset
+
+    def get_basic_blocks(self, loc_key):
+        """
+        get the basic blocks of the function represented by lock_key
+        """
+        block = self.cfg.loc_key_to_block(loc_key)
+        disassembler = self.machine.dis_engine(self.container.bin_stream, loc_db=self.loc_db, follow_call=False)
+        cfg = disassembler.dis_multiblock(self.block_offset(block))
+        return cfg.blocks
+
+    def extract_basic_block_features(self, _, bb):
+        for feature, va in capa.features.extractors.miasm.basicblock.extract_features(bb):
+            yield feature, va
+
+    def get_instructions(self, _, bb):
+        return bb.lines
+
+    def extract_insn_features(self, f, bb, insn):
+        for feature, va in capa.features.extractors.miasm.insn.extract_features(self, f, bb, insn):
+            yield feature, va
+
+    def _get_entry_points(self):
+        entry_points = {self.get_base_address()}
+
+        for _, va in miasm.jitter.loader.pe.get_export_name_addr_list(self.pe):
+            entry_points.add(va)
+
+        return entry_points
+
+    # This is more efficient that using the `blocks` argument in `dis_multiblock`
+    # See http://www.williballenthin.com/post/2020-01-12-miasm-part-2
+    # TODO: port this efficiency improvement to miasm
+    def _build_cfg(self):
+        loc_db = self.container.loc_db
+        disassembler = self.machine.dis_engine(self.container.bin_stream, follow_call=True, loc_db=loc_db)
+        job_done = set()
+        cfgs = {}
+
+        for va in self._get_entry_points():
+            cfgs[va] = disassembler.dis_multiblock(va, job_done=job_done)
+
+        complete_cfs = miasm.core.asmblock.AsmCFG(loc_db)
+        for cfg in cfgs.values():
+            complete_cfs.merge(cfg)
+
+        disassembler.apply_splitting(complete_cfs)
+        return complete_cfs

capa/features/extractors/miasm/basicblock.py

@@ -0,0 +1,134 @@
+# Copyright (C) 2020 FireEye, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: https://github.com/fireeye/capa/blob/master/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import sys
+import string
+import struct
+
+from capa.features import Characteristic
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+
+
+# TODO: Avoid this duplication (this code is in __init__ as well)
+def block_offset(bb):
+    return bb.lines[0].offset
+
+
+def extract_bb_tight_loop(bb):
+    """ check basic block for tight loop indicators """
+    if any(c.loc_key == bb.loc_key for c in bb.bto):
+        yield Characteristic("tight loop"), block_offset(bb)
+
+
+def is_mov_imm_to_stack(instr):
+    """
+    Return if instruction moves immediate onto stack
+    """
+    if not instr.name.startswith("MOV"):
+        return False
+
+    try:
+        dst, src = instr.args
+    except ValueError:
+        # not two operands
+        return False
+
+    if not src.is_int():
+        return False
+
+    if not dst.is_mem():
+        return False
+
+    # should detect things like `@8[ESP + 0x8]` and `EBP` and not fail in other cases
+    if any(register in str(dst) for register in ["EBP", "RBP", "ESP", "RSP"]):
+        return True
+
+    return False
+
+
+def is_printable_ascii(chars):
+    if sys.version_info >= (3, 0):
+        return all(c < 127 and chr(c) in string.printable for c in chars)
+    else:
+        return all(ord(c) < 127 and c in string.printable for c in chars)
+
+
+def is_printable_utf16le(chars):
+    if all(c == b"\x00" for c in chars[1::2]):
+        return is_printable_ascii(chars[::2])
+
+
+def get_printable_len(insn):
+    """
+    Return string length if all operand bytes are ascii or utf16-le printable
+    """
+    dst, src = insn.args
+
+    if not src.is_int():
+        return ValueError("unexpected operand type")
+
+    if not dst.is_mem():
+        return ValueError("unexpected operand type")
+
+    if isinstance(src.arg, int):
+        val = src.arg
+    else:
+        val = src.arg.arg
+
+    size = (val.bit_length() + 7) // 8
+
+    if size == 0:
+        return 0
+    elif size == 1:
+        chars = struct.pack("<B", val)
+    elif size == 2:
+        chars = struct.pack("<H", val)
+    elif size == 4:
+        chars = struct.pack("<I", val)
+    elif size == 8:
+        chars = struct.pack("<Q", val)
+
+    if is_printable_ascii(chars):
+        return size
+
+    if is_printable_utf16le(chars):
+        return size / 2
+
+    return 0
+
+
+def extract_stackstring(bb):
+    """ check basic block for stackstring indicators """
+    count = 0
+    for line in bb.lines:
+        if is_mov_imm_to_stack(line):
+            count += get_printable_len(line)
+        if count > MIN_STACKSTRING_LEN:
+            yield Characteristic("stack string"), block_offset(bb)
+            return
+
+
+def extract_features(bb):
+    """
+    extract features from the given basic block.
+    args:
+      bb (miasm.core.asmblock.AsmBlock): the basic block to process.
+    yields:
+      Feature, set[VA]: the features and their location found in this basic block.
+    """
+    yield BasicBlock(), block_offset(bb)
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, va in bb_handler(bb):
+            yield feature, va
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    extract_stackstring,
+)

capa/features/extractors/miasm/file.py

@@ -0,0 +1,102 @@
+# Copyright (C) 2020 FireEye, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: https://github.com/fireeye/capa/blob/master/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import re
+
+import miasm.analysis.binary
+
+import capa.features.extractors.strings
+from capa.features import String, Characteristic
+from capa.features.file import Export, Import, Section
+
+
+def extract_file_embedded_pe(extractor):
+    """
+    extract embedded PE features
+    """
+    buf = extractor.buf
+    for match in re.finditer(b"MZ", buf):
+        offset = match.start()
+        subcontainer = miasm.analysis.binary.ContainerPE.from_string(buf[offset:], loc_db=extractor.loc_db)
+        if isinstance(subcontainer, miasm.analysis.binary.ContainerPE):
+            yield Characteristic("embedded pe"), offset
+
+
+def extract_file_export_names(extractor):
+    """
+    extract file exports and their addresses
+    """
+    for symbol, va in miasm.jitter.loader.pe.get_export_name_addr_list(extractor.pe):
+        # Only use func names and not ordinals
+        if isinstance(symbol, str):
+            yield Export(symbol), va
+
+
+def extract_file_import_names(extractor):
+    """
+    extract imported function names and their addresses
+    1. imports by ordinal:
+     - modulename.#ordinal
+    2. imports by name, results in two features to support importname-only matching:
+     - modulename.importname
+     - importname
+    """
+    for ((dll, symbol), va_set) in miasm.jitter.loader.pe.get_import_address_pe(extractor.pe).items():
+        dll_name = dll[:-4]  # Remove .dll
+        for va in va_set:
+            if isinstance(symbol, int):
+                yield Import("%s.#%s" % (dll_name, symbol)), va
+            else:
+                yield Import("%s.%s" % (dll_name, symbol)), va
+                yield Import(symbol), va
+
+
+def extract_file_section_names(extractor):
+    """
+    extract file sections and their addresses
+    """
+    for section in extractor.pe.SHList.shlist:
+        name = section.name.partition(b"\x00")[0].decode("ascii")
+        va = section.addr
+        yield Section(name), va
+
+
+def extract_file_strings(extractor):
+    """
+    extract ASCII and UTF-16 LE strings from file
+    """
+    for s in capa.features.extractors.strings.extract_ascii_strings(extractor.buf):
+        yield String(s.s), s.offset
+
+    for s in capa.features.extractors.strings.extract_unicode_strings(extractor.buf):
+        yield String(s.s), s.offset
+
+
+def extract_file_features(extractor):
+    """
+    extract file features from given buffer and parsed binary
+
+    args:
+      buf (bytes): binary content
+      container (miasm.analysis.binary.ContainerPE): parsed binary returned by miasm
+
+    yields:
+      Tuple[Feature, VA]: a feature and its location.
+    """
+    for file_handler in FILE_HANDLERS:
+        for feature, va in file_handler(extractor):
+            yield feature, va
+
+
+FILE_HANDLERS = (
+    extract_file_embedded_pe,
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_section_names,
+    extract_file_strings,
+)

capa/features/extractors/miasm/function.py

@@ -0,0 +1,50 @@
+# Copyright (C) 2020 FireEye, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: https://github.com/fireeye/capa/blob/master/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from capa.features import Characteristic
+
+
+def extract_function_calls_to(extractor, loc_key):
+    for pred_key in extractor.cfg.predecessors(loc_key):
+        pred_block = extractor.cfg.loc_key_to_block(pred_key)
+        pred_insn = pred_block.get_subcall_instr()
+        if pred_insn and pred_insn.is_subcall():
+            dst = pred_insn.args[0]
+            if dst.is_loc() and dst.loc_key == loc_key:
+                yield Characteristic("calls to"), pred_insn.offset
+
+
+def extract_function_loop(extractor, loc_key):
+    """
+    returns if the function has a loop
+    """
+    block = extractor.cfg.loc_key_to_block(loc_key)
+    disassembler = extractor.machine.dis_engine(
+        extractor.container.bin_stream, loc_db=extractor.loc_db, follow_call=False
+    )
+    offset = extractor.block_offset(block)
+    cfg = disassembler.dis_multiblock(offset)
+    if cfg.has_loop():
+        yield Characteristic("loop"), offset
+
+
+def extract_features(extractor, loc_key):
+    """
+    extract features from the given function.
+    args:
+      cfg (AsmCFG): the CFG of the function from which to extract features
+      loc_key (LocKey): LocKey which represents the beginning of the function
+    yields:
+      Feature, set[VA]: the features and their location found in this function.
+    """
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, va in func_handler(extractor, loc_key):
+            yield feature, va
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)

capa/features/extractors/miasm/insn.py

@@ -0,0 +1,126 @@
+# Copyright (C) 2020 FireEye, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: https://github.com/fireeye/capa/blob/master/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import miasm.expression.expression
+
+import capa.features.extractors.helpers
+from capa.features.insn import Mnemonic
+
+
+# TODO: remove duplication (similar code in file.py)
+# TODO: this function should be cached
+def get_imports(pe):
+    imports = {}
+    for ((dll, symbol), va_set) in miasm.jitter.loader.pe.get_import_address_pe(pe).items():
+        dll_name = dll[:-4]
+        for va in va_set:
+            if isinstance(symbol, int):
+                imports[va] = "%s.#%s" % (dll_name, symbol)
+            else:
+                imports[va] = "%s.%s" % (dll_name, symbol)
+    return imports
+
+
+def extract_insn_api_features(extractor, _f, _bb, insn):
+    """parse API features from the given instruction."""
+    if insn.is_subcall():
+        arg = insn.args[0]
+        if isinstance(arg, miasm.expression.expression.ExprMem) and isinstance(
+            arg.ptr, miasm.expression.expression.ExprInt
+        ):
+            target = int(arg.ptr)
+            imports = get_imports(extractor.pe)
+            if target in imports:
+                dll, _, symbol = imports[target].rpartition(".")
+                for feature in capa.features.extractors.helpers.generate_symbols(dll, symbol):
+                    yield feature, insn.offset
+
+
+def extract_insn_number_features(extractor, f, bb, insn):
+    """parse number features from the given instruction."""
+    raise NotImplementedError()
+
+
+def extract_insn_string_features(extractor, f, bb, insn):
+    """parse string features from the given instruction."""
+    raise NotImplementedError()
+
+
+def extract_insn_offset_features(extractor, f, bb, insn):
+    """parse structure offset features from the given instruction."""
+    raise NotImplementedError()
+
+
+def extract_insn_nzxor_characteristic_features(extractor, f, bb, insn):
+    """
+    parse non-zeroing XOR instruction from the given instruction.
+    ignore expected non-zeroing XORs, e.g. security cookies.
+    """
+    raise NotImplementedError()
+
+
+def extract_insn_mnemonic_features(extractor, f, bb, insn):
+    """parse mnemonic features from the given instruction."""
+    yield Mnemonic(insn.name), insn.offset
+
+
+def extract_insn_peb_access_characteristic_features(extractor, f, bb, insn):
+    """
+    parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
+    """
+    raise NotImplementedError()
+
+
+def extract_insn_segment_access_features(extractor, f, bb, insn):
+    """ parse the instruction for access to fs or gs """
+    raise NotImplementedError()
+
+
+def extract_insn_cross_section_cflow(extractor, f, bb, insn):
+    """
+    inspect the instruction for a CALL or JMP that crosses section boundaries.
+    """
+    raise NotImplementedError()
+
+
+# this is a feature that's most relevant at the function scope,
+# however, its most efficient to extract at the instruction scope.
+def extract_function_calls_from(f, bb, insn):
+    raise NotImplementedError()
+
+
+def extract_features(extractor, f, bb, insn):
+    """
+    extract features from the given insn.
+    args:
+      extractor (MiasmFeatureExtractor)
+      f (miasm.expression.expression.LocKey): the function from which to extract features
+      bb (miasm.core.asmblock.AsmBlock): the basic block to process.
+      insn (Instruction): the instruction to process.
+    yields:
+      Feature, set[VA]: the features and their location found in this insn.
+    """
+    for insn_handler in INSTRUCTION_HANDLERS:
+        for feature, va in insn_handler(extractor, f, bb, insn):
+            yield feature, va
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    # extract_insn_number_features,
+    # extract_insn_string_features,
+    # extract_insn_bytes_features,
+    # extract_insn_offset_features,
+    # extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    # extract_insn_peb_access_characteristic_features,
+    # extract_insn_cross_section_cflow,
+    # extract_insn_segment_access_features,
+    # extract_function_calls_from,
+    # extract_function_indirect_call_characteristic_features,
+)

capa/features/extractors/null.py

@@ -1,170 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Dict, List, Tuple, Union
-from dataclasses import dataclass
-
-from typing_extensions import TypeAlias
-
-from capa.features.common import Feature
-from capa.features.address import NO_ADDRESS, Address, ThreadAddress, ProcessAddress, DynamicCallAddress
-from capa.features.extractors.base_extractor import (
-    BBHandle,
-    CallHandle,
-    InsnHandle,
-    SampleHashes,
-    ThreadHandle,
-    ProcessHandle,
-    FunctionHandle,
-    StaticFeatureExtractor,
-    DynamicFeatureExtractor,
-)
-
-
-@dataclass
-class InstructionFeatures:
-    features: List[Tuple[Address, Feature]]
-
-
-@dataclass
-class BasicBlockFeatures:
-    features: List[Tuple[Address, Feature]]
-    instructions: Dict[Address, InstructionFeatures]
-
-
-@dataclass
-class FunctionFeatures:
-    features: List[Tuple[Address, Feature]]
-    basic_blocks: Dict[Address, BasicBlockFeatures]
-
-
-@dataclass
-class NullStaticFeatureExtractor(StaticFeatureExtractor):
-    """
-    An extractor that extracts some user-provided features.
-
-    This is useful for testing, as we can provide expected values and see if matching works.
-    """
-
-    base_address: Address
-    sample_hashes: SampleHashes
-    global_features: List[Feature]
-    file_features: List[Tuple[Address, Feature]]
-    functions: Dict[Address, FunctionFeatures]
-
-    def get_base_address(self):
-        return self.base_address
-
-    def get_sample_hashes(self) -> SampleHashes:
-        return self.sample_hashes
-
-    def extract_global_features(self):
-        for feature in self.global_features:
-            yield feature, NO_ADDRESS
-
-    def extract_file_features(self):
-        for address, feature in self.file_features:
-            yield feature, address
-
-    def get_functions(self):
-        for address in sorted(self.functions.keys()):
-            yield FunctionHandle(address, None)
-
-    def extract_function_features(self, f):
-        for address, feature in self.functions[f.address].features:
-            yield feature, address
-
-    def get_basic_blocks(self, f):
-        for address in sorted(self.functions[f.address].basic_blocks.keys()):
-            yield BBHandle(address, None)
-
-    def extract_basic_block_features(self, f, bb):
-        for address, feature in self.functions[f.address].basic_blocks[bb.address].features:
-            yield feature, address
-
-    def get_instructions(self, f, bb):
-        for address in sorted(self.functions[f.address].basic_blocks[bb.address].instructions.keys()):
-            yield InsnHandle(address, None)
-
-    def extract_insn_features(self, f, bb, insn):
-        for address, feature in self.functions[f.address].basic_blocks[bb.address].instructions[insn.address].features:
-            yield feature, address
-
-
-@dataclass
-class CallFeatures:
-    name: str
-    features: List[Tuple[Address, Feature]]
-
-
-@dataclass
-class ThreadFeatures:
-    features: List[Tuple[Address, Feature]]
-    calls: Dict[Address, CallFeatures]
-
-
-@dataclass
-class ProcessFeatures:
-    features: List[Tuple[Address, Feature]]
-    threads: Dict[Address, ThreadFeatures]
-    name: str
-
-
-@dataclass
-class NullDynamicFeatureExtractor(DynamicFeatureExtractor):
-    base_address: Address
-    sample_hashes: SampleHashes
-    global_features: List[Feature]
-    file_features: List[Tuple[Address, Feature]]
-    processes: Dict[Address, ProcessFeatures]
-
-    def extract_global_features(self):
-        for feature in self.global_features:
-            yield feature, NO_ADDRESS
-
-    def get_sample_hashes(self) -> SampleHashes:
-        return self.sample_hashes
-
-    def extract_file_features(self):
-        for address, feature in self.file_features:
-            yield feature, address
-
-    def get_processes(self):
-        for address in sorted(self.processes.keys()):
-            assert isinstance(address, ProcessAddress)
-            yield ProcessHandle(address=address, inner={})
-
-    def extract_process_features(self, ph):
-        for addr, feature in self.processes[ph.address].features:
-            yield feature, addr
-
-    def get_process_name(self, ph) -> str:
-        return self.processes[ph.address].name
-
-    def get_threads(self, ph):
-        for address in sorted(self.processes[ph.address].threads.keys()):
-            assert isinstance(address, ThreadAddress)
-            yield ThreadHandle(address=address, inner={})
-
-    def extract_thread_features(self, ph, th):
-        for addr, feature in self.processes[ph.address].threads[th.address].features:
-            yield feature, addr
-
-    def get_calls(self, ph, th):
-        for address in sorted(self.processes[ph.address].threads[th.address].calls.keys()):
-            assert isinstance(address, DynamicCallAddress)
-            yield CallHandle(address=address, inner={})
-
-    def extract_call_features(self, ph, th, ch):
-        for address, feature in self.processes[ph.address].threads[th.address].calls[ch.address].features:
-            yield feature, address
-
-    def get_call_name(self, ph, th, ch) -> str:
-        return self.processes[ph.address].threads[th.address].calls[ch.address].name
-
-
-NullFeatureExtractor: TypeAlias = Union[NullStaticFeatureExtractor, NullDynamicFeatureExtractor]

capa/features/extractors/pefile.py

@@ -1,229 +0,0 @@
-# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import logging
-from pathlib import Path
-
-import pefile
-
-import capa.features.common
-import capa.features.extractors
-import capa.features.extractors.common
-import capa.features.extractors.helpers
-import capa.features.extractors.strings
-from capa.features.file import Export, Import, Section
-from capa.features.common import OS, ARCH_I386, FORMAT_PE, ARCH_AMD64, OS_WINDOWS, Arch, Format, Characteristic
-from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import SampleHashes, StaticFeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_embedded_pe(buf, **kwargs):
-    for offset, _ in capa.features.extractors.helpers.carve_pe(buf, 1):
-        yield Characteristic("embedded pe"), FileOffsetAddress(offset)
-
-
-def extract_file_export_names(pe, **kwargs):
-    base_address = pe.OPTIONAL_HEADER.ImageBase
-
-    if hasattr(pe, "DIRECTORY_ENTRY_EXPORT"):
-        for export in pe.DIRECTORY_ENTRY_EXPORT.symbols:
-            if not export.name:
-                continue
-            try:
-                name = export.name.partition(b"\x00")[0].decode("ascii")
-            except UnicodeDecodeError:
-                continue
-
-            if export.forwarder is None:
-                va = base_address + export.address
-                yield Export(name), AbsoluteVirtualAddress(va)
-
-            else:
-                try:
-                    forwarded_name = export.forwarder.partition(b"\x00")[0].decode("ascii")
-                except UnicodeDecodeError:
-                    continue
-                forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(forwarded_name)
-                va = base_address + export.address
-                yield Export(forwarded_name), AbsoluteVirtualAddress(va)
-                yield Characteristic("forwarded export"), AbsoluteVirtualAddress(va)
-
-
-def extract_file_import_names(pe, **kwargs):
-    """
-    extract imported function names
-    1. imports by ordinal:
-     - modulename.#ordinal
-    2. imports by name, results in two features to support importname-only matching:
-     - modulename.importname
-     - importname
-    """
-    if hasattr(pe, "DIRECTORY_ENTRY_IMPORT"):
-        for dll in pe.DIRECTORY_ENTRY_IMPORT:
-            try:
-                modname = dll.dll.partition(b"\x00")[0].decode("ascii")
-            except UnicodeDecodeError:
-                continue
-
-            # strip extension
-            modname = modname.rpartition(".")[0].lower()
-
-            for imp in dll.imports:
-                if imp.import_by_ordinal:
-                    impname = f"#{imp.ordinal}"
-                else:
-                    try:
-                        impname = imp.name.partition(b"\x00")[0].decode("ascii")
-                    except UnicodeDecodeError:
-                        continue
-
-                for name in capa.features.extractors.helpers.generate_symbols(modname, impname, include_dll=True):
-                    yield Import(name), AbsoluteVirtualAddress(imp.address)
-
-
-def extract_file_section_names(pe, **kwargs):
-    base_address = pe.OPTIONAL_HEADER.ImageBase
-
-    for section in pe.sections:
-        try:
-            name = section.Name.partition(b"\x00")[0].decode("ascii")
-        except UnicodeDecodeError:
-            continue
-
-        yield Section(name), AbsoluteVirtualAddress(base_address + section.VirtualAddress)
-
-
-def extract_file_strings(buf, **kwargs):
-    yield from capa.features.extractors.common.extract_file_strings(buf)
-
-
-def extract_file_function_names(**kwargs):
-    """
-    extract the names of statically-linked library functions.
-    """
-    if False:
-        # using a `yield` here to force this to be a generator, not function.
-        yield NotImplementedError("pefile doesn't have library matching")
-    return
-
-
-def extract_file_os(**kwargs):
-    # assuming PE -> Windows
-    # though i suppose they're also used by UEFI
-    yield OS(OS_WINDOWS), NO_ADDRESS
-
-
-def extract_file_format(**kwargs):
-    yield Format(FORMAT_PE), NO_ADDRESS
-
-
-def extract_file_arch(pe, **kwargs):
-    if pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_I386"]:
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_AMD64"]:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        logger.warning("unsupported architecture: %s", pefile.MACHINE_TYPE[pe.FILE_HEADER.Machine])
-
-
-def extract_file_features(pe, buf):
-    """
-    extract file features from given workspace
-
-    args:
-      pe (pefile.PE): the parsed PE
-      buf: the raw sample bytes
-
-    yields:
-      Tuple[Feature, VA]: a feature and its location.
-    """
-
-    for file_handler in FILE_HANDLERS:
-        # file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
-        for feature, va in file_handler(pe=pe, buf=buf):  # type: ignore
-            yield feature, va
-
-
-FILE_HANDLERS = (
-    extract_file_embedded_pe,
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_section_names,
-    extract_file_strings,
-    extract_file_function_names,
-    extract_file_format,
-)
-
-
-def extract_global_features(pe, buf):
-    """
-    extract global features from given workspace
-
-    args:
-      pe (pefile.PE): the parsed PE
-      buf: the raw sample bytes
-
-    yields:
-      Tuple[Feature, VA]: a feature and its location.
-    """
-    for handler in GLOBAL_HANDLERS:
-        # file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
-        for feature, va in handler(pe=pe, buf=buf):  # type: ignore
-            yield feature, va
-
-
-GLOBAL_HANDLERS = (
-    extract_file_os,
-    extract_file_arch,
-)
-
-
-class PefileFeatureExtractor(StaticFeatureExtractor):
-    def __init__(self, path: Path):
-        super().__init__(hashes=SampleHashes.from_bytes(path.read_bytes()))
-        self.path: Path = path
-        self.pe = pefile.PE(str(path))
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(self.pe.OPTIONAL_HEADER.ImageBase)
-
-    def extract_global_features(self):
-        buf = Path(self.path).read_bytes()
-
-        yield from extract_global_features(self.pe, buf)
-
-    def extract_file_features(self):
-        buf = Path(self.path).read_bytes()
-
-        yield from extract_file_features(self.pe, buf)
-
-    def get_functions(self):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def extract_function_features(self, f):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def get_basic_blocks(self, f):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def extract_basic_block_features(self, f, bb):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def get_instructions(self, f, bb):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def extract_insn_features(self, f, bb, insn):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def is_library_function(self, va):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def get_function_name(self, va):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")

capa/features/extractors/smda/__init__.py

@@ -0,0 +1,52 @@
+import sys
+import types
+
+from smda.common.SmdaReport import SmdaReport
+from smda.common.SmdaInstruction import SmdaInstruction
+
+import capa.features.extractors.smda.file
+import capa.features.extractors.smda.insn
+import capa.features.extractors.smda.function
+import capa.features.extractors.smda.basicblock
+from capa.main import UnsupportedRuntimeError
+from capa.features.extractors import FeatureExtractor
+
+
+class SmdaFeatureExtractor(FeatureExtractor):
+    def __init__(self, smda_report: SmdaReport, path):
+        super(SmdaFeatureExtractor, self).__init__()
+        if sys.version_info < (3, 0):
+            raise UnsupportedRuntimeError("SMDA should only be used with Python 3.")
+        self.smda_report = smda_report
+        self.path = path
+
+    def get_base_address(self):
+        return self.smda_report.base_addr
+
+    def extract_file_features(self):
+        for feature, va in capa.features.extractors.smda.file.extract_features(self.smda_report, self.path):
+            yield feature, va
+
+    def get_functions(self):
+        for function in self.smda_report.getFunctions():
+            yield function
+
+    def extract_function_features(self, f):
+        for feature, va in capa.features.extractors.smda.function.extract_features(f):
+            yield feature, va
+
+    def get_basic_blocks(self, f):
+        for bb in f.getBlocks():
+            yield bb
+
+    def extract_basic_block_features(self, f, bb):
+        for feature, va in capa.features.extractors.smda.basicblock.extract_features(f, bb):
+            yield feature, va
+
+    def get_instructions(self, f, bb):
+        for smda_ins in bb.getInstructions():
+            yield smda_ins
+
+    def extract_insn_features(self, f, bb, insn):
+        for feature, va in capa.features.extractors.smda.insn.extract_features(f, bb, insn):
+            yield feature, va

capa/features/extractors/smda/basicblock.py

@@ -0,0 +1,131 @@
+import sys
+import string
+import struct
+
+from capa.features import Characteristic
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+
+
+def _bb_has_tight_loop(f, bb):
+    """
+    parse tight loops, true if last instruction in basic block branches to bb start
+    """
+    return bb.offset in f.blockrefs[bb.offset] if bb.offset in f.blockrefs else False
+
+
+def extract_bb_tight_loop(f, bb):
+    """ check basic block for tight loop indicators """
+    if _bb_has_tight_loop(f, bb):
+        yield Characteristic("tight loop"), bb.offset
+
+
+def _bb_has_stackstring(f, bb):
+    """
+    extract potential stackstring creation, using the following heuristics:
+      - basic block contains enough moves of constant bytes to the stack
+    """
+    count = 0
+    for instr in bb.getInstructions():
+        if is_mov_imm_to_stack(instr):
+            count += get_printable_len(instr.getDetailed())
+        if count > MIN_STACKSTRING_LEN:
+            return True
+    return False
+
+
+def get_operands(smda_ins):
+    return [o.strip() for o in smda_ins.operands.split(",")]
+
+
+def extract_stackstring(f, bb):
+    """ check basic block for stackstring indicators """
+    if _bb_has_stackstring(f, bb):
+        yield Characteristic("stack string"), bb.offset
+
+
+def is_mov_imm_to_stack(smda_ins):
+    """
+    Return if instruction moves immediate onto stack
+    """
+    if not smda_ins.mnemonic.startswith("mov"):
+        return False
+
+    try:
+        dst, src = get_operands(smda_ins)
+    except ValueError:
+        # not two operands
+        return False
+
+    try:
+        int(src, 16)
+    except ValueError:
+        return False
+
+    if not any(regname in dst for regname in ["ebp", "rbp", "esp", "rsp"]):
+        return False
+
+    return True
+
+
+def is_printable_ascii(chars):
+    return all(c < 127 and chr(c) in string.printable for c in chars)
+
+
+def is_printable_utf16le(chars):
+    if all(c == 0x00 for c in chars[1::2]):
+        return is_printable_ascii(chars[::2])
+
+
+def get_printable_len(instr):
+    """
+    Return string length if all operand bytes are ascii or utf16-le printable
+
+    Works on a capstone instruction
+    """
+    # should have exactly two operands for mov immediate
+    if len(instr.operands) != 2:
+        return 0
+
+    op_value = instr.operands[1].value.imm
+
+    if instr.imm_size == 1:
+        chars = struct.pack("<B", op_value & 0xFF)
+    elif instr.imm_size == 2:
+        chars = struct.pack("<H", op_value & 0xFFFF)
+    elif instr.imm_size == 4:
+        chars = struct.pack("<I", op_value & 0xFFFFFFFF)
+    elif instr.imm_size == 8:
+        chars = struct.pack("<Q", op_value & 0xFFFFFFFFFFFFFFFF)
+    else:
+        raise ValueError("Unhandled operand data type 0x%x." % instr.imm_size)
+
+    if is_printable_ascii(chars):
+        return instr.imm_size
+    if is_printable_utf16le(chars):
+        return instr.imm_size // 2
+
+    return 0
+
+
+def extract_features(f, bb):
+    """
+    extract features from the given basic block.
+
+    args:
+      f (smda.common.SmdaFunction): the function from which to extract features
+      bb (smda.common.SmdaBasicBlock): the basic block to process.
+
+    yields:
+      Feature, set[VA]: the features and their location found in this basic block.
+    """
+    yield BasicBlock(), bb.offset
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, va in bb_handler(f, bb):
+            yield feature, va
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    extract_stackstring,
+)

capa/features/extractors/smda/file.py

@@ -0,0 +1,139 @@
+import struct
+
+# if we have SMDA we definitely have lief
+import lief
+
+import capa.features.extractors.helpers
+import capa.features.extractors.strings
+from capa.features import String, Characteristic
+from capa.features.file import Export, Import, Section
+
+
+def carve(pbytes, offset=0):
+    """
+    Return a list of (offset, size, xor) tuples of embedded PEs
+
+    Based on the version from vivisect:
+    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
+    And its IDA adaptation:
+    capa/features/extractors/ida/file.py
+    """
+    mz_xor = [
+        (
+            capa.features.extractors.helpers.xor_static(b"MZ", i),
+            capa.features.extractors.helpers.xor_static(b"PE", i),
+            i,
+        )
+        for i in range(256)
+    ]
+
+    pblen = len(pbytes)
+    todo = [(pbytes.find(mzx, offset), mzx, pex, i) for mzx, pex, i in mz_xor]
+    todo = [(off, mzx, pex, i) for (off, mzx, pex, i) in todo if off != -1]
+
+    while len(todo):
+
+        off, mzx, pex, i = todo.pop()
+
+        # The MZ header has one field we will check
+        # e_lfanew is at 0x3c
+        e_lfanew = off + 0x3C
+        if pblen < (e_lfanew + 4):
+            continue
+
+        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(pbytes[e_lfanew : e_lfanew + 4], i))[0]
+
+        nextres = pbytes.find(mzx, off + 1)
+        if nextres != -1:
+            todo.append((nextres, mzx, pex, i))
+
+        peoff = off + newoff
+        if pblen < (peoff + 2):
+            continue
+
+        if pbytes[peoff : peoff + 2] == pex:
+            yield (off, i)
+
+
+def extract_file_embedded_pe(smda_report, file_path):
+    with open(file_path, "rb") as f:
+        fbytes = f.read()
+
+    for offset, i in carve(fbytes, 1):
+        yield Characteristic("embedded pe"), offset
+
+
+def extract_file_export_names(smda_report, file_path):
+    lief_binary = lief.parse(file_path)
+    if lief_binary is not None:
+        for function in lief_binary.exported_functions:
+            yield Export(function.name), function.address
+
+
+def extract_file_import_names(smda_report, file_path):
+    # extract import table info via LIEF
+    lief_binary = lief.parse(file_path)
+    if not isinstance(lief_binary, lief.PE.Binary):
+        return
+    for imported_library in lief_binary.imports:
+        library_name = imported_library.name.lower()
+        library_name = library_name[:-4] if library_name.endswith(".dll") else library_name
+        for func in imported_library.entries:
+            if func.name:
+                va = func.iat_address + smda_report.base_addr
+                for name in capa.features.extractors.helpers.generate_symbols(library_name, func.name):
+                    yield Import(name), va
+            elif func.is_ordinal:
+                for name in capa.features.extractors.helpers.generate_symbols(library_name, "#%s" % func.ordinal):
+                    yield Import(name), va
+
+
+def extract_file_section_names(smda_report, file_path):
+    lief_binary = lief.parse(file_path)
+    if not isinstance(lief_binary, lief.PE.Binary):
+        return
+    if lief_binary and lief_binary.sections:
+        base_address = lief_binary.optional_header.imagebase
+        for section in lief_binary.sections:
+            yield Section(section.name), base_address + section.virtual_address
+
+
+def extract_file_strings(smda_report, file_path):
+    """
+    extract ASCII and UTF-16 LE strings from file
+    """
+    with open(file_path, "rb") as f:
+        b = f.read()
+
+    for s in capa.features.extractors.strings.extract_ascii_strings(b):
+        yield String(s.s), s.offset
+
+    for s in capa.features.extractors.strings.extract_unicode_strings(b):
+        yield String(s.s), s.offset
+
+
+def extract_features(smda_report, file_path):
+    """
+    extract file features from given workspace
+
+    args:
+      smda_report (smda.common.SmdaReport): a SmdaReport
+      file_path: path to the input file
+
+    yields:
+      Tuple[Feature, VA]: a feature and its location.
+    """
+
+    for file_handler in FILE_HANDLERS:
+        result = file_handler(smda_report, file_path)
+        for feature, va in file_handler(smda_report, file_path):
+            yield feature, va
+
+
+FILE_HANDLERS = (
+    extract_file_embedded_pe,
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_section_names,
+    extract_file_strings,
+)

capa/features/extractors/smda/function.py

@@ -0,0 +1,38 @@
+from capa.features import Characteristic
+from capa.features.extractors import loops
+
+
+def extract_function_calls_to(f):
+    for inref in f.inrefs:
+        yield Characteristic("calls to"), inref
+
+
+def extract_function_loop(f):
+    """
+    parse if a function has a loop
+    """
+    edges = []
+    for bb_from, bb_tos in f.blockrefs.items():
+        for bb_to in bb_tos:
+            edges.append((bb_from, bb_to))
+
+    if edges and loops.has_loop(edges):
+        yield Characteristic("loop"), f.offset
+
+
+def extract_features(f):
+    """
+    extract features from the given function.
+
+    args:
+      f (smda.common.SmdaFunction): the function from which to extract features
+
+    yields:
+      Feature, set[VA]: the features and their location found in this function.
+    """
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, va in func_handler(f):
+            yield feature, va
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)

capa/features/extractors/smda/insn.py

@@ -0,0 +1,393 @@
+import re
+import string
+import struct
+
+from smda.common.SmdaReport import SmdaReport
+
+import capa.features.extractors.helpers
+from capa.features import (
+    ARCH_X32,
+    ARCH_X64,
+    MAX_BYTES_FEATURE_SIZE,
+    THUNK_CHAIN_DEPTH_DELTA,
+    Bytes,
+    String,
+    Characteristic,
+)
+from capa.features.insn import API, Number, Offset, Mnemonic
+
+# security cookie checks may perform non-zeroing XORs, these are expected within a certain
+# byte range within the first and returning basic blocks, this helps to reduce FP features
+SECURITY_COOKIE_BYTES_DELTA = 0x40
+PATTERN_HEXNUM = re.compile(r"[+\-] (?P<num>0x[a-fA-F0-9]+)")
+PATTERN_SINGLENUM = re.compile(r"[+\-] (?P<num>[0-9])")
+
+
+def get_arch(smda_report):
+    if smda_report.architecture == "intel":
+        if smda_report.bitness == 32:
+            return ARCH_X32
+        elif smda_report.bitness == 64:
+            return ARCH_X64
+    else:
+        raise NotImplementedError
+
+
+def extract_insn_api_features(f, bb, insn):
+    """parse API features from the given instruction."""
+    if insn.offset in f.apirefs:
+        api_entry = f.apirefs[insn.offset]
+        # reformat
+        dll_name, api_name = api_entry.split("!")
+        dll_name = dll_name.split(".")[0]
+        dll_name = dll_name.lower()
+        for name in capa.features.extractors.helpers.generate_symbols(dll_name, api_name):
+            yield API(name), insn.offset
+    elif insn.offset in f.outrefs:
+        current_function = f
+        current_instruction = insn
+        for index in range(THUNK_CHAIN_DEPTH_DELTA):
+            if current_function and len(current_function.outrefs[current_instruction.offset]) == 1:
+                target = current_function.outrefs[current_instruction.offset][0]
+                referenced_function = current_function.smda_report.getFunction(target)
+                if referenced_function:
+                    # TODO SMDA: implement this function for both jmp and call, checking if function has 1 instruction which refs an API
+                    if referenced_function.isApiThunk():
+                        api_entry = (
+                            referenced_function.apirefs[target] if target in referenced_function.apirefs else None
+                        )
+                        if api_entry:
+                            # reformat
+                            dll_name, api_name = api_entry.split("!")
+                            dll_name = dll_name.split(".")[0]
+                            dll_name = dll_name.lower()
+                            for name in capa.features.extractors.helpers.generate_symbols(dll_name, api_name):
+                                yield API(name), insn.offset
+                    elif referenced_function.num_instructions == 1 and referenced_function.num_outrefs == 1:
+                        current_function = referenced_function
+                        current_instruction = [i for i in referenced_function.getInstructions()][0]
+                else:
+                    return
+
+
+def extract_insn_number_features(f, bb, insn):
+    """parse number features from the given instruction."""
+    # example:
+    #
+    #     push    3136B0h         ; dwControlCode
+    operands = [o.strip() for o in insn.operands.split(",")]
+    if insn.mnemonic == "add" and operands[0] in ["esp", "rsp"]:
+        # skip things like:
+        #
+        #    .text:00401140                 call    sub_407E2B
+        #    .text:00401145                 add     esp, 0Ch
+        return
+    for operand in operands:
+        try:
+            yield Number(int(operand, 16)), insn.offset
+            yield Number(int(operand, 16), arch=get_arch(f.smda_report)), insn.offset
+        except:
+            continue
+
+
+def read_bytes(smda_report, va, num_bytes=None):
+    """
+    read up to MAX_BYTES_FEATURE_SIZE from the given address.
+    """
+
+    rva = va - smda_report.base_addr
+    if smda_report.buffer is None:
+        return
+    buffer_end = len(smda_report.buffer)
+    max_bytes = num_bytes if num_bytes is not None else MAX_BYTES_FEATURE_SIZE
+    if rva + max_bytes > buffer_end:
+        return smda_report.buffer[rva:]
+    else:
+        return smda_report.buffer[rva : rva + max_bytes]
+
+
+def derefs(smda_report, p):
+    """
+    recursively follow the given pointer, yielding the valid memory addresses along the way.
+    useful when you may have a pointer to string, or pointer to pointer to string, etc.
+
+    this is a "do what i mean" type of helper function.
+
+    based on the implementation in viv/insn.py
+    """
+    depth = 0
+    while True:
+        if not smda_report.isAddrWithinMemoryImage(p):
+            return
+        yield p
+
+        bytes_ = read_bytes(smda_report, p, num_bytes=4)
+        val = struct.unpack("I", bytes_)[0]
+
+        # sanity: pointer points to self
+        if val == p:
+            return
+
+        # sanity: avoid chains of pointers that are unreasonably deep
+        depth += 1
+        if depth > 10:
+            return
+
+        p = val
+
+
+def extract_insn_bytes_features(f, bb, insn):
+    """
+    parse byte sequence features from the given instruction.
+    example:
+        #     push    offset iid_004118d4_IShellLinkA ; riid
+    """
+    for data_ref in insn.getDataRefs():
+        for v in derefs(f.smda_report, data_ref):
+            bytes_read = read_bytes(f.smda_report, v)
+            if bytes_read is None:
+                continue
+            if capa.features.extractors.helpers.all_zeros(bytes_read):
+                continue
+
+            yield Bytes(bytes_read), insn.offset
+
+
+def detect_ascii_len(smda_report, offset):
+    if smda_report.buffer is None:
+        return 0
+    ascii_len = 0
+    rva = offset - smda_report.base_addr
+    char = smda_report.buffer[rva]
+    while char < 127 and chr(char) in string.printable:
+        ascii_len += 1
+        rva += 1
+        char = smda_report.buffer[rva]
+    if char == 0:
+        return ascii_len
+    return 0
+
+
+def detect_unicode_len(smda_report, offset):
+    if smda_report.buffer is None:
+        return 0
+    unicode_len = 0
+    rva = offset - smda_report.base_addr
+    char = smda_report.buffer[rva]
+    second_char = smda_report.buffer[rva + 1]
+    while char < 127 and chr(char) in string.printable and second_char == 0:
+        unicode_len += 2
+        rva += 2
+        char = smda_report.buffer[rva]
+        second_char = smda_report.buffer[rva + 1]
+    if char == 0 and second_char == 0:
+        return unicode_len
+    return 0
+
+
+def read_string(smda_report, offset):
+    alen = detect_ascii_len(smda_report, offset)
+    if alen > 1:
+        return read_bytes(smda_report, offset, alen).decode("utf-8")
+    ulen = detect_unicode_len(smda_report, offset)
+    if ulen > 2:
+        return read_bytes(smda_report, offset, ulen).decode("utf-16")
+
+
+def extract_insn_string_features(f, bb, insn):
+    """parse string features from the given instruction."""
+    # example:
+    #
+    #     push    offset aAcr     ; "ACR  > "
+    for data_ref in insn.getDataRefs():
+        for v in derefs(f.smda_report, data_ref):
+            string_read = read_string(f.smda_report, v)
+            if string_read:
+                yield String(string_read.rstrip("\x00")), insn.offset
+
+
+def extract_insn_offset_features(f, bb, insn):
+    """parse structure offset features from the given instruction."""
+    # examples:
+    #
+    #     mov eax, [esi + 4]
+    #     mov eax, [esi + ecx + 16384]
+    operands = [o.strip() for o in insn.operands.split(",")]
+    for operand in operands:
+        if not "ptr" in operand:
+            continue
+        if "esp" in operand or "ebp" in operand or "rbp" in operand:
+            continue
+        number = 0
+        number_hex = re.search(PATTERN_HEXNUM, operand)
+        number_int = re.search(PATTERN_SINGLENUM, operand)
+        if number_hex:
+            number = int(number_hex.group("num"), 16)
+            number = -1 * number if number_hex.group().startswith("-") else number
+        elif number_int:
+            number = int(number_int.group("num"))
+            number = -1 * number if number_int.group().startswith("-") else number
+        yield Offset(number), insn.offset
+        yield Offset(number, arch=get_arch(f.smda_report)), insn.offset
+
+
+def is_security_cookie(f, bb, insn):
+    """
+    check if an instruction is related to security cookie checks
+    """
+    # security cookie check should use SP or BP
+    operands = [o.strip() for o in insn.operands.split(",")]
+    if operands[1] not in ["esp", "ebp", "rsp", "rbp"]:
+        return False
+    for index, block in enumerate(f.getBlocks()):
+        # expect security cookie init in first basic block within first bytes (instructions)
+        block_instructions = [i for i in block.getInstructions()]
+        if index == 0 and insn.offset < (block_instructions[0].offset + SECURITY_COOKIE_BYTES_DELTA):
+            return True
+        # ... or within last bytes (instructions) before a return
+        if block_instructions[-1].mnemonic.startswith("ret") and insn.offset > (
+            block_instructions[-1].offset - SECURITY_COOKIE_BYTES_DELTA
+        ):
+            return True
+    return False
+
+
+def extract_insn_nzxor_characteristic_features(f, bb, insn):
+    """
+    parse non-zeroing XOR instruction from the given instruction.
+    ignore expected non-zeroing XORs, e.g. security cookies.
+    """
+
+    if insn.mnemonic not in ("xor", "xorpd", "xorps", "pxor"):
+        return
+
+    operands = [o.strip() for o in insn.operands.split(",")]
+    if operands[0] == operands[1]:
+        return
+
+    if is_security_cookie(f, bb, insn):
+        return
+
+    yield Characteristic("nzxor"), insn.offset
+
+
+def extract_insn_mnemonic_features(f, bb, insn):
+    """parse mnemonic features from the given instruction."""
+    yield Mnemonic(insn.mnemonic), insn.offset
+
+
+def extract_insn_peb_access_characteristic_features(f, bb, insn):
+    """
+    parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
+    """
+
+    if insn.mnemonic not in ["push", "mov"]:
+        return
+
+    operands = [o.strip() for o in insn.operands.split(",")]
+    for operand in operands:
+        if "fs:" in operand and "0x30" in operand:
+            yield Characteristic("peb access"), insn.offset
+        elif "gs:" in operand and "0x60" in operand:
+            yield Characteristic("peb access"), insn.offset
+
+
+def extract_insn_segment_access_features(f, bb, insn):
+    """ parse the instruction for access to fs or gs """
+    operands = [o.strip() for o in insn.operands.split(",")]
+    for operand in operands:
+        if "fs:" in operand:
+            yield Characteristic("fs access"), insn.offset
+        elif "gs:" in operand:
+            yield Characteristic("gs access"), insn.offset
+
+
+def extract_insn_cross_section_cflow(f, bb, insn):
+    """
+    inspect the instruction for a CALL or JMP that crosses section boundaries.
+    """
+    if insn.mnemonic in ["call", "jmp"]:
+        if insn.offset in f.apirefs:
+            return
+
+        smda_report = insn.smda_function.smda_report
+        if insn.offset in f.outrefs:
+            for target in f.outrefs[insn.offset]:
+                if smda_report.getSection(insn.offset) != smda_report.getSection(target):
+                    yield Characteristic("cross section flow"), insn.offset
+        elif insn.operands.startswith("0x"):
+            target = int(insn.operands, 16)
+            if smda_report.getSection(insn.offset) != smda_report.getSection(target):
+                yield Characteristic("cross section flow"), insn.offset
+
+
+# this is a feature that's most relevant at the function scope,
+# however, its most efficient to extract at the instruction scope.
+def extract_function_calls_from(f, bb, insn):
+    if insn.mnemonic != "call":
+        return
+
+    if insn.offset in f.outrefs:
+        for outref in f.outrefs[insn.offset]:
+            yield Characteristic("calls from"), outref
+
+            if outref == f.offset:
+                # if we found a jump target and it's the function address
+                # mark as recursive
+                yield Characteristic("recursive call"), outref
+    if insn.offset in f.apirefs:
+        yield Characteristic("calls from"), f.apirefs[insn.offset]
+
+
+# this is a feature that's most relevant at the function or basic block scope,
+# however, its most efficient to extract at the instruction scope.
+def extract_function_indirect_call_characteristic_features(f, bb, insn):
+    """
+    extract indirect function call characteristic (e.g., call eax or call dword ptr [edx+4])
+    does not include calls like => call ds:dword_ABD4974
+    """
+    if insn.mnemonic != "call":
+        return
+    if insn.operands.startswith("0x"):
+        return False
+    if "qword ptr" in insn.operands and "rip" in insn.operands:
+        return False
+    if insn.operands.startswith("dword ptr [0x"):
+        return False
+    # call edx
+    # call dword ptr [eax+50h]
+    # call qword ptr [rsp+78h]
+    yield Characteristic("indirect call"), insn.offset
+
+
+def extract_features(f, bb, insn):
+    """
+    extract features from the given insn.
+
+    args:
+      f (smda.common.SmdaFunction): the function to process.
+      bb (smda.common.SmdaBasicBlock): the basic block to process.
+      insn (smda.common.SmdaInstruction): the instruction to process.
+
+    yields:
+      Feature, set[VA]: the features and their location found in this insn.
+    """
+    for insn_handler in INSTRUCTION_HANDLERS:
+        for feature, va in insn_handler(f, bb, insn):
+            yield feature, va
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_string_features,
+    extract_insn_bytes_features,
+    extract_insn_offset_features,
+    extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    extract_insn_peb_access_characteristic_features,
+    extract_insn_cross_section_cflow,
+    extract_insn_segment_access_features,
+    extract_function_calls_from,
+    extract_function_indirect_call_characteristic_features,
+)