unpack: add can_unpack routine

.gitattributes

@@ -1,9 +0,0 @@
-# Set the default behavior, in case people don't have core.autocrlf set.
-* text=auto
-
-# Explicitly declare text files you want to always be normalized and converted
-# to native line endings on checkout.
-*.py text
-*.yml text
-*.md text
-*.txt text

.github/CODE_OF_CONDUCT.md

@@ -1,46 +1,46 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version]
-
-[homepage]: https://contributor-covenant.org
-[version]: https://contributor-covenant.org/version/1/4/
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

.github/CONTRIBUTING.md

@@ -1,197 +1,197 @@
-# Contributing to Capa
-
-First off, thanks for taking the time to contribute!
-
-The following is a set of guidelines for contributing to capa and its packages, which are hosted in the [Mandiant Organization](https://github.com/mandiant) on GitHub. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
-
-#### Table Of Contents
-
-[Code of Conduct](#code-of-conduct)
-
-[What should I know before I get started?](#what-should-i-know-before-i-get-started)
-  * [Capa and its Repositories](#capa-and-its-repositories)
-  * [Capa Design Decisions](#design-decisions)
-
-[How Can I Contribute?](#how-can-i-contribute)
-  * [Reporting Bugs](#reporting-bugs)
-  * [Suggesting Enhancements](#suggesting-enhancements)
-  * [Your First Code Contribution](#your-first-code-contribution)
-  * [Pull Requests](#pull-requests)
-
-[Styleguides](#styleguides)
-  * [Git Commit Messages](#git-commit-messages)
-  * [Python Styleguide](#python-styleguide)
-  * [Rules Styleguide](#rules-styleguide)
-
-## Code of Conduct
-
-This project and everyone participating in it is governed by the [Capa Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers.
-
-## What should I know before I get started?
-
-### Capa and its repositories
-
-We host the capa project as three Github repositories:
-  - [capa](https://github.com/mandiant/capa)
-  - [capa-rules](https://github.com/mandiant/capa-rules)
-  - [capa-testfiles](https://github.com/mandiant/capa-testfiles)
-  
-The command line tools, logic engine, and other Python source code are found in the `capa` repository.
-This is the repository to fork when you want to enhance the features, performance, or user interface of capa.
-Do *not* push rules directly to this repository, instead...
-
-The standard rules contributed by the community are found in the `capa-rules` repository.
-When you have an idea for a new rule, you should open a PR against `capa-rules`.
-We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and discussions should happen.
-If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`.
-Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way.
-
-Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository.
-These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites;
- however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate,
- a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth.
-
-### Design Decisions
-
-When we make a significant decision in how we maintain the project and what we can or cannot support,
- we will document it in the [capa issues tracker](https://github.com/mandiant/capa/issues).
-This is the best place review our discussions about what/how/why we do things in the project.
-If you have a question, check to see if it is documented there.
-If it is *not* documented there, or you can't find an answer, please open a issue.
-We'll link to existing issues when appropriate to keep discussions in one place.
-
-## How Can I Contribute?
-
-### Reporting Bugs
-
-This section guides you through submitting a bug report for capa.
-Following these guidelines helps maintainers and the community understand your report, reproduce the behavior, and find related reports.
-
-Before creating bug reports, please check [this list](#before-submitting-a-bug-report)
- as you might find out that you don't need to create one.
-When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report).
-Fill out [the required template](./ISSUE_TEMPLATE/bug_report.md),
- the information it asks for helps us resolve issues faster.
-
-> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one.
-
-#### Before Submitting A Bug Report
-
-* **Determine [which repository the problem should be reported in](#capa-and-its-repositories)**.
-* **Perform a [cursory search](https://github.com/mandiant/capa/issues?q=is%3Aissue)** to see if the problem has already been reported. If it has **and the issue is still open**, add a comment to the existing issue instead of opening a new one.
-
-#### How Do I Submit A (Good) Bug Report?
-
-Bugs are tracked as [GitHub issues](https://guides.github.com/features/issues/).
-After you've determined [which repository](#capa-and-its-repositories) your bug is related to,
- create an issue on that repository and provide the following information by filling in 
- [the template](./ISSUE_TEMPLATE/bug_report.md).
-
-Explain the problem and include additional details to help maintainers reproduce the problem:
-
-* **Use a clear and descriptive title** for the issue to identify the problem.
-* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started capa, e.g. which command exactly you used in the terminal, or how you started capa otherwise.
-* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
-* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior.
-* **Explain which behavior you expected to see instead and why.**
-* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
-* **If you're reporting that capa crashed**, include the stack trace from the terminal. Include the stack trace in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist.
-* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below.
-
-Provide more context by answering these questions:
-
-* **Did the problem start happening recently** (e.g. after updating to a new version of capa) or was this always a problem?
-* If the problem started happening recently, **can you reproduce the problem in an older version of capa?** What's the most recent version in which the problem doesn't happen? You can download older versions of capa from [the releases page](https://github.com/mandiant/capa/releases).
-* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens.
-* If the problem is related to working with files (e.g. opening and editing files), **does the problem happen for all files and projects or only some?** Does the problem happen only when working with local or remote files (e.g. on network drives), with files of a specific type (e.g. only JavaScript or Python files), with large files or files with very long lines, or with files in a specific encoding? Is there anything else special about the files you are using?
-
-Include details about your configuration and environment:
-
-* **Which version of capa are you using?** You can get the exact version by running `capa --version` in your terminal.
-* **What's the name and version of the OS you're using**?
-
-### Suggesting Enhancements
-
-This section guides you through submitting an enhancement suggestion for capa, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion and find related suggestions.
-
-Before creating enhancement suggestions, please check [this list](#before-submitting-an-enhancement-suggestion) as you might find out that you don't need to create one. When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](./ISSUE_TEMPLATE/feature_request.md), including the steps that you imagine you would take if the feature you're requesting existed.
-
-#### Before Submitting An Enhancement Suggestion
-
-* **Determine [which repository the enhancement should be suggested in](#capa-and-its-repositories).**
-* **Perform a [cursory search](https://github.com/mandiant/capa/issues?q=is%3Aissue)** to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one.
-
-#### How Do I Submit A (Good) Enhancement Suggestion?
-
-Enhancement suggestions are tracked as [GitHub issues](https://guides.github.com/features/issues/). After you've determined [which repository](#capa-and-its-repositories) your enhancement suggestion is related to, create an issue on that repository and provide the following information:
-
-* **Use a clear and descriptive title** for the issue to identify the suggestion.
-* **Provide a step-by-step description of the suggested enhancement** in as many details as possible.
-* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
-* **Describe the current behavior** and **explain which behavior you expected to see instead** and why.
-* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of capa which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
-* **Explain why this enhancement would be useful** to most capa users and isn't something that can or should be implemented as an external tool that uses capa as a library.
-* **Specify which version of capa you're using.** You can get the exact version by running `capa --version` in your terminal.
-* **Specify the name and version of the OS you're using.**
-
-### Your First Code Contribution
-
-Unsure where to begin contributing to capa? You can start by looking through these `good-first-issue` and `rule-idea` issues:
-
-* [good-first-issue](https://github.com/mandiant/capa/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code, and a test or two.
-* [rule-idea](https://github.com/mandiant/capa-rules/issues?q=is%3Aissue+is%3Aopen+label%3A%22rule+idea%22) - issues that describe potential new rule ideas.
-
-Both issue lists are sorted by total number of comments. While not perfect, number of comments is a reasonable proxy for impact a given change will have.
-
-#### Local development
-
-capa and all its resources can be developed locally. 
-For instructions on how to do this, see the "Method 3" section of the [installation guide](https://github.com/mandiant/capa/blob/master/doc/installation.md).
-
-### Pull Requests
-
-The process described here has several goals:
-
-- Maintain capa's quality
-- Fix problems that are important to users
-- Engage the community in working toward the best possible capa
-- Enable a sustainable system for capa's maintainers to review contributions
-
-Please follow these steps to have your contribution considered by the maintainers:
-
-1. Follow the [styleguides](#styleguides)
-2. Update the CHANGELOG and add tests and documentation. In case they are not needed, indicate it in [the PR template](pull_request_template.md).
-3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing? </summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
-
-While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
-
-## Styleguides
-
-### Git Commit Messages
-
-* Use the present tense ("Add feature" not "Added feature")
-* Use the imperative mood ("Move cursor to..." not "Moves cursor to...")
-* Prefix the first line with the component in question ("rules: ..." or "render: ...")
-* Reference issues and pull requests liberally after the first line
-
-### Python Styleguide
-
-All Python code must adhere to the style guide used by capa: 
-
-  1. [PEP8](https://www.python.org/dev/peps/pep-0008/), with clarifications from
-  2. [Willi's style guide](https://docs.google.com/document/d/1iRpeg-w4DtibwytUyC_dDT7IGhNGBP25-nQfuBa-Fyk/edit?usp=sharing), formatted with
-  3. [isort](https://pypi.org/project/isort/) (with line width 120 and ordered by line length), and formatted with
-  4. [black](https://github.com/psf/black) (with line width 120), and formatted with
-  5. [dos2unix](https://linux.die.net/man/1/dos2unix)
-  
-Our CI pipeline will reformat and enforce the Python styleguide.
- 
-### Rules Styleguide
-
-All (non-nursery) capa rules must:
-
-  1. pass the [linter](https://github.com/mandiant/capa/blob/master/scripts/lint.py), and
-  2. be formatted with [capafmt](https://github.com/mandiant/capa/blob/master/scripts/capafmt.py)
-  
-This ensures that all rules meet the same minimum level of quality and are structured in a consistent way.
-Our CI pipeline will reformat and enforce the capa rules styleguide.
+# Contributing to Capa
+
+First off, thanks for taking the time to contribute!
+
+The following is a set of guidelines for contributing to capa and its packages, which are hosted in the [FireEye Organization](https://github.com/fireeye) on GitHub. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
+
+#### Table Of Contents
+
+[Code of Conduct](#code-of-conduct)
+
+[What should I know before I get started?](#what-should-i-know-before-i-get-started)
+  * [Capa and its Repositories](#capa-and-its-repositories)
+  * [Capa Design Decisions](#design-decisions)
+
+[How Can I Contribute?](#how-can-i-contribute)
+  * [Reporting Bugs](#reporting-bugs)
+  * [Suggesting Enhancements](#suggesting-enhancements)
+  * [Your First Code Contribution](#your-first-code-contribution)
+  * [Pull Requests](#pull-requests)
+
+[Styleguides](#styleguides)
+  * [Git Commit Messages](#git-commit-messages)
+  * [Python Styleguide](#python-styleguide)
+  * [Rules Styleguide](#rules-styleguide)
+
+## Code of Conduct
+
+This project and everyone participating in it is governed by the [Capa Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers.
+
+## What should I know before I get started?
+
+### Capa and its repositories
+
+We host the capa project as three Github repositories:
+  - [capa](https://github.com/fireeye/capa)
+  - [capa-rules](https://github.com/fireeye/capa-rules)
+  - [capa-testfiles](https://github.com/fireeye/capa-testfiles)
+  
+The command line tools, logic engine, and other Python source code are found in the `capa` repository.
+This is the repository to fork when you want to enhance the features, performance, or user interface of capa.
+Do *not* push rules directly to this repository, instead...
+
+The standard rules contributed by the community are found in the `capa-rules` repository.
+When you have an idea for a new rule, you should open a PR against `capa-rules`.
+We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and discussions should happen.
+If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`.
+Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way.
+
+Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository.
+These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites;
+ however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate,
+ a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth.
+
+### Design Decisions
+
+When we make a significant decision in how we maintain the project and what we can or cannot support,
+ we will document it in the [capa issues tracker](https://github.com/fireeye/capa/issues).
+This is the best place review our discussions about what/how/why we do things in the project.
+If you have a question, check to see if it is documented there.
+If it is *not* documented there, or you can't find an answer, please open a issue.
+We'll link to existing issues when appropriate to keep discussions in one place.
+
+## How Can I Contribute?
+
+### Reporting Bugs
+
+This section guides you through submitting a bug report for capa.
+Following these guidelines helps maintainers and the community understand your report, reproduce the behavior, and find related reports.
+
+Before creating bug reports, please check [this list](#before-submitting-a-bug-report)
+ as you might find out that you don't need to create one.
+When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report).
+Fill out [the required template](./ISSUE_TEMPLATE/bug_report.md),
+ the information it asks for helps us resolve issues faster.
+
+> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one.
+
+#### Before Submitting A Bug Report
+
+* **Determine [which repository the problem should be reported in](#capa-and-its-repositories)**.
+* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the problem has already been reported. If it has **and the issue is still open**, add a comment to the existing issue instead of opening a new one.
+
+#### How Do I Submit A (Good) Bug Report?
+
+Bugs are tracked as [GitHub issues](https://guides.github.com/features/issues/).
+After you've determined [which repository](#capa-and-its-repositories) your bug is related to,
+ create an issue on that repository and provide the following information by filling in 
+ [the template](./ISSUE_TEMPLATE/bug_report.md).
+
+Explain the problem and include additional details to help maintainers reproduce the problem:
+
+* **Use a clear and descriptive title** for the issue to identify the problem.
+* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started capa, e.g. which command exactly you used in the terminal, or how you started capa otherwise.
+* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
+* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior.
+* **Explain which behavior you expected to see instead and why.**
+* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
+* **If you're reporting that capa crashed**, include the stack trace from the terminal. Include the stack trace in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist.
+* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below.
+
+Provide more context by answering these questions:
+
+* **Did the problem start happening recently** (e.g. after updating to a new version of capa) or was this always a problem?
+* If the problem started happening recently, **can you reproduce the problem in an older version of capa?** What's the most recent version in which the problem doesn't happen? You can download older versions of capa from [the releases page](https://github.com/fireeye/capa/releases).
+* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens.
+* If the problem is related to working with files (e.g. opening and editing files), **does the problem happen for all files and projects or only some?** Does the problem happen only when working with local or remote files (e.g. on network drives), with files of a specific type (e.g. only JavaScript or Python files), with large files or files with very long lines, or with files in a specific encoding? Is there anything else special about the files you are using?
+
+Include details about your configuration and environment:
+
+* **Which version of capa are you using?** You can get the exact version by running `capa --version` in your terminal.
+* **What's the name and version of the OS you're using**?
+
+### Suggesting Enhancements
+
+This section guides you through submitting an enhancement suggestion for capa, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion and find related suggestions.
+
+Before creating enhancement suggestions, please check [this list](#before-submitting-an-enhancement-suggestion) as you might find out that you don't need to create one. When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](./ISSUE_TEMPLATE/feature_request.md), including the steps that you imagine you would take if the feature you're requesting existed.
+
+#### Before Submitting An Enhancement Suggestion
+
+* **Determine [which repository the enhancement should be suggested in](#capa-and-its-repositories).**
+* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one.
+
+#### How Do I Submit A (Good) Enhancement Suggestion?
+
+Enhancement suggestions are tracked as [GitHub issues](https://guides.github.com/features/issues/). After you've determined [which repository](#capa-and-its-repositories) your enhancement suggestion is related to, create an issue on that repository and provide the following information:
+
+* **Use a clear and descriptive title** for the issue to identify the suggestion.
+* **Provide a step-by-step description of the suggested enhancement** in as many details as possible.
+* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines).
+* **Describe the current behavior** and **explain which behavior you expected to see instead** and why.
+* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of capa which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux.
+* **Explain why this enhancement would be useful** to most capa users and isn't something that can or should be implemented as an external tool that uses capa as a library.
+* **Specify which version of capa you're using.** You can get the exact version by running `capa --version` in your terminal.
+* **Specify the name and version of the OS you're using.**
+
+### Your First Code Contribution
+
+Unsure where to begin contributing to capa? You can start by looking through these `good-first-issue` and `rule-idea` issues:
+
+* [good-first-issue](https://github.com/fireeye/capa/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code, and a test or two.
+* [rule-idea](https://github.com/fireeye/capa-rules/issues?q=is%3Aissue+is%3Aopen+label%3A%22rule+idea%22) - issues that describe potential new rule ideas.
+
+Both issue lists are sorted by total number of comments. While not perfect, number of comments is a reasonable proxy for impact a given change will have.
+
+#### Local development
+
+capa and all its resources can be developed locally. 
+For instructions on how to do this, see the "Method 3" section of the [installation guide](https://github.com/fireeye/capa/blob/master/doc/installation.md).
+
+### Pull Requests
+
+The process described here has several goals:
+
+- Maintain capa's quality
+- Fix problems that are important to users
+- Engage the community in working toward the best possible capa
+- Enable a sustainable system for capa's maintainers to review contributions
+
+Please follow these steps to have your contribution considered by the maintainers:
+
+1. Follow all instructions in [the template](PULL_REQUEST_TEMPLATE.md)
+2. Follow the [styleguides](#styleguides)
+3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing? </summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
+
+While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
+
+## Styleguides
+
+### Git Commit Messages
+
+* Use the present tense ("Add feature" not "Added feature")
+* Use the imperative mood ("Move cursor to..." not "Moves cursor to...")
+* Prefix the first line with the component in question ("rules: ..." or "render: ...")
+* Reference issues and pull requests liberally after the first line
+
+### Python Styleguide
+
+All Python code must adhere to the style guide used by capa: 
+
+  1. [PEP8](https://www.python.org/dev/peps/pep-0008/), with clarifications from
+  2. [Willi's style guide](https://docs.google.com/document/d/1iRpeg-w4DtibwytUyC_dDT7IGhNGBP25-nQfuBa-Fyk/edit?usp=sharing), formatted with
+  3. [isort](https://pypi.org/project/isort/) (with line width 120 and ordered by line length), and formatted with
+  4. [black](https://github.com/psf/black) (with line width 120), and formatted with
+  5. [dos2unix](https://linux.die.net/man/1/dos2unix)
+  
+Our CI pipeline will reformat and enforce the Python styleguide.
+ 
+### Rules Styleguide
+
+All (non-nursery) capa rules must:
+
+  1. pass the [linter](https://github.com/fireeye/capa/blob/master/scripts/lint.py), and
+  2. be formatted with [capafmt](https://github.com/fireeye/capa/blob/master/scripts/capafmt.py)
+  
+This ensures that all rules meet the same minimum level of quality and are structured in a consistent way.
+Our CI pipeline will reformat and enforce the capa rules styleguide.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,47 +1,47 @@
----
-name: Bug report
-about: Create a report to help us improve
-
----
-<!--
-# Is your bug report related to capa rules (for example a false positive)?
-We use submodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/mandiant/capa-rules/issues.
-
-# Have you checked that your issue isn't already filed?
-Please search if there is a similar issue at https://github.com/mandiant/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
-
-# Have you read capa's Code of Conduct?
-By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/mandiant/capa/blob/master/.github/CODE_OF_CONDUCT.md
-
-# Have you read capa's CONTRIBUTING guide?
-It contains helpful information about how to contribute to capa. Check https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md#reporting-bugs
--->
-
-### Description
-
-<!-- Description of the issue -->
-
-### Steps to Reproduce
-
-<!-- 1. First Step -->
-<!-- 2. Second Step -->
-<!-- 3. and so on… -->
-
-**Expected behavior:**
-
-<!-- What you expect to happen -->
-
-**Actual behavior:**
-
-<!-- What actually happens -->
-
-### Versions
-
-<!-- You can get this information from copy and pasting the output of `capa --version` from the command line.
- Please specify the component you're using (e.g. standalone tool or IDA Pro integration) and your Python version.
- Also, please include the OS and what version of the OS you're running. -->
-
-### Additional Information
-
-<!-- Any additional information, configuration or data that might be necessary to reproduce the issue. -->
-
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+<!--
+# Is your bug report related to capa rules (for example a false positive)?
+We use sybmodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/fireeye/capa-rules/issues.
+
+# Have you checked that your issue isn't already filed?
+Please search if there is a similar issue at https://github.com/fireeye/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
+
+# Have you read capa's Code of Conduct?
+By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/fireeye/capa/blob/master/.github/CODE_OF_CONDUCT.md
+
+# Have you read capa's CONTRIBUTING guide?
+It contains helpful information about how to contribute to capa. Check https://github.com/fireeye/capa/blob/master/.github/CONTRIBUTING.md#reporting-bugs
+-->
+
+### Description
+
+<!-- Description of the issue -->
+
+### Steps to Reproduce
+
+<!-- 1. First Step -->
+<!-- 2. Second Step -->
+<!-- 3. and so on… -->
+
+**Expected behavior:**
+
+<!-- What you expect to happen -->
+
+**Actual behavior:**
+
+<!-- What actually happens -->
+
+### Versions
+
+<!-- You can get this information from copy and pasting the output of `capa --version` from the command line.
+ Please specify the component you're using (e.g. standalone tool or IDA Pro integration) and your Python version.
+ Also, please include the OS and what version of the OS you're running. -->
+
+### Additional Information
+
+<!-- Any additional information, configuration or data that might be necessary to reproduce the issue. -->
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,35 +1,35 @@
----
-name: Feature request
-about: Suggest an idea for capa
-
----
-<!--
-# Is your issue related to capa rules (for example an idea for a new rule)?
-We use submodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/mandiant/capa-rules/issues.
-
-# Have you checked that your issue isn't already filed?
-Please search if there is a similar issue at https://github.com/mandiant/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
-
-# Have you read capa's Code of Conduct?
-By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/mandiant/capa/blob/master/.github/CODE_OF_CONDUCT.md
-
-# Have you read capa's CONTRIBUTING guide?
-It contains helpful information about how to contribute to capa. Check https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md#suggesting-enhancements
--->
-
-### Summary
-
-<!-- One paragraph explanation of the feature. -->
-
-### Motivation
-
-<!-- Why are we doing this? What use cases does it support? What is the expected outcome? -->
-
-### Describe alternatives you've considered
-
-<!-- A clear and concise description of the alternative solutions you've considered. -->
-
-## Additional context
-
-<!-- Add any other context or screenshots about the feature request here. -->
-
+---
+name: Feature request
+about: Suggest an idea for capa
+
+---
+<!--
+# Is your issue related to capa rules (for example an idea for a new rule)?
+We use sybmodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/fireeye/capa-rules/issues.
+
+# Have you checked that your issue isn't already filed?
+Please search if there is a similar issue at https://github.com/fireeye/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
+
+# Have you read capa's Code of Conduct?
+By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/fireeye/capa/blob/master/.github/CODE_OF_CONDUCT.md
+
+# Have you read capa's CONTRIBUTING guide?
+It contains helpful information about how to contribute to capa. Check https://github.com/fireeye/capa/blob/master/.github/CONTRIBUTING.md#suggesting-enhancements
+-->
+
+### Summary
+
+<!-- One paragraph explanation of the feature. -->
+
+### Motivation
+
+<!-- Why are we doing this? What use cases does it support? What is the expected outcome? -->
+
+### Describe alternatives you've considered
+
+<!-- A clear and concise description of the alternative solutions you've considered. -->
+
+## Additional context
+
+<!-- Add any other context or screenshots about the feature request here. -->
+

.github/dependabot.yml

@@ -1,6 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "weekly"

.github/mypy/mypy.ini

@@ -1,79 +0,0 @@
-[mypy]
-
-[mypy-halo.*]
-ignore_missing_imports = True
-
-[mypy-tqdm.*]
-ignore_missing_imports = True
-
-[mypy-ruamel.*]
-ignore_missing_imports = True
-
-[mypy-networkx.*]
-ignore_missing_imports = True
-
-[mypy-pefile.*]
-ignore_missing_imports = True
-
-[mypy-viv_utils.*]
-ignore_missing_imports = True
-
-[mypy-flirt.*]
-ignore_missing_imports = True
-
-[mypy-smda.*]
-ignore_missing_imports = True
-
-[mypy-lief.*]
-ignore_missing_imports = True
-
-[mypy-idc.*]
-ignore_missing_imports = True
-
-[mypy-vivisect.*]
-ignore_missing_imports = True
-
-[mypy-envi.*]
-ignore_missing_imports = True
-
-[mypy-PE.*]
-ignore_missing_imports = True
-
-[mypy-idaapi.*]
-ignore_missing_imports = True
-
-[mypy-idautils.*]
-ignore_missing_imports = True
-
-[mypy-ida_bytes.*]
-ignore_missing_imports = True
-
-[mypy-ida_kernwin.*]
-ignore_missing_imports = True
-
-[mypy-ida_settings.*]
-ignore_missing_imports = True
-
-[mypy-ida_funcs.*]
-ignore_missing_imports = True
-
-[mypy-ida_loader.*]
-ignore_missing_imports = True
-
-[mypy-PyQt5.*]
-ignore_missing_imports = True
-
-[mypy-binaryninja.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-devtools.*]
-ignore_missing_imports = True
-
-[mypy-elftools.*]
-ignore_missing_imports = True
-
-[mypy-dncil.*]
-ignore_missing_imports = True

.github/pull_request_template.md

@@ -1,22 +0,0 @@
-
-<!--
-Thank you for contributing to capa! <3
-
-Please read capa's CONTRIBUTING guide if you haven't done so already.
-It contains helpful information about how to contribute to capa. Check https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md
-
-Please describe the changes in this pull request (PR). Include your motivation and context to help us review.
-
-Please mention the issue your PR addresses (if any):
-closes #issue_number
--->
-
-
-### Checklist
-
-<!-- CHANGELOG.md has a `master (unreleased)` section. Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning in the release notes to this file. -->
-- [ ] No CHANGELOG update needed
-<!-- Tests prove that your fix/work as expected and ensure it doesn't break on the feature. -->
-- [ ] No new tests needed
-<!-- Please help us keeping capa documentation up-to-date -->
-- [ ] No documentation update needed

.github/pyinstaller/hooks/hook-smda.py

@@ -1,5 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-import PyInstaller.utils.hooks
-
-# ref: https://groups.google.com/g/pyinstaller/c/amWi0-66uZI/m/miPoKfWjBAAJ
-binaries = PyInstaller.utils.hooks.collect_dynamic_libs("capstone")

.github/pyinstaller/hooks/hook-vivisect.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 
 from PyInstaller.utils.hooks import copy_metadata
 
@@ -13,144 +13,3 @@ from PyInstaller.utils.hooks import copy_metadata
 #
 # ref: https://github.com/pyinstaller/pyinstaller/issues/1713#issuecomment-162682084
 datas = copy_metadata("vivisect")
-
-excludedimports = [
-    # viv gui requires these heavy libraries,
-    # but viv as a library doesn't.
-    # they shouldn't be installed in our configuration,
-    # but we'll ensure they don't slip in here (such as on developers' systems).
-    "PyQt5",
-    "qt5",
-    "pyqtwebengine",
-    # the above are imported by these viv modules.
-    # so really, we'd want to exclude these submodules of viv.
-    # but i dont think this works.
-    "vqt",
-    "vdb.qt",
-    "envi.qt",
-    # unused by capa
-    "pyasn1",
-]
-
-hiddenimports = [
-    # vivisect does manual/runtime importing of its modules,
-    # so declare the things that could be imported here.
-    "vivisect",
-    "vivisect.analysis",
-    "vivisect.analysis.amd64",
-    "vivisect.analysis.amd64",
-    "vivisect.analysis.amd64.emulation",
-    "vivisect.analysis.amd64.golang",
-    "vivisect.analysis.crypto",
-    "vivisect.analysis.crypto",
-    "vivisect.analysis.crypto.constants",
-    "vivisect.analysis.elf",
-    "vivisect.analysis.elf.elfplt",
-    "vivisect.analysis.elf.elfplt_late",
-    "vivisect.analysis.elf.libc_start_main",
-    "vivisect.analysis.generic",
-    "vivisect.analysis.generic",
-    "vivisect.analysis.generic.codeblocks",
-    "vivisect.analysis.generic.emucode",
-    "vivisect.analysis.generic.entrypoints",
-    "vivisect.analysis.generic.funcentries",
-    "vivisect.analysis.generic.impapi",
-    "vivisect.analysis.generic.mkpointers",
-    "vivisect.analysis.generic.pointers",
-    "vivisect.analysis.generic.pointertables",
-    "vivisect.analysis.generic.relocations",
-    "vivisect.analysis.generic.strconst",
-    "vivisect.analysis.generic.switchcase",
-    "vivisect.analysis.generic.thunks",
-    "vivisect.analysis.generic.noret",
-    "vivisect.analysis.i386",
-    "vivisect.analysis.i386",
-    "vivisect.analysis.i386.calling",
-    "vivisect.analysis.i386.golang",
-    "vivisect.analysis.i386.importcalls",
-    "vivisect.analysis.i386.instrhook",
-    "vivisect.analysis.i386.thunk_bx",
-    "vivisect.analysis.ms",
-    "vivisect.analysis.ms",
-    "vivisect.analysis.ms.hotpatch",
-    "vivisect.analysis.ms.localhints",
-    "vivisect.analysis.ms.msvc",
-    "vivisect.analysis.ms.msvcfunc",
-    "vivisect.analysis.ms.vftables",
-    "vivisect.analysis.pe",
-    "vivisect.impapi.posix.amd64",
-    "vivisect.impapi.posix.i386",
-    "vivisect.impapi.windows",
-    "vivisect.impapi.windows.amd64",
-    "vivisect.impapi.windows.i386",
-    "vivisect.impapi.winkern.i386",
-    "vivisect.impapi.winkern.amd64",
-    "vivisect.parsers.blob",
-    "vivisect.parsers.elf",
-    "vivisect.parsers.ihex",
-    "vivisect.parsers.macho",
-    "vivisect.parsers.pe",
-    "vivisect.storage",
-    "vivisect.storage.basicfile",
-    "vstruct.constants",
-    "vstruct.constants.ntstatus",
-    "vstruct.defs",
-    "vstruct.defs.arm7",
-    "vstruct.defs.bmp",
-    "vstruct.defs.dns",
-    "vstruct.defs.elf",
-    "vstruct.defs.gif",
-    "vstruct.defs.ihex",
-    "vstruct.defs.inet",
-    "vstruct.defs.java",
-    "vstruct.defs.kdcom",
-    "vstruct.defs.macho",
-    "vstruct.defs.macho.const",
-    "vstruct.defs.macho.fat",
-    "vstruct.defs.macho.loader",
-    "vstruct.defs.macho.stabs",
-    "vstruct.defs.minidump",
-    "vstruct.defs.pcap",
-    "vstruct.defs.pe",
-    "vstruct.defs.pptp",
-    "vstruct.defs.rar",
-    "vstruct.defs.swf",
-    "vstruct.defs.win32",
-    "vstruct.defs.windows",
-    "vstruct.defs.windows.win_5_1_i386",
-    "vstruct.defs.windows.win_5_1_i386.ntdll",
-    "vstruct.defs.windows.win_5_1_i386.ntoskrnl",
-    "vstruct.defs.windows.win_5_1_i386.win32k",
-    "vstruct.defs.windows.win_5_2_i386",
-    "vstruct.defs.windows.win_5_2_i386.ntdll",
-    "vstruct.defs.windows.win_5_2_i386.ntoskrnl",
-    "vstruct.defs.windows.win_5_2_i386.win32k",
-    "vstruct.defs.windows.win_6_1_amd64",
-    "vstruct.defs.windows.win_6_1_amd64.ntdll",
-    "vstruct.defs.windows.win_6_1_amd64.ntoskrnl",
-    "vstruct.defs.windows.win_6_1_amd64.win32k",
-    "vstruct.defs.windows.win_6_1_i386",
-    "vstruct.defs.windows.win_6_1_i386.ntdll",
-    "vstruct.defs.windows.win_6_1_i386.ntoskrnl",
-    "vstruct.defs.windows.win_6_1_i386.win32k",
-    "vstruct.defs.windows.win_6_1_wow64",
-    "vstruct.defs.windows.win_6_1_wow64.ntdll",
-    "vstruct.defs.windows.win_6_2_amd64",
-    "vstruct.defs.windows.win_6_2_amd64.ntdll",
-    "vstruct.defs.windows.win_6_2_amd64.ntoskrnl",
-    "vstruct.defs.windows.win_6_2_amd64.win32k",
-    "vstruct.defs.windows.win_6_2_i386",
-    "vstruct.defs.windows.win_6_2_i386.ntdll",
-    "vstruct.defs.windows.win_6_2_i386.ntoskrnl",
-    "vstruct.defs.windows.win_6_2_i386.win32k",
-    "vstruct.defs.windows.win_6_2_wow64",
-    "vstruct.defs.windows.win_6_2_wow64.ntdll",
-    "vstruct.defs.windows.win_6_3_amd64",
-    "vstruct.defs.windows.win_6_3_amd64.ntdll",
-    "vstruct.defs.windows.win_6_3_amd64.ntoskrnl",
-    "vstruct.defs.windows.win_6_3_i386",
-    "vstruct.defs.windows.win_6_3_i386.ntdll",
-    "vstruct.defs.windows.win_6_3_i386.ntoskrnl",
-    "vstruct.defs.windows.win_6_3_wow64",
-    "vstruct.defs.windows.win_6_3_wow64.ntdll",
-]

.github/pyinstaller/pyinstaller.spec

@@ -1,5 +1,5 @@
 # -*- mode: python -*-
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 import os.path
 import subprocess
 
@@ -16,10 +16,9 @@ with open('./capa/version.py', 'wb') as f:
     #                 - commits since
     #                   g------- git hash fragment
     version = (subprocess.check_output(["git", "describe", "--always", "--tags", "--long"])
-               .decode("utf-8")
                .strip()
                .replace("tags/", ""))
-    f.write(("__version__ = '%s'" % version).encode("utf-8"))
+    f.write("__version__ = '%s'" % version)
 
 a = Analysis(
     # when invoking pyinstaller from the project root,
@@ -33,7 +32,6 @@ a = Analysis(
         # this gets invoked from the directory of the spec file,
         # i.e. ./.github/pyinstaller
         ('../../rules', 'rules'),
-        ('../../sigs', 'sigs'),
 
         # capa.render.default uses tabulate that depends on wcwidth.
         # it seems wcwidth uses a json file `version.json`
@@ -43,6 +41,127 @@ a = Analysis(
         # ref: https://stackoverflow.com/a/62278462/87207
         (os.path.dirname(wcwidth.__file__), 'wcwidth')
     ],
+    hiddenimports=[
+        # vivisect does manual/runtime importing of its modules,
+        # so declare the things that could be imported here.
+        "pycparser",
+        "vivisect",
+        "vivisect.analysis",
+        "vivisect.analysis.amd64",
+        "vivisect.analysis.amd64",
+        "vivisect.analysis.amd64.emulation",
+        "vivisect.analysis.amd64.golang",
+        "vivisect.analysis.crypto",
+        "vivisect.analysis.crypto",
+        "vivisect.analysis.crypto.constants",
+        "vivisect.analysis.elf",
+        "vivisect.analysis.elf",
+        "vivisect.analysis.elf.elfplt",
+        "vivisect.analysis.elf.libc_start_main",
+        "vivisect.analysis.generic",
+        "vivisect.analysis.generic",
+        "vivisect.analysis.generic.codeblocks",
+        "vivisect.analysis.generic.emucode",
+        "vivisect.analysis.generic.entrypoints",
+        "vivisect.analysis.generic.funcentries",
+        "vivisect.analysis.generic.impapi",
+        "vivisect.analysis.generic.mkpointers",
+        "vivisect.analysis.generic.pointers",
+        "vivisect.analysis.generic.pointertables",
+        "vivisect.analysis.generic.relocations",
+        "vivisect.analysis.generic.strconst",
+        "vivisect.analysis.generic.switchcase",
+        "vivisect.analysis.generic.thunks",
+        "vivisect.analysis.i386",
+        "vivisect.analysis.i386",
+        "vivisect.analysis.i386.calling",
+        "vivisect.analysis.i386.golang",
+        "vivisect.analysis.i386.importcalls",
+        "vivisect.analysis.i386.instrhook",
+        "vivisect.analysis.i386.thunk_bx",
+        "vivisect.analysis.ms",
+        "vivisect.analysis.ms",
+        "vivisect.analysis.ms.hotpatch",
+        "vivisect.analysis.ms.localhints",
+        "vivisect.analysis.ms.msvc",
+        "vivisect.analysis.ms.msvcfunc",
+        "vivisect.analysis.ms.vftables",
+        "vivisect.analysis.pe",
+        "vivisect.impapi.posix.amd64",
+        "vivisect.impapi.posix.i386",
+        "vivisect.impapi.windows",
+        "vivisect.impapi.windows.amd64",
+        "vivisect.impapi.windows.i386",
+        "vivisect.parsers.blob",
+        "vivisect.parsers.elf",
+        "vivisect.parsers.ihex",
+        "vivisect.parsers.macho",
+        "vivisect.parsers.parse_pe",
+        "vivisect.parsers.utils",
+        "vivisect.storage",
+        "vivisect.storage.basicfile",
+        "vstruct.constants",
+        "vstruct.constants.ntstatus",
+        "vstruct.defs",
+        "vstruct.defs.arm7",
+        "vstruct.defs.bmp",
+        "vstruct.defs.dns",
+        "vstruct.defs.elf",
+        "vstruct.defs.gif",
+        "vstruct.defs.ihex",
+        "vstruct.defs.inet",
+        "vstruct.defs.java",
+        "vstruct.defs.kdcom",
+        "vstruct.defs.macho",
+        "vstruct.defs.macho.const",
+        "vstruct.defs.macho.fat",
+        "vstruct.defs.macho.loader",
+        "vstruct.defs.macho.stabs",
+        "vstruct.defs.minidump",
+        "vstruct.defs.pcap",
+        "vstruct.defs.pe",
+        "vstruct.defs.pptp",
+        "vstruct.defs.rar",
+        "vstruct.defs.swf",
+        "vstruct.defs.win32",
+        "vstruct.defs.windows",
+        "vstruct.defs.windows.win_5_1_i386",
+        "vstruct.defs.windows.win_5_1_i386.ntdll",
+        "vstruct.defs.windows.win_5_1_i386.ntoskrnl",
+        "vstruct.defs.windows.win_5_1_i386.win32k",
+        "vstruct.defs.windows.win_5_2_i386",
+        "vstruct.defs.windows.win_5_2_i386.ntdll",
+        "vstruct.defs.windows.win_5_2_i386.ntoskrnl",
+        "vstruct.defs.windows.win_5_2_i386.win32k",
+        "vstruct.defs.windows.win_6_1_amd64",
+        "vstruct.defs.windows.win_6_1_amd64.ntdll",
+        "vstruct.defs.windows.win_6_1_amd64.ntoskrnl",
+        "vstruct.defs.windows.win_6_1_amd64.win32k",
+        "vstruct.defs.windows.win_6_1_i386",
+        "vstruct.defs.windows.win_6_1_i386.ntdll",
+        "vstruct.defs.windows.win_6_1_i386.ntoskrnl",
+        "vstruct.defs.windows.win_6_1_i386.win32k",
+        "vstruct.defs.windows.win_6_1_wow64",
+        "vstruct.defs.windows.win_6_1_wow64.ntdll",
+        "vstruct.defs.windows.win_6_2_amd64",
+        "vstruct.defs.windows.win_6_2_amd64.ntdll",
+        "vstruct.defs.windows.win_6_2_amd64.ntoskrnl",
+        "vstruct.defs.windows.win_6_2_amd64.win32k",
+        "vstruct.defs.windows.win_6_2_i386",
+        "vstruct.defs.windows.win_6_2_i386.ntdll",
+        "vstruct.defs.windows.win_6_2_i386.ntoskrnl",
+        "vstruct.defs.windows.win_6_2_i386.win32k",
+        "vstruct.defs.windows.win_6_2_wow64",
+        "vstruct.defs.windows.win_6_2_wow64.ntdll",
+        "vstruct.defs.windows.win_6_3_amd64",
+        "vstruct.defs.windows.win_6_3_amd64.ntdll",
+        "vstruct.defs.windows.win_6_3_amd64.ntoskrnl",
+        "vstruct.defs.windows.win_6_3_i386",
+        "vstruct.defs.windows.win_6_3_i386.ntdll",
+        "vstruct.defs.windows.win_6_3_i386.ntoskrnl",
+        "vstruct.defs.windows.win_6_3_wow64",
+        "vstruct.defs.windows.win_6_3_wow64.ntdll",
+    ],
     # when invoking pyinstaller from the project root,
     # this gets run from the project root.
     hookspath=['.github/pyinstaller/hooks'],
@@ -60,25 +179,6 @@ a = Analysis(
         # since we don't spawn a notebook, we can safely remove these.
         "IPython",
         "ipywidgets",
-
-        # these are pulled in by networkx
-        # but we don't need to compute the strongly connected components.
-        "numpy",
-        "scipy",
-        "matplotlib",
-        "pandas",
-        "pytest",
-
-        # deps from viv that we don't use.
-        # this duplicates the entries in `hook-vivisect`,
-        # but works better this way.
-        "vqt",
-        "vdb.qt",
-        "envi.qt",
-        "PyQt5",
-        "qt5",
-        "pyqtwebengine",
-        "pyasn1"
     ])
 
 a.binaries = a.binaries - TOC([
@@ -95,7 +195,7 @@ exe = EXE(pyz,
           a.datas,
           exclude_binaries=False,
           name='capa',
-          # TODO not working anymore for unknown reason icon='logo.ico',
+          icon='logo.ico',
           debug=False,
           strip=None,
           upx=True,
@@ -109,4 +209,5 @@ exe = EXE(pyz,
 #               a.datas,
 #               strip=None,
 #               upx=True,
-#               name='capa-dat')
+#               name='capa-dat')
+

.github/workflows/build.yml

@@ -1,118 +1,77 @@
-name: build
-
-on:
-  pull_request:
-    branches: [ master ]
-  release:
-    types: [edited, published]
-
-jobs:
-  build:
-    name: PyInstaller for ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      # set to false for debugging
-      fail-fast: true
-      matrix:
-        include:
-          - os: ubuntu-18.04
-            # use old linux so that the shared library versioning is more portable
-            artifact_name: capa
-            asset_name: linux
-          - os: windows-2022
-            artifact_name: capa.exe
-            asset_name: windows
-          - os: macos-10.15
-            artifact_name: capa
-            asset_name: macos
-    steps:
-      - name: Checkout capa
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-      # using Python 3.8 to support running across multiple operating systems including Windows 7
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-      - if: matrix.os == 'ubuntu-18.04'
-        run: sudo apt-get install -y libyaml-dev
-      - name: Upgrade pip, setuptools
-        run: pip install --upgrade pip setuptools
-      - name: Install capa with build requirements
-        run: pip install -e .[build]
-      - name: Build standalone executable
-        run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
-      - name: Does it run (PE)?
-        run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_"
-      - name: Does it run (Shellcode)?
-        run: dist/capa "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
-      - name: Does it run (ELF)?
-        run: dist/capa "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
-      - uses: actions/upload-artifact@v2
-        with:
-          name: ${{ matrix.asset_name }}
-          path: dist/${{ matrix.artifact_name }}
-
-  test_run:
-    # test that binaries run on push to master
-    if: github.event_name == 'push'
-    name: Test run on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    needs: [build]
-    strategy:
-      matrix:
-        include:
-          # OSs not already tested above
-          - os: ubuntu-18.04
-            artifact_name: capa
-            asset_name: linux
-          - os: ubuntu-20.04
-            artifact_name: capa
-            asset_name: linux
-          - os: windows-2022
-            artifact_name: capa.exe
-            asset_name: windows
-    steps:
-      - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@v2
-        with:
-          name: ${{ matrix.asset_name }}
-      - name: Set executable flag
-        if: matrix.os != 'windows-2022'      
-        run: chmod +x ${{ matrix.artifact_name }}
-      - name: Run capa
-        run: ./${{ matrix.artifact_name }} -h
-
-  zip_and_upload:
-    # upload zipped binaries to Release page
-    if: github.event_name == 'release'
-    name: zip and upload ${{ matrix.asset_name }}
-    runs-on: ubuntu-20.04
-    needs: [build]
-    strategy:
-      matrix:
-        include:
-          - asset_name: linux
-            artifact_name: capa
-          - asset_name: windows
-            artifact_name: capa.exe
-          - asset_name: macos
-            artifact_name: capa
-    steps:
-      - name: Download ${{ matrix.asset_name }}
-        uses: actions/download-artifact@v2
-        with:
-          name: ${{ matrix.asset_name }}
-      - name: Set executable flag
-        run: chmod +x ${{ matrix.artifact_name }}
-      - name: Set zip name
-        run: echo "zip_name=capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV
-      - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
-        run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
-      - name: Upload ${{ env.zip_name }} to GH Release
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN}}
-          file: ${{ env.zip_name }}
-          tag: ${{ github.ref }}
+name: build
+
+on:
+  release:
+    types: [created, edited]
+
+jobs:
+  build:
+    name: PyInstaller for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-16.04
+            # use old linux so that the shared library versioning is more portable
+            artifact_name: capa
+            asset_name: linux
+          - os: windows-latest
+            artifact_name: capa.exe
+            asset_name: windows
+          - os: macos-latest
+            artifact_name: capa
+            asset_name: macos
+    steps:
+      - name: Checkout capa
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Set up Python 2.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 2.7
+      - name: Install PyInstaller
+        # pyinstaller 4 doesn't support Python 2.7
+        run: pip install 'pyinstaller==3.*'
+      - name: Install capa
+        run: pip install -e .
+      - name: Build standalone executable
+        run: pyinstaller .github/pyinstaller/pyinstaller.spec
+      - name: Does it run?
+        run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_"
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.asset_name }}
+          path: dist/${{ matrix.artifact_name }}
+
+  zip:
+    name: zip ${{ matrix.asset_name }}
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      matrix:
+        include:
+          - asset_name: linux
+            artifact_name: capa
+          - asset_name: windows
+            artifact_name: capa.exe
+          - asset_name: macos
+            artifact_name: capa
+    steps:
+      - name: Download ${{ matrix.asset_name }}
+        uses: actions/download-artifact@v2
+        with:
+          name: ${{ matrix.asset_name }}
+      - name: Set executable flag
+        run: chmod +x ${{ matrix.artifact_name }}
+      - name: Set zip name
+        run: echo ::set-env name=zip_name::capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip
+      - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }}
+        run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }}
+      - name: Upload ${{ env.zip_name }} to GH Release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN}}
+          file: ${{ env.zip_name }}
+          tag: ${{ github.ref }}
+

.github/workflows/changelog.yml

@@ -1,41 +0,0 @@
-name: changelog
-
-on:
-  # We need pull_request_target instead of pull_request because a write
-  # repository token is needed to add a review to a PR. DO NOT BUILD
-  # OR RUN UNTRUSTED CODE FROM PRs IN THIS ACTION
-  pull_request_target:
-    types: [opened, edited, synchronize]
-
-jobs:
-  check_changelog:
-    # no need to check for dependency updates via dependabot
-    if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]'
-    runs-on: ubuntu-20.04
-    env:
-      NO_CHANGELOG: '[x] No CHANGELOG update needed'
-    steps:
-    - name: Get changed files
-      id: files
-      uses: Ana06/get-changed-files@v1.2
-    - name: check changelog updated
-      id: changelog_updated
-      env:
-        PR_BODY: ${{ github.event.pull_request.body }}
-        FILES: ${{ steps.files.outputs.modified }}
-      run: |
-        echo $FILES | grep -qF 'CHANGELOG.md' || echo $PR_BODY | grep -qiF "$NO_CHANGELOG"
-    - name: Reject pull request if no CHANGELOG update
-      if: ${{ always() && steps.changelog_updated.outcome == 'failure' }}
-      uses: Ana06/automatic-pull-request-review@v0.1.0
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        event: REQUEST_CHANGES
-        body: "Please add bug fixes, new features, breaking changes and anything else you think is worthwhile mentioning to the `master (unreleased)` section of CHANGELOG.md. If no CHANGELOG update is needed add the following to the PR description: `${{ env.NO_CHANGELOG }}`"
-        allow_duplicate: false
-    - name: Dismiss previous review if CHANGELOG update
-      uses: Ana06/automatic-pull-request-review@v0.1.0
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        event: DISMISS
-        body: "CHANGELOG updated or no update needed, thanks! :smile:"

.github/workflows/publish.yml

@@ -1,30 +0,0 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
-name: publish to pypi
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  deploy:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: |
-          python setup.py sdist bdist_wheel
-          twine upload --skip-existing dist/*
-

.github/workflows/tag.yml

@@ -1,29 +0,0 @@
-name: tag
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  tag:
-    name: Tag capa rules
-    runs-on: ubuntu-20.04
-    steps:
-    - name: Checkout capa-rules
-      uses: actions/checkout@v2
-      with:
-        repository: mandiant/capa-rules
-        token: ${{ secrets.CAPA_TOKEN }}
-    - name: Tag capa-rules
-      run: |
-        # user information is needed to create annotated tags (with a message)
-        git config user.email 'capa-dev@mandiant.com'
-        git config user.name 'Capa Bot'
-        name=${{ github.event.release.tag_name }}
-        git tag $name -m "https://github.com/mandiant/capa/releases/$name"
-    - name: Push tag to capa-rules
-      uses: ad-m/github-push-action@master
-      with:
-        repository: mandiant/capa-rules
-        github_token: ${{ secrets.CAPA_TOKEN }}
-        tags: true

.github/workflows/tests.yml

@@ -2,92 +2,68 @@ name: CI
 
 on:
   push:
-    branches: [ master, dotnet-main ]
+    branches: [ master ]
   pull_request:
-    # TODO cleanup these
-    branches: [ master, dotnet-main, feature-981 ]
-
-# save workspaces to speed up testing
-env:
-  CAPA_SAVE_WORKSPACE: "True"
+    branches: [ master ]
 
 jobs:
-  changelog_format:
-    runs-on: ubuntu-20.04
-    steps:
-    - name: Checkout capa
-      uses: actions/checkout@v2
-    # The sync GH action in capa-rules relies on a single '- *$' in the CHANGELOG file
-    - name: Ensure CHANGELOG has '- *$'
-      run: |
-        number=$(grep '\- *$' CHANGELOG.md | wc -l)
-        if [ $number != 1 ]; then exit 1; fi
-
   code_style:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout capa
       uses: actions/checkout@v2
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
-        python-version: "3.8"
+        python-version: 3.8
     - name: Install dependencies
-      run: pip install -e .[dev]
+      run: pip install 'isort==5.*' black
     - name: Lint with isort
       run: isort --profile black --length-sort --line-width 120 -c .
     - name: Lint with black
       run: black -l 120 --check .
-    - name: Lint with pycodestyle
-      run: pycodestyle --show-source capa/ scripts/ tests/
-    - name: Check types with mypy
-      run: mypy --config-file .github/mypy/mypy.ini capa/ scripts/ tests/
 
   rule_linter:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
-    - name: Checkout capa with submodules
+    - name: Checkout capa with rules submodule
       uses: actions/checkout@v2
       with:
-        submodules: recursive
+        submodules: true
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
-        python-version: "3.8"
+        python-version: 3.8
+    # We don't need vivisect, so we can install capa using Python3
     - name: Install capa
       run: pip install -e .
     - name: Run rule linter
       run: python scripts/lint.py rules/
 
   tests:
-    name: Tests in ${{ matrix.python-version }} on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
+    name: Tests in ${{ matrix.python }}
+    runs-on: ubuntu-latest
     needs: [code_style, rule_linter]
     strategy:
-      fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-10.15]
-        # across all operating systems
-        python-version: ["3.7", "3.10"]
         include:
-          # on Ubuntu run these as well
-          - os: ubuntu-20.04
-            python-version: "3.8"
-          - os: ubuntu-20.04
-            python-version: "3.9"
+          - python: 2.7
+          - python: 3.6
+          - python: 3.7
+          - python: 3.8
+          - python: '3.9.0-alpha - 3.9.x' # Python latest
     steps:
     - name: Checkout capa with submodules
       uses: actions/checkout@v2
       with:
-        submodules: recursive
-    - name: Set up Python ${{ matrix.python-version }}
+        submodules: true
+    - name: Set up Python ${{ matrix.python }}
       uses: actions/setup-python@v2
       with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install pyyaml
-      if: matrix.os == 'ubuntu-20.04'
-      run: sudo apt-get install -y libyaml-dev
+        python-version: ${{ matrix.python }}
     - name: Install capa
-      run: pip install -e .[dev]
+      # TODO: remove `pefile` when we bump lancelot >= 0.3.7
+      run: pip install -e .[dev] pefile
     - name: Run tests
-      run: pytest -v tests/
+      run: pytest tests/
+

.gitignore

@@ -114,11 +114,3 @@ venv.bak/
 isort-output.log
 black-output.log
 rule-linter-output.log
-.vscode
-scripts/perf/*.txt
-scripts/perf/*.svg
-scripts/perf/*.zip
-.direnv
-.envrc
-.DS_Store
-*/.DS_Store

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "rules"]
 	path = rules
-	url = ../capa-rules.git
+	url = git@github.com:fireeye/capa-rules.git
 [submodule "tests/data"]
 	path = tests/data
-	url = ../capa-testfiles.git
+	url = git@github.com:fireeye/capa-testfiles.git

LICENSE.txt

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (C) 2020 Mandiant, Inc.
+   Copyright (C) 2020 FireEye, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,20 +1,14 @@
-![capa](https://github.com/mandiant/capa/blob/master/.github/logo.png)
+![capa](.github/logo.png)
 
-[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
-[![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-665-blue.svg)](https://github.com/mandiant/capa-rules)
-[![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
-[![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
+[![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
+[![Number of rules](https://img.shields.io/badge/rules-303-blue.svg)](https://github.com/fireeye/capa-rules)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
 capa detects capabilities in executable files.
-You run it against a PE, ELF, or shellcode file and it tells you what it thinks the program can do.
+You run it against a PE file or shellcode and it tells you what it thinks the program can do.
 For example, it might suggest that the file is a backdoor, is capable of installing services, or relies on HTTP to communicate.
 
-Check out:
-- the overview in our first [capa blog post](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html)
-- the major version 2.0 updates described in our [second blog post](https://www.fireeye.com/blog/threat-research/2021/07/capa-2-better-stronger-faster.html)
-- the major version 3.0 (ELF support) described in the [third blog post](https://www.fireeye.com/blog/threat-research/2021/09/elfant-in-the-room-capa-v3.html)
+Check out the overview in our first [capa blog post](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html).
 
 ```
 $ capa.exe suspicious.exe
@@ -66,11 +60,18 @@ $ capa.exe suspicious.exe
 
 # download and usage
 
-Download stable releases of the standalone capa binaries [here](https://github.com/mandiant/capa/releases). You can run the standalone binaries without installation. capa is a command line tool that should be run from the terminal.
+Download stable releases of the standalone capa binaries [here](https://github.com/fireeye/capa/releases). You can run the standalone binaries without installation. capa is a command line tool that should be run from the terminal.
 
-To use capa as a library or integrate with another tool, see [doc/installation.md](https://github.com/mandiant/capa/blob/master/doc/installation.md) for further setup instructions.
+<!--
+Alternatively, you can fetch a nightly build of a standalone binary from one of the following links. These are built using the latest development branch.
+- Windows 64bit: TODO
+- Linux: TODO
+- OSX: TODO
+-->
 
-For more information about how to use capa, see [doc/usage.md](https://github.com/mandiant/capa/blob/master/doc/usage.md).
+To use capa as a library or integrate with another tool, see [doc/installation.md](doc/installation.md) for further setup instructions.
+
+For more information about how to use capa, including running it as an IDA script/plugin see [doc/usage.md](doc/usage.md).
 
 # example
 
@@ -87,11 +88,11 @@ This is useful for at least two reasons:
   - it shows where within the binary an experienced analyst might study with IDA Pro
 
 ```
-$ capa.exe suspicious.exe -vv
+λ capa.exe suspicious.exe -vv
 ...
 execute shell command and capture output
 namespace   c2/shell
-author      matthew.williams@mandiant.com
+author      matthew.williams@fireeye.com
 scope       function
 att&ck      Execution::Command and Scripting Interpreter::Windows Command Shell [T1059.003]
 references  https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/ns-processthreadsapi-startupinfoa
@@ -127,7 +128,7 @@ rule:
   meta:
     name: hash data with CRC32
     namespace: data-manipulation/checksum/crc32
-    author: moritz.raabe@mandiant.com
+    author: moritz.raabe@fireeye.com
     scope: function
     examples:
       - 2D3EDC218A90F03089CC01715A9F047F:0x403CBD
@@ -142,24 +143,23 @@ rule:
       - api: RtlComputeCrc32
 ```
 
-The [github.com/mandiant/capa-rules](https://github.com/mandiant/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
+The [github.com/fireeye/capa-rules](https://github.com/fireeye/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
 Please learn to write rules and contribute new entries as you find interesting techniques in malware.
 
-If you use IDA Pro, then you can use the [capa explorer](https://github.com/mandiant/capa/tree/master/capa/ida/plugin) plugin.
-capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted directly from your IDA Pro database.
+If you use IDA Pro, then you use can use the [IDA Pro plugin for capa](./capa/ida/ida_capa_explorer.py).
+This script adds new user interface elements to IDA, including an interactive tree view of rule matches and their locations within the current database.
+As you select the checkboxes, the plugin will highlight the addresses associated with the features.
+We use this plugin all the time to quickly jump to interesting parts of a program.
 
-![capa + IDA Pro integration](https://github.com/mandiant/capa/blob/master/doc/img/explorer_expanded.png)
+![capa + IDA Pro integration](.github/capa-ida.jpg)
 
 # further information
 ## capa
-- [Installation](https://github.com/mandiant/capa/blob/master/doc/installation.md)
-- [Usage](https://github.com/mandiant/capa/blob/master/doc/usage.md)
-- [Limitations](https://github.com/mandiant/capa/blob/master/doc/limitations.md)
-- [Contributing Guide](https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md)
+- [doc/installation](doc/installation.md)
+- [doc/usage](doc/usage.md)
+- [doc/limitations](doc/limitations.md)
+- [Contributing Guide](.github/CONTRIBUTING.md)
 
 ## capa rules
-- [capa-rules repository](https://github.com/mandiant/capa-rules)
-- [capa-rules rule format](https://github.com/mandiant/capa-rules/blob/master/doc/format.md)
-
-## capa testfiles
-The [capa-testfiles repository](https://github.com/mandiant/capa-testfiles) contains the data we use to test capa's code and rules
+- [capa-rules repository](https://github.com/fireeye/capa-rules)
+- [capa-rules rule format](https://github.com/fireeye/capa-rules/blob/master/doc/format.md)

capa/engine.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,31 +6,14 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+import sys
 import copy
 import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Mapping, Iterable
 
-import capa.perf
-import capa.features.common
-from capa.features.common import Result, Feature
-from capa.features.address import Address
-
-if TYPE_CHECKING:
-    # circular import, otherwise
-    import capa.rules
+import capa.features
 
 
-# a collection of features and the locations at which they are found.
-#
-# used throughout matching as the context in which features are searched:
-# to check if a feature exists, do: `Number(0x10) in features`.
-# to collect the locations of a feature, do: `features[Number(0x10)]`
-#
-# aliased here so that the type can be documented and xref'd.
-FeatureSet = Dict[Feature, Set[Address]]
-
-
-class Statement:
+class Statement(object):
     """
     superclass for structural nodes, such as and/or/not.
     this exists to provide a default impl for `__str__` and `__repr__`,
@@ -51,12 +34,15 @@ class Statement:
     def __repr__(self):
         return str(self)
 
-    def evaluate(self, features: FeatureSet, short_circuit=True) -> Result:
+    def evaluate(self, ctx):
         """
         classes that inherit `Statement` must implement `evaluate`
 
         args:
-            short_circuit (bool): if true, then statements like and/or/some may short circuit.
+          ctx (defaultdict[Feature, set[VA]])
+
+        returns:
+          Result
         """
         raise NotImplementedError()
 
@@ -65,7 +51,7 @@ class Statement:
             yield self.child
 
         if hasattr(self, "children"):
-            for child in getattr(self, "children"):
+            for child in self.children:
                 yield child
 
     def replace_child(self, existing, new):
@@ -74,76 +60,75 @@ class Statement:
                 self.child = new
 
         if hasattr(self, "children"):
-            children = getattr(self, "children")
-            for i, child in enumerate(children):
+            for i, child in enumerate(self.children):
                 if child is existing:
-                    children[i] = new
+                    self.children[i] = new
+
+
+class Result(object):
+    """
+    represents the results of an evaluation of statements against features.
+
+    instances of this class should behave like a bool,
+    e.g. `assert Result(True, ...) == True`
+
+    instances track additional metadata about evaluation results.
+    they contain references to the statement node (e.g. an And statement),
+     as well as the children Result instances.
+
+    we need this so that we can render the tree of expressions and their results.
+    """
+
+    def __init__(self, success, statement, children, locations=None):
+        """
+        args:
+          success (bool)
+          statement (capa.engine.Statement or capa.features.Feature)
+          children (list[Result])
+          locations (iterable[VA])
+        """
+        super(Result, self).__init__()
+        self.success = success
+        self.statement = statement
+        self.children = children
+        self.locations = locations if locations is not None else ()
+
+    def __eq__(self, other):
+        if isinstance(other, bool):
+            return self.success == other
+        return False
+
+    def __bool__(self):
+        return self.success
+
+    def __nonzero__(self):
+        return self.success
 
 
 class And(Statement):
-    """
-    match if all of the children evaluate to True.
-
-    the order of evaluation is dictated by the property
-    `And.children` (type: List[Statement|Feature]).
-    a query optimizer may safely manipulate the order of these children.
-    """
+    """match if all of the children evaluate to True."""
 
     def __init__(self, children, description=None):
         super(And, self).__init__(description=description)
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.and"] += 1
-
-        if short_circuit:
-            results = []
-            for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
-                results.append(result)
-                if not result:
-                    # short circuit
-                    return Result(False, self, results)
-
-            return Result(True, self, results)
-        else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
-            success = all(results)
-            return Result(success, self, results)
+    def evaluate(self, ctx):
+        results = [child.evaluate(ctx) for child in self.children]
+        success = all(results)
+        return Result(success, self, results)
 
 
 class Or(Statement):
-    """
-    match if any of the children evaluate to True.
-
-    the order of evaluation is dictated by the property
-    `Or.children` (type: List[Statement|Feature]).
-    a query optimizer may safely manipulate the order of these children.
-    """
+    """match if any of the children evaluate to True."""
 
     def __init__(self, children, description=None):
         super(Or, self).__init__(description=description)
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.or"] += 1
-
-        if short_circuit:
-            results = []
-            for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
-                results.append(result)
-                if result:
-                    # short circuit as soon as we hit one match
-                    return Result(True, self, results)
-
-            return Result(False, self, results)
-        else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
-            success = any(results)
-            return Result(success, self, results)
+    def evaluate(self, ctx):
+        results = [child.evaluate(ctx) for child in self.children]
+        success = any(results)
+        return Result(success, self, results)
 
 
 class Not(Statement):
@@ -153,55 +138,28 @@ class Not(Statement):
         super(Not, self).__init__(description=description)
         self.child = child
 
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.not"] += 1
-
-        results = [self.child.evaluate(ctx, short_circuit=short_circuit)]
+    def evaluate(self, ctx):
+        results = [self.child.evaluate(ctx)]
         success = not results[0]
         return Result(success, self, results)
 
 
 class Some(Statement):
-    """
-    match if at least N of the children evaluate to True.
-
-    the order of evaluation is dictated by the property
-    `Some.children` (type: List[Statement|Feature]).
-    a query optimizer may safely manipulate the order of these children.
-    """
+    """match if at least N of the children evaluate to True."""
 
     def __init__(self, count, children, description=None):
         super(Some, self).__init__(description=description)
         self.count = count
         self.children = children
 
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.some"] += 1
-
-        if short_circuit:
-            results = []
-            satisfied_children_count = 0
-            for child in self.children:
-                result = child.evaluate(ctx, short_circuit=short_circuit)
-                results.append(result)
-                if result:
-                    satisfied_children_count += 1
-
-                if satisfied_children_count >= self.count:
-                    # short circuit as soon as we hit the threshold
-                    return Result(True, self, results)
-
-            return Result(False, self, results)
-        else:
-            results = [child.evaluate(ctx, short_circuit=short_circuit) for child in self.children]
-            # note that here we cast the child result as a bool
-            # because we've overridden `__bool__` above.
-            #
-            # we can't use `if child is True` because the instance is not True.
-            success = sum([1 for child in results if bool(child) is True]) >= self.count
-            return Result(success, self, results)
+    def evaluate(self, ctx):
+        results = [child.evaluate(ctx) for child in self.children]
+        # note that here we cast the child result as a bool
+        # because we've overridden `__bool__` above.
+        #
+        # we can't use `if child is True` because the instance is not True.
+        success = sum([1 for child in results if bool(child) is True]) >= self.count
+        return Result(success, self, results)
 
 
 class Range(Statement):
@@ -213,10 +171,7 @@ class Range(Statement):
         self.min = min if min is not None else 0
         self.max = max if max is not None else (1 << 64 - 1)
 
-    def evaluate(self, ctx, **kwargs):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.range"] += 1
-
+    def evaluate(self, ctx):
         count = len(ctx.get(self.child, []))
         if self.min == 0 and count == 0:
             return Result(True, self, [])
@@ -236,66 +191,59 @@ class Subscope(Statement):
     the engine should preprocess rules to extract subscope statements into their own rules.
     """
 
-    def __init__(self, scope, child, description=None):
-        super(Subscope, self).__init__(description=description)
+    def __init__(self, scope, child):
+        super(Subscope, self).__init__()
         self.scope = scope
         self.child = child
 
-    def evaluate(self, ctx, **kwargs):
+    def evaluate(self, ctx):
         raise ValueError("cannot evaluate a subscope directly!")
 
 
-# mapping from rule name to list of: (location of match, result object)
-#
-# used throughout matching and rendering to collection the results
-#  of statement evaluation and their locations.
-#
-# to check if a rule matched, do: `"TCP client" in matches`.
-# to find where a rule matched, do: `map(first, matches["TCP client"])`
-# to see how a rule matched, do:
-#
-#     for address, match_details in matches["TCP client"]:
-#         inspect(match_details)
-#
-# aliased here so that the type can be documented and xref'd.
-MatchResults = Mapping[str, List[Tuple[Address, Result]]]
-
-
-def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations: Iterable[Address]):
+def topologically_order_rules(rules):
     """
-    record into the given featureset that the given rule matched at the given locations.
+    order the given rules such that dependencies show up before dependents.
+    this means that as we match rules, we can add features for the matches, and these
+     will be matched by subsequent rules if they follow this order.
 
-    naively, this is just adding a MatchedRule feature;
-    however, we also want to record matches for the rule's namespaces.
-
-    updates `features` in-place. doesn't modify the remaining arguments.
+    assumes that the rule dependency graph is a DAG.
     """
-    features[capa.features.common.MatchedRule(rule.name)].update(locations)
-    namespace = rule.meta.get("namespace")
-    if namespace:
-        while namespace:
-            features[capa.features.common.MatchedRule(namespace)].update(locations)
-            namespace, _, _ = namespace.rpartition("/")
+    # we evaluate `rules` multiple times, so if its a generator, realize it into a list.
+    rules = list(rules)
+    namespaces = capa.rules.index_rules_by_namespace(rules)
+    rules = {rule.name: rule for rule in rules}
+    seen = set([])
+    ret = []
+
+    def rec(rule):
+        if rule.name in seen:
+            return
+
+        for dep in rule.get_dependencies(namespaces):
+            rec(rules[dep])
+
+        ret.append(rule)
+        seen.add(rule.name)
+
+    for rule in rules.values():
+        rec(rule)
+
+    return ret
 
 
-def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -> Tuple[FeatureSet, MatchResults]:
+def match(rules, features, va):
     """
-    match the given rules against the given features,
-    returning an updated set of features and the matches.
+    Args:
+      rules (List[capa.rules.Rule]): these must already be ordered topologically by dependency.
+      features (Mapping[capa.features.Feature, int]):
+      va (int): location of the features
 
-    the updated features are just like the input,
-    but extended to include the match features (e.g. names of rules that matched).
-    the given feature set is not modified; an updated copy is returned.
-
-    the given list of rules must be ordered topologically by dependency,
-    or else `match` statements will not be handled correctly.
-
-    this routine should be fairly optimized, but is not guaranteed to be the fastest matcher possible.
-    it has a particularly convenient signature: (rules, features) -> matches
-    other strategies can be imagined that match differently; implement these elsewhere.
-    specifically, this routine does "top down" matching of the given rules against the feature set.
+    Returns:
+      Tuple[List[capa.features.Feature], Dict[str, Tuple[int, capa.engine.Result]]]: two-tuple with entries:
+        - list of features used for matching (which may be greater than argument, due to rule match features), and
+        - mapping from rule name to (location of match, result object)
     """
-    results = collections.defaultdict(list)  # type: MatchResults
+    results = collections.defaultdict(list)
 
     # copy features so that we can modify it
     # without affecting the caller (keep this function pure)
@@ -304,22 +252,15 @@ def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -
     features = collections.defaultdict(set, copy.copy(features))
 
     for rule in rules:
-        res = rule.evaluate(features, short_circuit=True)
+        res = rule.evaluate(features)
         if res:
-            # we first matched the rule with short circuiting enabled.
-            # this is much faster than without short circuiting.
-            # however, we want to collect all results thoroughly,
-            # so once we've found a match quickly,
-            # go back and capture results without short circuiting.
-            res = rule.evaluate(features, short_circuit=False)
+            results[rule.name].append((va, res))
+            features[capa.features.MatchedRule(rule.name)].add(va)
 
-            # sanity check
-            assert bool(res) is True
-
-            results[rule.name].append((addr, res))
-            # we need to update the current `features`
-            # because subsequent iterations of this loop may use newly added features,
-            # such as rule or namespace matches.
-            index_rule_matches(features, rule, [addr])
+            namespace = rule.meta.get("namespace")
+            if namespace:
+                while namespace:
+                    features[capa.features.MatchedRule(namespace)].add(va)
+                    namespace, _, _ = namespace.rpartition("/")
 
     return (features, results)

capa/exceptions.py

@@ -1,14 +0,0 @@
-class UnsupportedRuntimeError(RuntimeError):
-    pass
-
-
-class UnsupportedFormatError(ValueError):
-    pass
-
-
-class UnsupportedArchError(ValueError):
-    pass
-
-
-class UnsupportedOSError(ValueError):
-    pass

capa/features/__init__.py

@@ -0,0 +1,192 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import re
+import sys
+import codecs
+import logging
+
+import capa.engine
+
+logger = logging.getLogger(__name__)
+MAX_BYTES_FEATURE_SIZE = 0x100
+
+# identifiers for supported architectures names that tweak a feature
+# for example, offset/x32
+ARCH_X32 = "x32"
+ARCH_X64 = "x64"
+VALID_ARCH = (ARCH_X32, ARCH_X64)
+
+
+def bytes_to_str(b):
+    if sys.version_info[0] >= 3:
+        return str(codecs.encode(b, "hex").decode("utf-8"))
+    else:
+        return codecs.encode(b, "hex")
+
+
+def hex_string(h):
+    """ render hex string e.g. "0a40b1" as "0A 40 B1" """
+    return " ".join(h[i : i + 2] for i in range(0, len(h), 2)).upper()
+
+
+class Feature(object):
+    def __init__(self, value, arch=None, description=None):
+        """
+        Args:
+          value (any): the value of the feature, such as the number or string.
+          arch (str): one of the VALID_ARCH values, or None.
+            When None, then the feature applies to any architecture.
+            Modifies the feature name from `feature` to `feature/arch`, like `offset/x32`.
+          description (str): a human-readable description that explains the feature value.
+        """
+        super(Feature, self).__init__()
+
+        if arch is not None:
+            if arch not in VALID_ARCH:
+                raise ValueError("arch '%s' must be one of %s" % (arch, VALID_ARCH))
+            self.name = self.__class__.__name__.lower() + "/" + arch
+        else:
+            self.name = self.__class__.__name__.lower()
+
+        self.value = value
+        self.arch = arch
+        self.description = description
+
+    def __hash__(self):
+        return hash((self.name, self.value, self.arch))
+
+    def __eq__(self, other):
+        return self.name == other.name and self.value == other.value and self.arch == other.arch
+
+    def get_value_str(self):
+        """
+        render the value of this feature, for use by `__str__` and friends.
+        subclasses should override to customize the rendering.
+
+        Returns: any
+        """
+        return self.value
+
+    def __str__(self):
+        if self.value:
+            if self.description:
+                return "%s(%s = %s)" % (self.name, self.get_value_str(), self.description)
+            else:
+                return "%s(%s)" % (self.name, self.get_value_str())
+        else:
+            return "%s" % self.name
+
+    def __repr__(self):
+        return str(self)
+
+    def evaluate(self, ctx):
+        return capa.engine.Result(self in ctx, self, [], locations=ctx.get(self, []))
+
+    def freeze_serialize(self):
+        if self.arch is not None:
+            return (self.__class__.__name__, [self.value, {"arch": self.arch}])
+        else:
+            return (self.__class__.__name__, [self.value])
+
+    @classmethod
+    def freeze_deserialize(cls, args):
+        # as you can see below in code,
+        # if the last argument is a dictionary,
+        # consider it to be kwargs passed to the feature constructor.
+        if len(args) == 1:
+            return cls(*args)
+        elif isinstance(args[-1], dict):
+            kwargs = args[-1]
+            args = args[:-1]
+            return cls(*args, **kwargs)
+
+
+class MatchedRule(Feature):
+    def __init__(self, value, description=None):
+        super(MatchedRule, self).__init__(value, description=description)
+        self.name = "match"
+
+
+class Characteristic(Feature):
+    def __init__(self, value, description=None):
+        super(Characteristic, self).__init__(value, description=description)
+
+
+class String(Feature):
+    def __init__(self, value, description=None):
+        super(String, self).__init__(value, description=description)
+
+
+class Regex(String):
+    def __init__(self, value, description=None):
+        super(Regex, self).__init__(value, description=description)
+        pat = self.value[len("/") : -len("/")]
+        flags = re.DOTALL
+        if value.endswith("/i"):
+            pat = self.value[len("/") : -len("/i")]
+            flags |= re.IGNORECASE
+        try:
+            self.re = re.compile(pat, flags)
+        except re.error:
+            if value.endswith("/i"):
+                value = value[: -len("i")]
+            raise ValueError(
+                "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value
+            )
+        self.match = None
+
+    def evaluate(self, ctx):
+        for feature, locations in ctx.items():
+            if not isinstance(feature, (capa.features.String,)):
+                continue
+
+            # `re.search` finds a match anywhere in the given string
+            # which implies leading and/or trailing whitespace.
+            # using this mode cleans is more convenient for rule authors,
+            # so that they don't have to prefix/suffix their terms like: /.*foo.*/.
+            if self.re.search(feature.value):
+                self.match = feature.value
+                return capa.engine.Result(True, self, [], locations=locations)
+
+        return capa.engine.Result(False, self, [])
+
+    def __str__(self):
+        return 'regex(string =~ %s, matched = "%s")' % (self.value, self.match)
+
+
+class StringFactory(object):
+    def __new__(self, value, description=None):
+        if value.startswith("/") and (value.endswith("/") or value.endswith("/i")):
+            return Regex(value, description=description)
+        return String(value, description=description)
+
+
+class Bytes(Feature):
+    def __init__(self, value, description=None):
+        super(Bytes, self).__init__(value, description=description)
+
+    def evaluate(self, ctx):
+        for feature, locations in ctx.items():
+            if not isinstance(feature, (capa.features.Bytes,)):
+                continue
+
+            if feature.value.startswith(self.value):
+                return capa.engine.Result(True, self, [], locations=locations)
+
+        return capa.engine.Result(False, self, [])
+
+    def get_value_str(self):
+        return hex_string(bytes_to_str(self.value))
+
+    def freeze_serialize(self):
+        return (self.__class__.__name__, [bytes_to_str(self.value).upper()])
+
+    @classmethod
+    def freeze_deserialize(cls, args):
+        return cls(*[codecs.decode(x, "hex") for x in args])

capa/features/address.py

@@ -1,96 +0,0 @@
-import abc
-
-from dncil.clr.token import Token
-
-
-class Address(abc.ABC):
-    @abc.abstractmethod
-    def __lt__(self, other):
-        # implement < so that addresses can be sorted from low to high
-        ...
-
-    @abc.abstractmethod
-    def __hash__(self):
-        # implement hash so that addresses can be used in sets and dicts
-        ...
-
-    @abc.abstractmethod
-    def __str__(self):
-        # implement str so the address can be rendered in capa output
-        ...
-
-
-class AbsoluteVirtualAddress(int, Address):
-    """an absolute memory address"""
-
-    def __new__(cls, v):
-        assert v >= 0
-        return int.__new__(cls, v)
-
-    def __str__(self):
-        return str(self)
-
-
-class RelativeVirtualAddress(int, Address):
-    """a memory address relative to a base address"""
-
-    pass
-
-
-class FileOffsetAddress(int, Address):
-    """an address relative to the start of a file"""
-
-    def __new__(cls, v):
-        assert v >= 0
-        return int.__new__(cls, v)
-
-    def __str__(self):
-        return str(self)
-
-
-class DNTokenAddress(Address):
-    """a .NET token"""
-
-    def __init__(self, token: Token):
-        self.token = token
-
-    def __lt__(self, other):
-        return self.token.value < other.token.value
-
-    def __hash__(self):
-        return hash(self.token.value)
-
-    def __str__(self):
-        return str(self.token)
-
-
-class DNTokenOffsetAddress(Address):
-    """an offset into an object specified by a .NET token"""
-
-    def __init__(self, token: Token, offset: int):
-        assert offset >= 0
-        self.token = token
-        self.offset = offset
-
-    def __lt__(self, other):
-        return (self.token.value, self.offset) < (other.token.value, other.offset)
-
-    def __hash__(self):
-        return hash((self.token.value, self.offset))
-
-    def __str__(self):
-        return f"{self.token:s}+{hex(self.offset)}"
-
-
-class _NoAddress(Address):
-    def __lt__(self, other):
-        return False
-
-    def __hash__(self):
-        return hash(0)
-
-    def __str__(self):
-        return "no address"
-
-
-NO_ADDRESS = _NoAddress()

capa/features/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,7 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from capa.features.common import Feature
+from capa.features import Feature
 
 
 class BasicBlock(Feature):
@@ -20,7 +20,7 @@ class BasicBlock(Feature):
         return ""
 
     def freeze_serialize(self):
-        return (self.__class__.__name__, ())
+        return (self.__class__.__name__, [])
 
     @classmethod
     def freeze_deserialize(cls, args):

capa/features/common.py

@@ -1,439 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import re
-import abc
-import codecs
-import logging
-import collections
-from typing import TYPE_CHECKING, Set, Dict, List, Union, Optional, Sequence
-
-if TYPE_CHECKING:
-    # circular import, otherwise
-    import capa.engine
-
-import capa.perf
-import capa.features
-import capa.features.extractors.elf
-from capa.features.address import Address
-
-logger = logging.getLogger(__name__)
-MAX_BYTES_FEATURE_SIZE = 0x100
-
-# thunks may be chained so we specify a delta to control the depth to which these chains are explored
-THUNK_CHAIN_DEPTH_DELTA = 5
-
-
-def bytes_to_str(b: bytes) -> str:
-    return str(codecs.encode(b, "hex").decode("utf-8"))
-
-
-def hex_string(h: str) -> str:
-    """render hex string e.g. "0a40b1" as "0A 40 B1" """
-    return " ".join(h[i : i + 2] for i in range(0, len(h), 2)).upper()
-
-
-def escape_string(s: str) -> str:
-    """escape special characters"""
-    s = repr(s)
-    if not s.startswith(('"', "'")):
-        # u'hello\r\nworld' -> hello\\r\\nworld
-        s = s[2:-1]
-    else:
-        # 'hello\r\nworld' -> hello\\r\\nworld
-        s = s[1:-1]
-    s = s.replace("\\'", "'")  # repr() may escape "'" in some edge cases, remove
-    s = s.replace('"', '\\"')  # repr() does not escape '"', add
-    return s
-
-
-class Result:
-    """
-    represents the results of an evaluation of statements against features.
-
-    instances of this class should behave like a bool,
-    e.g. `assert Result(True, ...) == True`
-
-    instances track additional metadata about evaluation results.
-    they contain references to the statement node (e.g. an And statement),
-     as well as the children Result instances.
-
-    we need this so that we can render the tree of expressions and their results.
-    """
-
-    def __init__(
-        self,
-        success: bool,
-        statement: Union["capa.engine.Statement", "Feature"],
-        children: List["Result"],
-        locations: Optional[Set[Address]] = None,
-    ):
-        super(Result, self).__init__()
-        self.success = success
-        self.statement = statement
-        self.children = children
-        self.locations = locations if locations is not None else set()
-
-    def __eq__(self, other):
-        if isinstance(other, bool):
-            return self.success == other
-        return False
-
-    def __bool__(self):
-        return self.success
-
-    def __nonzero__(self):
-        return self.success
-
-
-class Feature(abc.ABC):
-    def __init__(self, value: Union[str, int, float, bytes], description=None):
-        """
-        Args:
-          value (any): the value of the feature, such as the number or string.
-          description (str): a human-readable description that explains the feature value.
-        """
-        super(Feature, self).__init__()
-        self.name = self.__class__.__name__.lower()
-
-        self.value = value
-        self.description = description
-
-    def __hash__(self):
-        return hash((self.name, self.value))
-
-    def __lt__(self, other):
-        return self.freeze_serialize() < other.freeze_serialize()
-
-    def __eq__(self, other):
-        return self.name == other.name and self.value == other.value
-
-    def get_value_str(self) -> str:
-        """
-        render the value of this feature, for use by `__str__` and friends.
-        subclasses should override to customize the rendering.
-
-        Returns: any
-        """
-        return str(self.value)
-
-    def __str__(self):
-        if self.value is not None:
-            if self.description:
-                return "%s(%s = %s)" % (self.name, self.get_value_str(), self.description)
-            else:
-                return "%s(%s)" % (self.name, self.get_value_str())
-        else:
-            return "%s" % self.name
-
-    def __repr__(self):
-        return str(self)
-
-    def evaluate(self, ctx: Dict["Feature", Set[Address]], **kwargs) -> Result:
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature." + self.name] += 1
-        return Result(self in ctx, self, [], locations=ctx.get(self, set()))
-
-    def freeze_serialize(self):
-        return (self.__class__.__name__, (self.value,))
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        # as you can see below in code,
-        # if the last argument is a dictionary,
-        # consider it to be kwargs passed to the feature constructor.
-        if len(args) == 1:
-            return cls(*args)
-        elif isinstance(args[-1], dict):
-            kwargs = args[-1]
-            args = args[:-1]
-            return cls(*args, **kwargs)
-        else:
-            return cls(*args)
-
-
-class MatchedRule(Feature):
-    def __init__(self, value: str, description=None):
-        super(MatchedRule, self).__init__(value, description=description)
-        self.name = "match"
-
-
-class Characteristic(Feature):
-    def __init__(self, value: str, description=None):
-        super(Characteristic, self).__init__(value, description=description)
-
-
-class String(Feature):
-    def __init__(self, value: str, description=None):
-        super(String, self).__init__(value, description=description)
-
-
-class Substring(String):
-    def __init__(self, value: str, description=None):
-        super(Substring, self).__init__(value, description=description)
-        self.value = value
-
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.substring"] += 1
-
-        # mapping from string value to list of locations.
-        # will unique the locations later on.
-        matches = collections.defaultdict(list)
-
-        for feature, locations in ctx.items():
-            if not isinstance(feature, (String,)):
-                continue
-
-            if not isinstance(feature.value, str):
-                # this is a programming error: String should only contain str
-                raise ValueError("unexpected feature value type")
-
-            if self.value in feature.value:
-                matches[feature.value].extend(locations)
-                if short_circuit:
-                    # we found one matching string, thats sufficient to match.
-                    # don't collect other matching strings in this mode.
-                    break
-
-        if matches:
-            # finalize: defaultdict -> dict
-            # which makes json serialization easier
-            matches = dict(matches)
-
-            # collect all locations
-            locations = set()
-            for s in matches.keys():
-                matches[s] = list(set(matches[s]))
-                locations.update(matches[s])
-
-            # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
-            # this is because `self` may match on many strings, so we can't stuff the matched value into it.
-            # instead, return a new instance that has a reference to both the substring and the matched values.
-            return Result(True, _MatchedSubstring(self, matches), [], locations=locations)
-        else:
-            return Result(False, _MatchedSubstring(self, None), [])
-
-    def __str__(self):
-        return "substring(%s)" % self.value
-
-
-class _MatchedSubstring(Substring):
-    """
-    this represents specific match instances of a substring feature.
-    treat it the same as a `Substring` except it has the `matches` field that contains the complete strings that matched.
-
-    note: this type should only ever be constructed by `Substring.evaluate()`. it is not part of the public API.
-    """
-
-    def __init__(self, substring: Substring, matches):
-        """
-        args:
-          substring (Substring): the substring feature that matches.
-          match (Dict[string, List[int]]|None): mapping from matching string to its locations.
-        """
-        super(_MatchedSubstring, self).__init__(str(substring.value), description=substring.description)
-        # we want this to collide with the name of `Substring` above,
-        # so that it works nicely with the renderers.
-        self.name = "substring"
-        # this may be None if the substring doesn't match
-        self.matches = matches
-
-    def __str__(self):
-        return 'substring("%s", matches = %s)' % (
-            self.value,
-            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
-        )
-
-
-class Regex(String):
-    def __init__(self, value: str, description=None):
-        super(Regex, self).__init__(value, description=description)
-        self.value = value
-
-        pat = self.value[len("/") : -len("/")]
-        flags = re.DOTALL
-        if value.endswith("/i"):
-            pat = self.value[len("/") : -len("/i")]
-            flags |= re.IGNORECASE
-        try:
-            self.re = re.compile(pat, flags)
-        except re.error:
-            if value.endswith("/i"):
-                value = value[: -len("i")]
-            raise ValueError(
-                "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value
-            )
-
-    def evaluate(self, ctx, short_circuit=True):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.regex"] += 1
-
-        # mapping from string value to list of locations.
-        # will unique the locations later on.
-        matches = collections.defaultdict(list)
-
-        for feature, locations in ctx.items():
-            if not isinstance(feature, (String,)):
-                continue
-
-            if not isinstance(feature.value, str):
-                # this is a programming error: String should only contain str
-                raise ValueError("unexpected feature value type")
-
-            # `re.search` finds a match anywhere in the given string
-            # which implies leading and/or trailing whitespace.
-            # using this mode cleans is more convenient for rule authors,
-            # so that they don't have to prefix/suffix their terms like: /.*foo.*/.
-            if self.re.search(feature.value):
-                matches[feature.value].extend(locations)
-                if short_circuit:
-                    # we found one matching string, thats sufficient to match.
-                    # don't collect other matching strings in this mode.
-                    break
-
-        if matches:
-            # finalize: defaultdict -> dict
-            # which makes json serialization easier
-            matches = dict(matches)
-
-            # collect all locations
-            locations = set()
-            for s in matches.keys():
-                matches[s] = list(set(matches[s]))
-                locations.update(matches[s])
-
-            # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
-            # this is because `self` may match on many strings, so we can't stuff the matched value into it.
-            # instead, return a new instance that has a reference to both the regex and the matched values.
-            # see #262.
-            return Result(True, _MatchedRegex(self, matches), [], locations=locations)
-        else:
-            return Result(False, _MatchedRegex(self, None), [])
-
-    def __str__(self):
-        return "regex(string =~ %s)" % self.value
-
-
-class _MatchedRegex(Regex):
-    """
-    this represents specific match instances of a regular expression feature.
-    treat it the same as a `Regex` except it has the `matches` field that contains the complete strings that matched.
-
-    note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
-    """
-
-    def __init__(self, regex: Regex, matches):
-        """
-        args:
-          regex (Regex): the regex feature that matches.
-          match (Dict[string, List[int]]|None): mapping from matching string to its locations.
-        """
-        super(_MatchedRegex, self).__init__(str(regex.value), description=regex.description)
-        # we want this to collide with the name of `Regex` above,
-        # so that it works nicely with the renderers.
-        self.name = "regex"
-        # this may be None if the regex doesn't match
-        self.matches = matches
-
-    def __str__(self):
-        return "regex(string =~ %s, matches = %s)" % (
-            self.value,
-            ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys())),
-        )
-
-
-class StringFactory:
-    def __new__(cls, value: str, description=None):
-        if value.startswith("/") and (value.endswith("/") or value.endswith("/i")):
-            return Regex(value, description=description)
-        return String(value, description=description)
-
-
-class Bytes(Feature):
-    def __init__(self, value: bytes, description=None):
-        super(Bytes, self).__init__(value, description=description)
-        self.value = value
-
-    def evaluate(self, ctx, **kwargs):
-        capa.perf.counters["evaluate.feature"] += 1
-        capa.perf.counters["evaluate.feature.bytes"] += 1
-
-        for feature, locations in ctx.items():
-            if not isinstance(feature, (Bytes,)):
-                continue
-
-            if feature.value.startswith(self.value):
-                return Result(True, self, [], locations=locations)
-
-        return Result(False, self, [])
-
-    def get_value_str(self):
-        return hex_string(bytes_to_str(self.value))
-
-    def freeze_serialize(self):
-        return (self.__class__.__name__, (bytes_to_str(self.value).upper(),))
-
-    @classmethod
-    def freeze_deserialize(cls, args):
-        return cls(*[codecs.decode(x, "hex") for x in args])
-
-
-# other candidates here: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types
-ARCH_I386 = "i386"
-ARCH_AMD64 = "amd64"
-# dotnet
-ARCH_ANY = "any"
-VALID_ARCH = (ARCH_I386, ARCH_AMD64, ARCH_ANY)
-
-
-class Arch(Feature):
-    def __init__(self, value: str, description=None):
-        super(Arch, self).__init__(value, description=description)
-        self.name = "arch"
-
-
-OS_WINDOWS = "windows"
-OS_LINUX = "linux"
-OS_MACOS = "macos"
-# dotnet
-OS_ANY = "any"
-VALID_OS = {os.value for os in capa.features.extractors.elf.OS}
-VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS, OS_ANY})
-
-
-class OS(Feature):
-    def __init__(self, value: str, description=None):
-        super(OS, self).__init__(value, description=description)
-        self.name = "os"
-
-
-FORMAT_PE = "pe"
-FORMAT_ELF = "elf"
-FORMAT_DOTNET = "dotnet"
-VALID_FORMAT = (FORMAT_PE, FORMAT_ELF, FORMAT_DOTNET)
-# internal only, not to be used in rules
-FORMAT_AUTO = "auto"
-FORMAT_SC32 = "sc32"
-FORMAT_SC64 = "sc64"
-FORMAT_FREEZE = "freeze"
-FORMAT_UNKNOWN = "unknown"
-
-
-class Format(Feature):
-    def __init__(self, value: str, description=None):
-        super(Format, self).__init__(value, description=description)
-        self.name = "format"
-
-
-def is_global_feature(feature):
-    """
-    is this a feature that is extracted at every scope?
-    today, these are OS and arch features.
-    """
-    return isinstance(feature, (OS, Arch))

capa/features/extractors/__init__.py

@@ -0,0 +1,286 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import abc
+
+
+class FeatureExtractor(object):
+    """
+    FeatureExtractor defines the interface for fetching features from a sample.
+
+    There may be multiple backends that support fetching features for capa.
+    For example, we use vivisect by default, but also want to support saving
+     and restoring features from a JSON file.
+    When we restore the features, we'd like to use exactly the same matching logic
+     to find matching rules.
+    Therefore, we can define a FeatureExtractor that provides features from the
+     serialized JSON file and do matching without a binary analysis pass.
+    Also, this provides a way to hook in an IDA backend.
+
+    This class is not instantiated directly; it is the base class for other implementations.
+    """
+
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self):
+        #
+        # note: a subclass should define ctor parameters for its own use.
+        #  for example, the Vivisect feature extract might require the vw and/or path.
+        # this base class doesn't know what to do with that info, though.
+        #
+        super(FeatureExtractor, self).__init__()
+
+    @abc.abstractmethod
+    def get_base_address(self):
+        """
+        fetch the preferred load address at which the sample was analyzed.
+
+        returns: int
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def extract_file_features(self):
+        """
+        extract file-scope features.
+
+        example::
+
+            extractor = VivisectFeatureExtractor(vw, path)
+            for feature, va in extractor.get_file_features():
+                print('0x%x: %s', va, feature)
+
+        yields:
+          Tuple[capa.features.Feature, int]: feature and its location
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def get_functions(self):
+        """
+        enumerate the functions and provide opaque values that will
+         subsequently be provided to `.extract_function_features()`, etc.
+
+        by "opaque value", we mean that this can be any object, as long as it
+         provides enough context to `.extract_function_features()`.
+
+        the opaque value should support casting to int (`__int__`) for the function start address.
+
+        yields:
+          any: the opaque function value.
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def extract_function_features(self, f):
+        """
+        extract function-scope features.
+        the arguments are opaque values previously provided by `.get_functions()`, etc.
+
+        example::
+
+            extractor = VivisectFeatureExtractor(vw, path)
+            for function in extractor.get_functions():
+                for feature, va in extractor.extract_function_features(function):
+                    print('0x%x: %s', va, feature)
+
+        args:
+          f [any]: an opaque value previously fetched from `.get_functions()`.
+
+        yields:
+          Tuple[capa.features.Feature, int]: feature and its location
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def get_basic_blocks(self, f):
+        """
+        enumerate the basic blocks in the given function and provide opaque values that will
+         subsequently be provided to `.extract_basic_block_features()`, etc.
+
+        by "opaque value", we mean that this can be any object, as long as it
+         provides enough context to `.extract_basic_block_features()`.
+
+        the opaque value should support casting to int (`__int__`) for the basic block start address.
+
+        yields:
+          any: the opaque basic block value.
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def extract_basic_block_features(self, f, bb):
+        """
+        extract basic block-scope features.
+        the arguments are opaque values previously provided by `.get_functions()`, etc.
+
+        example::
+
+            extractor = VivisectFeatureExtractor(vw, path)
+            for function in extractor.get_functions():
+                for bb in extractor.get_basic_blocks(function):
+                    for feature, va in extractor.extract_basic_block_features(function, bb):
+                        print('0x%x: %s', va, feature)
+
+        args:
+          f [any]: an opaque value previously fetched from `.get_functions()`.
+          bb [any]: an opaque value previously fetched from `.get_basic_blocks()`.
+
+        yields:
+          Tuple[capa.features.Feature, int]: feature and its location
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def get_instructions(self, f, bb):
+        """
+        enumerate the instructions in the given basic block and provide opaque values that will
+         subsequently be provided to `.extract_insn_features()`, etc.
+
+        by "opaque value", we mean that this can be any object, as long as it
+         provides enough context to `.extract_insn_features()`.
+
+        the opaque value should support casting to int (`__int__`) for the instruction address.
+
+        yields:
+          any: the opaque function value.
+        """
+        raise NotImplemented
+
+    @abc.abstractmethod
+    def extract_insn_features(self, f, bb, insn):
+        """
+        extract instruction-scope features.
+        the arguments are opaque values previously provided by `.get_functions()`, etc.
+
+        example::
+
+            extractor = VivisectFeatureExtractor(vw, path)
+            for function in extractor.get_functions():
+                for bb in extractor.get_basic_blocks(function):
+                    for insn in extractor.get_instructions(function, bb):
+                        for feature, va in extractor.extract_insn_features(function, bb, insn):
+                            print('0x%x: %s', va, feature)
+
+        args:
+          f [any]: an opaque value previously fetched from `.get_functions()`.
+          bb [any]: an opaque value previously fetched from `.get_basic_blocks()`.
+          insn [any]: an opaque value previously fetched from `.get_instructions()`.
+
+        yields:
+          Tuple[capa.features.Feature, int]: feature and its location
+        """
+        raise NotImplemented
+
+
+class NullFeatureExtractor(FeatureExtractor):
+    """
+    An extractor that extracts some user-provided features.
+    The structure of the single parameter is demonstrated in the example below.
+
+    This is useful for testing, as we can provide expected values and see if matching works.
+    Also, this is how we represent features deserialized from a freeze file.
+
+    example::
+
+        extractor = NullFeatureExtractor({
+            'base address: 0x401000,
+            'file features': [
+                (0x402345, capa.features.Characteristic('embedded pe')),
+            ],
+            'functions': {
+                0x401000: {
+                    'features': [
+                        (0x401000, capa.features.Characteristic('nzxor')),
+                    ],
+                    'basic blocks': {
+                        0x401000: {
+                            'features': [
+                                (0x401000, capa.features.Characteristic('tight-loop')),
+                            ],
+                            'instructions': {
+                                0x401000: {
+                                    'features': [
+                                        (0x401000, capa.features.Characteristic('nzxor')),
+                                    ],
+                                },
+                                0x401002: ...
+                            }
+                        },
+                        0x401005: ...
+                    }
+                },
+                0x40200: ...
+            }
+        )
+    """
+
+    def __init__(self, features):
+        super(NullFeatureExtractor, self).__init__()
+        self.features = features
+
+    def get_base_address(self):
+        return self.features["base address"]
+
+    def extract_file_features(self):
+        for p in self.features.get("file features", []):
+            va, feature = p
+            yield feature, va
+
+    def get_functions(self):
+        for va in sorted(self.features["functions"].keys()):
+            yield va
+
+    def extract_function_features(self, f):
+        for p in self.features.get("functions", {}).get(f, {}).get("features", []):  # noqa: E127 line over-indented
+            va, feature = p
+            yield feature, va
+
+    def get_basic_blocks(self, f):
+        for va in sorted(
+            self.features.get("functions", {})  # noqa: E127 line over-indented
+            .get(f, {})
+            .get("basic blocks", {})
+            .keys()
+        ):
+            yield va
+
+    def extract_basic_block_features(self, f, bb):
+        for p in (
+            self.features.get("functions", {})  # noqa: E127 line over-indented
+            .get(f, {})
+            .get("basic blocks", {})
+            .get(bb, {})
+            .get("features", [])
+        ):
+            va, feature = p
+            yield feature, va
+
+    def get_instructions(self, f, bb):
+        for va in sorted(
+            self.features.get("functions", {})  # noqa: E127 line over-indented
+            .get(f, {})
+            .get("basic blocks", {})
+            .get(bb, {})
+            .get("instructions", {})
+            .keys()
+        ):
+            yield va
+
+    def extract_insn_features(self, f, bb, insn):
+        for p in (
+            self.features.get("functions", {})  # noqa: E127 line over-indented
+            .get(f, {})
+            .get("basic blocks", {})
+            .get(bb, {})
+            .get("instructions", {})
+            .get(insn, {})
+            .get("features", [])
+        ):
+            va, feature = p
+            yield feature, va

capa/features/extractors/base_extractor.py

@@ -1,259 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import abc
-import dataclasses
-from typing import Any, Dict, Tuple, Iterator
-from dataclasses import dataclass
-
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-
-# feature extractors may reference functions, BBs, insns by opaque handle values.
-# you can use the `.address` property to get and render the address of the feature.
-#
-# these handles are only consumed by routines on
-# the feature extractor from which they were created.
-
-
-@dataclass
-class FunctionHandle:
-    """reference to a function recognized by a feature extractor.
-
-    Attributes:
-        address: the address of the function.
-        inner: extractor-specific data.
-        ctx: a context object for the extractor.
-    """
-
-    address: Address
-    inner: Any
-    ctx: Dict[str, Any] = dataclasses.field(default_factory=dict)
-
-
-@dataclass
-class BBHandle:
-    """reference to a basic block recognized by a feature extractor.
-
-    Attributes:
-        address: the address of the basic block start address.
-        inner: extractor-specific data.
-    """
-
-    address: Address
-    inner: Any
-
-
-@dataclass
-class InsnHandle:
-    """reference to a instruction recognized by a feature extractor.
-
-    Attributes:
-        address: the address of the instruction address.
-        inner: extractor-specific data.
-    """
-
-    address: Address
-    inner: Any
-
-
-class FeatureExtractor:
-    """
-    FeatureExtractor defines the interface for fetching features from a sample.
-
-    There may be multiple backends that support fetching features for capa.
-    For example, we use vivisect by default, but also want to support saving
-     and restoring features from a JSON file.
-    When we restore the features, we'd like to use exactly the same matching logic
-     to find matching rules.
-    Therefore, we can define a FeatureExtractor that provides features from the
-     serialized JSON file and do matching without a binary analysis pass.
-    Also, this provides a way to hook in an IDA backend.
-
-    This class is not instantiated directly; it is the base class for other implementations.
-    """
-
-    __metaclass__ = abc.ABCMeta
-
-    def __init__(self):
-        #
-        # note: a subclass should define ctor parameters for its own use.
-        #  for example, the Vivisect feature extract might require the vw and/or path.
-        # this base class doesn't know what to do with that info, though.
-        #
-        super(FeatureExtractor, self).__init__()
-
-    @abc.abstractmethod
-    def get_base_address(self) -> AbsoluteVirtualAddress:
-        """
-        fetch the preferred load address at which the sample was analyzed.
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract features found at every scope ("global").
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for feature, va in extractor.get_global_features():
-                print('0x%x: %s', va, feature)
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract file-scope features.
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for feature, va in extractor.get_file_features():
-                print('0x%x: %s', va, feature)
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        """
-        enumerate the functions and provide opaque values that will
-         subsequently be provided to `.extract_function_features()`, etc.
-        """
-        raise NotImplementedError()
-
-    def is_library_function(self, addr: Address) -> bool:
-        """
-        is the given address a library function?
-        the backend may implement its own function matching algorithm, or none at all.
-        we accept an address here, rather than function object,
-         to handle addresses identified in instructions.
-
-        this information is used to:
-          - filter out matches in library functions (by default), and
-          - recognize when to fetch symbol names for called (non-API) functions
-
-        args:
-          addr (Address): the address of a function.
-
-        returns:
-          bool: True if the given address is the start of a library function.
-        """
-        return False
-
-    def get_function_name(self, addr: Address) -> str:
-        """
-        fetch any recognized name for the given address.
-        this is only guaranteed to return a value when the given function is a recognized library function.
-        we accept a VA here, rather than function object, to handle addresses identified in instructions.
-
-        args:
-          addr (Address): the address of a function.
-
-        returns:
-          str: the function name
-
-        raises:
-          KeyError: when the given function does not have a name.
-        """
-        raise KeyError(addr)
-
-    @abc.abstractmethod
-    def extract_function_features(self, f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract function-scope features.
-        the arguments are opaque values previously provided by `.get_functions()`, etc.
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for function in extractor.get_functions():
-                for feature, address in extractor.extract_function_features(function):
-                    print('0x%x: %s', address, feature)
-
-        args:
-          f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_basic_blocks(self, f: FunctionHandle) -> Iterator[BBHandle]:
-        """
-        enumerate the basic blocks in the given function and provide opaque values that will
-         subsequently be provided to `.extract_basic_block_features()`, etc.
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract basic block-scope features.
-        the arguments are opaque values previously provided by `.get_functions()`, etc.
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for function in extractor.get_functions():
-                for bb in extractor.get_basic_blocks(function):
-                    for feature, address in extractor.extract_basic_block_features(function, bb):
-                        print('0x%x: %s', address, feature)
-
-        args:
-          f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
-          bb [BBHandle]: an opaque value previously fetched from `.get_basic_blocks()`.
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def get_instructions(self, f: FunctionHandle, bb: BBHandle) -> Iterator[InsnHandle]:
-        """
-        enumerate the instructions in the given basic block and provide opaque values that will
-         subsequently be provided to `.extract_insn_features()`, etc.
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def extract_insn_features(
-        self, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
-    ) -> Iterator[Tuple[Feature, Address]]:
-        """
-        extract instruction-scope features.
-        the arguments are opaque values previously provided by `.get_functions()`, etc.
-
-        example::
-
-            extractor = VivisectFeatureExtractor(vw, path)
-            for function in extractor.get_functions():
-                for bb in extractor.get_basic_blocks(function):
-                    for insn in extractor.get_instructions(function, bb):
-                        for feature, address in extractor.extract_insn_features(function, bb, insn):
-                            print('0x%x: %s', address, feature)
-
-        args:
-          f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
-          bb [BBHandle]: an opaque value previously fetched from `.get_basic_blocks()`.
-          insn [InsnHandle]: an opaque value previously fetched from `.get_instructions()`.
-
-        yields:
-          Tuple[Feature, Address]: feature and its location
-        """
-        raise NotImplementedError()

capa/features/extractors/common.py

@@ -1,100 +0,0 @@
-import io
-import logging
-import binascii
-import contextlib
-from typing import Tuple, Iterator
-
-import pefile
-
-import capa.features
-import capa.features.extractors.elf
-import capa.features.extractors.pefile
-from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, FORMAT_FREEZE, Arch, Format, String, Feature
-from capa.features.freeze import is_freeze
-from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[String, Address]]:
-    """
-    extract ASCII and UTF-16 LE strings from file
-    """
-    for s in capa.features.extractors.strings.extract_ascii_strings(buf):
-        yield String(s.s), FileOffsetAddress(s.offset)
-
-    for s in capa.features.extractors.strings.extract_unicode_strings(buf):
-        yield String(s.s), FileOffsetAddress(s.offset)
-
-
-def extract_format(buf) -> Iterator[Tuple[Feature, Address]]:
-    if buf.startswith(b"MZ"):
-        yield Format(FORMAT_PE), NO_ADDRESS
-    elif buf.startswith(b"\x7fELF"):
-        yield Format(FORMAT_ELF), NO_ADDRESS
-    elif is_freeze(buf):
-        yield Format(FORMAT_FREEZE), NO_ADDRESS
-    else:
-        # we likely end up here:
-        #  1. handling a file format (e.g. macho)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s", binascii.hexlify(buf[:4]).decode("ascii"))
-        return
-
-
-def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
-    if buf.startswith(b"MZ"):
-        yield from capa.features.extractors.pefile.extract_file_arch(pe=pefile.PE(data=buf))
-
-    elif buf.startswith(b"\x7fELF"):
-        with contextlib.closing(io.BytesIO(buf)) as f:
-            arch = capa.features.extractors.elf.detect_elf_arch(f)
-
-        if arch not in capa.features.common.VALID_ARCH:
-            logger.debug("unsupported arch: %s", arch)
-            return
-
-        yield Arch(arch), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the arch,
-        # but i think this would be rarely used.
-        # rules that rely on arch conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess Arch", binascii.hexlify(buf[:4]).decode("ascii"))
-        return
-
-
-def extract_os(buf) -> Iterator[Tuple[Feature, Address]]:
-    if buf.startswith(b"MZ"):
-        yield OS(OS_WINDOWS), NO_ADDRESS
-    elif buf.startswith(b"\x7fELF"):
-        with contextlib.closing(io.BytesIO(buf)) as f:
-            os = capa.features.extractors.elf.detect_elf_os(f)
-
-        if os not in capa.features.common.VALID_OS:
-            logger.debug("unsupported os: %s", os)
-            return
-
-        yield OS(os), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the OS,
-        # but i think this would be rarely used.
-        # rules that rely on OS conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess OS", binascii.hexlify(buf[:4]).decode("ascii"))
-        return

capa/features/extractors/dnfile/extractor.py

@@ -1,71 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, List, Tuple, Iterator
-
-from capa.features.address import Address, DNTokenAddress, DNTokenOffsetAddress, AbsoluteVirtualAddress
-
-if TYPE_CHECKING:
-    from capa.features.common import Feature
-
-import dnfile
-from dncil.clr.token import Token
-
-import capa.features.extractors
-import capa.features.extractors.dnfile.file
-import capa.features.extractors.dnfile.insn
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
-from capa.features.extractors.dnfile.helpers import get_dotnet_managed_method_bodies
-
-
-class DnfileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
-        super(DnfileFeatureExtractor, self).__init__()
-        self.pe: dnfile.dnPE = dnfile.dnPE(path)
-
-        # pre-compute these because we'll yield them at *every* scope.
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_os(pe=self.pe))
-        self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_arch(pe=self.pe))
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(0x0)
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.dnfile.file.extract_features(self.pe)
-
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        for token, f in get_dotnet_managed_method_bodies(self.pe):
-            yield FunctionHandle(address=DNTokenAddress(Token(token)), inner=f, ctx={"pe": self.pe})
-
-    def extract_function_features(self, f):
-        # TODO
-        yield from []
-
-    def get_basic_blocks(self, f) -> Iterator[BBHandle]:
-        # each dotnet method is considered 1 basic block
-        yield BBHandle(
-            address=f.address,
-            inner=f.inner,
-        )
-
-    def extract_basic_block_features(self, fh, bbh):
-        # we don't support basic block features
-        yield from []
-
-    def get_instructions(self, fh, bbh):
-        for insn in bbh.inner.instructions:
-            yield InsnHandle(address=DNTokenOffsetAddress(bbh.address.token, insn.offset - fh.inner.offset), inner=insn)
-
-    def extract_insn_features(self, fh, bbh, ih) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.dnfile.insn.extract_features(fh, bbh, ih)

capa/features/extractors/dnfile/file.py

@@ -1,54 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Tuple, Iterator
-
-if TYPE_CHECKING:
-    import dnfile
-    from capa.features.common import Feature, Format, String, Characteristic
-    from capa.features.file import Import, FunctionName
-    from capa.features.address import Address
-
-import capa.features.extractors
-
-
-def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[Tuple[Import, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_import_names(pe=pe)
-
-
-def extract_file_format(pe: dnfile.dnPE) -> Iterator[Tuple[Format, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_format(pe=pe)
-
-
-def extract_file_function_names(pe: dnfile.dnPE) -> Iterator[Tuple[FunctionName, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_function_names(pe=pe)
-
-
-def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe)
-
-
-def extract_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[Tuple[Characteristic, Address]]:
-    yield from capa.features.extractors.dotnetfile.extract_mixed_mode_characteristic_features(pe=pe)
-
-
-def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for file_handler in FILE_HANDLERS:
-        for (feature, address) in file_handler(pe):
-            yield feature, address
-
-
-FILE_HANDLERS = (
-    extract_file_import_names,
-    extract_file_function_names,
-    extract_file_strings,
-    extract_file_format,
-    extract_mixed_mode_characteristic_features,
-)

capa/features/extractors/dnfile/helpers.py

@@ -1,213 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-import logging
-from typing import Any, Tuple, Iterator, Optional
-
-import dnfile
-from dncil.cil.body import CilMethodBody
-from dncil.cil.error import MethodBodyFormatError
-from dncil.clr.token import Token, StringToken, InvalidToken
-from dncil.cil.body.reader import CilMethodBodyReaderBase
-
-logger = logging.getLogger(__name__)
-
-# key indexes to dotnet metadata tables
-DOTNET_META_TABLES_BY_INDEX = {table.value: table.name for table in dnfile.enums.MetadataTables}
-
-
-class DnfileMethodBodyReader(CilMethodBodyReaderBase):
-    def __init__(self, pe: dnfile.dnPE, row: dnfile.mdtable.MethodDefRow):
-        self.pe: dnfile.dnPE = pe
-        self.offset: int = self.pe.get_offset_from_rva(row.Rva)
-
-    def read(self, n: int) -> bytes:
-        data: bytes = self.pe.get_data(self.pe.get_rva_from_offset(self.offset), n)
-        self.offset += n
-        return data
-
-    def tell(self) -> int:
-        return self.offset
-
-    def seek(self, offset: int) -> int:
-        self.offset = offset
-        return self.offset
-
-
-def calculate_dotnet_token_value(table: int, rid: int) -> int:
-    return ((table & 0xFF) << Token.TABLE_SHIFT) | (rid & Token.RID_MASK)
-
-
-def resolve_dotnet_token(pe: dnfile.dnPE, token: Token) -> Any:
-    """map generic token to string or table row"""
-    if isinstance(token, StringToken):
-        user_string: Optional[str] = read_dotnet_user_string(pe, token)
-        if user_string is None:
-            return InvalidToken(token.value)
-        return user_string
-
-    table_name: str = DOTNET_META_TABLES_BY_INDEX.get(token.table, "")
-    if not table_name:
-        # table_index is not valid
-        return InvalidToken(token.value)
-
-    table: Any = getattr(pe.net.mdtables, table_name, None)
-    if table is None:
-        # table index is valid but table is not present
-        return InvalidToken(token.value)
-
-    try:
-        return table.rows[token.rid - 1]
-    except IndexError:
-        # table index is valid but row index is not valid
-        return InvalidToken(token.value)
-
-
-def read_dotnet_method_body(pe: dnfile.dnPE, row: dnfile.mdtable.MethodDefRow) -> Optional[CilMethodBody]:
-    """read dotnet method body"""
-    try:
-        return CilMethodBody(DnfileMethodBodyReader(pe, row))
-    except MethodBodyFormatError as e:
-        logger.warn("failed to parse managed method body @ 0x%08x (%s)" % (row.Rva, e))
-        return None
-
-
-def read_dotnet_user_string(pe: dnfile.dnPE, token: StringToken) -> Optional[str]:
-    """read user string from #US stream"""
-    try:
-        user_string: Optional[dnfile.stream.UserString] = pe.net.user_strings.get_us(token.rid)
-    except UnicodeDecodeError as e:
-        logger.warn("failed to decode #US stream index 0x%08x (%s)" % (token.rid, e))
-        return None
-    if user_string is None:
-        return None
-    return user_string.value
-
-
-def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
-    """get managed imports from MemberRef table
-
-    see https://www.ntcore.com/files/dotnetformat.htm
-
-    10 - MemberRef Table
-        Each row represents an imported method
-            Class (index into the TypeRef, ModuleRef, MethodDef, TypeSpec or TypeDef tables)
-            Name (index into String heap)
-    01 - TypeRef Table
-        Each row represents an imported class, its namespace and the assembly which contains it
-            TypeName (index into String heap)
-            TypeNamespace (index into String heap)
-    """
-    if not is_dotnet_table_valid(pe, "MemberRef"):
-        return
-
-    for (rid, row) in enumerate(pe.net.mdtables.MemberRef):
-        if not isinstance(row.Class.row, (dnfile.mdtable.TypeRefRow,)):
-            continue
-
-        # like File::OpenRead
-        name = f"{row.Class.row.TypeName}::{row.Name}"
-
-        # ECMA II.22.38: TypeNamespace can be null or non-null
-        if row.Class.row.TypeNamespace:
-            # like System.IO.File::OpenRead
-            name = f"{row.Class.row.TypeNamespace}.{name}"
-
-        token: int = calculate_dotnet_token_value(pe.net.mdtables.MemberRef.number, rid + 1)
-
-        yield token, name
-
-
-def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
-    """get unmanaged imports from ImplMap table
-
-    see https://www.ntcore.com/files/dotnetformat.htm
-
-    28 - ImplMap Table
-        ImplMap table holds information about unmanaged methods that can be reached from managed code, using PInvoke dispatch
-            MemberForwarded (index into the Field or MethodDef table; more precisely, a MemberForwarded coded index)
-            ImportName (index into the String heap)
-            ImportScope (index into the ModuleRef table)
-    """
-    if not is_dotnet_table_valid(pe, "ImplMap"):
-        return
-
-    for row in pe.net.mdtables.ImplMap:
-        dll: str = row.ImportScope.row.Name
-        symbol: str = row.ImportName
-
-        # ECMA says "Each row of the ImplMap table associates a row in the MethodDef table (MemberForwarded) with the
-        # name of a routine (ImportName) in some unmanaged DLL (ImportScope)"; so we calculate and map the MemberForwarded
-        # MethodDef table token to help us later record native import method calls made from CIL
-        token: int = calculate_dotnet_token_value(row.MemberForwarded.table.number, row.MemberForwarded.row_index)
-
-        # like Kernel32.dll
-        if dll and "." in dll:
-            dll = dll.split(".")[0]
-
-        # like kernel32.CreateFileA
-        name: str = f"{dll}.{symbol}"
-
-        yield token, name
-
-
-def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
-    """get managed methods from MethodDef table"""
-    if not is_dotnet_table_valid(pe, "MethodDef"):
-        return
-
-    for (rid, row) in enumerate(pe.net.mdtables.MethodDef):
-        if not row.ImplFlags.miIL or any((row.Flags.mdAbstract, row.Flags.mdPinvokeImpl)):
-            # skip methods that do not have a method body
-            continue
-
-        body: Optional[CilMethodBody] = read_dotnet_method_body(pe, row)
-        if body is None:
-            continue
-
-        token: int = calculate_dotnet_token_value(dnfile.enums.MetadataTables.MethodDef.value, rid + 1)
-        yield token, body
-
-
-def is_dotnet_table_valid(pe: dnfile.dnPE, table_name: str) -> bool:
-    return bool(getattr(pe.net.mdtables, table_name, None))
-
-
-def get_dotnet_managed_method_names(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
-    """get managed method names from TypeDef table
-
-    see https://www.ntcore.com/files/dotnetformat.htm
-
-    02 - TypeDef Table
-        Each row represents a class in the current assembly.
-            TypeName (index into String heap)
-            TypeNamespace (index into String heap)
-            MethodList (index into MethodDef table; it marks the first of a continguous run of Methods owned by this Type)
-    """
-    if not is_dotnet_table_valid(pe, "TypeDef"):
-        return
-
-    for row in pe.net.mdtables.TypeDef:
-        for index in row.MethodList:
-            # like File::OpenRead
-            name = f"{row.TypeName}::{index.row.Name}"
-
-            # ECMA II.22.37: TypeNamespace can be null or non-null
-            if row.TypeNamespace:
-                # like System.IO.File::OpenRead
-                name = f"{row.TypeNamespace}.{name}"
-
-            token = calculate_dotnet_token_value(index.table.number, index.row_index)
-
-            yield token, name
-
-
-def is_dotnet_mixed_mode(pe: dnfile.dnPE) -> bool:
-    return not bool(pe.net.Flags.CLR_ILONLY)

capa/features/extractors/dnfile/insn.py

@@ -1,149 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any, Dict, Tuple, Iterator, Optional
-
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-if TYPE_CHECKING:
-    from dncil.cil.instruction import Instruction
-    from dncil.cil.body import CilMethodBody
-    from capa.features.common import Feature
-    from capa.features.address import Address
-
-import dnfile
-from dncil.clr.token import StringToken, InvalidToken
-from dncil.cil.opcode import OpCodes
-
-import capa.features.extractors.helpers
-from capa.features.insn import API, Number
-from capa.features.common import String, Characteristic
-from capa.features.extractors.dnfile.helpers import (
-    resolve_dotnet_token,
-    read_dotnet_user_string,
-    get_dotnet_managed_imports,
-    get_dotnet_unmanaged_imports,
-    get_dotnet_managed_method_names,
-)
-
-
-def get_managed_imports(ctx: Dict) -> Dict:
-    if "managed_imports_cache" not in ctx:
-        ctx["managed_imports_cache"] = {}
-        for (token, name) in get_dotnet_managed_imports(ctx["pe"]):
-            ctx["managed_imports_cache"][token] = name
-    return ctx["managed_imports_cache"]
-
-
-def get_unmanaged_imports(ctx: Dict) -> Dict:
-    if "unmanaged_imports_cache" not in ctx:
-        ctx["unmanaged_imports_cache"] = {}
-        for (token, name) in get_dotnet_unmanaged_imports(ctx["pe"]):
-            ctx["unmanaged_imports_cache"][token] = name
-    return ctx["unmanaged_imports_cache"]
-
-
-def get_methods(ctx: Dict) -> Dict:
-    if "methods_cache" not in ctx:
-        ctx["methods_cache"] = {}
-        for (token, name) in get_dotnet_managed_method_names(ctx["pe"]):
-            ctx["methods_cache"][token] = name
-    return ctx["methods_cache"]
-
-
-def get_callee_name(ctx: Dict, token: int) -> str:
-    """map dotnet token to method name"""
-    name: str = get_managed_imports(ctx).get(token, "")
-    if not name:
-        # we must check unmanaged imports before managed methods because we map forwarded managed methods
-        # to their unmanaged imports; we prefer a forwarded managed method be mapped to its unmanaged import for analysis
-        name = get_unmanaged_imports(ctx).get(token, "")
-        if not name:
-            name = get_methods(ctx).get(token, "")
-    return name
-
-
-def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction API features"""
-    insn: Instruction = ih.inner
-
-    if insn.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp, OpCodes.Calli):
-        return
-
-    name: str = get_callee_name(fh.ctx, insn.operand.value)
-    if not name:
-        return
-
-    if "::" in name:
-        # like System.IO.File::OpenRead
-        yield API(name), ih.address
-    else:
-        # like kernel32.CreateFileA
-        dll, _, symbol = name.rpartition(".")
-        for name_variant in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-            yield API(name_variant), ih.address
-
-
-def extract_insn_number_features(fh, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction number features"""
-    insn: Instruction = ih.inner
-
-    if insn.is_ldc():
-        yield Number(insn.get_ldc()), ih.address
-
-
-def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction string features"""
-    f: CilMethodBody = fh.inner
-    insn: Instruction = ih.inner
-
-    if not insn.is_ldstr():
-        return
-
-    if not isinstance(insn.operand, StringToken):
-        return
-
-    user_string: Optional[str] = read_dotnet_user_string(fh.ctx["pe"], insn.operand)
-    if user_string is None:
-        return
-
-    yield String(user_string), ih.address
-
-
-def extract_unmanaged_call_characteristic_features(
-    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Characteristic, Address]]:
-    insn: Instruction = ih.inner
-    if insn.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp, OpCodes.Calli):
-        return
-
-    token: Any = resolve_dotnet_token(fh.ctx["pe"], insn.operand)
-    if isinstance(token, InvalidToken):
-        return
-    if not isinstance(token, dnfile.mdtable.MethodDefRow):
-        return
-
-    if any((token.Flags.mdPinvokeImpl, token.ImplFlags.miUnmanaged, token.ImplFlags.miNative)):
-        yield Characteristic("unmanaged call"), insn.offset
-
-
-def extract_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract instruction features"""
-    for inst_handler in INSTRUCTION_HANDLERS:
-        for (feature, addr) in inst_handler(fh, bbh, ih):
-            yield feature, addr
-
-
-INSTRUCTION_HANDLERS = (
-    extract_insn_api_features,
-    extract_insn_number_features,
-    extract_insn_string_features,
-    extract_unmanaged_call_characteristic_features,
-)

capa/features/extractors/dnfile_.py

@@ -1,116 +0,0 @@
-import logging
-from typing import Tuple, Iterator
-
-import dnfile
-import pefile
-
-from capa.features.common import OS, OS_ANY, ARCH_ANY, ARCH_I386, ARCH_AMD64, FORMAT_DOTNET, Arch, Format, Feature
-from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_format(**kwargs) -> Iterator[Tuple[Feature, Address]]:
-    yield Format(FORMAT_DOTNET), NO_ADDRESS
-
-
-def extract_file_os(**kwargs) -> Iterator[Tuple[Feature, Address]]:
-    yield OS(OS_ANY), NO_ADDRESS
-
-
-def extract_file_arch(pe, **kwargs) -> Iterator[Tuple[Feature, Address]]:
-    # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
-    # .NET 4.5 added option: any CPU, 32-bit preferred
-    if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        yield Arch(ARCH_ANY), NO_ADDRESS
-
-
-def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for file_handler in FILE_HANDLERS:
-        for feature, address in file_handler(pe=pe):  # type: ignore
-            yield feature, address
-
-
-FILE_HANDLERS = (
-    # extract_file_export_names,
-    # extract_file_import_names,
-    # extract_file_section_names,
-    # extract_file_strings,
-    # extract_file_function_names,
-    extract_file_format,
-)
-
-
-def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for handler in GLOBAL_HANDLERS:
-        for feature, addr in handler(pe=pe):  # type: ignore
-            yield feature, addr
-
-
-GLOBAL_HANDLERS = (
-    extract_file_os,
-    extract_file_arch,
-)
-
-
-class DnfileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
-        super(DnfileFeatureExtractor, self).__init__()
-        self.path: str = path
-        self.pe: dnfile.dnPE = dnfile.dnPE(path)
-
-    def get_base_address(self) -> AbsoluteVirtualAddress:
-        return AbsoluteVirtualAddress(0x0)
-
-    def get_entry_point(self) -> int:
-        # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
-        #  True: native EP: Token
-        #  False: managed EP: RVA
-        return self.pe.net.struct.EntryPointTokenOrRva
-
-    def extract_global_features(self):
-        yield from extract_global_features(self.pe)
-
-    def extract_file_features(self):
-        yield from extract_file_features(self.pe)
-
-    def is_dotnet_file(self) -> bool:
-        return bool(self.pe.net)
-
-    def is_mixed_mode(self) -> bool:
-        return not bool(self.pe.net.Flags.CLR_ILONLY)
-
-    def get_runtime_version(self) -> Tuple[int, int]:
-        return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
-
-    def get_meta_version_string(self) -> str:
-        return self.pe.net.metadata.struct.Version.rstrip(b"\x00").decode("utf-8")
-
-    def get_functions(self):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def extract_function_features(self, f):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def get_basic_blocks(self, f):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def extract_basic_block_features(self, f, bb):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def get_instructions(self, f, bb):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def extract_insn_features(self, f, bb, insn):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def is_library_function(self, va):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")
-
-    def get_function_name(self, va):
-        raise NotImplementedError("DnfileFeatureExtractor can only be used to extract file features")

capa/features/extractors/dotnetfile.py

@@ -1,163 +0,0 @@
-import logging
-from typing import Tuple, Iterator
-
-import dnfile
-import pefile
-from dncil.clr.token import Token
-
-import capa.features.extractors.helpers
-from capa.features.file import Import, FunctionName
-from capa.features.common import (
-    OS,
-    OS_ANY,
-    ARCH_ANY,
-    ARCH_I386,
-    ARCH_AMD64,
-    FORMAT_DOTNET,
-    Arch,
-    Format,
-    String,
-    Feature,
-    Characteristic,
-)
-from capa.features.address import NO_ADDRESS, Address, DNTokenAddress, DNTokenOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
-from capa.features.extractors.dnfile.helpers import (
-    is_dotnet_mixed_mode,
-    get_dotnet_managed_imports,
-    calculate_dotnet_token_value,
-    get_dotnet_unmanaged_imports,
-    get_dotnet_managed_method_names,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_format(**kwargs) -> Iterator[Tuple[Format, Address]]:
-    yield Format(FORMAT_DOTNET), NO_ADDRESS
-
-
-def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Import, Address]]:
-    for (token, name) in get_dotnet_managed_imports(pe):
-        # like System.IO.File::OpenRead
-        yield Import(name), DNTokenAddress(Token(token))
-
-    for (token, name) in get_dotnet_unmanaged_imports(pe):
-        # like kernel32.CreateFileA
-        dll, _, symbol = name.rpartition(".")
-        for name_variant in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-            yield Import(name_variant), DNTokenAddress(Token(token))
-
-
-def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[FunctionName, Address]]:
-    for (token, name) in get_dotnet_managed_method_names(pe):
-        yield FunctionName(name), DNTokenAddress(Token(token))
-
-
-def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
-    yield OS(OS_ANY), NO_ADDRESS
-
-
-def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Arch, Address]]:
-    # to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
-    # .NET 4.5 added option: any CPU, 32-bit preferred
-    if pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE:
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif not pe.net.Flags.CLR_32BITREQUIRED and pe.PE_TYPE == pefile.OPTIONAL_HEADER_MAGIC_PE_PLUS:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        yield Arch(ARCH_ANY), NO_ADDRESS
-
-
-def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[String, Address]]:
-    yield from capa.features.extractors.common.extract_file_strings(pe.__data__)
-
-
-def extract_mixed_mode_characteristic_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Characteristic, Address]]:
-    if is_dotnet_mixed_mode(pe):
-        yield Characteristic("mixed mode"), NO_ADDRESS
-
-
-def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler(pe=pe):  # type: ignore
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_file_import_names,
-    extract_file_function_names,
-    extract_file_strings,
-    extract_file_format,
-    extract_mixed_mode_characteristic_features,
-)
-
-
-def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
-    for handler in GLOBAL_HANDLERS:
-        for feature, va in handler(pe=pe):  # type: ignore
-            yield feature, va
-
-
-GLOBAL_HANDLERS = (
-    extract_file_os,
-    extract_file_arch,
-)
-
-
-class DotnetFileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
-        super(DotnetFileFeatureExtractor, self).__init__()
-        self.path: str = path
-        self.pe: dnfile.dnPE = dnfile.dnPE(path)
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(0x0)
-
-    def get_entry_point(self) -> int:
-        # self.pe.net.Flags.CLT_NATIVE_ENTRYPOINT
-        #  True: native EP: Token
-        #  False: managed EP: RVA
-        return self.pe.net.struct.EntryPointTokenOrRva
-
-    def extract_global_features(self):
-        yield from extract_global_features(self.pe)
-
-    def extract_file_features(self):
-        yield from extract_file_features(self.pe)
-
-    def is_dotnet_file(self) -> bool:
-        return bool(self.pe.net)
-
-    def is_mixed_mode(self) -> bool:
-        return is_dotnet_mixed_mode(self.pe)
-
-    def get_runtime_version(self) -> Tuple[int, int]:
-        return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion
-
-    def get_meta_version_string(self) -> str:
-        return self.pe.net.metadata.struct.Version.rstrip(b"\x00").decode("utf-8")
-
-    def get_functions(self):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def extract_function_features(self, f):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def get_basic_blocks(self, f):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def extract_basic_block_features(self, f, bb):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def get_instructions(self, f, bb):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def extract_insn_features(self, f, bb, insn):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def is_library_function(self, va):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")
-
-    def get_function_name(self, va):
-        raise NotImplementedError("DotnetFileFeatureExtractor can only be used to extract file features")

capa/features/extractors/elf.py

@@ -1,326 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import struct
-import logging
-from enum import Enum
-from typing import BinaryIO
-
-logger = logging.getLogger(__name__)
-
-
-def align(v, alignment):
-    remainder = v % alignment
-    if remainder == 0:
-        return v
-    else:
-        return v + (alignment - remainder)
-
-
-class CorruptElfFile(ValueError):
-    pass
-
-
-class OS(str, Enum):
-    HPUX = "hpux"
-    NETBSD = "netbsd"
-    LINUX = "linux"
-    HURD = "hurd"
-    _86OPEN = "86open"
-    SOLARIS = "solaris"
-    AIX = "aix"
-    IRIX = "irix"
-    FREEBSD = "freebsd"
-    TRU64 = "tru64"
-    MODESTO = "modesto"
-    OPENBSD = "openbsd"
-    OPENVMS = "openvms"
-    NSK = "nsk"
-    AROS = "aros"
-    FENIXOS = "fenixos"
-    CLOUD = "cloud"
-    SYLLABLE = "syllable"
-    NACL = "nacl"
-
-
-def detect_elf_os(f) -> str:
-    """
-    f: type Union[BinaryIO, IDAIO]
-    """
-    f.seek(0x0)
-    file_header = f.read(0x40)
-
-    # we'll set this to the detected OS
-    # prefer the first heuristics,
-    # but rather than short circuiting,
-    # we'll still parse out the remainder, for debugging.
-    ret = None
-
-    if not file_header.startswith(b"\x7fELF"):
-        raise CorruptElfFile("missing magic header")
-
-    ei_class, ei_data = struct.unpack_from("BB", file_header, 4)
-    logger.debug("ei_class: 0x%02x ei_data: 0x%02x", ei_class, ei_data)
-    if ei_class == 1:
-        bitness = 32
-    elif ei_class == 2:
-        bitness = 64
-    else:
-        raise CorruptElfFile("invalid ei_class: 0x%02x" % ei_class)
-
-    if ei_data == 1:
-        endian = "<"
-    elif ei_data == 2:
-        endian = ">"
-    else:
-        raise CorruptElfFile("not an ELF file: invalid ei_data: 0x%02x" % ei_data)
-
-    if bitness == 32:
-        (e_phoff, e_shoff) = struct.unpack_from(endian + "II", file_header, 0x1C)
-        e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x2A)
-        e_shentsize, e_shnum = struct.unpack_from(endian + "HH", file_header, 0x2E)
-    elif bitness == 64:
-        (e_phoff, e_shoff) = struct.unpack_from(endian + "QQ", file_header, 0x20)
-        e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x36)
-        e_shentsize, e_shnum = struct.unpack_from(endian + "HH", file_header, 0x3A)
-    else:
-        raise NotImplementedError()
-
-    logger.debug("e_phoff: 0x%02x e_phentsize: 0x%02x e_phnum: %d", e_phoff, e_phentsize, e_phnum)
-
-    (ei_osabi,) = struct.unpack_from(endian + "B", file_header, 7)
-    OSABI = {
-        # via pyelftools: https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/elftools/elf/enums.py#L35-L58
-        # some candidates are commented out because the are not useful values,
-        # at least when guessing OSes
-        # 0: "SYSV",  # too often used when OS is not SYSV
-        1: OS.HPUX,
-        2: OS.NETBSD,
-        3: OS.LINUX,
-        4: OS.HURD,
-        5: OS._86OPEN,
-        6: OS.SOLARIS,
-        7: OS.AIX,
-        8: OS.IRIX,
-        9: OS.FREEBSD,
-        10: OS.TRU64,
-        11: OS.MODESTO,
-        12: OS.OPENBSD,
-        13: OS.OPENVMS,
-        14: OS.NSK,
-        15: OS.AROS,
-        16: OS.FENIXOS,
-        17: OS.CLOUD,
-        # 53: "SORTFIX",      # i can't find any reference to this OS, i dont think it exists
-        # 64: "ARM_AEABI",    # not an OS
-        # 97: "ARM",          # not an OS
-        # 255: "STANDALONE",  # not an OS
-    }
-    logger.debug("ei_osabi: 0x%02x (%s)", ei_osabi, OSABI.get(ei_osabi, "unknown"))
-
-    # os_osabi == 0 is commonly set even when the OS is not SYSV.
-    # other values are unused or unknown.
-    if ei_osabi in OSABI and ei_osabi != 0x0:
-        # subsequent strategies may overwrite this value
-        ret = OSABI[ei_osabi]
-
-    f.seek(e_phoff)
-    program_header_size = e_phnum * e_phentsize
-    program_headers = f.read(program_header_size)
-    if len(program_headers) != program_header_size:
-        logger.warning("failed to read program headers")
-        e_phnum = 0
-
-    # search for PT_NOTE sections that specify an OS
-    # for example, on Linux there is a GNU section with minimum kernel version
-    for i in range(e_phnum):
-        offset = i * e_phentsize
-        phent = program_headers[offset : offset + e_phentsize]
-
-        PT_NOTE = 0x4
-
-        (p_type,) = struct.unpack_from(endian + "I", phent, 0x0)
-        logger.debug("p_type: 0x%04x", p_type)
-        if p_type != PT_NOTE:
-            continue
-
-        if bitness == 32:
-            p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4)
-        elif bitness == 64:
-            p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8)
-        else:
-            raise NotImplementedError()
-
-        logger.debug("p_offset: 0x%02x p_filesz: 0x%04x", p_offset, p_filesz)
-
-        f.seek(p_offset)
-        note = f.read(p_filesz)
-        if len(note) != p_filesz:
-            logger.warning("failed to read note content")
-            continue
-
-        namesz, descsz, type_ = struct.unpack_from(endian + "III", note, 0x0)
-        name_offset = 0xC
-        desc_offset = name_offset + align(namesz, 0x4)
-
-        logger.debug("namesz: 0x%02x descsz: 0x%02x type: 0x%04x", namesz, descsz, type_)
-
-        name = note[name_offset : name_offset + namesz].partition(b"\x00")[0].decode("ascii")
-        logger.debug("name: %s", name)
-
-        if type_ != 1:
-            continue
-
-        if name == "GNU":
-            if descsz < 16:
-                continue
-
-            desc = note[desc_offset : desc_offset + descsz]
-            abi_tag, kmajor, kminor, kpatch = struct.unpack_from(endian + "IIII", desc, 0x0)
-            # via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658
-            # and here: https://github.com/bminor/binutils-gdb/blob/34c54daa337da9fadf87d2706d6a590ae1f88f4d/include/elf/common.h#L933-L939
-            GNU_ABI_TAG = {
-                0: OS.LINUX,
-                1: OS.HURD,
-                2: OS.SOLARIS,
-                3: OS.FREEBSD,
-                4: OS.NETBSD,
-                5: OS.SYLLABLE,
-                6: OS.NACL,
-            }
-            logger.debug("GNU_ABI_TAG: 0x%02x", abi_tag)
-
-            if abi_tag in GNU_ABI_TAG:
-                # update only if not set
-                # so we can get the debugging output of subsequent strategies
-                ret = GNU_ABI_TAG[abi_tag] if not ret else ret
-                logger.debug("abi tag: %s earliest compatible kernel: %d.%d.%d", ret, kmajor, kminor, kpatch)
-        elif name == "OpenBSD":
-            logger.debug("note owner: %s", "OPENBSD")
-            ret = OS.OPENBSD if not ret else ret
-        elif name == "NetBSD":
-            logger.debug("note owner: %s", "NETBSD")
-            ret = OS.NETBSD if not ret else ret
-        elif name == "FreeBSD":
-            logger.debug("note owner: %s", "FREEBSD")
-            ret = OS.FREEBSD if not ret else ret
-
-    # search for recognizable dynamic linkers (interpreters)
-    # for example, on linux, we see file paths like: /lib64/ld-linux-x86-64.so.2
-    for i in range(e_phnum):
-        offset = i * e_phentsize
-        phent = program_headers[offset : offset + e_phentsize]
-
-        PT_INTERP = 0x3
-
-        (p_type,) = struct.unpack_from(endian + "I", phent, 0x0)
-        if p_type != PT_INTERP:
-            continue
-
-        if bitness == 32:
-            p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4)
-        elif bitness == 64:
-            p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8)
-        else:
-            raise NotImplementedError()
-
-        f.seek(p_offset)
-        interp = f.read(p_filesz)
-        if len(interp) != p_filesz:
-            logger.warning("failed to read interp content")
-            continue
-
-        linker = interp.partition(b"\x00")[0].decode("ascii")
-        logger.debug("linker: %s", linker)
-        if "ld-linux" in linker:
-            # update only if not set
-            # so we can get the debugging output of subsequent strategies
-            ret = OS.LINUX if ret is None else ret
-
-    f.seek(e_shoff)
-    section_header_size = e_shnum * e_shentsize
-    section_headers = f.read(section_header_size)
-    if len(section_headers) != section_header_size:
-        logger.warning("failed to read section headers")
-        e_shnum = 0
-
-    # search for notes stored in sections that aren't visible in program headers.
-    # e.g. .note.Linux in Linux kernel modules.
-    for i in range(e_shnum):
-        offset = i * e_shentsize
-        shent = section_headers[offset : offset + e_shentsize]
-
-        if bitness == 32:
-            sh_name, sh_type, _, sh_addr, sh_offset, sh_size = struct.unpack_from(endian + "IIIIII", shent, 0x0)
-        elif bitness == 64:
-            sh_name, sh_type, _, sh_addr, sh_offset, sh_size = struct.unpack_from(endian + "IIQQQQ", shent, 0x0)
-        else:
-            raise NotImplementedError()
-
-        SHT_NOTE = 0x7
-        if sh_type != SHT_NOTE:
-            continue
-
-        logger.debug("sh_offset: 0x%02x sh_size: 0x%04x", sh_offset, sh_size)
-
-        f.seek(sh_offset)
-        note = f.read(sh_size)
-        if len(note) != sh_size:
-            logger.warning("failed to read note content")
-            continue
-
-        namesz, descsz, type_ = struct.unpack_from(endian + "III", note, 0x0)
-        name_offset = 0xC
-        desc_offset = name_offset + align(namesz, 0x4)
-
-        logger.debug("namesz: 0x%02x descsz: 0x%02x type: 0x%04x", namesz, descsz, type_)
-
-        name = note[name_offset : name_offset + namesz].partition(b"\x00")[0].decode("ascii")
-        logger.debug("name: %s", name)
-
-        if name == "Linux":
-            logger.debug("note owner: %s", "LINUX")
-            ret = OS.LINUX if not ret else ret
-
-    return ret.value if ret is not None else "unknown"
-
-
-class Arch(str, Enum):
-    I386 = "i386"
-    AMD64 = "amd64"
-
-
-def detect_elf_arch(f: BinaryIO) -> str:
-    f.seek(0x0)
-    file_header = f.read(0x40)
-
-    if not file_header.startswith(b"\x7fELF"):
-        raise CorruptElfFile("missing magic header")
-
-    (ei_data,) = struct.unpack_from("B", file_header, 5)
-    logger.debug("ei_data: 0x%02x", ei_data)
-
-    if ei_data == 1:
-        endian = "<"
-    elif ei_data == 2:
-        endian = ">"
-    else:
-        raise CorruptElfFile("not an ELF file: invalid ei_data: 0x%02x" % ei_data)
-
-    (ei_machine,) = struct.unpack_from(endian + "H", file_header, 0x12)
-    logger.debug("ei_machine: 0x%02x", ei_machine)
-
-    EM_386 = 0x3
-    EM_X86_64 = 0x3E
-    if ei_machine == EM_386:
-        return Arch.I386
-    elif ei_machine == EM_X86_64:
-        return Arch.AMD64
-    else:
-        # not really unknown, but unsupport at the moment:
-        # https://github.com/eliben/pyelftools/blob/ab444d982d1849191e910299a985989857466620/elftools/elf/enums.py#L73
-        return "unknown"

capa/features/extractors/elffile.py

@@ -1,160 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import io
-import logging
-import contextlib
-from typing import Tuple, Iterator
-
-from elftools.elf.elffile import ELFFile, SymbolTableSection
-
-import capa.features.extractors.common
-from capa.features.file import Import, Section
-from capa.features.common import OS, FORMAT_ELF, Arch, Format, Feature
-from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.elf import Arch as ElfArch
-from capa.features.extractors.base_extractor import FeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_import_names(elf, **kwargs):
-    # see https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/scripts/readelf.py#L372
-    symbol_tables = [(idx, s) for idx, s in enumerate(elf.iter_sections()) if isinstance(s, SymbolTableSection)]
-
-    for section_index, section in symbol_tables:
-        if not isinstance(section, SymbolTableSection):
-            continue
-
-        if section["sh_entsize"] == 0:
-            logger.debug("Symbol table '%s' has a sh_entsize of zero!" % (section.name))
-            continue
-
-        logger.debug("Symbol table '%s' contains %s entries:" % (section.name, section.num_symbols()))
-
-        for nsym, symbol in enumerate(section.iter_symbols()):
-            if symbol.name and symbol.entry.st_info.type == "STT_FUNC":
-                # TODO symbol address
-                # TODO symbol version info?
-                yield Import(symbol.name), FileOffsetAddress(0x0)
-
-
-def extract_file_section_names(elf, **kwargs):
-    for section in elf.iter_sections():
-        if section.name:
-            yield Section(section.name), AbsoluteVirtualAddress(section.header.sh_addr)
-        elif section.is_null():
-            yield Section("NULL"), AbsoluteVirtualAddress(section.header.sh_addr)
-
-
-def extract_file_strings(buf, **kwargs):
-    yield from capa.features.extractors.common.extract_file_strings(buf)
-
-
-def extract_file_os(elf, buf, **kwargs):
-    # our current approach does not always get an OS value, e.g. for packed samples
-    # for file limitation purposes, we're more lax here
-    try:
-        os_tuple = next(capa.features.extractors.common.extract_os(buf))
-        yield os_tuple
-    except StopIteration:
-        yield OS("unknown"), NO_ADDRESS
-
-
-def extract_file_format(**kwargs):
-    yield Format(FORMAT_ELF), NO_ADDRESS
-
-
-def extract_file_arch(elf, **kwargs):
-    # TODO merge with capa.features.extractors.elf.detect_elf_arch()
-    arch = elf.get_machine_arch()
-    if arch == "x86":
-        yield Arch(ElfArch.I386), NO_ADDRESS
-    elif arch == "x64":
-        yield Arch(ElfArch.AMD64), NO_ADDRESS
-    else:
-        logger.warning("unsupported architecture: %s", arch)
-
-
-def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler(elf=elf, buf=buf):  # type: ignore
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    # TODO extract_file_export_names,
-    extract_file_import_names,
-    extract_file_section_names,
-    extract_file_strings,
-    # no library matching
-    extract_file_format,
-)
-
-
-def extract_global_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
-    for global_handler in GLOBAL_HANDLERS:
-        for feature, addr in global_handler(elf=elf, buf=buf):  # type: ignore
-            yield feature, addr
-
-
-GLOBAL_HANDLERS = (
-    extract_file_os,
-    extract_file_arch,
-)
-
-
-class ElfFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
-        super(ElfFeatureExtractor, self).__init__()
-        self.path = path
-        with open(self.path, "rb") as f:
-            self.elf = ELFFile(io.BytesIO(f.read()))
-
-    def get_base_address(self):
-        # virtual address of the first segment with type LOAD
-        for segment in self.elf.iter_segments():
-            if segment.header.p_type == "PT_LOAD":
-                return AbsoluteVirtualAddress(segment.header.p_vaddr)
-
-    def extract_global_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
-
-        for feature, addr in extract_global_features(self.elf, buf):
-            yield feature, addr
-
-    def extract_file_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
-
-        for feature, addr in extract_file_features(self.elf, buf):
-            yield feature, addr
-
-    def get_functions(self):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def extract_function_features(self, f):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def get_basic_blocks(self, f):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def extract_basic_block_features(self, f, bb):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def get_instructions(self, f, bb):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def extract_insn_features(self, f, bb, insn):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def is_library_function(self, va):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")
-
-    def get_function_name(self, va):
-        raise NotImplementedError("ElfFeatureExtractor can only be used to extract file features")

capa/features/extractors/helpers.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,44 +6,40 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-import struct
+import sys
 import builtins
-from typing import Tuple, Iterator
+
+from capa.features.insn import API
 
 MIN_STACKSTRING_LEN = 8
 
 
-def xor_static(data: bytes, i: int) -> bytes:
-    return bytes(c ^ i for c in data)
+def xor_static(data, i):
+    if sys.version_info >= (3, 0):
+        return bytes(c ^ i for c in data)
+    else:
+        return "".join(chr(ord(c) ^ i) for c in data)
 
 
-def is_aw_function(symbol: str) -> bool:
+def is_aw_function(function_name):
     """
     is the given function name an A/W function?
     these are variants of functions that, on Windows, accept either a narrow or wide string.
     """
-    if len(symbol) < 2:
+    if len(function_name) < 2:
         return False
 
     # last character should be 'A' or 'W'
-    if symbol[-1] not in ("A", "W"):
+    if function_name[-1] not in ("A", "W"):
         return False
 
-    return True
+    # second to last character should be lowercase letter
+    return "a" <= function_name[-2] <= "z" or "0" <= function_name[-2] <= "9"
 
 
-def is_ordinal(symbol: str) -> bool:
+def generate_api_features(apiname, va):
     """
-    is the given symbol an ordinal that is prefixed by "#"?
-    """
-    if symbol:
-        return symbol[0] == "#"
-    return False
-
-
-def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
-    """
-    for a given dll and symbol name, generate variants.
+    for a given function name and address, generate API names.
     we over-generate features to make matching easier.
     these include:
       - kernel32.CreateFileA
@@ -51,30 +47,29 @@ def generate_symbols(dll: str, symbol: str) -> Iterator[str]:
       - CreateFileA
       - CreateFile
     """
-    # normalize dll name
-    dll = dll.lower()
+    # (kernel32.CreateFileA, 0x401000)
+    yield API(apiname), va
 
-    # kernel32.CreateFileA
-    yield "%s.%s" % (dll, symbol)
+    if is_aw_function(apiname):
+        # (kernel32.CreateFile, 0x401000)
+        yield API(apiname[:-1]), va
 
-    if not is_ordinal(symbol):
-        # CreateFileA
-        yield symbol
+    if "." in apiname:
+        modname, impname = apiname.split(".")
+        # strip modname to support importname-only matching
+        # (CreateFileA, 0x401000)
+        yield API(impname), va
 
-    if is_aw_function(symbol):
-        # kernel32.CreateFile
-        yield "%s.%s" % (dll, symbol[:-1])
-
-        if not is_ordinal(symbol):
-            # CreateFile
-            yield symbol[:-1]
+        if is_aw_function(impname):
+            # (CreateFile, 0x401000)
+            yield API(impname[:-1]), va
 
 
-def all_zeros(bytez: bytes) -> bool:
+def all_zeros(bytez):
     return all(b == 0 for b in builtins.bytes(bytez))
 
 
-def twos_complement(val: int, bits: int) -> int:
+def twos_complement(val, bits):
     """
     compute the 2's complement of int value val
 
@@ -87,49 +82,3 @@ def twos_complement(val: int, bits: int) -> int:
     else:
         # return positive value as is
         return val
-
-
-def carve_pe(pbytes: bytes, offset: int = 0) -> Iterator[Tuple[int, int]]:
-    """
-    Generate (offset, key) tuples of embedded PEs
-
-    Based on the version from vivisect:
-      https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
-    And its IDA adaptation:
-      capa/features/extractors/ida/file.py
-    """
-    mz_xor = [
-        (
-            xor_static(b"MZ", key),
-            xor_static(b"PE", key),
-            key,
-        )
-        for key in range(256)
-    ]
-
-    pblen = len(pbytes)
-    todo = [(pbytes.find(mzx, offset), mzx, pex, key) for mzx, pex, key in mz_xor]
-    todo = [(off, mzx, pex, key) for (off, mzx, pex, key) in todo if off != -1]
-
-    while len(todo):
-
-        off, mzx, pex, key = todo.pop()
-
-        # The MZ header has one field we will check
-        # e_lfanew is at 0x3c
-        e_lfanew = off + 0x3C
-        if pblen < (e_lfanew + 4):
-            continue
-
-        newoff = struct.unpack("<I", xor_static(pbytes[e_lfanew : e_lfanew + 4], key))[0]
-
-        nextres = pbytes.find(mzx, off + 1)
-        if nextres != -1:
-            todo.append((nextres, mzx, pex, key))
-
-        peoff = off + newoff
-        if pblen < (peoff + 2):
-            continue
-
-        if pbytes[peoff : peoff + 2] == pex:
-            yield (off, key)

capa/features/extractors/ida/__init__.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import sys
+import types
+
+import idaapi
+
+import capa.features.extractors.ida.file
+import capa.features.extractors.ida.insn
+import capa.features.extractors.ida.function
+import capa.features.extractors.ida.basicblock
+from capa.features.extractors import FeatureExtractor
+
+
+def get_ea(self):
+    """ """
+    if isinstance(self, (idaapi.BasicBlock, idaapi.func_t)):
+        return self.start_ea
+    if isinstance(self, idaapi.insn_t):
+        return self.ea
+    raise TypeError
+
+
+def add_ea_int_cast(o):
+    """
+    dynamically add a cast-to-int (`__int__`) method to the given object
+    that returns the value of the `.ea` property.
+    this bit of skullduggery lets use cast viv-utils objects as ints.
+    the correct way of doing this is to update viv-utils (or subclass the objects here).
+    """
+    if sys.version_info[0] >= 3:
+        setattr(o, "__int__", types.MethodType(get_ea, o))
+    else:
+        setattr(o, "__int__", types.MethodType(get_ea, o, type(o)))
+    return o
+
+
+class IdaFeatureExtractor(FeatureExtractor):
+    def __init__(self):
+        super(IdaFeatureExtractor, self).__init__()
+
+    def get_base_address(self):
+        return idaapi.get_imagebase()
+
+    def extract_file_features(self):
+        for (feature, ea) in capa.features.extractors.ida.file.extract_features():
+            yield feature, ea
+
+    def get_functions(self):
+        import capa.features.extractors.ida.helpers as ida_helpers
+
+        # data structure shared across functions yielded here.
+        # useful for caching analysis relevant across a single workspace.
+        ctx = {}
+
+        # ignore library functions and thunk functions as identified by IDA
+        for f in ida_helpers.get_functions(skip_thunks=True, skip_libs=True):
+            setattr(f, "ctx", ctx)
+            yield add_ea_int_cast(f)
+
+    @staticmethod
+    def get_function(ea):
+        f = idaapi.get_func(ea)
+        setattr(f, "ctx", {})
+        return add_ea_int_cast(f)
+
+    def extract_function_features(self, f):
+        for (feature, ea) in capa.features.extractors.ida.function.extract_features(f):
+            yield feature, ea
+
+    def get_basic_blocks(self, f):
+        for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
+            yield add_ea_int_cast(bb)
+
+    def extract_basic_block_features(self, f, bb):
+        for (feature, ea) in capa.features.extractors.ida.basicblock.extract_features(f, bb):
+            yield feature, ea
+
+    def get_instructions(self, f, bb):
+        import capa.features.extractors.ida.helpers as ida_helpers
+
+        for insn in ida_helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
+            yield add_ea_int_cast(insn)
+
+    def extract_insn_features(self, f, bb, insn):
+        for (feature, ea) in capa.features.extractors.ida.insn.extract_features(f, bb, insn):
+            yield feature, ea

capa/features/extractors/ida/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,23 +6,25 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+import sys
 import string
 import struct
-from typing import Tuple, Iterator
 
 import idaapi
 
 import capa.features.extractors.ida.helpers
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address
+from capa.features import Characteristic
 from capa.features.basicblock import BasicBlock
 from capa.features.extractors.ida import helpers
 from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
 
 
-def get_printable_len(op: idaapi.op_t) -> int:
-    """Return string length if all operand bytes are ascii or utf16-le printable"""
+def get_printable_len(op):
+    """ Return string length if all operand bytes are ascii or utf16-le printable
+
+        args:
+            op (IDA op_t)
+    """
     op_val = capa.features.extractors.ida.helpers.mask_op_val(op)
 
     if op.dtype == idaapi.dt_byte:
@@ -36,12 +38,19 @@ def get_printable_len(op: idaapi.op_t) -> int:
     else:
         raise ValueError("Unhandled operand data type 0x%x." % op.dtype)
 
-    def is_printable_ascii(chars_: bytes):
-        return all(c < 127 and chr(c) in string.printable for c in chars_)
+    def is_printable_ascii(chars):
+        if sys.version_info[0] >= 3:
+            return all(c < 127 and chr(c) in string.printable for c in chars)
+        else:
+            return all(ord(c) < 127 and c in string.printable for c in chars)
 
-    def is_printable_utf16le(chars_: bytes):
-        if all(c == 0x00 for c in chars_[1::2]):
-            return is_printable_ascii(chars_[::2])
+    def is_printable_utf16le(chars):
+        if sys.version_info[0] >= 3:
+            if all(c == 0x00 for c in chars[1::2]):
+                return is_printable_ascii(chars[::2])
+        else:
+            if all(c == "\x00" for c in chars[1::2]):
+                return is_printable_ascii(chars[::2])
 
     if is_printable_ascii(chars):
         return idaapi.get_dtype_size(op.dtype)
@@ -52,8 +61,12 @@ def get_printable_len(op: idaapi.op_t) -> int:
     return 0
 
 
-def is_mov_imm_to_stack(insn: idaapi.insn_t) -> bool:
-    """verify instruction moves immediate onto stack"""
+def is_mov_imm_to_stack(insn):
+    """ verify instruction moves immediate onto stack
+
+        args:
+            insn (IDA insn_t)
+    """
     if insn.Op2.type != idaapi.o_imm:
         return False
 
@@ -66,10 +79,14 @@ def is_mov_imm_to_stack(insn: idaapi.insn_t) -> bool:
     return True
 
 
-def bb_contains_stackstring(f: idaapi.func_t, bb: idaapi.BasicBlock) -> bool:
-    """check basic block for stackstring indicators
+def bb_contains_stackstring(f, bb):
+    """ check basic block for stackstring indicators
 
-    true if basic block contains enough moves of constant bytes to the stack
+        true if basic block contains enough moves of constant bytes to the stack
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
     """
     count = 0
     for insn in capa.features.extractors.ida.helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
@@ -80,24 +97,39 @@ def bb_contains_stackstring(f: idaapi.func_t, bb: idaapi.BasicBlock) -> bool:
     return False
 
 
-def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract stackstring indicators from basic block"""
-    if bb_contains_stackstring(fh.inner, bbh.inner):
-        yield Characteristic("stack string"), bbh.address
+def extract_bb_stackstring(f, bb):
+    """ extract stackstring indicators from basic block
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+    """
+    if bb_contains_stackstring(f, bb):
+        yield Characteristic("stack string"), bb.start_ea
 
 
-def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract tight loop indicators from a basic block"""
-    if capa.features.extractors.ida.helpers.is_basic_block_tight_loop(bbh.inner):
-        yield Characteristic("tight loop"), bbh.address
+def extract_bb_tight_loop(f, bb):
+    """ extract tight loop indicators from a basic block
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+    """
+    if capa.features.extractors.ida.helpers.is_basic_block_tight_loop(bb):
+        yield Characteristic("tight loop"), bb.start_ea
 
 
-def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract basic block features"""
+def extract_features(f, bb):
+    """ extract basic block features
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+    """
     for bb_handler in BASIC_BLOCK_HANDLERS:
-        for (feature, addr) in bb_handler(fh, bbh):
-            yield feature, addr
-    yield BasicBlock(), bbh.address
+        for (feature, ea) in bb_handler(f, bb):
+            yield feature, ea
+    yield BasicBlock(), bb.start_ea
 
 
 BASIC_BLOCK_HANDLERS = (
@@ -108,10 +140,9 @@ BASIC_BLOCK_HANDLERS = (
 
 def main():
     features = []
-    for fhandle in helpers.get_functions(skip_thunks=True, skip_libs=True):
-        f: idaapi.func_t = fhandle.inner
+    for f in helpers.get_functions(skip_thunks=True, skip_libs=True):
         for bb in idaapi.FlowChart(f, flags=idaapi.FC_PREDS):
-            features.extend(list(extract_features(fhandle, bb)))
+            features.extend(list(extract_features(f, bb)))
 
     import pprint
 

capa/features/extractors/ida/extractor.py

@@ -1,70 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import List, Tuple, Iterator
-
-import idaapi
-
-import capa.ida.helpers
-import capa.features.extractors.elf
-import capa.features.extractors.ida.file
-import capa.features.extractors.ida.insn
-import capa.features.extractors.ida.global_
-import capa.features.extractors.ida.function
-import capa.features.extractors.ida.basicblock
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
-
-
-class IdaFeatureExtractor(FeatureExtractor):
-    def __init__(self):
-        super(IdaFeatureExtractor, self).__init__()
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.ida.global_.extract_os())
-        self.global_features.extend(capa.features.extractors.ida.global_.extract_arch())
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(idaapi.get_imagebase())
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.ida.file.extract_features()
-
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        import capa.features.extractors.ida.helpers as ida_helpers
-
-        # ignore library functions and thunk functions as identified by IDA
-        yield from ida_helpers.get_functions(skip_thunks=True, skip_libs=True)
-
-    @staticmethod
-    def get_function(ea: int) -> FunctionHandle:
-        f = idaapi.get_func(ea)
-        return FunctionHandle(address=AbsoluteVirtualAddress(f.start_ea), inner=f)
-
-    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.ida.function.extract_features(fh)
-
-    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
-        import capa.features.extractors.ida.helpers as ida_helpers
-
-        for bb in ida_helpers.get_function_blocks(fh.inner):
-            yield BBHandle(address=AbsoluteVirtualAddress(bb.start_ea), inner=bb)
-
-    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.ida.basicblock.extract_features(fh, bbh)
-
-    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
-        import capa.features.extractors.ida.helpers as ida_helpers
-
-        for insn in ida_helpers.get_instructions_in_range(bbh.inner.start_ea, bbh.inner.end_ea):
-            yield InsnHandle(address=AbsoluteVirtualAddress(insn.ea), inner=insn)
-
-    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
-        yield from capa.features.extractors.ida.insn.extract_features(fh, bbh, ih)

capa/features/extractors/ida/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,26 +7,26 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import struct
-from typing import Tuple, Iterator
 
 import idc
 import idaapi
 import idautils
 
-import capa.features.extractors.common
 import capa.features.extractors.helpers
 import capa.features.extractors.strings
 import capa.features.extractors.ida.helpers
-from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, Feature, Characteristic
-from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
+from capa.features import String, Characteristic
+from capa.features.file import Export, Import, Section
 
 
-def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
-    """check segment for embedded PE
+def check_segment_for_pe(seg):
+    """ check segment for embedded PE
 
-    adapted for IDA from:
-    https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
+        adapted for IDA from:
+        https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
+
+        args:
+            seg (IDA segment_t)
     """
     seg_max = seg.end_ea
     mz_xor = [
@@ -37,11 +37,11 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
         )
         for i in range(256)
     ]
-
-    todo = []
-    for (mzx, pex, i) in mz_xor:
-        for off in capa.features.extractors.ida.helpers.find_byte_sequence(seg.start_ea, seg.end_ea, mzx):
-            todo.append((off, mzx, pex, i))
+    todo = [
+        (capa.features.extractors.ida.helpers.find_byte_sequence(seg.start_ea, seg.end_ea, mzx), mzx, pex, i)
+        for mzx, pex, i in mz_xor
+    ]
+    todo = [(off, mzx, pex, i) for (off, mzx, pex, i) in todo if off != idaapi.BADADDR]
 
     while len(todo):
         off, mzx, pex, i = todo.pop()
@@ -59,122 +59,83 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
             continue
 
         if idc.get_bytes(peoff, 2) == pex:
-            yield off, i
+            yield (off, i)
 
-        for nextres in capa.features.extractors.ida.helpers.find_byte_sequence(off + 1, seg.end_ea, mzx):
+        nextres = capa.features.extractors.ida.helpers.find_byte_sequence(off + 1, seg.end_ea, mzx)
+        if nextres != -1:
             todo.append((nextres, mzx, pex, i))
 
 
-def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
-    """extract embedded PE features
+def extract_file_embedded_pe():
+    """ extract embedded PE features
 
-    IDA must load resource sections for this to be complete
-        - '-R' from console
-        - Check 'Load resource sections' when opening binary in IDA manually
+        IDA must load resource sections for this to be complete
+            - '-R' from console
+            - Check 'Load resource sections' when opening binary in IDA manually
     """
     for seg in capa.features.extractors.ida.helpers.get_segments(skip_header_segments=True):
         for (ea, _) in check_segment_for_pe(seg):
-            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
+            yield Characteristic("embedded pe"), ea
 
 
-def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract function exports"""
+def extract_file_export_names():
+    """ extract function exports """
     for (_, _, ea, name) in idautils.Entries():
-        yield Export(name), AbsoluteVirtualAddress(ea)
+        yield Export(name), ea
 
 
-def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract function imports
+def extract_file_import_names():
+    """ extract function imports
 
-    1. imports by ordinal:
-     - modulename.#ordinal
+        1. imports by ordinal:
+         - modulename.#ordinal
 
-    2. imports by name, results in two features to support importname-only
-       matching:
-     - modulename.importname
-     - importname
+        2. imports by name, results in two features to support importname-only
+           matching:
+         - modulename.importname
+         - importname
     """
     for (ea, info) in capa.features.extractors.ida.helpers.get_file_imports().items():
-        addr = AbsoluteVirtualAddress(ea)
-        if info[1] and info[2]:
-            # e.g. in mimikatz: ('cabinet', 'FCIAddFile', 11L)
-            # extract by name here and by ordinal below
-            for name in capa.features.extractors.helpers.generate_symbols(info[0], info[1]):
-                yield Import(name), addr
-            dll = info[0]
-            symbol = "#%d" % (info[2])
-        elif info[1]:
-            dll = info[0]
-            symbol = info[1]
-        elif info[2]:
-            dll = info[0]
-            symbol = "#%d" % (info[2])
-        else:
-            continue
-
-        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-            yield Import(name), addr
+        if info[1]:
+            yield Import("%s.%s" % (info[0], info[1])), ea
+            yield Import(info[1]), ea
+        if info[2]:
+            yield Import("%s.#%s" % (info[0], str(info[2]))), ea
 
 
-def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
-    """extract section names
+def extract_file_section_names():
+    """ extract section names
 
-    IDA must load resource sections for this to be complete
-        - '-R' from console
-        - Check 'Load resource sections' when opening binary in IDA manually
+        IDA must load resource sections for this to be complete
+            - '-R' from console
+            - Check 'Load resource sections' when opening binary in IDA manually
     """
     for seg in capa.features.extractors.ida.helpers.get_segments(skip_header_segments=True):
-        yield Section(idaapi.get_segm_name(seg)), AbsoluteVirtualAddress(seg.start_ea)
+        yield Section(idaapi.get_segm_name(seg)), seg.start_ea
 
 
-def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
-    """extract ASCII and UTF-16 LE strings
+def extract_file_strings():
+    """ extract ASCII and UTF-16 LE strings
 
-    IDA must load resource sections for this to be complete
-        - '-R' from console
-        - Check 'Load resource sections' when opening binary in IDA manually
+        IDA must load resource sections for this to be complete
+            - '-R' from console
+            - Check 'Load resource sections' when opening binary in IDA manually
     """
     for seg in capa.features.extractors.ida.helpers.get_segments():
         seg_buff = capa.features.extractors.ida.helpers.get_segment_buffer(seg)
-        yield from capa.features.extractors.common.extract_file_strings(seg_buff)
+
+        for s in capa.features.extractors.strings.extract_ascii_strings(seg_buff):
+            yield String(s.s), (seg.start_ea + s.offset)
+
+        for s in capa.features.extractors.strings.extract_unicode_strings(seg_buff):
+            yield String(s.s), (seg.start_ea + s.offset)
 
 
-def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract the names of statically-linked library functions.
-    """
-    for ea in idautils.Functions():
-        addr = AbsoluteVirtualAddress(ea)
-        if idaapi.get_func(ea).flags & idaapi.FUNC_LIB:
-            name = idaapi.get_name(ea)
-            yield FunctionName(name), addr
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                yield FunctionName(name[1:]), addr
-
-
-def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
-    file_info = idaapi.get_inf_structure()
-
-    if file_info.filetype == idaapi.f_PE:
-        yield Format(FORMAT_PE), NO_ADDRESS
-    elif file_info.filetype == idaapi.f_ELF:
-        yield Format(FORMAT_ELF), NO_ADDRESS
-    elif file_info.filetype == idaapi.f_BIN:
-        # no file type to return when processing a binary file, but we want to continue processing
-        return
-    else:
-        raise NotImplementedError("file format: %d" % file_info.filetype)
-
-
-def extract_features() -> Iterator[Tuple[Feature, Address]]:
-    """extract file features"""
+def extract_features():
+    """ extract file features """
     for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler():
-            yield feature, addr
+        for feature, va in file_handler():
+            yield feature, va
 
 
 FILE_HANDLERS = (
@@ -183,8 +144,6 @@ FILE_HANDLERS = (
     extract_file_strings,
     extract_file_section_names,
     extract_file_embedded_pe,
-    extract_file_function_names,
-    extract_file_format,
 )
 
 

capa/features/extractors/ida/function.py

@@ -1,31 +1,35 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Tuple, Iterator
 
 import idaapi
 import idautils
 
 import capa.features.extractors.ida.helpers
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features import Characteristic
 from capa.features.extractors import loops
-from capa.features.extractors.base_extractor import FunctionHandle
 
 
-def extract_function_calls_to(fh: FunctionHandle):
-    """extract callers to a function"""
-    for ea in idautils.CodeRefsTo(fh.inner.start_ea, True):
-        yield Characteristic("calls to"), AbsoluteVirtualAddress(ea)
+def extract_function_calls_to(f):
+    """ extract callers to a function
+
+        args:
+            f (IDA func_t)
+    """
+    for ea in idautils.CodeRefsTo(f.start_ea, True):
+        yield Characteristic("calls to"), ea
 
 
-def extract_function_loop(fh: FunctionHandle):
-    """extract loop indicators from a function"""
-    f: idaapi.func_t = fh.inner
+def extract_function_loop(f):
+    """ extract loop indicators from a function
+
+        args:
+            f (IDA func_t)
+    """
     edges = []
 
     # construct control flow graph
@@ -37,16 +41,25 @@ def extract_function_loop(fh: FunctionHandle):
         yield Characteristic("loop"), f.start_ea
 
 
-def extract_recursive_call(fh: FunctionHandle):
-    """extract recursive function call"""
-    if capa.features.extractors.ida.helpers.is_function_recursive(fh.inner):
-        yield Characteristic("recursive call"), fh.address
+def extract_recursive_call(f):
+    """ extract recursive function call
+
+        args:
+            f (IDA func_t)
+    """
+    if capa.features.extractors.ida.helpers.is_function_recursive(f):
+        yield Characteristic("recursive call"), f.start_ea
 
 
-def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_features(f):
+    """ extract function features
+
+        arg:
+            f (IDA func_t)
+    """
     for func_handler in FUNCTION_HANDLERS:
-        for (feature, addr) in func_handler(fh):
-            yield feature, addr
+        for (feature, ea) in func_handler(f):
+            yield feature, ea
 
 
 FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
@@ -55,8 +68,8 @@ FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_r
 def main():
     """ """
     features = []
-    for fhandle in capa.features.extractors.ida.helpers.get_functions(skip_thunks=True, skip_libs=True):
-        features.extend(list(extract_features(fhandle)))
+    for f in capa.features.extractors.ida.get_functions(skip_thunks=True, skip_libs=True):
+        features.extend(list(extract_features(f)))
 
     import pprint
 

capa/features/extractors/ida/global_.py

@@ -1,58 +0,0 @@
-import logging
-import contextlib
-from typing import Tuple, Iterator
-
-import idaapi
-import ida_loader
-
-import capa.ida.helpers
-import capa.features.extractors.elf
-from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
-from capa.features.address import NO_ADDRESS, Address
-
-logger = logging.getLogger(__name__)
-
-
-def extract_os() -> Iterator[Tuple[Feature, Address]]:
-    format_name: str = ida_loader.get_file_type_name()
-
-    if "PE" in format_name:
-        yield OS(OS_WINDOWS), NO_ADDRESS
-
-    elif "ELF" in format_name:
-        with contextlib.closing(capa.ida.helpers.IDAIO()) as f:
-            os = capa.features.extractors.elf.detect_elf_os(f)
-
-        yield OS(os), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling shellcode, or
-        #  2. handling a new file format (e.g. macho)
-        #
-        # for (1) we can't do much - its shellcode and all bets are off.
-        # we could maybe accept a futher CLI argument to specify the OS,
-        # but i think this would be rarely used.
-        # rules that rely on OS conditions will fail to match on shellcode.
-        #
-        # for (2), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported file format: %s, will not guess OS", format_name)
-        return
-
-
-def extract_arch() -> Iterator[Tuple[Feature, Address]]:
-    info: idaapi.idainfo = idaapi.get_inf_structure()
-    if info.procname == "metapc" and info.is_64bit():
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    elif info.procname == "metapc" and info.is_32bit():
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif info.procname == "metapc":
-        logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel")
-        return
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", info.procname)
-        return

capa/features/extractors/ida/helpers.py

@@ -1,59 +1,54 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Any, Dict, Tuple, Iterator
+
+import sys
+import string
 
 import idc
 import idaapi
 import idautils
-import ida_bytes
-
-from capa.features.address import AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FunctionHandle
 
 
-def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
-    """yield all ea of a given byte sequence
+def find_byte_sequence(start, end, seq):
+    """ find byte sequence
 
-    args:
-        start: min virtual address
-        end: max virtual address
-        seq: bytes to search e.g. b"\x01\x03"
+        args:
+            start: min virtual address
+            end: max virtual address
+            seq: bytes to search e.g. b'\x01\x03'
     """
-    seqstr = " ".join(["%02x" % b for b in seq])
-    while True:
-        # TODO find_binary: Deprecated. Please use ida_bytes.bin_search() instead.
-        ea = idaapi.find_binary(start, end, seqstr, 0, idaapi.SEARCH_DOWN)
-        if ea == idaapi.BADADDR:
-            break
-        start = ea + 1
-        yield ea
+    if sys.version_info[0] >= 3:
+        return idaapi.find_binary(start, end, " ".join(["%02x" % b for b in seq]), 0, idaapi.SEARCH_DOWN)
+    else:
+        return idaapi.find_binary(start, end, " ".join(["%02x" % ord(b) for b in seq]), 0, idaapi.SEARCH_DOWN)
 
 
-def get_functions(
-    start: int = None, end: int = None, skip_thunks: bool = False, skip_libs: bool = False
-) -> Iterator[FunctionHandle]:
-    """get functions, range optional
+def get_functions(start=None, end=None, skip_thunks=False, skip_libs=False):
+    """ get functions, range optional
 
-    args:
-        start: min virtual address
-        end: max virtual address
+        args:
+            start: min virtual address
+            end: max virtual address
+
+        ret:
+            yield func_t*
     """
     for ea in idautils.Functions(start=start, end=end):
         f = idaapi.get_func(ea)
         if not (skip_thunks and (f.flags & idaapi.FUNC_THUNK) or skip_libs and (f.flags & idaapi.FUNC_LIB)):
-            yield FunctionHandle(address=AbsoluteVirtualAddress(ea), inner=f)
+            yield f
 
 
-def get_segments(skip_header_segments=False) -> Iterator[idaapi.segment_t]:
-    """get list of segments (sections) in the binary image
+def get_segments(skip_header_segments=False):
+    """ get list of segments (sections) in the binary image
 
-    args:
-        skip_header_segments: IDA may load header segments - skip if set
+        args:
+            skip_header_segments: IDA may load header segments - skip if set
     """
     for n in range(idaapi.get_segm_qty()):
         seg = idaapi.getnseg(n)
@@ -61,10 +56,10 @@ def get_segments(skip_header_segments=False) -> Iterator[idaapi.segment_t]:
             yield seg
 
 
-def get_segment_buffer(seg: idaapi.segment_t) -> bytes:
-    """return bytes stored in a given segment
+def get_segment_buffer(seg):
+    """ return bytes stored in a given segment
 
-    decrease buffer size until IDA is able to read bytes from the segment
+        decrease buffer size until IDA is able to read bytes from the segment
     """
     buff = b""
     sz = seg.end_ea - seg.start_ea
@@ -79,8 +74,8 @@ def get_segment_buffer(seg: idaapi.segment_t) -> bytes:
     return buff if buff else b""
 
 
-def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
-    """get file imports"""
+def get_file_imports():
+    """ get file imports """
     imports = {}
 
     for idx in range(idaapi.get_import_module_qty()):
@@ -89,18 +84,10 @@ def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
         if not library:
             continue
 
-        # IDA uses section names for the library of ELF imports, like ".dynsym"
-        library = library.lstrip(".")
-
         def inspect_import(ea, function, ordinal):
             if function and function.startswith("__imp_"):
-                # handle mangled PE imports
+                # handle mangled names starting
                 function = function[len("__imp_") :]
-
-            if function and "@@" in function:
-                # handle mangled ELF imports, like "fopen@@glibc_2.2.5"
-                function, _, _ = function.partition("@@")
-
             imports[ea] = (library.lower(), function, ordinal)
             return True
 
@@ -109,12 +96,14 @@ def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
     return imports
 
 
-def get_instructions_in_range(start: int, end: int) -> Iterator[idaapi.insn_t]:
-    """yield instructions in range
+def get_instructions_in_range(start, end):
+    """ yield instructions in range
 
-    args:
-        start: virtual address (inclusive)
-        end: virtual address (exclusive)
+        args:
+            start: virtual address (inclusive)
+            end: virtual address (exclusive)
+        yield:
+            (insn_t*)
     """
     for head in idautils.Heads(start, end):
         insn = idautils.DecodeInstruction(head)
@@ -122,8 +111,8 @@ def get_instructions_in_range(start: int, end: int) -> Iterator[idaapi.insn_t]:
             yield insn
 
 
-def is_operand_equal(op1: idaapi.op_t, op2: idaapi.op_t) -> bool:
-    """compare two IDA op_t"""
+def is_operand_equal(op1, op2):
+    """ compare two IDA op_t """
     if op1.flags != op2.flags:
         return False
 
@@ -148,8 +137,8 @@ def is_operand_equal(op1: idaapi.op_t, op2: idaapi.op_t) -> bool:
     return True
 
 
-def is_basic_block_equal(bb1: idaapi.BasicBlock, bb2: idaapi.BasicBlock) -> bool:
-    """compare two IDA BasicBlock"""
+def is_basic_block_equal(bb1, bb2):
+    """ compare two IDA BasicBlock """
     if bb1.start_ea != bb2.start_ea:
         return False
 
@@ -162,17 +151,13 @@ def is_basic_block_equal(bb1: idaapi.BasicBlock, bb2: idaapi.BasicBlock) -> bool
     return True
 
 
-def basic_block_size(bb: idaapi.BasicBlock) -> int:
-    """calculate size of basic block"""
+def basic_block_size(bb):
+    """ calculate size of basic block """
     return bb.end_ea - bb.start_ea
 
 
-def read_bytes_at(ea: int, count: int) -> bytes:
+def read_bytes_at(ea, count):
     """ """
-    # check if byte has a value, see get_wide_byte doc
-    if not idc.is_loaded(ea):
-        return b""
-
     segm_end = idc.get_segm_end(ea)
     if ea + count > segm_end:
         return idc.get_bytes(ea, segm_end - ea)
@@ -180,10 +165,10 @@ def read_bytes_at(ea: int, count: int) -> bytes:
         return idc.get_bytes(ea, count)
 
 
-def find_string_at(ea: int, min_: int = 4) -> str:
-    """check if ASCII string exists at a given virtual address"""
+def find_string_at(ea, min=4):
+    """ check if ASCII string exists at a given virtual address """
     found = idaapi.get_strlit_contents(ea, -1, idaapi.STRTYPE_C)
-    if found and len(found) > min_:
+    if found and len(found) > min:
         try:
             found = found.decode("ascii")
             # hacky check for IDA bug; get_strlit_contents also reads Unicode as
@@ -197,11 +182,11 @@ def find_string_at(ea: int, min_: int = 4) -> str:
     return ""
 
 
-def get_op_phrase_info(op: idaapi.op_t) -> Dict:
-    """parse phrase features from operand
+def get_op_phrase_info(op):
+    """ parse phrase features from operand
 
-    Pretty much dup of sark's implementation:
-        https://github.com/tmr232/Sark/blob/master/sark/code/instruction.py#L28-L73
+        Pretty much dup of sark's implementation:
+            https://github.com/tmr232/Sark/blob/master/sark/code/instruction.py#L28-L73
     """
     if op.type not in (idaapi.o_phrase, idaapi.o_displ):
         return {}
@@ -234,24 +219,18 @@ def get_op_phrase_info(op: idaapi.op_t) -> Dict:
     return {"base": base, "index": index, "scale": scale, "offset": offset}
 
 
-def is_op_write(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
-    """Check if an operand is written to (destination operand)"""
+def is_op_write(insn, op):
+    """ Check if an operand is written to (destination operand) """
     return idaapi.has_cf_chg(insn.get_canon_feature(), op.n)
 
 
-def is_op_read(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
-    """Check if an operand is read from (source operand)"""
+def is_op_read(insn, op):
+    """ Check if an operand is read from (source operand) """
     return idaapi.has_cf_use(insn.get_canon_feature(), op.n)
 
 
-def is_op_offset(insn: idaapi.insn_t, op: idaapi.op_t) -> bool:
-    """Check is an operand has been marked as an offset (by auto-analysis or manually)"""
-    flags = idaapi.get_flags(insn.ea)
-    return ida_bytes.is_off(flags, op.n)
-
-
-def is_sp_modified(insn: idaapi.insn_t) -> bool:
-    """determine if instruction modifies SP, ESP, RSP"""
+def is_sp_modified(insn):
+    """ determine if instruction modifies SP, ESP, RSP """
     for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
         if op.reg == idautils.procregs.sp.reg and is_op_write(insn, op):
             # register is stack and written
@@ -259,8 +238,8 @@ def is_sp_modified(insn: idaapi.insn_t) -> bool:
     return False
 
 
-def is_bp_modified(insn: idaapi.insn_t) -> bool:
-    """check if instruction modifies BP, EBP, RBP"""
+def is_bp_modified(insn):
+    """ check if instruction modifies BP, EBP, RBP """
     for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)):
         if op.reg == idautils.procregs.bp.reg and is_op_write(insn, op):
             # register is base and written
@@ -268,13 +247,13 @@ def is_bp_modified(insn: idaapi.insn_t) -> bool:
     return False
 
 
-def is_frame_register(reg: int) -> bool:
-    """check if register is sp or bp"""
+def is_frame_register(reg):
+    """ check if register is sp or bp """
     return reg in (idautils.procregs.sp.reg, idautils.procregs.bp.reg)
 
 
-def get_insn_ops(insn: idaapi.insn_t, target_ops: Tuple[Any] = None) -> idaapi.op_t:
-    """yield op_t for instruction, filter on type if specified"""
+def get_insn_ops(insn, target_ops=()):
+    """ yield op_t for instruction, filter on type if specified """
     for op in insn.ops:
         if op.type == idaapi.o_void:
             # avoid looping all 6 ops if only subset exists
@@ -284,21 +263,21 @@ def get_insn_ops(insn: idaapi.insn_t, target_ops: Tuple[Any] = None) -> idaapi.o
         yield op
 
 
-def is_op_stack_var(ea: int, index: int) -> bool:
-    """check if operand is a stack variable"""
+def is_op_stack_var(ea, index):
+    """ check if operand is a stack variable """
     return idaapi.is_stkvar(idaapi.get_flags(ea), index)
 
 
-def mask_op_val(op: idaapi.op_t) -> int:
-    """mask value by data type
+def mask_op_val(op):
+    """ mask value by data type
 
-    necessary due to a bug in AMD64
+        necessary due to a bug in AMD64
 
-    Example:
-        .rsrc:0054C12C mov [ebp+var_4], 0FFFFFFFFh
+        Example:
+            .rsrc:0054C12C mov [ebp+var_4], 0FFFFFFFFh
 
-        insn.Op2.dtype == idaapi.dt_dword
-        insn.Op2.value == 0xffffffffffffffff
+            insn.Op2.dtype == idaapi.dt_dword
+            insn.Op2.value == 0xffffffffffffffff
     """
     masks = {
         idaapi.dt_byte: 0xFF,
@@ -309,18 +288,26 @@ def mask_op_val(op: idaapi.op_t) -> int:
     return masks.get(op.dtype, op.value) & op.value
 
 
-def is_function_recursive(f: idaapi.func_t) -> bool:
-    """check if function is recursive"""
+def is_function_recursive(f):
+    """ check if function is recursive
+
+        args:
+            f (IDA func_t)
+    """
     for ref in idautils.CodeRefsTo(f.start_ea, True):
         if f.contains(ref):
             return True
     return False
 
 
-def is_basic_block_tight_loop(bb: idaapi.BasicBlock) -> bool:
-    """check basic block loops to self
+def is_basic_block_tight_loop(bb):
+    """ check basic block loops to self
 
-    true if last instruction in basic block branches to basic block start
+        true if last instruction in basic block branches to basic block start
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
     """
     bb_end = idc.prev_head(bb.end_ea)
     if bb.start_ea < bb_end:
@@ -330,8 +317,8 @@ def is_basic_block_tight_loop(bb: idaapi.BasicBlock) -> bool:
     return False
 
 
-def find_data_reference_from_insn(insn: idaapi.insn_t, max_depth: int = 10) -> int:
-    """search for data reference from instruction, return address of instruction if no reference exists"""
+def find_data_reference_from_insn(insn, max_depth=10):
+    """ search for data reference from instruction, return address of instruction if no reference exists """
     depth = 0
     ea = insn.ea
 
@@ -346,10 +333,6 @@ def find_data_reference_from_insn(insn: idaapi.insn_t, max_depth: int = 10) -> i
             # break if circular reference
             break
 
-        if not idaapi.is_mapped(data_refs[0]):
-            # break if address is not mapped
-            break
-
         depth += 1
         if depth > max_depth:
             # break if max depth
@@ -358,20 +341,3 @@ def find_data_reference_from_insn(insn: idaapi.insn_t, max_depth: int = 10) -> i
         ea = data_refs[0]
 
     return ea
-
-
-def get_function_blocks(f: idaapi.func_t) -> Iterator[idaapi.BasicBlock]:
-    """yield basic blocks contained in specified function"""
-    # leverage idaapi.FC_NOEXT flag to ignore useless external blocks referenced by the function
-    for block in idaapi.FlowChart(f, flags=(idaapi.FC_PREDS | idaapi.FC_NOEXT)):
-        yield block
-
-
-def is_basic_block_return(bb: idaapi.BasicBlock) -> bool:
-    """check if basic block is return block"""
-    return bb.type == idaapi.fcb_ret
-
-
-def has_sib(oper: idaapi.op_t) -> bool:
-    # via: https://reverseengineering.stackexchange.com/a/14300
-    return oper.specflag1 == 1

capa/features/extractors/ida/insn.py

@@ -1,11 +1,10 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Any, Dict, Tuple, Iterator
 
 import idc
 import idaapi
@@ -13,100 +12,82 @@ import idautils
 
 import capa.features.extractors.helpers
 import capa.features.extractors.ida.helpers
-from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
-from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-# security cookie checks may perform non-zeroing XORs, these are expected within a certain
-# byte range within the first and returning basic blocks, this helps to reduce FP features
-SECURITY_COOKIE_BYTES_DELTA = 0x40
+from capa.features import ARCH_X32, ARCH_X64, MAX_BYTES_FEATURE_SIZE, Bytes, String, Characteristic
+from capa.features.insn import Number, Offset, Mnemonic
 
 
-def get_imports(ctx: Dict[str, Any]) -> Dict[str, Any]:
+def get_arch(ctx):
+    """
+    fetch the ARCH_* constant for the currently open workspace.
+
+    via Tamir Bahar/@tmr232
+    https://reverseengineering.stackexchange.com/a/11398/17194
+    """
+    if "arch" not in ctx:
+        info = idaapi.get_inf_structure()
+        if info.is_64bit():
+            ctx["arch"] = ARCH_X64
+        elif info.is_32bit():
+            ctx["arch"] = ARCH_X32
+        else:
+            raise ValueError("unexpected architecture")
+    return ctx["arch"]
+
+
+def get_imports(ctx):
     if "imports_cache" not in ctx:
         ctx["imports_cache"] = capa.features.extractors.ida.helpers.get_file_imports()
     return ctx["imports_cache"]
 
 
-def check_for_api_call(ctx: Dict[str, Any], insn: idaapi.insn_t) -> Iterator[str]:
-    """check instruction for API call"""
-    info = ()
-    ref = insn.ea
-
-    # attempt to resolve API calls by following chained thunks to a reasonable depth
-    for _ in range(THUNK_CHAIN_DEPTH_DELTA):
-        # assume only one code/data ref when resolving "call" or "jmp"
-        try:
-            ref = tuple(idautils.CodeRefsFrom(ref, False))[0]
-        except IndexError:
-            try:
-                # thunks may be marked as data refs
-                ref = tuple(idautils.DataRefsFrom(ref))[0]
-            except IndexError:
-                break
+def check_for_api_call(ctx, insn):
+    """ check instruction for API call """
+    if not idaapi.is_call_insn(insn):
+        return
 
+    for ref in idautils.CodeRefsFrom(insn.ea, False):
         info = get_imports(ctx).get(ref, ())
         if info:
-            break
-
-        f = idaapi.get_func(ref)
-        if not f or not (f.flags & idaapi.FUNC_THUNK):
-            break
-
-    if info:
-        yield "%s.%s" % (info[0], info[1])
+            yield "%s.%s" % (info[0], info[1])
+        else:
+            f = idaapi.get_func(ref)
+            # check if call to thunk
+            # TODO: first instruction might not always be the thunk
+            if f and (f.flags & idaapi.FUNC_THUNK):
+                for thunk_ref in idautils.DataRefsFrom(ref):
+                    # TODO: always data ref for thunk??
+                    info = get_imports(ctx).get(thunk_ref, ())
+                    if info:
+                        yield "%s.%s" % (info[0], info[1])
 
 
-def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_api_features(f, bb, insn):
+    """ parse instruction API features
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
+
+        example:
+            call dword [0x00473038]
     """
-    parse instruction API features
+    for api in check_for_api_call(f.ctx, insn):
+        for (feature, ea) in capa.features.extractors.helpers.generate_api_features(api, insn.ea):
+            yield feature, ea
 
-    example:
-       call dword [0x00473038]
+
+def extract_insn_number_features(f, bb, insn):
+    """ parse instruction number features
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
+
+        example:
+            push    3136B0h         ; dwControlCode
     """
-    insn: idaapi.insn_t = ih.inner
-
-    if not insn.get_canon_mnem() in ("call", "jmp"):
-        return
-
-    for api in check_for_api_call(fh.ctx, insn):
-        dll, _, symbol = api.rpartition(".")
-        for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-            yield API(name), ih.address
-
-    # extract IDA/FLIRT recognized API functions
-    targets = tuple(idautils.CodeRefsFrom(insn.ea, False))
-    if not targets:
-        return
-
-    target = targets[0]
-    target_func = idaapi.get_func(target)
-    if not target_func or target_func.start_ea != target:
-        # not a function (start)
-        return
-
-    if target_func.flags & idaapi.FUNC_LIB:
-        name = idaapi.get_name(target_func.start_ea)
-        yield API(name), ih.address
-        if name.startswith("_"):
-            # some linkers may prefix linked routines with a `_` to avoid name collisions.
-            # extract features for both the mangled and un-mangled representations.
-            # e.g. `_fwrite` -> `fwrite`
-            # see: https://stackoverflow.com/a/2628384/87207
-            yield API(name[1:]), ih.address
-
-
-def extract_insn_number_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction number features
-    example:
-        push    3136B0h         ; dwControlCode
-    """
-    insn: idaapi.insn_t = ih.inner
-
     if idaapi.is_ret_insn(insn):
         # skip things like:
         #   .text:0042250E retn 8
@@ -117,89 +98,66 @@ def extract_insn_number_features(
         #   .text:00401145 add esp, 0Ch
         return
 
-    for i, op in enumerate(insn.ops):
-        if op.type == idaapi.o_void:
-            break
-        if op.type not in (idaapi.o_imm, idaapi.o_mem):
-            continue
-        # skip things like:
-        #   .text:00401100 shr eax, offset loc_C
-        if capa.features.extractors.ida.helpers.is_op_offset(insn, op):
-            continue
-
+    for op in capa.features.extractors.ida.helpers.get_insn_ops(insn, target_ops=(idaapi.o_imm, idaapi.o_mem)):
         if op.type == idaapi.o_imm:
             const = capa.features.extractors.ida.helpers.mask_op_val(op)
         else:
             const = op.addr
-
-        yield Number(const), ih.address
-        yield OperandNumber(i, const), ih.address
-
-        if insn.itype == idaapi.NN_add and 0 < const < MAX_STRUCTURE_SIZE and op.type == idaapi.o_imm:
-            # for pattern like:
-            #
-            #     add eax, 0x10
-            #
-            # assume 0x10 is also an offset (imagine eax is a pointer).
-            yield Offset(const), ih.address
-            yield OperandOffset(i, const), ih.address
+        if not idaapi.is_mapped(const):
+            yield Number(const), insn.ea
+            yield Number(const, arch=get_arch(f.ctx)), insn.ea
 
 
-def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_bytes_features(f, bb, insn):
+    """ parse referenced byte sequences
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
+
+        example:
+            push    offset iid_004118d4_IShellLinkA ; riid
     """
-    parse referenced byte sequences
-    example:
-        push    offset iid_004118d4_IShellLinkA ; riid
-    """
-    insn: idaapi.insn_t = ih.inner
-
-    if idaapi.is_call_insn(insn):
-        return
-
     ref = capa.features.extractors.ida.helpers.find_data_reference_from_insn(insn)
     if ref != insn.ea:
         extracted_bytes = capa.features.extractors.ida.helpers.read_bytes_at(ref, MAX_BYTES_FEATURE_SIZE)
         if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
-            yield Bytes(extracted_bytes), ih.address
+            yield Bytes(extracted_bytes), insn.ea
 
 
-def extract_insn_string_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_string_features(f, bb, insn):
+    """ parse instruction string features
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
+
+        example:
+            push offset aAcr     ; "ACR  > "
     """
-    parse instruction string features
-
-    example:
-        push offset aAcr     ; "ACR  > "
-    """
-    insn: idaapi.insn_t = ih.inner
-
     ref = capa.features.extractors.ida.helpers.find_data_reference_from_insn(insn)
     if ref != insn.ea:
         found = capa.features.extractors.ida.helpers.find_string_at(ref)
         if found:
-            yield String(found), ih.address
+            yield String(found), insn.ea
 
 
-def extract_insn_offset_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_offset_features(f, bb, insn):
+    """ parse instruction structure offset features
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
+
+        example:
+            .text:0040112F cmp [esi+4], ebx
     """
-    parse instruction structure offset features
-
-    example:
-        .text:0040112F cmp [esi+4], ebx
-    """
-    insn: idaapi.insn_t = ih.inner
-
-    for i, op in enumerate(insn.ops):
-        if op.type == idaapi.o_void:
-            break
-        if op.type not in (idaapi.o_phrase, idaapi.o_displ):
-            continue
+    for op in capa.features.extractors.ida.helpers.get_insn_ops(insn, target_ops=(idaapi.o_phrase, idaapi.o_displ)):
         if capa.features.extractors.ida.helpers.is_op_stack_var(insn.ea, op.n):
             continue
-
         p_info = capa.features.extractors.ida.helpers.get_op_phrase_info(op)
         op_off = p_info.get("offset", 0)
         if idaapi.is_mapped(op_off):
@@ -212,36 +170,16 @@ def extract_insn_offset_features(
         # https://stackoverflow.com/questions/31853189/x86-64-assembly-why-displacement-not-64-bits
         op_off = capa.features.extractors.helpers.twos_complement(op_off, 32)
 
-        yield Offset(op_off), ih.address
-        yield OperandOffset(i, op_off), ih.address
-
-        if (
-            insn.itype == idaapi.NN_lea
-            and i == 1
-            # o_displ is used for both:
-            #   [eax+1]
-            #   [eax+ebx+2]
-            and op.type == idaapi.o_displ
-            # but the SIB is only present for [eax+ebx+2]
-            # which we don't want
-            and not capa.features.extractors.ida.helpers.has_sib(op)
-        ):
-            # for pattern like:
-            #
-            #     lea eax, [ebx + 1]
-            #
-            # assume 1 is also an offset (imagine ebx is a zero register).
-            yield Number(op_off), ih.address
-            yield OperandNumber(i, op_off), ih.address
+        yield Offset(op_off), insn.ea
+        yield Offset(op_off, arch=get_arch(f.ctx)), insn.ea
 
 
-def contains_stack_cookie_keywords(s: str) -> bool:
-    """
-    check if string contains stack cookie keywords
+def contains_stack_cookie_keywords(s):
+    """ check if string contains stack cookie keywords
 
-    Examples:
-        xor     ecx, ebp ; StackCookie
-        mov     eax, ___security_cookie
+        Examples:
+            xor     ecx, ebp ; StackCookie
+            mov     eax, ___security_cookie
     """
     if not s:
         return False
@@ -251,31 +189,31 @@ def contains_stack_cookie_keywords(s: str) -> bool:
     return any(keyword in s for keyword in ("stack", "security"))
 
 
-def bb_stack_cookie_registers(bb: idaapi.BasicBlock) -> Iterator[int]:
-    """scan basic block for stack cookie operations
+def bb_stack_cookie_registers(bb):
+    """ scan basic block for stack cookie operations
 
-    yield registers ids that may have been used for stack cookie operations
+        yield registers ids that may have been used for stack cookie operations
 
-    assume instruction that sets stack cookie and nzxor exist in same block
-    and stack cookie register is not modified prior to nzxor
+        assume instruction that sets stack cookie and nzxor exist in same block
+        and stack cookie register is not modified prior to nzxor
 
-    Example:
-        .text:004062DA mov     eax, ___security_cookie <-- stack cookie
-        .text:004062DF mov     ecx, eax
-        .text:004062E1 mov     ebx, [esi]
-        .text:004062E3 and     ecx, 1Fh
-        .text:004062E6 mov     edi, [esi+4]
-        .text:004062E9 xor     ebx, eax
-        .text:004062EB mov     esi, [esi+8]
-        .text:004062EE xor     edi, eax <-- ignore
-        .text:004062F0 xor     esi, eax <-- ignore
-        .text:004062F2 ror     edi, cl
-        .text:004062F4 ror     esi, cl
-        .text:004062F6 ror     ebx, cl
-        .text:004062F8 cmp     edi, esi
-        .text:004062FA jnz     loc_40639D
+        Example:
+            .text:004062DA mov     eax, ___security_cookie <-- stack cookie
+            .text:004062DF mov     ecx, eax
+            .text:004062E1 mov     ebx, [esi]
+            .text:004062E3 and     ecx, 1Fh
+            .text:004062E6 mov     edi, [esi+4]
+            .text:004062E9 xor     ebx, eax
+            .text:004062EB mov     esi, [esi+8]
+            .text:004062EE xor     edi, eax <-- ignore
+            .text:004062F0 xor     esi, eax <-- ignore
+            .text:004062F2 ror     edi, cl
+            .text:004062F4 ror     esi, cl
+            .text:004062F6 ror     ebx, cl
+            .text:004062F8 cmp     edi, esi
+            .text:004062FA jnz     loc_40639D
 
-    TODO: this is expensive, but necessary?...
+        TODO: this is expensive, but necessary?...
     """
     for insn in capa.features.extractors.ida.helpers.get_instructions_in_range(bb.start_ea, bb.end_ea):
         if contains_stack_cookie_keywords(idc.GetDisasm(insn.ea)):
@@ -285,37 +223,12 @@ def bb_stack_cookie_registers(bb: idaapi.BasicBlock) -> Iterator[int]:
                     yield op.reg
 
 
-def is_nzxor_stack_cookie_delta(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.insn_t) -> bool:
-    """check if nzxor exists within stack cookie delta"""
-    # security cookie check should use SP or BP
-    if not capa.features.extractors.ida.helpers.is_frame_register(insn.Op2.reg):
-        return False
-
-    f_bbs = tuple(capa.features.extractors.ida.helpers.get_function_blocks(f))
-
-    # expect security cookie init in first basic block within first bytes (instructions)
-    if capa.features.extractors.ida.helpers.is_basic_block_equal(bb, f_bbs[0]) and insn.ea < (
-        bb.start_ea + SECURITY_COOKIE_BYTES_DELTA
-    ):
-        return True
-
-    # ... or within last bytes (instructions) before a return
-    if capa.features.extractors.ida.helpers.is_basic_block_return(bb) and insn.ea > (
-        bb.start_ea + capa.features.extractors.ida.helpers.basic_block_size(bb) - SECURITY_COOKIE_BYTES_DELTA
-    ):
-        return True
-
-    return False
-
-
-def is_nzxor_stack_cookie(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.insn_t) -> bool:
-    """check if nzxor is related to stack cookie"""
+def is_nzxor_stack_cookie(f, bb, insn):
+    """ check if nzxor is related to stack cookie """
     if contains_stack_cookie_keywords(idaapi.get_cmt(insn.ea, False)):
         # Example:
         #   xor     ecx, ebp        ; StackCookie
         return True
-    if is_nzxor_stack_cookie_delta(f, bb, insn):
-        return True
     stack_cookie_regs = tuple(bb_stack_cookie_registers(bb))
     if any(op_reg in stack_cookie_regs for op_reg in (insn.Op1.reg, insn.Op2.reg)):
         # Example:
@@ -325,58 +238,44 @@ def is_nzxor_stack_cookie(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.
     return False
 
 
-def extract_insn_nzxor_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse instruction non-zeroing XOR instruction
-    ignore expected non-zeroing XORs, e.g. security cookies
-    """
-    insn: idaapi.insn_t = ih.inner
+def extract_insn_nzxor_characteristic_features(f, bb, insn):
+    """ parse instruction non-zeroing XOR instruction
 
-    if insn.itype not in (idaapi.NN_xor, idaapi.NN_xorpd, idaapi.NN_xorps, idaapi.NN_pxor):
+        ignore expected non-zeroing XORs, e.g. security cookies
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
+    """
+    if insn.itype != idaapi.NN_xor:
         return
     if capa.features.extractors.ida.helpers.is_operand_equal(insn.Op1, insn.Op2):
         return
-    if is_nzxor_stack_cookie(fh.inner, bbh.inner, insn):
+    if is_nzxor_stack_cookie(f, bb, insn):
         return
-    yield Characteristic("nzxor"), ih.address
+    yield Characteristic("nzxor"), insn.ea
 
 
-def extract_insn_mnemonic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction mnemonic features"""
-    yield Mnemonic(idc.print_insn_mnem(ih.inner.ea)), ih.address
+def extract_insn_mnemonic_features(f, bb, insn):
+    """ parse instruction mnemonic features
 
-
-def extract_insn_obfs_call_plus_5_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
     """
-    parse call $+5 instruction from the given instruction.
+    yield Mnemonic(insn.get_canon_mnem()), insn.ea
+
+
+def extract_insn_peb_access_characteristic_features(f, bb, insn):
+    """ parse instruction peb access
+
+        fs:[0x30] on x86, gs:[0x60] on x64
+
+        TODO:
+            IDA should be able to do this..
     """
-    insn: idaapi.insn_t = ih.inner
-
-    if not idaapi.is_call_insn(insn):
-        return
-
-    if insn.ea + 5 == idc.get_operand_value(insn.ea, 0):
-        yield Characteristic("call $+5"), ih.address
-
-
-def extract_insn_peb_access_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction peb access
-
-    fs:[0x30] on x86, gs:[0x60] on x64
-
-    TODO:
-        IDA should be able to do this..
-    """
-    insn: idaapi.insn_t = ih.inner
-
     if insn.itype not in (idaapi.NN_push, idaapi.NN_mov):
         return
 
@@ -388,19 +287,15 @@ def extract_insn_peb_access_characteristic_features(
 
     if " fs:30h" in disasm or " gs:60h" in disasm:
         # TODO: replace above with proper IDA
-        yield Characteristic("peb access"), ih.address
+        yield Characteristic("peb access"), insn.ea
 
 
-def extract_insn_segment_access_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse instruction fs or gs access
+def extract_insn_segment_access_features(f, bb, insn):
+    """ parse instruction fs or gs access
 
-    TODO:
-        IDA should be able to do this...
+        TODO:
+            IDA should be able to do this...
     """
-    insn: idaapi.insn_t = ih.inner
-
     if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
         # try to optimize for only memory references
         return
@@ -409,21 +304,23 @@ def extract_insn_segment_access_features(
 
     if " fs:" in disasm:
         # TODO: replace above with proper IDA
-        yield Characteristic("fs access"), ih.address
+        yield Characteristic("fs access"), insn.ea
 
     if " gs:" in disasm:
         # TODO: replace above with proper IDA
-        yield Characteristic("gs access"), ih.address
+        yield Characteristic("gs access"), insn.ea
 
 
-def extract_insn_cross_section_cflow(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
-    insn: idaapi.insn_t = ih.inner
+def extract_insn_cross_section_cflow(f, bb, insn):
+    """ inspect the instruction for a CALL or JMP that crosses section boundaries
 
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
+    """
     for ref in idautils.CodeRefsFrom(insn.ea, False):
-        if ref in get_imports(fh.ctx).keys():
+        if ref in get_imports(f.ctx).keys():
             # ignore API calls
             continue
         if not idaapi.getseg(ref):
@@ -431,40 +328,50 @@ def extract_insn_cross_section_cflow(
             continue
         if idaapi.getseg(ref) == idaapi.getseg(insn.ea):
             continue
-        yield Characteristic("cross section flow"), ih.address
+        yield Characteristic("cross section flow"), insn.ea
 
 
-def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract functions calls from features
+def extract_function_calls_from(f, bb, insn):
+    """ extract functions calls from features
 
-    most relevant at the function scope, however, its most efficient to extract at the instruction scope
+        most relevant at the function scope, however, its most efficient to extract at the instruction scope
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
     """
-    insn: idaapi.insn_t = ih.inner
-
     if idaapi.is_call_insn(insn):
         for ref in idautils.CodeRefsFrom(insn.ea, False):
-            yield Characteristic("calls from"), AbsoluteVirtualAddress(ref)
+            yield Characteristic("calls from"), ref
 
 
-def extract_function_indirect_call_characteristic_features(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
-    does not include calls like => call ds:dword_ABD4974
+def extract_function_indirect_call_characteristic_features(f, bb, insn):
+    """ extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
+        does not include calls like => call ds:dword_ABD4974
 
-    most relevant at the function or basic block scope;
-    however, its most efficient to extract at the instruction scope
+        most relevant at the function or basic block scope;
+        however, its most efficient to extract at the instruction scope
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
     """
-    insn: idaapi.insn_t = ih.inner
-
     if idaapi.is_call_insn(insn) and idc.get_operand_type(insn.ea, 0) in (idc.o_reg, idc.o_phrase, idc.o_displ):
-        yield Characteristic("indirect call"), ih.address
+        yield Characteristic("indirect call"), insn.ea
 
 
-def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """extract instruction features"""
+def extract_features(f, bb, insn):
+    """ extract instruction features
+
+        args:
+            f (IDA func_t)
+            bb (IDA BasicBlock)
+            insn (IDA insn_t)
+    """
     for inst_handler in INSTRUCTION_HANDLERS:
-        for (feature, ea) in inst_handler(f, bbh, insn):
+        for (feature, ea) in inst_handler(f, bb, insn):
             yield feature, ea
 
 
@@ -476,7 +383,6 @@ INSTRUCTION_HANDLERS = (
     extract_insn_offset_features,
     extract_insn_nzxor_characteristic_features,
     extract_insn_mnemonic_features,
-    extract_insn_obfs_call_plus_5_characteristic_features,
     extract_insn_peb_access_characteristic_features,
     extract_insn_cross_section_cflow,
     extract_insn_segment_access_features,

capa/features/extractors/lancelot/__init__.py

@@ -0,0 +1,92 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+
+import lancelot
+
+import capa.features.extractors
+import capa.features.extractors.lancelot.file
+import capa.features.extractors.lancelot.insn
+import capa.features.extractors.lancelot.function
+import capa.features.extractors.lancelot.basicblock
+
+__all__ = ["file", "function", "basicblock", "insn"]
+logger = logging.getLogger(__name__)
+
+
+class BB(object):
+    """extend the lancelot.BasicBlock with an __int__ method to access the address"""
+
+    def __init__(self, ws, bb):
+        super(BB, self).__init__()
+        self.ws = ws
+        self.address = bb.address
+        self.length = bb.length
+        self.predecessors = bb.predecessors
+        self.successors = bb.successors
+
+    def __int__(self):
+        return self.address
+
+    @property
+    def instructions(self):
+        va = self.address
+        while va < self.address + self.length:
+            try:
+                insn = self.ws.read_insn(va)
+            except ValueError:
+                logger.warning("failed to read instruction at 0x%x", va)
+                return
+
+            yield insn
+            va += insn.length
+
+
+class LancelotFeatureExtractor(capa.features.extractors.FeatureExtractor):
+    def __init__(self, buf):
+        super(LancelotFeatureExtractor, self).__init__()
+        self.buf = buf
+        self.ws = lancelot.from_bytes(buf)
+        self.ctx = {}
+
+    def get_base_address(self):
+        return self.ws.base_address
+
+    def extract_file_features(self):
+        for feature, va in capa.features.extractors.lancelot.file.extract_file_features(self.buf):
+            yield feature, va
+
+    def get_functions(self):
+        for va in self.ws.get_functions():
+            # this is just the address of the function
+            yield va
+
+    def extract_function_features(self, f):
+        for feature, va in capa.features.extractors.lancelot.function.extract_function_features(self.ws, f):
+            yield feature, va
+
+    def get_basic_blocks(self, f):
+        try:
+            cfg = self.ws.build_cfg(f)
+        except:
+            logger.warning("failed to build CFG for 0x%x", f)
+            return
+        else:
+            for bb in cfg.basic_blocks.values():
+                yield BB(self.ws, bb)
+
+    def extract_basic_block_features(self, f, bb):
+        for feature, va in capa.features.extractors.lancelot.basicblock.extract_basic_block_features(self.ws, bb):
+            yield feature, va
+
+    def get_instructions(self, f, bb):
+        return bb.instructions
+
+    def extract_insn_features(self, f, bb, insn):
+        for feature, va in capa.features.extractors.lancelot.insn.extract_insn_features(self, f, bb, insn):
+            yield feature, va

capa/features/extractors/lancelot/basicblock.py

@@ -0,0 +1,120 @@
+import string
+import struct
+import logging
+
+from lancelot import (
+    FLOW_VA,
+    OPERAND_SIZE,
+    OPERAND_TYPE,
+    MEMORY_OPERAND_BASE,
+    OPERAND_TYPE_MEMORY,
+    OPERAND_TYPE_IMMEDIATE,
+    IMMEDIATE_OPERAND_VALUE,
+)
+
+from capa.features import Characteristic
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+
+logger = logging.getLogger(__name__)
+
+
+def extract_bb_tight_loop(ws, bb):
+    """ check basic block for tight loop indicators """
+    if bb.address in map(lambda flow: flow[FLOW_VA], bb.successors):
+        yield Characteristic("tight loop"), bb.address
+
+
+def is_mov_imm_to_stack(insn):
+    if not insn.mnemonic.startswith("mov"):
+        return False
+
+    try:
+        dst, src = insn.operands
+    except ValueError:
+        # not two operands
+        return False
+
+    if src[OPERAND_TYPE] != OPERAND_TYPE_IMMEDIATE:
+        return False
+
+    if src[IMMEDIATE_OPERAND_VALUE] < 0:
+        return False
+
+    if dst[OPERAND_TYPE] != OPERAND_TYPE_MEMORY:
+        return False
+
+    if dst[MEMORY_OPERAND_BASE] not in ("ebp", "rbp", "esp", "rsp"):
+        return False
+
+    return True
+
+
+def is_printable_ascii(chars):
+    return all(c < 127 and chr(c) in string.printable for c in chars)
+
+
+def is_printable_utf16le(chars):
+    if all(c == b"\x00" for c in chars[1::2]):
+        return is_printable_ascii(chars[::2])
+
+
+def get_printable_len(operand):
+    """
+    Return string length if all operand bytes are ascii or utf16-le printable
+    """
+    operand_size = operand[OPERAND_SIZE]
+    if operand_size == 8:
+        chars = struct.pack("<B", operand[IMMEDIATE_OPERAND_VALUE])
+    elif operand_size == 16:
+        chars = struct.pack("<H", operand[IMMEDIATE_OPERAND_VALUE])
+    elif operand_size == 32:
+        chars = struct.pack("<I", operand[IMMEDIATE_OPERAND_VALUE])
+    elif operand_size == 64:
+        chars = struct.pack("<Q", operand[IMMEDIATE_OPERAND_VALUE])
+    else:
+        raise ValueError("unexpected operand size: " + str(operand_size))
+
+    if is_printable_ascii(chars):
+        return operand_size / 8
+    if is_printable_utf16le(chars):
+        return operand_size / 16
+
+    return 0
+
+
+def _bb_has_stackstring(ws, bb):
+    """
+    extract potential stackstring creation, using the following heuristics:
+      - basic block contains enough moves of constant bytes to the stack
+    """
+    count = 0
+    for insn in bb.instructions:
+        if is_mov_imm_to_stack(insn):
+            # add number of operand bytes
+            src = insn.operands[1]
+            count += get_printable_len(src)
+
+        if count > MIN_STACKSTRING_LEN:
+            return True
+
+    return False
+
+
+def extract_stackstring(ws, bb):
+    """ check basic block for stackstring indicators """
+    if _bb_has_stackstring(ws, bb):
+        yield Characteristic("stack string"), bb.address
+
+
+def extract_basic_block_features(ws, bb):
+    yield BasicBlock(), bb.address
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, va in bb_handler(ws, bb):
+            yield feature, va
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    extract_stackstring,
+)

capa/features/extractors/lancelot/file.py

@@ -0,0 +1,81 @@
+import pefile
+
+import capa.features.extractors.strings
+from capa.features import String, Characteristic
+from capa.features.file import Export, Import, Section
+
+
+def extract_file_embedded_pe(buf, pe):
+    buf = buf[2:]
+
+    total_offset = 2
+    while True:
+        try:
+            offset = buf.index(b"MZ")
+        except ValueError:
+            return
+        else:
+            rest = buf[offset:]
+            total_offset += offset
+
+            try:
+                _ = pefile.PE(data=rest)
+            except:
+                pass
+            else:
+                yield Characteristic("embedded pe"), total_offset
+
+            buf = rest[2:]
+            total_offset += 2
+
+
+def extract_file_export_names(buf, pe):
+    if not hasattr(pe, "DIRECTORY_ENTRY_EXPORT"):
+        return
+
+    base_address = pe.OPTIONAL_HEADER.ImageBase
+    for exp in pe.DIRECTORY_ENTRY_EXPORT.symbols:
+        yield Export(exp.name.decode("ascii")), base_address + exp.address
+
+
+def extract_file_import_names(buf, pe):
+    base_address = pe.OPTIONAL_HEADER.ImageBase
+    for entry in pe.DIRECTORY_ENTRY_IMPORT:
+        libname = entry.dll.decode("ascii").lower().partition(".")[0]
+        for imp in entry.imports:
+            if imp.ordinal:
+                yield Import("%s.#%s" % (libname, imp.ordinal)), imp.address
+            else:
+                impname = imp.name.decode("ascii")
+                yield Import("%s.%s" % (libname, impname)), imp.address
+                yield Import("%s" % (impname)), imp.address
+
+
+def extract_file_section_names(buf, pe):
+    base_address = pe.OPTIONAL_HEADER.ImageBase
+    for section in pe.sections:
+        yield Section(section.Name.partition(b"\x00")[0].decode("ascii")), base_address + section.VirtualAddress
+
+
+def extract_file_strings(buf, pe):
+    for s in capa.features.extractors.strings.extract_ascii_strings(buf):
+        yield String(s.s), s.offset
+
+    for s in capa.features.extractors.strings.extract_unicode_strings(buf):
+        yield String(s.s), s.offset
+
+
+def extract_file_features(buf):
+    pe = pefile.PE(data=buf)
+    for file_handler in FILE_HANDLERS:
+        for feature, va in file_handler(buf, pe):
+            yield feature, va
+
+
+FILE_HANDLERS = (
+    extract_file_embedded_pe,
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_section_names,
+    extract_file_strings,
+)

capa/features/extractors/lancelot/function.py

@@ -0,0 +1,64 @@
+import logging
+
+try:
+    from functools import lru_cache
+except ImportError:
+    from backports.functools_lru_cache import lru_cache
+
+from lancelot import (
+    FLOW_VA,
+    FLOW_TYPE,
+    FLOW_TYPE_CONDITIONAL_JUMP,
+    FLOW_TYPE_CONDITIONAL_MOVE,
+    FLOW_TYPE_UNCONDITIONAL_JUMP,
+)
+
+from capa.features import Characteristic
+from capa.features.extractors import loops
+
+logger = logging.getLogger(__name__)
+
+
+@lru_cache
+def get_call_graph(ws):
+    return ws.build_call_graph()
+
+
+def extract_function_calls_to(ws, f):
+    cg = get_call_graph(ws)
+
+    for caller in cg.calls_to.get(f, []):
+        yield Characteristic("calls to"), caller
+
+
+def extract_function_loop(ws, f):
+    edges = []
+    for bb in ws.build_cfg(f).basic_blocks.values():
+        for flow in bb.successors:
+            if flow[FLOW_TYPE] in (
+                FLOW_TYPE_UNCONDITIONAL_JUMP,
+                FLOW_TYPE_CONDITIONAL_JUMP,
+                FLOW_TYPE_CONDITIONAL_MOVE,
+            ):
+                edges.append((bb.address, flow[FLOW_VA]))
+                continue
+
+    if edges and loops.has_loop(edges):
+        yield Characteristic("loop"), f
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)
+
+
+_not_implemented = set([])
+
+
+def extract_function_features(ws, f):
+    for func_handler in FUNCTION_HANDLERS:
+        try:
+            for feature, va in func_handler(ws, f):
+                yield feature, va
+        except NotImplementedError:
+            if func_handler.__name__ not in _not_implemented:
+                logger.warning("not implemented: %s", func_handler.__name__)
+                _not_implemented.add(func_handler.__name__)

capa/features/extractors/lancelot/helpers.py

@@ -0,0 +1,33 @@
+from lancelot import (
+    OPERAND_TYPE,
+    MEMORY_OPERAND_BASE,
+    MEMORY_OPERAND_DISP,
+    OPERAND_TYPE_MEMORY,
+    OPERAND_TYPE_IMMEDIATE,
+    IMMEDIATE_OPERAND_VALUE,
+    IMMEDIATE_OPERAND_IS_RELATIVE,
+)
+
+
+def get_operand_target(insn, op):
+    if op[OPERAND_TYPE] == OPERAND_TYPE_MEMORY:
+        # call direct, x64
+        # rip relative
+        # kernel32-64:180001041    call    cs:__imp_RtlVirtualUnwind_0
+        if op[MEMORY_OPERAND_BASE] == "rip":
+            return op[MEMORY_OPERAND_DISP] + insn.address + insn.length
+
+        # call direct, x32
+        # mimikatz:0x403BD3  call    ds:CryptAcquireContextW
+        elif op[MEMORY_OPERAND_BASE] == None:
+            return op[MEMORY_OPERAND_DISP]
+
+    # call via thunk
+    # mimikatz:0x455A41  call    LsaQueryInformationPolicy
+    elif op[OPERAND_TYPE] == OPERAND_TYPE_IMMEDIATE and op[IMMEDIATE_OPERAND_IS_RELATIVE]:
+        return op[IMMEDIATE_OPERAND_VALUE] + insn.address + insn.length
+
+    elif op[OPERAND_TYPE] == OPERAND_TYPE_IMMEDIATE:
+        return op[IMMEDIATE_OPERAND_VALUE]
+
+    raise ValueError("memory operand has no target")

capa/features/extractors/lancelot/indirect_calls.py

@@ -0,0 +1,149 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import collections
+
+from lancelot import (
+    FLOW_VA,
+    OPERAND_TYPE,
+    PERMISSION_READ,
+    MEMORY_OPERAND_BASE,
+    MEMORY_OPERAND_DISP,
+    OPERAND_TYPE_MEMORY,
+    MEMORY_OPERAND_INDEX,
+    OPERAND_TYPE_REGISTER,
+    MEMORY_OPERAND_SEGMENT,
+    OPERAND_TYPE_IMMEDIATE,
+    IMMEDIATE_OPERAND_VALUE,
+    REGISTER_OPERAND_REGISTER,
+    IMMEDIATE_OPERAND_IS_RELATIVE,
+)
+
+from capa.features.extractors.lancelot.helpers import get_operand_target
+
+DESTRUCTIVE_MNEMONICS = ("mov", "lea", "pop", "xor")
+
+
+class NotFoundError(Exception):
+    pass
+
+
+def read_instructions(ws, bb):
+    va = bb.address
+    while va < bb.address + bb.length:
+        try:
+            insn = ws.read_insn(va)
+        except ValueError:
+            return
+
+        yield insn
+        va += insn.length
+
+
+def build_instruction_predecessors(ws, cfg):
+    preds = collections.defaultdict(set)
+
+    for bb in cfg.basic_blocks.values():
+        insns = list(read_instructions(ws, bb))
+
+        for i, insn in enumerate(insns):
+            if i == 0:
+                for pred in bb.predecessors:
+                    pred_bb = cfg.basic_blocks[pred[FLOW_VA]]
+                    preds[insn.address].add(list(read_instructions(ws, pred_bb))[-1].address)
+            else:
+                preds[insn.address].add(insns[i - 1].address)
+
+    return preds
+
+
+def find_definition(ws, f, insn):
+    """
+    scan backwards from the given address looking for assignments to the given register.
+    if a constant, return that value.
+    args:
+      ws (lancelot.PE)
+      f (int): the function start address
+      insn (lancelot.Instruction): call instruction to resolve
+    returns:
+      (va: int, value?: int|None): the address of the assignment and the value, if a constant.
+    raises:
+      NotFoundError: when the definition cannot be found.
+    """
+    assert insn.mnemonic == "call"
+    op0 = insn.operands[0]
+    assert op0[OPERAND_TYPE] == OPERAND_TYPE_REGISTER
+    reg = op0[REGISTER_OPERAND_REGISTER]
+
+    cfg = ws.build_cfg(f)
+    preds = build_instruction_predecessors(ws, cfg)
+
+    q = collections.deque()
+    seen = set([])
+    q.extend(preds[insn.address])
+    while q:
+        cur = q.popleft()
+
+        # skip if we've already processed this location
+        if cur in seen:
+            continue
+        seen.add(cur)
+
+        insn = ws.read_insn(cur)
+        operands = insn.operands
+
+        if len(operands) == 0:
+            q.extend(preds[cur])
+            continue
+
+        op0 = operands[0]
+        if not (
+            op0[OPERAND_TYPE] == OPERAND_TYPE_REGISTER
+            and op0[REGISTER_OPERAND_REGISTER] == reg
+            and insn.mnemonic in DESTRUCTIVE_MNEMONICS
+        ):
+            q.extend(preds[cur])
+            continue
+
+        # if we reach here, the instruction is destructive to our target register.
+
+        # we currently only support extracting the constant from something like: `mov $reg, IAT`
+        # so, any other pattern results in an unknown value, represented by None.
+        # this is a good place to extend in the future, if we need more robust support.
+        if insn.mnemonic != "mov":
+            return (cur, None)
+        else:
+            op1 = operands[1]
+            try:
+                target = get_operand_target(insn, op1)
+            except ValueError:
+                return (cur, None)
+            else:
+                return (cur, target)
+
+    raise NotFoundError()
+
+
+def is_indirect_call(insn):
+    return insn.mnemonic == "call" and insn.operands[0][OPERAND_TYPE] == OPERAND_TYPE_REGISTER
+
+
+def resolve_indirect_call(ws, f, insn):
+    """
+    inspect the given indirect call instruction and attempt to resolve the target address.
+    args:
+      ws (lancelot.PE): the analysis workspace
+      f (int): the address of the function to analyze
+      insn (lancelot.Instruction): the instruction at which to start analysis
+    returns:
+      (va: int, value?: int|None): the address of the assignment and the value, if a constant.
+    raises:
+      NotFoundError: when the definition cannot be found.
+    """
+    assert is_indirect_call(insn)
+    return find_definition(ws, f, insn)

capa/features/extractors/lancelot/insn.py

@@ -0,0 +1,487 @@
+import logging
+import itertools
+
+import pefile
+
+try:
+    from functools import lru_cache
+except ImportError:
+    from backports.functools_lru_cache import lru_cache
+
+from lancelot import (
+    OPERAND_TYPE,
+    PERMISSION_READ,
+    MEMORY_OPERAND_BASE,
+    MEMORY_OPERAND_DISP,
+    OPERAND_TYPE_MEMORY,
+    MEMORY_OPERAND_INDEX,
+    OPERAND_TYPE_REGISTER,
+    MEMORY_OPERAND_SEGMENT,
+    OPERAND_TYPE_IMMEDIATE,
+    IMMEDIATE_OPERAND_VALUE,
+    REGISTER_OPERAND_REGISTER,
+    IMMEDIATE_OPERAND_IS_RELATIVE,
+)
+
+import capa.features.extractors.helpers
+from capa.features import ARCH_X32, ARCH_X64, MAX_BYTES_FEATURE_SIZE, Bytes, String, Characteristic
+from capa.features.insn import Number, Offset, Mnemonic
+from capa.features.extractors.lancelot.helpers import get_operand_target
+from capa.features.extractors.lancelot.function import get_call_graph
+from capa.features.extractors.lancelot.indirect_calls import NotFoundError, resolve_indirect_call
+
+logger = logging.getLogger(__name__)
+
+
+# security cookie checks may perform non-zeroing XORs, these are expected within a certain
+# byte range within the first and returning basic blocks, this helps to reduce FP features
+SECURITY_COOKIE_BYTES_DELTA = 0x40
+
+
+def get_arch(ws):
+    if ws.arch == "x32":
+        return ARCH_X32
+    elif ws.arch == "x64":
+        return ARCH_X64
+    else:
+        raise ValueError("unexpected architecture")
+
+
+@lru_cache
+def get_pefile(xtor):
+    return pefile.PE(data=xtor.buf)
+
+
+@lru_cache
+def get_imports(xtor):
+    pe = get_pefile(xtor)
+
+    imports = {}
+    for entry in pe.DIRECTORY_ENTRY_IMPORT:
+        libname = entry.dll.decode("ascii").lower().partition(".")[0]
+        for imp in entry.imports:
+            if imp.ordinal:
+                imports[imp.address] = "%s.#%s" % (libname, imp.ordinal)
+            else:
+                impname = imp.name.decode("ascii")
+                imports[imp.address] = "%s.%s" % (libname, impname)
+    return imports
+
+
+@lru_cache
+def get_thunks(xtor):
+    thunks = {}
+    for va in xtor.ws.get_functions():
+        try:
+            insn = xtor.ws.read_insn(va)
+        except ValueError:
+            continue
+
+        if insn.mnemonic != "jmp":
+            continue
+
+        op0 = insn.operands[0]
+
+        try:
+            target = get_operand_target(insn, op0)
+        except ValueError:
+            continue
+
+        imports = get_imports(xtor)
+        if target not in imports:
+            continue
+
+        thunks[va] = imports[target]
+
+    return thunks
+
+
+def extract_insn_api_features(xtor, f, bb, insn):
+    """parse API features from the given instruction."""
+
+    if insn.mnemonic != "call":
+        return
+
+    op0 = insn.operands[0]
+
+    if op0[OPERAND_TYPE] == OPERAND_TYPE_REGISTER:
+        try:
+            (_, target) = resolve_indirect_call(xtor.ws, f, insn)
+        except NotFoundError:
+            return
+        if target is None:
+            return
+    else:
+        try:
+            target = get_operand_target(insn, op0)
+        except ValueError:
+            return
+
+    imports = get_imports(xtor)
+    if target in imports:
+        for feature, va in capa.features.extractors.helpers.generate_api_features(imports[target], insn.address):
+            yield feature, va
+        return
+
+    thunks = get_thunks(xtor)
+    if target in thunks:
+        for feature, va in capa.features.extractors.helpers.generate_api_features(thunks[target], insn.address):
+            yield feature, va
+
+
+def extract_insn_mnemonic_features(xtor, f, bb, insn):
+    """parse mnemonic features from the given instruction."""
+    yield Mnemonic(insn.mnemonic), insn.address
+
+
+def extract_insn_number_features(xtor, f, bb, insn):
+    """parse number features from the given instruction."""
+    operands = insn.operands
+
+    for operand in operands:
+        if operand[OPERAND_TYPE] != OPERAND_TYPE_IMMEDIATE:
+            continue
+
+        v = operand[IMMEDIATE_OPERAND_VALUE]
+
+        if xtor.ws.probe(v) & PERMISSION_READ:
+            # v is a valid address
+            # therefore, assume its not also a constant.
+            continue
+
+        if (
+            insn.mnemonic == "add"
+            and operands[0][OPERAND_TYPE] == OPERAND_TYPE_REGISTER
+            and operands[0][REGISTER_OPERAND_REGISTER] == "esp"
+        ):
+            # skip things like:
+            #
+            #    .text:00401140                 call    sub_407E2B
+            #    .text:00401145                 add     esp, 0Ch
+            return
+
+        yield Number(v), insn.address
+        yield Number(v, arch=get_arch(xtor.ws)), insn.address
+
+
+def extract_insn_offset_features(xtor, f, bb, insn):
+    """parse structure offset features from the given instruction."""
+    operands = insn.operands
+
+    for operand in operands:
+        if operand[OPERAND_TYPE] != OPERAND_TYPE_MEMORY:
+            continue
+
+        if operand[MEMORY_OPERAND_BASE] in ("esp", "ebp", "rbp"):
+            continue
+
+        # lancelot provides `None` when the displacement is not present.
+        v = operand[MEMORY_OPERAND_DISP] or 0
+
+        yield Offset(v), insn.address
+        yield Offset(v, arch=get_arch(xtor.ws)), insn.address
+
+
+def derefs(xtor, p):
+    """
+    recursively follow the given pointer, yielding the valid memory addresses along the way.
+    useful when you may have a pointer to string, or pointer to pointer to string, etc.
+    this is a "do what i mean" type of helper function.
+    """
+
+    depth = 0
+    while True:
+        if not xtor.ws.probe(p) & PERMISSION_READ:
+            return
+        yield p
+
+        next = xtor.ws.read_pointer(p)
+
+        # sanity: pointer points to self
+        if next == p:
+            return
+
+        # sanity: avoid chains of pointers that are unreasonably deep
+        depth += 1
+        if depth > 10:
+            return
+
+        p = next
+
+
+def read_bytes(xtor, va):
+    """
+    read up to MAX_BYTES_FEATURE_SIZE from the given address.
+
+    raises:
+      ValueError: if the given address is not valid.
+    """
+    start = va
+    end = va + MAX_BYTES_FEATURE_SIZE
+    pe = get_pefile(xtor)
+
+    for section in pe.sections:
+        section_start = pe.OPTIONAL_HEADER.ImageBase + section.VirtualAddress
+        section_end = pe.OPTIONAL_HEADER.ImageBase + section.VirtualAddress + section.Misc_VirtualSize
+
+        if section_start <= start < section_end:
+            end = min(end, section_end)
+            return xtor.ws.read_bytes(start, end - start)
+
+    raise ValueError("invalid address")
+
+
+# these are mnemonics that may flow (jump) elsewhere
+FLOW_MNEMONICS = set(
+    [
+        "call",
+        "jb",
+        "jbe",
+        "jcxz",
+        "jecxz",
+        "jknzd",
+        "jkzd",
+        "jl",
+        "jle",
+        "jmp",
+        "jnb",
+        "jnbe",
+        "jnl",
+        "jnle",
+        "jno",
+        "jnp",
+        "jns",
+        "jnz",
+        "jo",
+        "jp",
+        "jrcxz",
+        "js",
+        "jz",
+    ]
+)
+
+
+def extract_insn_bytes_features(xtor, f, bb, insn):
+    """
+    parse byte sequence features from the given instruction.
+    """
+    if insn.mnemonic in FLOW_MNEMONICS:
+        return
+
+    for operand in insn.operands:
+        try:
+            target = get_operand_target(insn, operand)
+        except ValueError:
+            continue
+
+        for ptr in derefs(xtor, target):
+            try:
+                buf = read_bytes(xtor, ptr)
+            except ValueError:
+                continue
+
+            if capa.features.extractors.helpers.all_zeros(buf):
+                continue
+
+            yield Bytes(buf), insn.address
+
+
+def first(s):
+    """enumerate the first element in the sequence"""
+    for i in s:
+        yield i
+        break
+
+
+def extract_insn_string_features(xtor, f, bb, insn):
+    """parse string features from the given instruction."""
+    for bytez, va in extract_insn_bytes_features(xtor, f, bb, insn):
+        buf = bytez.value
+
+        for s in itertools.chain(
+            first(capa.features.extractors.strings.extract_ascii_strings(buf)),
+            first(capa.features.extractors.strings.extract_unicode_strings(buf)),
+        ):
+            if s.offset == 0:
+                yield String(s.s), va
+
+
+def is_security_cookie(xtor, f, bb, insn):
+    """
+    check if an instruction is related to security cookie checks
+    """
+    op1 = insn.operands[1]
+    if op1[OPERAND_TYPE] == OPERAND_TYPE_REGISTER and op1[REGISTER_OPERAND_REGISTER] not in (
+        "esp",
+        "ebp",
+        "rbp",
+        "rsp",
+    ):
+        return False
+
+    # expect security cookie init in first basic block within first bytes (instructions)
+    if f == bb.address and insn.address < (bb.address + SECURITY_COOKIE_BYTES_DELTA):
+        return True
+
+    # ... or within last bytes (instructions) before a return
+    insns = list(xtor.get_instructions(f, bb))
+    if insns[-1].mnemonic in ("ret", "retn") and insn.address > (bb.address + bb.length - SECURITY_COOKIE_BYTES_DELTA):
+        return True
+
+    return False
+
+
+def extract_insn_nzxor_characteristic_features(xtor, f, bb, insn):
+    """
+    parse non-zeroing XOR instruction from the given instruction.
+    ignore expected non-zeroing XORs, e.g. security cookies.
+    """
+    if insn.mnemonic != "xor":
+        return
+
+    operands = insn.operands
+    if operands[0] == operands[1]:
+        return
+
+    if is_security_cookie(xtor, f, bb, insn):
+        return
+
+    yield Characteristic("nzxor"), insn.address
+
+
+def extract_insn_peb_access_characteristic_features(xtor, f, bb, insn):
+    """
+    parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
+    """
+    for operand in insn.operands:
+        if (
+            operand[OPERAND_TYPE] == OPERAND_TYPE_MEMORY
+            and operand[MEMORY_OPERAND_SEGMENT] == "gs"
+            and operand[MEMORY_OPERAND_DISP] == 0x60
+        ):
+            yield Characteristic("peb access"), insn.address
+
+        if (
+            operand[OPERAND_TYPE] == OPERAND_TYPE_MEMORY
+            and operand[MEMORY_OPERAND_SEGMENT] == "fs"
+            and operand[MEMORY_OPERAND_DISP] == 0x30
+        ):
+            yield Characteristic("peb access"), insn.address
+
+
+def extract_insn_segment_access_features(xtor, f, bb, insn):
+    """ parse the instruction for access to fs or gs """
+    for operand in insn.operands:
+        if operand[OPERAND_TYPE] == OPERAND_TYPE_MEMORY and operand[MEMORY_OPERAND_SEGMENT] == "gs":
+            yield Characteristic("gs access"), insn.address
+
+        if operand[OPERAND_TYPE] == OPERAND_TYPE_MEMORY and operand[MEMORY_OPERAND_SEGMENT] == "fs":
+            yield Characteristic("fs access"), insn.address
+
+
+def get_section(xtor, va):
+    pe = get_pefile(xtor)
+
+    for i, section in enumerate(pe.sections):
+        section_start = pe.OPTIONAL_HEADER.ImageBase + section.VirtualAddress
+        section_end = pe.OPTIONAL_HEADER.ImageBase + section.VirtualAddress + section.Misc_VirtualSize
+
+        if section_start <= va < section_end:
+            return i
+
+    raise ValueError("invalid address")
+
+
+def extract_insn_cross_section_cflow(xtor, f, bb, insn):
+    """
+    inspect the instruction for a CALL or JMP that crosses section boundaries.
+    """
+    if insn.mnemonic not in FLOW_MNEMONICS:
+        return
+
+    try:
+        target = get_operand_target(insn, insn.operands[0])
+    except ValueError:
+        return
+
+    if target in get_imports(xtor):
+        return
+
+    try:
+        if get_section(xtor, insn.address) != get_section(xtor, target):
+            yield Characteristic("cross section flow"), insn.address
+    except ValueError:
+        return
+
+
+def extract_function_calls_from(xtor, f, bb, insn):
+    cg = get_call_graph(xtor.ws)
+
+    for callee in cg.calls_from.get(insn.address, []):
+        yield Characteristic("calls from"), callee
+
+        if callee == f:
+            yield Characteristic("recursive call"), insn.address
+
+    # lancelot doesn't count API calls when constructing the call graph
+    # so we still have to scan for calls to an import
+    if insn.mnemonic != "call":
+        return
+
+    try:
+        target = get_operand_target(insn, insn.operands[0])
+    except ValueError:
+        return
+
+    imports = get_imports(xtor)
+    if target in imports:
+        yield Characteristic("calls from"), target
+
+
+# this is a feature that's most relevant at the function or basic block scope,
+# however, its most efficient to extract at the instruction scope.
+def extract_function_indirect_call_characteristic_features(xtor, f, bb, insn):
+    """
+    extract indirect function call characteristic (e.g., call eax or call dword ptr [edx+4])
+    does not include calls like => call ds:dword_ABD4974
+    """
+    if insn.mnemonic != "call":
+        return
+
+    op0 = insn.operands[0]
+    if op0[OPERAND_TYPE] == OPERAND_TYPE_REGISTER:
+        yield Characteristic("indirect call"), insn.address
+    elif op0[OPERAND_TYPE] == OPERAND_TYPE_MEMORY and op0[MEMORY_OPERAND_BASE] is not None:
+        yield Characteristic("indirect call"), insn.address
+    elif op0[OPERAND_TYPE] == OPERAND_TYPE_MEMORY and op0[MEMORY_OPERAND_INDEX] is not None:
+        yield Characteristic("indirect call"), insn.address
+
+
+_not_implemented = set([])
+
+
+def extract_insn_features(xtor, f, bb, insn):
+    for insn_handler in INSTRUCTION_HANDLERS:
+        try:
+            for feature, va in insn_handler(xtor, f, bb, insn):
+                yield feature, va
+        except NotImplementedError:
+            if insn_handler.__name__ not in _not_implemented:
+                logger.warning("not implemented: %s", insn_handler.__name__)
+                _not_implemented.add(insn_handler.__name__)
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_string_features,
+    extract_insn_bytes_features,
+    extract_insn_offset_features,
+    extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    extract_insn_peb_access_characteristic_features,
+    extract_insn_cross_section_cflow,
+    extract_insn_segment_access_features,
+    extract_function_calls_from,
+    extract_function_indirect_call_characteristic_features,
+)

capa/features/extractors/loops.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,20 +6,20 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-import networkx
+from networkx import nx
 from networkx.algorithms.components import strongly_connected_components
 
 
 def has_loop(edges, threshold=2):
-    """check if a list of edges representing a directed graph contains a loop
+    """ check if a list of edges representing a directed graph contains a loop
 
-    args:
-        edges: list of edge sets representing a directed graph i.e. [(1, 2), (2, 1)]
-        threshold: min number of nodes contained in loop
+        args:
+            edges: list of edge sets representing a directed graph i.e. [(1, 2), (2, 1)]
+            threshold: min number of nodes contained in loop
 
-    returns:
-        bool
+        returns:
+            bool
     """
-    g = networkx.DiGraph()
+    g = nx.DiGraph()
     g.add_edges_from(edges)
     return any(len(comp) >= threshold for comp in strongly_connected_components(g))

capa/features/extractors/null.py

@@ -1,77 +0,0 @@
-from typing import Dict, List, Tuple
-from dataclasses import dataclass
-
-from capa.features.common import Feature
-from capa.features.address import Address
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
-
-
-@dataclass
-class InstructionFeatures:
-    features: List[Tuple[Address, Feature]]
-
-
-@dataclass
-class BasicBlockFeatures:
-    features: List[Tuple[Address, Feature]]
-    instructions: Dict[Address, InstructionFeatures]
-
-
-@dataclass
-class FunctionFeatures:
-    features: List[Tuple[Address, Feature]]
-    basic_blocks: Dict[Address, BasicBlockFeatures]
-
-
-@dataclass
-class NullFeatureExtractor(FeatureExtractor):
-    """
-    An extractor that extracts some user-provided features.
-
-    This is useful for testing, as we can provide expected values and see if matching works.
-    """
-
-    base_address: Address
-    global_features: List[Feature]
-    file_features: List[Tuple[Address, Feature]]
-    functions: Dict[Address, FunctionFeatures]
-
-    def get_base_address(self):
-        return self.base_address
-
-    def extract_global_features(self):
-        for address, feature in self.global_features:
-            yield feature, address
-
-    def extract_file_features(self):
-        for address, feature in self.file_features:
-            yield feature, address
-
-    def get_functions(self):
-        for address in sorted(self.functions.keys()):
-            yield FunctionHandle(address, None)
-
-    def extract_function_features(self, f):
-        for address, feature in self.functions.get(f.address, {}).features:
-            yield feature, address
-
-    def get_basic_blocks(self, f):
-        for address in sorted(self.functions.get(f.address, {}).basic_blocks.keys()):
-            yield BBHandle(address, None)
-
-    def extract_basic_block_features(self, f, bb):
-        for address, feature in self.functions.get(f.address, {}).basic_blocks.get(bb.address, {}).features:
-            yield feature, address
-
-    def get_instructions(self, f, bb):
-        for address in sorted(self.functions.get(f.address, {}).basic_blocks.get(bb.address, {}).instructions.keys()):
-            yield InsnHandle(address, None)
-
-    def extract_insn_features(self, f, bb, insn):
-        for address, feature in (
-            self.functions.get(f.address, {})
-            .basic_blocks.get(bb.address, {})
-            .instructions.get(insn.address, {})
-            .features
-        ):
-            yield feature, address

capa/features/extractors/pefile.py

@@ -1,216 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import logging
-
-import pefile
-
-import capa.features.common
-import capa.features.extractors
-import capa.features.extractors.common
-import capa.features.extractors.helpers
-import capa.features.extractors.strings
-from capa.features.file import Export, Import, Section
-from capa.features.common import OS, ARCH_I386, FORMAT_PE, ARCH_AMD64, OS_WINDOWS, Arch, Format, Characteristic
-from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import FeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-def extract_file_embedded_pe(buf, **kwargs):
-    for offset, _ in capa.features.extractors.helpers.carve_pe(buf, 1):
-        yield Characteristic("embedded pe"), FileOffsetAddress(offset)
-
-
-def extract_file_export_names(pe, **kwargs):
-    base_address = pe.OPTIONAL_HEADER.ImageBase
-
-    if hasattr(pe, "DIRECTORY_ENTRY_EXPORT"):
-        for export in pe.DIRECTORY_ENTRY_EXPORT.symbols:
-            if not export.name:
-                continue
-            try:
-                name = export.name.partition(b"\x00")[0].decode("ascii")
-            except UnicodeDecodeError:
-                continue
-            va = base_address + export.address
-            yield Export(name), AbsoluteVirtualAddress(va)
-
-
-def extract_file_import_names(pe, **kwargs):
-    """
-    extract imported function names
-    1. imports by ordinal:
-     - modulename.#ordinal
-    2. imports by name, results in two features to support importname-only matching:
-     - modulename.importname
-     - importname
-    """
-    if hasattr(pe, "DIRECTORY_ENTRY_IMPORT"):
-        for dll in pe.DIRECTORY_ENTRY_IMPORT:
-            try:
-                modname = dll.dll.partition(b"\x00")[0].decode("ascii")
-            except UnicodeDecodeError:
-                continue
-
-            # strip extension
-            modname = modname.rpartition(".")[0].lower()
-
-            for imp in dll.imports:
-                if imp.import_by_ordinal:
-                    impname = "#%s" % imp.ordinal
-                else:
-                    try:
-                        impname = imp.name.partition(b"\x00")[0].decode("ascii")
-                    except UnicodeDecodeError:
-                        continue
-
-                for name in capa.features.extractors.helpers.generate_symbols(modname, impname):
-                    yield Import(name), AbsoluteVirtualAddress(imp.address)
-
-
-def extract_file_section_names(pe, **kwargs):
-    base_address = pe.OPTIONAL_HEADER.ImageBase
-
-    for section in pe.sections:
-        try:
-            name = section.Name.partition(b"\x00")[0].decode("ascii")
-        except UnicodeDecodeError:
-            continue
-
-        yield Section(name), AbsoluteVirtualAddress(base_address + section.VirtualAddress)
-
-
-def extract_file_strings(buf, **kwargs):
-    yield from capa.features.extractors.common.extract_file_strings(buf)
-
-
-def extract_file_function_names(**kwargs):
-    """
-    extract the names of statically-linked library functions.
-    """
-    if False:
-        # using a `yield` here to force this to be a generator, not function.
-        yield NotImplementedError("pefile doesn't have library matching")
-    return
-
-
-def extract_file_os(**kwargs):
-    # assuming PE -> Windows
-    # though i suppose they're also used by UEFI
-    yield OS(OS_WINDOWS), NO_ADDRESS
-
-
-def extract_file_format(**kwargs):
-    yield Format(FORMAT_PE), NO_ADDRESS
-
-
-def extract_file_arch(pe, **kwargs):
-    if pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_I386"]:
-        yield Arch(ARCH_I386), NO_ADDRESS
-    elif pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_AMD64"]:
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        logger.warning("unsupported architecture: %s", pefile.MACHINE_TYPE[pe.FILE_HEADER.Machine])
-
-
-def extract_file_features(pe, buf):
-    """
-    extract file features from given workspace
-
-    args:
-      pe (pefile.PE): the parsed PE
-      buf: the raw sample bytes
-
-    yields:
-      Tuple[Feature, VA]: a feature and its location.
-    """
-
-    for file_handler in FILE_HANDLERS:
-        for feature, va in file_handler(pe=pe, buf=buf):
-            yield feature, va
-
-
-FILE_HANDLERS = (
-    extract_file_embedded_pe,
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_section_names,
-    extract_file_strings,
-    extract_file_function_names,
-    extract_file_format,
-)
-
-
-def extract_global_features(pe, buf):
-    """
-    extract global features from given workspace
-
-    args:
-      pe (pefile.PE): the parsed PE
-      buf: the raw sample bytes
-
-    yields:
-      Tuple[Feature, VA]: a feature and its location.
-    """
-    for handler in GLOBAL_HANDLERS:
-        for feature, va in handler(pe=pe, buf=buf):
-            yield feature, va
-
-
-GLOBAL_HANDLERS = (
-    extract_file_os,
-    extract_file_arch,
-)
-
-
-class PefileFeatureExtractor(FeatureExtractor):
-    def __init__(self, path: str):
-        super(PefileFeatureExtractor, self).__init__()
-        self.path = path
-        self.pe = pefile.PE(path)
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(self.pe.OPTIONAL_HEADER.ImageBase)
-
-    def extract_global_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
-
-        yield from extract_global_features(self.pe, buf)
-
-    def extract_file_features(self):
-        with open(self.path, "rb") as f:
-            buf = f.read()
-
-        yield from extract_file_features(self.pe, buf)
-
-    def get_functions(self):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def extract_function_features(self, f):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def get_basic_blocks(self, f):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def extract_basic_block_features(self, f, bb):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def get_instructions(self, f, bb):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def extract_insn_features(self, f, bb, insn):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def is_library_function(self, va):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")
-
-    def get_function_name(self, va):
-        raise NotImplementedError("PefileFeatureExtract can only be used to extract file features")

capa/features/extractors/smda/basicblock.py

@@ -1,133 +0,0 @@
-import string
-import struct
-from typing import Tuple, Iterator
-
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address
-from capa.features.basicblock import BasicBlock
-from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
-
-
-def _bb_has_tight_loop(f, bb):
-    """
-    parse tight loops, true if last instruction in basic block branches to bb start
-    """
-    return bb.offset in f.blockrefs[bb.offset] if bb.offset in f.blockrefs else False
-
-
-def extract_bb_tight_loop(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """check basic block for tight loop indicators"""
-    if _bb_has_tight_loop(f.inner, bb.inner):
-        yield Characteristic("tight loop"), bb.address
-
-
-def _bb_has_stackstring(f, bb):
-    """
-    extract potential stackstring creation, using the following heuristics:
-      - basic block contains enough moves of constant bytes to the stack
-    """
-    count = 0
-    for instr in bb.getInstructions():
-        if is_mov_imm_to_stack(instr):
-            count += get_printable_len(instr.getDetailed())
-        if count > MIN_STACKSTRING_LEN:
-            return True
-    return False
-
-
-def get_operands(smda_ins):
-    return [o.strip() for o in smda_ins.operands.split(",")]
-
-
-def extract_stackstring(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """check basic block for stackstring indicators"""
-    if _bb_has_stackstring(f.inner, bb.inner):
-        yield Characteristic("stack string"), bb.address
-
-
-def is_mov_imm_to_stack(smda_ins):
-    """
-    Return if instruction moves immediate onto stack
-    """
-    if not smda_ins.mnemonic.startswith("mov"):
-        return False
-
-    try:
-        dst, src = get_operands(smda_ins)
-    except ValueError:
-        # not two operands
-        return False
-
-    try:
-        int(src, 16)
-    except ValueError:
-        return False
-
-    if not any(regname in dst for regname in ["ebp", "rbp", "esp", "rsp"]):
-        return False
-
-    return True
-
-
-def is_printable_ascii(chars):
-    return all(c < 127 and chr(c) in string.printable for c in chars)
-
-
-def is_printable_utf16le(chars):
-    if all(c == 0x00 for c in chars[1::2]):
-        return is_printable_ascii(chars[::2])
-
-
-def get_printable_len(instr):
-    """
-    Return string length if all operand bytes are ascii or utf16-le printable
-
-    Works on a capstone instruction
-    """
-    # should have exactly two operands for mov immediate
-    if len(instr.operands) != 2:
-        return 0
-
-    op_value = instr.operands[1].value.imm
-
-    if instr.imm_size == 1:
-        chars = struct.pack("<B", op_value & 0xFF)
-    elif instr.imm_size == 2:
-        chars = struct.pack("<H", op_value & 0xFFFF)
-    elif instr.imm_size == 4:
-        chars = struct.pack("<I", op_value & 0xFFFFFFFF)
-    elif instr.imm_size == 8:
-        chars = struct.pack("<Q", op_value & 0xFFFFFFFFFFFFFFFF)
-    else:
-        raise ValueError("Unhandled operand data type 0x%x." % instr.imm_size)
-
-    if is_printable_ascii(chars):
-        return instr.imm_size
-    if is_printable_utf16le(chars):
-        return instr.imm_size // 2
-
-    return 0
-
-
-def extract_features(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract features from the given basic block.
-
-    args:
-      f: the function from which to extract features
-      bb: the basic block to process.
-
-    yields:
-      Tuple[Feature, Address]: the features and their location found in this basic block.
-    """
-    yield BasicBlock(), bb.address
-    for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, addr in bb_handler(f, bb):
-            yield feature, addr
-
-
-BASIC_BLOCK_HANDLERS = (
-    extract_bb_tight_loop,
-    extract_stackstring,
-)

capa/features/extractors/smda/extractor.py

@@ -1,57 +0,0 @@
-from typing import List, Tuple
-
-from smda.common.SmdaReport import SmdaReport
-
-import capa.features.extractors.common
-import capa.features.extractors.smda.file
-import capa.features.extractors.smda.insn
-import capa.features.extractors.smda.global_
-import capa.features.extractors.smda.function
-import capa.features.extractors.smda.basicblock
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
-
-
-class SmdaFeatureExtractor(FeatureExtractor):
-    def __init__(self, smda_report: SmdaReport, path):
-        super(SmdaFeatureExtractor, self).__init__()
-        self.smda_report = smda_report
-        self.path = path
-        with open(self.path, "rb") as f:
-            self.buf = f.read()
-
-        # pre-compute these because we'll yield them at *every* scope.
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf))
-        self.global_features.extend(capa.features.extractors.smda.global_.extract_arch(self.smda_report))
-
-    def get_base_address(self):
-        return AbsoluteVirtualAddress(self.smda_report.base_addr)
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.smda.file.extract_features(self.smda_report, self.buf)
-
-    def get_functions(self):
-        for function in self.smda_report.getFunctions():
-            yield FunctionHandle(address=AbsoluteVirtualAddress(function.offset), inner=function)
-
-    def extract_function_features(self, fh):
-        yield from capa.features.extractors.smda.function.extract_features(fh)
-
-    def get_basic_blocks(self, fh):
-        for bb in fh.inner.getBlocks():
-            yield BBHandle(address=AbsoluteVirtualAddress(bb.offset), inner=bb)
-
-    def extract_basic_block_features(self, fh, bbh):
-        yield from capa.features.extractors.smda.basicblock.extract_features(fh, bbh)
-
-    def get_instructions(self, fh, bbh):
-        for smda_ins in bbh.inner.getInstructions():
-            yield InsnHandle(address=AbsoluteVirtualAddress(smda_ins.offset), inner=smda_ins)
-
-    def extract_insn_features(self, fh, bbh, ih):
-        yield from capa.features.extractors.smda.insn.extract_features(fh, bbh, ih)

capa/features/extractors/smda/file.py

@@ -1,103 +0,0 @@
-# if we have SMDA we definitely have lief
-import lief
-
-import capa.features.extractors.common
-import capa.features.extractors.helpers
-import capa.features.extractors.strings
-from capa.features.file import Export, Import, Section
-from capa.features.common import String, Characteristic
-from capa.features.address import FileOffsetAddress, AbsoluteVirtualAddress
-
-
-def extract_file_embedded_pe(buf, **kwargs):
-    for offset, _ in capa.features.extractors.helpers.carve_pe(buf, 1):
-        yield Characteristic("embedded pe"), FileOffsetAddress(offset)
-
-
-def extract_file_export_names(buf, **kwargs):
-    lief_binary = lief.parse(buf)
-
-    if lief_binary is not None:
-        for function in lief_binary.exported_functions:
-            yield Export(function.name), AbsoluteVirtualAddress(function.address)
-
-
-def extract_file_import_names(smda_report, buf):
-    # extract import table info via LIEF
-    lief_binary = lief.parse(buf)
-    if not isinstance(lief_binary, lief.PE.Binary):
-        return
-    for imported_library in lief_binary.imports:
-        library_name = imported_library.name.lower()
-        library_name = library_name[:-4] if library_name.endswith(".dll") else library_name
-        for func in imported_library.entries:
-            va = func.iat_address + smda_report.base_addr
-            if func.name:
-                for name in capa.features.extractors.helpers.generate_symbols(library_name, func.name):
-                    yield Import(name), AbsoluteVirtualAddress(va)
-            elif func.is_ordinal:
-                for name in capa.features.extractors.helpers.generate_symbols(library_name, "#%s" % func.ordinal):
-                    yield Import(name), AbsoluteVirtualAddress(va)
-
-
-def extract_file_section_names(buf, **kwargs):
-    lief_binary = lief.parse(buf)
-    if not isinstance(lief_binary, lief.PE.Binary):
-        return
-    if lief_binary and lief_binary.sections:
-        base_address = lief_binary.optional_header.imagebase
-        for section in lief_binary.sections:
-            yield Section(section.name), AbsoluteVirtualAddress(base_address + section.virtual_address)
-
-
-def extract_file_strings(buf, **kwargs):
-    """
-    extract ASCII and UTF-16 LE strings from file
-    """
-    for s in capa.features.extractors.strings.extract_ascii_strings(buf):
-        yield String(s.s), FileOffsetAddress(s.offset)
-
-    for s in capa.features.extractors.strings.extract_unicode_strings(buf):
-        yield String(s.s), FileOffsetAddress(s.offset)
-
-
-def extract_file_function_names(smda_report, **kwargs):
-    """
-    extract the names of statically-linked library functions.
-    """
-    if False:
-        # using a `yield` here to force this to be a generator, not function.
-        yield NotImplementedError("SMDA doesn't have library matching")
-    return
-
-
-def extract_file_format(buf, **kwargs):
-    yield from capa.features.extractors.common.extract_format(buf)
-
-
-def extract_features(smda_report, buf):
-    """
-    extract file features from given workspace
-
-    args:
-      smda_report (smda.common.SmdaReport): a SmdaReport
-      buf: the raw bytes of the sample
-
-    yields:
-      Tuple[Feature, VA]: a feature and its location.
-    """
-
-    for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler(smda_report=smda_report, buf=buf):
-            yield feature, addr
-
-
-FILE_HANDLERS = (
-    extract_file_embedded_pe,
-    extract_file_export_names,
-    extract_file_import_names,
-    extract_file_section_names,
-    extract_file_strings,
-    extract_file_function_names,
-    extract_file_format,
-)

capa/features/extractors/smda/function.py

@@ -1,42 +0,0 @@
-from typing import Tuple, Iterator
-
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors import loops
-from capa.features.extractors.base_extractor import FunctionHandle
-
-
-def extract_function_calls_to(f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    for inref in f.inner.inrefs:
-        yield Characteristic("calls to"), AbsoluteVirtualAddress(inref)
-
-
-def extract_function_loop(f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse if a function has a loop
-    """
-    edges = []
-    for bb_from, bb_tos in f.inner.blockrefs.items():
-        for bb_to in bb_tos:
-            edges.append((bb_from, bb_to))
-
-    if edges and loops.has_loop(edges):
-        yield Characteristic("loop"), f.address
-
-
-def extract_features(f: FunctionHandle):
-    """
-    extract features from the given function.
-
-    args:
-      f: the function from which to extract features
-
-    yields:
-      Tuple[Feature, Address]: the features and their location found in this function.
-    """
-    for func_handler in FUNCTION_HANDLERS:
-        for feature, addr in func_handler(f):
-            yield feature, addr
-
-
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)

capa/features/extractors/smda/global_.py

@@ -1,21 +0,0 @@
-import logging
-
-from capa.features.common import ARCH_I386, ARCH_AMD64, Arch
-from capa.features.address import NO_ADDRESS
-
-logger = logging.getLogger(__name__)
-
-
-def extract_arch(smda_report):
-    if smda_report.architecture == "intel":
-        if smda_report.bitness == 32:
-            yield Arch(ARCH_I386), NO_ADDRESS
-        elif smda_report.bitness == 64:
-            yield Arch(ARCH_AMD64), NO_ADDRESS
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", smda_report.architecture)
-        return

capa/features/extractors/smda/insn.py

@@ -1,455 +0,0 @@
-import re
-import string
-import struct
-from typing import Tuple, Iterator
-
-import smda
-
-import capa.features.extractors.helpers
-from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
-from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
-
-# security cookie checks may perform non-zeroing XORs, these are expected within a certain
-# byte range within the first and returning basic blocks, this helps to reduce FP features
-SECURITY_COOKIE_BYTES_DELTA = 0x40
-PATTERN_HEXNUM = re.compile(r"[+\-] (?P<num>0x[a-fA-F0-9]+)")
-PATTERN_SINGLENUM = re.compile(r"[+\-] (?P<num>[0-9])")
-
-
-def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse API features from the given instruction."""
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    if ih.address in f.apirefs:
-        api_entry = f.apirefs[ih.address]
-        # reformat
-        dll_name, api_name = api_entry.split("!")
-        dll_name = dll_name.split(".")[0]
-        dll_name = dll_name.lower()
-        for name in capa.features.extractors.helpers.generate_symbols(dll_name, api_name):
-            yield API(name), ih.address
-    elif ih.address in f.outrefs:
-        current_function = f
-        current_instruction = insn
-        for index in range(THUNK_CHAIN_DEPTH_DELTA):
-            if current_function and len(current_function.outrefs[current_instruction.offset]) == 1:
-                target = current_function.outrefs[current_instruction.offset][0]
-                referenced_function = current_function.smda_report.getFunction(target)
-                if referenced_function:
-                    # TODO SMDA: implement this function for both jmp and call, checking if function has 1 instruction which refs an API
-                    if referenced_function.isApiThunk():
-                        api_entry = (
-                            referenced_function.apirefs[target] if target in referenced_function.apirefs else None
-                        )
-                        if api_entry:
-                            # reformat
-                            dll_name, api_name = api_entry.split("!")
-                            dll_name = dll_name.split(".")[0]
-                            dll_name = dll_name.lower()
-                            for name in capa.features.extractors.helpers.generate_symbols(dll_name, api_name):
-                                yield API(name), ih.address
-                    elif referenced_function.num_instructions == 1 and referenced_function.num_outrefs == 1:
-                        current_function = referenced_function
-                        current_instruction = [i for i in referenced_function.getInstructions()][0]
-                else:
-                    return
-
-
-def extract_insn_number_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse number features from the given instruction."""
-    # example:
-    #
-    #     push    3136B0h         ; dwControlCode
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    if insn.mnemonic == "add" and operands[0] in ["esp", "rsp"]:
-        # skip things like:
-        #
-        #    .text:00401140                 call    sub_407E2B
-        #    .text:00401145                 add     esp, 0Ch
-        return
-    for i, operand in enumerate(operands):
-        try:
-            # The result of bitwise operations is calculated as though carried out
-            # in two’s complement with an infinite number of sign bits
-            value = int(operand, 16) & ((1 << f.smda_report.bitness) - 1)
-        except ValueError:
-            continue
-        else:
-            yield Number(value), ih.address
-            yield OperandNumber(i, value), ih.address
-
-            if insn.mnemonic == "add" and 0 < value < MAX_STRUCTURE_SIZE:
-                # for pattern like:
-                #
-                #     add eax, 0x10
-                #
-                # assume 0x10 is also an offset (imagine eax is a pointer).
-                yield Offset(value), ih.address
-                yield OperandOffset(i, value), ih.address
-
-
-def read_bytes(smda_report, va, num_bytes=None):
-    """
-    read up to MAX_BYTES_FEATURE_SIZE from the given address.
-    """
-
-    rva = va - smda_report.base_addr
-    if smda_report.buffer is None:
-        raise ValueError("buffer is empty")
-    buffer_end = len(smda_report.buffer)
-    max_bytes = num_bytes if num_bytes is not None else MAX_BYTES_FEATURE_SIZE
-    if rva + max_bytes > buffer_end:
-        return smda_report.buffer[rva:]
-    else:
-        return smda_report.buffer[rva : rva + max_bytes]
-
-
-def derefs(smda_report, p):
-    """
-    recursively follow the given pointer, yielding the valid memory addresses along the way.
-    useful when you may have a pointer to string, or pointer to pointer to string, etc.
-
-    this is a "do what i mean" type of helper function.
-
-    based on the implementation in viv/insn.py
-    """
-    depth = 0
-    while True:
-        if not smda_report.isAddrWithinMemoryImage(p):
-            return
-        yield p
-
-        bytes_ = read_bytes(smda_report, p, num_bytes=4)
-        val = struct.unpack("I", bytes_)[0]
-
-        # sanity: pointer points to self
-        if val == p:
-            return
-
-        # sanity: avoid chains of pointers that are unreasonably deep
-        depth += 1
-        if depth > 10:
-            return
-
-        p = val
-
-
-def extract_insn_bytes_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse byte sequence features from the given instruction.
-    example:
-        #     push    offset iid_004118d4_IShellLinkA ; riid
-    """
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    for data_ref in insn.getDataRefs():
-        for v in derefs(f.smda_report, data_ref):
-            bytes_read = read_bytes(f.smda_report, v)
-            if bytes_read is None:
-                continue
-            if capa.features.extractors.helpers.all_zeros(bytes_read):
-                continue
-
-            yield Bytes(bytes_read), ih.address
-
-
-def detect_ascii_len(smda_report, offset):
-    if smda_report.buffer is None:
-        return 0
-    ascii_len = 0
-    rva = offset - smda_report.base_addr
-    char = smda_report.buffer[rva]
-    while char < 127 and chr(char) in string.printable:
-        ascii_len += 1
-        rva += 1
-        char = smda_report.buffer[rva]
-    if char == 0:
-        return ascii_len
-    return 0
-
-
-def detect_unicode_len(smda_report, offset):
-    if smda_report.buffer is None:
-        return 0
-    unicode_len = 0
-    rva = offset - smda_report.base_addr
-    char = smda_report.buffer[rva]
-    second_char = smda_report.buffer[rva + 1]
-    while char < 127 and chr(char) in string.printable and second_char == 0:
-        unicode_len += 2
-        rva += 2
-        char = smda_report.buffer[rva]
-        second_char = smda_report.buffer[rva + 1]
-    if char == 0 and second_char == 0:
-        return unicode_len
-    return 0
-
-
-def read_string(smda_report, offset):
-    alen = detect_ascii_len(smda_report, offset)
-    if alen > 1:
-        return read_bytes(smda_report, offset, alen).decode("utf-8")
-    ulen = detect_unicode_len(smda_report, offset)
-    if ulen > 2:
-        return read_bytes(smda_report, offset, ulen).decode("utf-16")
-
-
-def extract_insn_string_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse string features from the given instruction."""
-    # example:
-    #
-    #     push    offset aAcr     ; "ACR  > "
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    for data_ref in insn.getDataRefs():
-        for v in derefs(f.smda_report, data_ref):
-            string_read = read_string(f.smda_report, v)
-            if string_read:
-                yield String(string_read.rstrip("\x00")), ih.address
-
-
-def extract_insn_offset_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse structure offset features from the given instruction."""
-    # examples:
-    #
-    #     mov eax, [esi + 4]
-    #     mov eax, [esi + ecx + 16384]
-    insn: smda.Insn = ih.inner
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    for i, operand in enumerate(operands):
-        if "esp" in operand or "ebp" in operand or "rbp" in operand:
-            continue
-
-        number = 0
-        number_hex = re.search(PATTERN_HEXNUM, operand)
-        number_int = re.search(PATTERN_SINGLENUM, operand)
-        if number_hex:
-            number = int(number_hex.group("num"), 16)
-            number = -1 * number if number_hex.group().startswith("-") else number
-        elif number_int:
-            number = int(number_int.group("num"))
-            number = -1 * number if number_int.group().startswith("-") else number
-
-        if "ptr" not in operand:
-            if (
-                insn.mnemonic == "lea"
-                and i == 1
-                and (operand.count("+") + operand.count("-")) == 1
-                and operand.count("*") == 0
-            ):
-                # for pattern like:
-                #
-                #     lea eax, [ebx + 1]
-                #
-                # assume 1 is also an offset (imagine ebx is a zero register).
-                yield Number(number), ih.address
-                yield OperandNumber(i, number), ih.address
-
-            continue
-
-        yield Offset(number), ih.address
-        yield OperandOffset(i, number), ih.address
-
-
-def is_security_cookie(f, bb, insn):
-    """
-    check if an instruction is related to security cookie checks
-    """
-    # security cookie check should use SP or BP
-    operands = [o.strip() for o in insn.operands.split(",")]
-    if operands[1] not in ["esp", "ebp", "rsp", "rbp"]:
-        return False
-    for index, block in enumerate(f.getBlocks()):
-        # expect security cookie init in first basic block within first bytes (instructions)
-        block_instructions = [i for i in block.getInstructions()]
-        if index == 0 and insn.offset < (block_instructions[0].offset + SECURITY_COOKIE_BYTES_DELTA):
-            return True
-        # ... or within last bytes (instructions) before a return
-        if block_instructions[-1].mnemonic.startswith("ret") and insn.offset > (
-            block_instructions[-1].offset - SECURITY_COOKIE_BYTES_DELTA
-        ):
-            return True
-    return False
-
-
-def extract_insn_nzxor_characteristic_features(
-    fh: FunctionHandle, bh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse non-zeroing XOR instruction from the given instruction.
-    ignore expected non-zeroing XORs, e.g. security cookies.
-    """
-    f: smda.Function = fh.inner
-    bb: smda.BasicBlock = bh.inner
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic not in ("xor", "xorpd", "xorps", "pxor"):
-        return
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    if operands[0] == operands[1]:
-        return
-
-    if is_security_cookie(f, bb, insn):
-        return
-
-    yield Characteristic("nzxor"), ih.address
-
-
-def extract_insn_mnemonic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse mnemonic features from the given instruction."""
-    yield Mnemonic(ih.inner.mnemonic), ih.address
-
-
-def extract_insn_obfs_call_plus_5_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse call $+5 instruction from the given instruction.
-    """
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic != "call":
-        return
-
-    if not insn.operands.startswith("0x"):
-        return
-
-    if int(insn.operands, 16) == insn.offset + 5:
-        yield Characteristic("call $+5"), ih.address
-
-
-def extract_insn_peb_access_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
-    """
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic not in ["push", "mov"]:
-        return
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    for operand in operands:
-        if "fs:" in operand and "0x30" in operand:
-            yield Characteristic("peb access"), ih.address
-        elif "gs:" in operand and "0x60" in operand:
-            yield Characteristic("peb access"), ih.address
-
-
-def extract_insn_segment_access_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse the instruction for access to fs or gs"""
-    insn: smda.Insn = ih.inner
-
-    operands = [o.strip() for o in insn.operands.split(",")]
-    for operand in operands:
-        if "fs:" in operand:
-            yield Characteristic("fs access"), ih.address
-        elif "gs:" in operand:
-            yield Characteristic("gs access"), ih.address
-
-
-def extract_insn_cross_section_cflow(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    inspect the instruction for a CALL or JMP that crosses section boundaries.
-    """
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic in ["call", "jmp"]:
-        if ih.address in f.apirefs:
-            return
-
-        smda_report = insn.smda_function.smda_report
-        if ih.address in f.outrefs:
-            for target in f.outrefs[ih.address]:
-                if smda_report.getSection(ih.address) != smda_report.getSection(target):
-                    yield Characteristic("cross section flow"), ih.address
-        elif insn.operands.startswith("0x"):
-            target = int(insn.operands, 16)
-            if smda_report.getSection(ih.address) != smda_report.getSection(target):
-                yield Characteristic("cross section flow"), ih.address
-
-
-# this is a feature that's most relevant at the function scope,
-# however, its most efficient to extract at the instruction scope.
-def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    f: smda.Function = fh.inner
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic != "call":
-        return
-
-    if ih.address in f.outrefs:
-        for outref in f.outrefs[ih.address]:
-            yield Characteristic("calls from"), AbsoluteVirtualAddress(outref)
-
-            if outref == f.offset:
-                # if we found a jump target and it's the function address
-                # mark as recursive
-                yield Characteristic("recursive call"), AbsoluteVirtualAddress(outref)
-    if ih.address in f.apirefs:
-        yield Characteristic("calls from"), ih.address
-
-
-# this is a feature that's most relevant at the function or basic block scope,
-# however, its most efficient to extract at the instruction scope.
-def extract_function_indirect_call_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    extract indirect function call characteristic (e.g., call eax or call dword ptr [edx+4])
-    does not include calls like => call ds:dword_ABD4974
-    """
-    insn: smda.Insn = ih.inner
-
-    if insn.mnemonic != "call":
-        return
-    if insn.operands.startswith("0x"):
-        return False
-    if "qword ptr" in insn.operands and "rip" in insn.operands:
-        return False
-    if insn.operands.startswith("dword ptr [0x"):
-        return False
-    # call edx
-    # call dword ptr [eax+50h]
-    # call qword ptr [rsp+78h]
-    yield Characteristic("indirect call"), ih.address
-
-
-def extract_features(f, bb, insn):
-    """
-    extract features from the given insn.
-
-    args:
-      f: the function to process.
-      bb: the basic block to process.
-      insn: the instruction to process.
-
-    yields:
-      Tuple[Feature, Address]: the features and their location found in this insn.
-    """
-    for insn_handler in INSTRUCTION_HANDLERS:
-        for feature, addr in insn_handler(f, bb, insn):
-            yield feature, addr
-
-
-INSTRUCTION_HANDLERS = (
-    extract_insn_api_features,
-    extract_insn_number_features,
-    extract_insn_string_features,
-    extract_insn_bytes_features,
-    extract_insn_offset_features,
-    extract_insn_nzxor_characteristic_features,
-    extract_insn_mnemonic_features,
-    extract_insn_obfs_call_plus_5_characteristic_features,
-    extract_insn_peb_access_characteristic_features,
-    extract_insn_cross_section_cflow,
-    extract_insn_segment_access_features,
-    extract_function_calls_from,
-    extract_function_indirect_call_characteristic_features,
-)

capa/features/extractors/strings.py

@@ -1,6 +1,6 @@
-# strings code from FLOSS, https://github.com/mandiant/flare-floss
+# strings code from FLOSS, https://github.com/fireeye/flare-floss
 #
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt

capa/features/extractors/viv/__init__.py

@@ -0,0 +1,81 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import types
+
+import viv_utils
+
+import capa.features.extractors
+import capa.features.extractors.viv.file
+import capa.features.extractors.viv.insn
+import capa.features.extractors.viv.function
+import capa.features.extractors.viv.basicblock
+from capa.features.extractors import FeatureExtractor
+
+__all__ = ["file", "function", "basicblock", "insn"]
+
+
+def get_va(self):
+    try:
+        # vivisect type
+        return self.va
+    except AttributeError:
+        pass
+
+    raise TypeError()
+
+
+def add_va_int_cast(o):
+    """
+    dynamically add a cast-to-int (`__int__`) method to the given object
+    that returns the value of the `.va` property.
+
+    this bit of skullduggery lets use cast viv-utils objects as ints.
+    the correct way of doing this is to update viv-utils (or subclass the objects here).
+    """
+    setattr(o, "__int__", types.MethodType(get_va, o, type(o)))
+    return o
+
+
+class VivisectFeatureExtractor(FeatureExtractor):
+    def __init__(self, vw, path):
+        super(VivisectFeatureExtractor, self).__init__()
+        self.vw = vw
+        self.path = path
+
+    def get_base_address(self):
+        # assume there is only one file loaded into the vw
+        return list(self.vw.filemeta.values())[0]["imagebase"]
+
+    def extract_file_features(self):
+        for feature, va in capa.features.extractors.viv.file.extract_features(self.vw, self.path):
+            yield feature, va
+
+    def get_functions(self):
+        for va in sorted(self.vw.getFunctions()):
+            yield add_va_int_cast(viv_utils.Function(self.vw, va))
+
+    def extract_function_features(self, f):
+        for feature, va in capa.features.extractors.viv.function.extract_features(f):
+            yield feature, va
+
+    def get_basic_blocks(self, f):
+        for bb in f.basic_blocks:
+            yield add_va_int_cast(bb)
+
+    def extract_basic_block_features(self, f, bb):
+        for feature, va in capa.features.extractors.viv.basicblock.extract_features(f, bb):
+            yield feature, va
+
+    def get_instructions(self, f, bb):
+        for insn in bb.instructions:
+            yield add_va_int_cast(insn)
+
+    def extract_insn_features(self, f, bb, insn):
+        for feature, va in capa.features.extractors.viv.insn.extract_features(f, bb, insn):
+            yield feature, va

capa/features/extractors/viv/basicblock.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -8,30 +8,27 @@
 
 import string
 import struct
-from typing import Tuple, Iterator
 
 import envi
-import envi.archs.i386.disasm
+import vivisect.const
 
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features import Characteristic
 from capa.features.basicblock import BasicBlock
 from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
-from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
 
 
-def interface_extract_basic_block_XXX(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+def interface_extract_basic_block_XXX(f, bb):
     """
     parse features from the given basic block.
 
     args:
-      f: the function to process.
-      bb: the basic block to process.
+      f (viv_utils.Function): the function to process.
+      bb (viv_utils.BasicBlock): the basic block to process.
 
     yields:
-      (Feature, Address): the feature and the address at which its found.
+      (Feature, int): the feature and the address at which its found.
     """
-    ...
+    yield NotImplementedError("feature"), NotImplementedError("virtual address")
 
 
 def _bb_has_tight_loop(f, bb):
@@ -40,17 +37,17 @@ def _bb_has_tight_loop(f, bb):
     """
     if len(bb.instructions) > 0:
         for bva, bflags in bb.instructions[-1].getBranches():
-            if bflags & envi.BR_COND:
+            if bflags & vivisect.envi.BR_COND:
                 if bva == bb.va:
                     return True
 
     return False
 
 
-def extract_bb_tight_loop(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """check basic block for tight loop indicators"""
-    if _bb_has_tight_loop(f, bb.inner):
-        yield Characteristic("tight loop"), bb.address
+def extract_bb_tight_loop(f, bb):
+    """ check basic block for tight loop indicators """
+    if _bb_has_tight_loop(f, bb):
+        yield Characteristic("tight loop"), bb.va
 
 
 def _bb_has_stackstring(f, bb):
@@ -70,13 +67,13 @@ def _bb_has_stackstring(f, bb):
     return False
 
 
-def extract_stackstring(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
-    """check basic block for stackstring indicators"""
-    if _bb_has_stackstring(f, bb.inner):
-        yield Characteristic("stack string"), bb.address
+def extract_stackstring(f, bb):
+    """ check basic block for stackstring indicators """
+    if _bb_has_stackstring(f, bb):
+        yield Characteristic("stack string"), bb.va
 
 
-def is_mov_imm_to_stack(instr: envi.archs.i386.disasm.i386Opcode) -> bool:
+def is_mov_imm_to_stack(instr):
     """
     Return if instruction moves immediate onto stack
     """
@@ -108,7 +105,7 @@ def is_mov_imm_to_stack(instr: envi.archs.i386.disasm.i386Opcode) -> bool:
     return True
 
 
-def get_printable_len(oper: envi.archs.i386.disasm.i386ImmOper) -> int:
+def get_printable_len(oper):
     """
     Return string length if all operand bytes are ascii or utf16-le printable
     """
@@ -120,33 +117,23 @@ def get_printable_len(oper: envi.archs.i386.disasm.i386ImmOper) -> int:
         chars = struct.pack("<I", oper.imm)
     elif oper.tsize == 8:
         chars = struct.pack("<Q", oper.imm)
-    else:
-        raise ValueError("unexpected oper.tsize: %d" % (oper.tsize))
-
     if is_printable_ascii(chars):
         return oper.tsize
-    elif is_printable_utf16le(chars):
+    if is_printable_utf16le(chars):
         return oper.tsize / 2
-    else:
-        return 0
+    return 0
 
 
-def is_printable_ascii(chars: bytes) -> bool:
-    try:
-        chars_str = chars.decode("ascii")
-    except UnicodeDecodeError:
-        return False
-    else:
-        return all(c in string.printable for c in chars_str)
+def is_printable_ascii(chars):
+    return all(ord(c) < 127 and c in string.printable for c in chars)
 
 
-def is_printable_utf16le(chars: bytes) -> bool:
-    if all(c == b"\x00" for c in chars[1::2]):
+def is_printable_utf16le(chars):
+    if all(c == "\x00" for c in chars[1::2]):
         return is_printable_ascii(chars[::2])
-    return False
 
 
-def extract_features(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_features(f, bb):
     """
     extract features from the given basic block.
 
@@ -155,12 +142,12 @@ def extract_features(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature,
       bb (viv_utils.BasicBlock): the basic block to process.
 
     yields:
-      Tuple[Feature, int]: the features and their location found in this basic block.
+      Feature, set[VA]: the features and their location found in this basic block.
     """
-    yield BasicBlock(), AbsoluteVirtualAddress(bb.inner.va)
+    yield BasicBlock(), bb.va
     for bb_handler in BASIC_BLOCK_HANDLERS:
-        for feature, addr in bb_handler(f, bb):
-            yield feature, addr
+        for feature, va in bb_handler(f, bb):
+            yield feature, va
 
 
 BASIC_BLOCK_HANDLERS = (

capa/features/extractors/viv/extractor.py

@@ -1,92 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-from typing import List, Tuple, Iterator
-
-import viv_utils
-import viv_utils.flirt
-
-import capa.features.extractors.common
-import capa.features.extractors.viv.file
-import capa.features.extractors.viv.insn
-import capa.features.extractors.viv.global_
-import capa.features.extractors.viv.function
-import capa.features.extractors.viv.basicblock
-from capa.features.common import Feature
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
-
-logger = logging.getLogger(__name__)
-
-
-class VivInstructionHandle:
-    """this acts like a vivisect.Opcode but with an __int__() method"""
-
-    def __init__(self, inner):
-        self._inner = inner
-
-    def __int__(self):
-        return self.va
-
-    def __getattr__(self, name):
-        return getattr(self._inner, name)
-
-
-class VivisectFeatureExtractor(FeatureExtractor):
-    def __init__(self, vw, path):
-        super(VivisectFeatureExtractor, self).__init__()
-        self.vw = vw
-        self.path = path
-        with open(self.path, "rb") as f:
-            self.buf = f.read()
-
-        # pre-compute these because we'll yield them at *every* scope.
-        self.global_features: List[Tuple[Feature, Address]] = []
-        self.global_features.extend(capa.features.extractors.common.extract_os(self.buf))
-        self.global_features.extend(capa.features.extractors.viv.global_.extract_arch(self.vw))
-
-    def get_base_address(self):
-        # assume there is only one file loaded into the vw
-        return AbsoluteVirtualAddress(list(self.vw.filemeta.values())[0]["imagebase"])
-
-    def extract_global_features(self):
-        yield from self.global_features
-
-    def extract_file_features(self):
-        yield from capa.features.extractors.viv.file.extract_features(self.vw, self.buf)
-
-    def get_functions(self) -> Iterator[FunctionHandle]:
-        for va in sorted(self.vw.getFunctions()):
-            yield FunctionHandle(address=AbsoluteVirtualAddress(va), inner=viv_utils.Function(self.vw, va))
-
-    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.viv.function.extract_features(fh)
-
-    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
-        f: viv_utils.Function = fh.inner
-        for bb in f.basic_blocks:
-            yield BBHandle(address=AbsoluteVirtualAddress(bb.va), inner=bb)
-
-    def extract_basic_block_features(self, fh: FunctionHandle, bbh) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.viv.basicblock.extract_features(fh, bbh)
-
-    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
-        bb: viv_utils.BasicBlock = bbh.inner
-        for insn in bb.instructions:
-            yield InsnHandle(address=AbsoluteVirtualAddress(insn.va), inner=insn)
-
-    def extract_insn_features(
-        self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-    ) -> Iterator[Tuple[Feature, Address]]:
-        yield from capa.features.extractors.viv.insn.extract_features(fh, bbh, ih)
-
-    def is_library_function(self, addr):
-        return viv_utils.flirt.is_library_function(self.vw, addr)
-
-    def get_function_name(self, addr):
-        return viv_utils.get_function_name(self.vw, addr)

capa/features/extractors/viv/file.py

@@ -1,36 +1,32 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Tuple, Iterator
 
 import PE.carve as pe_carve  # vivisect PE
-import viv_utils
-import viv_utils.flirt
 
-import capa.features.insn
-import capa.features.extractors.common
-import capa.features.extractors.helpers
 import capa.features.extractors.strings
-from capa.features.file import Export, Import, Section, FunctionName
-from capa.features.common import String, Feature, Characteristic
-from capa.features.address import Address, FileOffsetAddress, AbsoluteVirtualAddress
+from capa.features import String, Characteristic
+from capa.features.file import Export, Import, Section
 
 
-def extract_file_embedded_pe(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]:
-    for offset, _ in pe_carve.carve(buf, 1):
-        yield Characteristic("embedded pe"), FileOffsetAddress(offset)
+def extract_file_embedded_pe(vw, file_path):
+    with open(file_path, "rb") as f:
+        fbytes = f.read()
+
+    for offset, i in pe_carve.carve(fbytes, 1):
+        yield Characteristic("embedded pe"), offset
 
 
-def extract_file_export_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
-    for va, _, name, _ in vw.getExports():
-        yield Export(name), AbsoluteVirtualAddress(va)
+def extract_file_export_names(vw, file_path):
+    for va, etype, name, _ in vw.getExports():
+        yield Export(name), va
 
 
-def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
+def extract_file_import_names(vw, file_path):
     """
     extract imported function names
     1. imports by ordinal:
@@ -41,17 +37,18 @@ def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]
     """
     for va, _, _, tinfo in vw.getImports():
         # vivisect source: tinfo = "%s.%s" % (libname, impname)
-        modname, impname = tinfo.split(".", 1)
+        modname, impname = tinfo.split(".")
         if is_viv_ord_impname(impname):
             # replace ord prefix with #
             impname = "#%s" % impname[len("ord") :]
-
-        addr = AbsoluteVirtualAddress(va)
-        for name in capa.features.extractors.helpers.generate_symbols(modname, impname):
-            yield Import(name), addr
+            tinfo = "%s.%s" % (modname, impname)
+            yield Import(tinfo), va
+        else:
+            yield Import(tinfo), va
+            yield Import(impname), va
 
 
-def is_viv_ord_impname(impname: str) -> bool:
+def is_viv_ord_impname(impname):
     """
     return if import name matches vivisect's ordinal naming scheme `'ord%d' % ord`
     """
@@ -65,51 +62,40 @@ def is_viv_ord_impname(impname: str) -> bool:
         return True
 
 
-def extract_file_section_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
+def extract_file_section_names(vw, file_path):
     for va, _, segname, _ in vw.getSegments():
-        yield Section(segname), AbsoluteVirtualAddress(va)
+        yield Section(segname), va
 
 
-def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]:
-    yield from capa.features.extractors.common.extract_file_strings(buf)
-
-
-def extract_file_function_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
+def extract_file_strings(vw, file_path):
     """
-    extract the names of statically-linked library functions.
+    extract ASCII and UTF-16 LE strings from file
     """
-    for va in sorted(vw.getFunctions()):
-        addr = AbsoluteVirtualAddress(va)
-        if viv_utils.flirt.is_library_function(vw, va):
-            name = viv_utils.get_function_name(vw, va)
-            yield FunctionName(name), addr
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                yield FunctionName(name[1:]), addr
+    with open(file_path, "rb") as f:
+        b = f.read()
+
+    for s in capa.features.extractors.strings.extract_ascii_strings(b):
+        yield String(s.s), s.offset
+
+    for s in capa.features.extractors.strings.extract_unicode_strings(b):
+        yield String(s.s), s.offset
 
 
-def extract_file_format(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]:
-    yield from capa.features.extractors.common.extract_format(buf)
-
-
-def extract_features(vw, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
+def extract_features(vw, file_path):
     """
     extract file features from given workspace
 
     args:
       vw (vivisect.VivWorkspace): the vivisect workspace
-      buf: the raw input file bytes
+      file_path: path to the input file
 
     yields:
-      Tuple[Feature, Address]: a feature and its location.
+      Tuple[Feature, VA]: a feature and its location.
     """
 
     for file_handler in FILE_HANDLERS:
-        for feature, addr in file_handler(vw=vw, buf=buf):  # type: ignore
-            yield feature, addr
+        for feature, va in file_handler(vw, file_path):
+            yield feature, va
 
 
 FILE_HANDLERS = (
@@ -118,6 +104,4 @@ FILE_HANDLERS = (
     extract_file_import_names,
     extract_file_section_names,
     extract_file_strings,
-    extract_file_function_names,
-    extract_file_format,
 )

capa/features/extractors/viv/function.py

@@ -1,47 +1,39 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import Tuple, Iterator
 
-import envi
-import viv_utils
 import vivisect.const
 
-from capa.features.common import Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features import Characteristic
 from capa.features.extractors import loops
-from capa.features.extractors.base_extractor import FunctionHandle
 
 
-def interface_extract_function_XXX(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+def interface_extract_function_XXX(f):
     """
     parse features from the given function.
 
     args:
-      f: the function to process.
+      f (viv_utils.Function): the function to process.
 
     yields:
-      (Feature, Address): the feature and the address at which its found.
+      (Feature, int): the feature and the address at which its found.
     """
-    ...
+    yield NotImplementedError("feature"), NotImplementedError("virtual address")
 
 
-def extract_function_calls_to(fhandle: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
-    f: viv_utils.Function = fhandle.inner
+def extract_function_calls_to(f):
     for src, _, _, _ in f.vw.getXrefsTo(f.va, rtype=vivisect.const.REF_CODE):
-        yield Characteristic("calls to"), AbsoluteVirtualAddress(src)
+        yield Characteristic("calls to"), src
 
 
-def extract_function_loop(fhandle: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_function_loop(f):
     """
     parse if a function has a loop
     """
-    f: viv_utils.Function = fhandle.inner
-
     edges = []
 
     for bb in f.basic_blocks:
@@ -49,30 +41,30 @@ def extract_function_loop(fhandle: FunctionHandle) -> Iterator[Tuple[Feature, Ad
             for bva, bflags in bb.instructions[-1].getBranches():
                 # vivisect does not set branch flags for non-conditional jmp so add explicit check
                 if (
-                    bflags & envi.BR_COND
-                    or bflags & envi.BR_FALL
-                    or bflags & envi.BR_TABLE
+                    bflags & vivisect.envi.BR_COND
+                    or bflags & vivisect.envi.BR_FALL
+                    or bflags & vivisect.envi.BR_TABLE
                     or bb.instructions[-1].mnem == "jmp"
                 ):
                     edges.append((bb.va, bva))
 
     if edges and loops.has_loop(edges):
-        yield Characteristic("loop"), fhandle.address
+        yield Characteristic("loop"), f.va
 
 
-def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_features(f):
     """
     extract features from the given function.
 
     args:
-      fh: the function handle from which to extract features
+      f (viv_utils.Function): the function from which to extract features
 
     yields:
-      Tuple[Feature, int]: the features and their location found in this function.
+      Feature, set[VA]: the features and their location found in this function.
     """
     for func_handler in FUNCTION_HANDLERS:
-        for feature, addr in func_handler(fh):
-            yield feature, addr
+        for feature, va in func_handler(f):
+            yield feature, va
 
 
 FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop)

capa/features/extractors/viv/global_.py

@@ -1,26 +0,0 @@
-import logging
-from typing import Tuple, Iterator
-
-import envi.archs.i386
-import envi.archs.amd64
-
-from capa.features.common import ARCH_I386, ARCH_AMD64, Arch, Feature
-from capa.features.address import NO_ADDRESS, Address
-
-logger = logging.getLogger(__name__)
-
-
-def extract_arch(vw) -> Iterator[Tuple[Feature, Address]]:
-    if isinstance(vw.arch, envi.archs.amd64.Amd64Module):
-        yield Arch(ARCH_AMD64), NO_ADDRESS
-
-    elif isinstance(vw.arch, envi.archs.i386.i386Module):
-        yield Arch(ARCH_I386), NO_ADDRESS
-
-    else:
-        # we likely end up here:
-        #  1. handling a new architecture (e.g. aarch64)
-        #
-        # for (1), this logic will need to be updated as the format is implemented.
-        logger.debug("unsupported architecture: %s", vw.arch.__class__.__name__)
-        return

capa/features/extractors/viv/helpers.py

@@ -1,23 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from typing import Optional
-
-from vivisect import VivWorkspace
-from vivisect.const import XR_TO, REF_CODE
-
-
-def get_coderef_from(vw: VivWorkspace, va: int) -> Optional[int]:
-    """
-    return first code `tova` whose origin is the specified va
-    return None if no code reference is found
-    """
-    xrefs = vw.getXrefsFrom(va, REF_CODE)
-    if len(xrefs) > 0:
-        return xrefs[0][XR_TO]
-    else:
-        return None

capa/features/extractors/viv/indirect_calls.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -7,16 +7,11 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import collections
-from typing import TYPE_CHECKING, Set, List, Deque, Tuple, Union, Optional
 
 import envi
 import vivisect.const
 import envi.archs.i386.disasm
 import envi.archs.amd64.disasm
-from vivisect import VivWorkspace
-
-if TYPE_CHECKING:
-    from capa.features.extractors.viv.extractor import VivInstructionHandle
 
 # pull out consts for lookup performance
 i386RegOper = envi.archs.i386.disasm.i386RegOper
@@ -31,7 +26,7 @@ FAR_BRANCH_MASK = envi.BR_PROC | envi.BR_DEREF | envi.BR_ARCH
 DESTRUCTIVE_MNEMONICS = ("mov", "lea", "pop", "xor")
 
 
-def get_previous_instructions(vw: VivWorkspace, va: int) -> List[int]:
+def get_previous_instructions(vw, va):
     """
     collect the instructions that flow to the given address, local to the current function.
 
@@ -48,14 +43,12 @@ def get_previous_instructions(vw: VivWorkspace, va: int) -> List[int]:
     # ensure that it fallsthrough to this one.
     loc = vw.getPrevLocation(va, adjacent=True)
     if loc is not None:
-        ploc = vw.getPrevLocation(va, adjacent=True)
-        if ploc is not None:
-            # from vivisect.const:
-            # location: (L_VA, L_SIZE, L_LTYPE, L_TINFO)
-            (pva, _, ptype, pinfo) = ploc
+        # from vivisect.const:
+        # location: (L_VA, L_SIZE, L_LTYPE, L_TINFO)
+        (pva, _, ptype, pinfo) = vw.getPrevLocation(va, adjacent=True)
 
-            if ptype == LOC_OP and not (pinfo & IF_NOFALL):
-                ret.append(pva)
+        if ptype == LOC_OP and not (pinfo & IF_NOFALL):
+            ret.append(pva)
 
     # find any code refs, e.g. jmp, to this location.
     # ignore any calls.
@@ -74,7 +67,7 @@ class NotFoundError(Exception):
     pass
 
 
-def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int, None]]:
+def find_definition(vw, va, reg):
     """
     scan backwards from the given address looking for assignments to the given register.
     if a constant, return that value.
@@ -90,8 +83,8 @@ def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int
     raises:
       NotFoundError: when the definition cannot be found.
     """
-    q = collections.deque()  # type: Deque[int]
-    seen = set([])  # type: Set[int]
+    q = collections.deque()
+    seen = set([])
 
     q.extend(get_previous_instructions(vw, va))
     while q:
@@ -135,16 +128,14 @@ def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int
     raise NotFoundError()
 
 
-def is_indirect_call(vw: VivWorkspace, va: int, insn: Optional["VivInstructionHandle"] = None) -> bool:
+def is_indirect_call(vw, va, insn=None):
     if insn is None:
         insn = vw.parseOpcode(va)
 
-    return insn.mnem in ("call", "jmp") and isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper)
+    return insn.mnem == "call" and isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper)
 
 
-def resolve_indirect_call(
-    vw: VivWorkspace, va: int, insn: Optional["VivInstructionHandle"] = None
-) -> Tuple[int, Optional[int]]:
+def resolve_indirect_call(vw, va, insn=None):
     """
     inspect the given indirect call instruction and attempt to resolve the target address.
 

capa/features/extractors/viv/insn.py

@@ -1,28 +1,18 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-from typing import List, Tuple, Callable, Iterator
 
-import envi
-import envi.exc
-import viv_utils
 import envi.memory
-import viv_utils.flirt
-import envi.archs.i386.regs
-import envi.archs.amd64.regs
+import vivisect.const
 import envi.archs.i386.disasm
-import envi.archs.amd64.disasm
 
 import capa.features.extractors.helpers
-import capa.features.extractors.viv.helpers
-from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
-from capa.features.common import MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, String, Feature, Characteristic
-from capa.features.address import Address, AbsoluteVirtualAddress
-from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+from capa.features import ARCH_X32, ARCH_X64, MAX_BYTES_FEATURE_SIZE, Bytes, String, Characteristic
+from capa.features.insn import Number, Offset, Mnemonic
 from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_indirect_call
 
 # security cookie checks may perform non-zeroing XORs, these are expected within a certain
@@ -30,109 +20,78 @@ from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_i
 SECURITY_COOKIE_BYTES_DELTA = 0x40
 
 
-def interface_extract_instruction_XXX(
-    fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
+def get_arch(vw):
+    arch = vw.getMeta("Architecture")
+    if arch == "i386":
+        return ARCH_X32
+    elif arch == "amd64":
+        return ARCH_X64
+
+
+def interface_extract_instruction_XXX(f, bb, insn):
     """
     parse features from the given instruction.
 
     args:
-      fh: the function handle to process.
-      bbh: the basic block handle to process.
-      ih: the instruction handle to process.
+      f (viv_utils.Function): the function to process.
+      bb (viv_utils.BasicBlock): the basic block to process.
+      insn (vivisect...Instruction): the instruction to process.
 
     yields:
-      (Feature, Address): the feature and the address at which its found.
+      (Feature, int): the feature and the address at which its found.
     """
-    ...
+    yield NotImplementedError("feature"), NotImplementedError("virtual address")
 
 
 def get_imports(vw):
     """
     caching accessor to vivisect workspace imports
     avoids performance issues in vivisect when collecting locations
-
-    returns: Dict[int, Tuple[str, str]]
     """
     if "imports" in vw.metadata:
         return vw.metadata["imports"]
     else:
-        imports = {
-            p[0]: (p[3].rpartition(".")[0], p[3].replace(".ord", ".#").rpartition(".")[2]) for p in vw.getImports()
-        }
+        imports = {p[0]: p[3] for p in vw.getImports()}
         vw.metadata["imports"] = imports
         return imports
 
 
-def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse API features from the given instruction.
+def extract_insn_api_features(f, bb, insn):
+    """parse API features from the given instruction."""
 
-    example:
-       call dword [0x00473038]
-    """
-    insn: envi.Opcode = ih.inner
-    f: viv_utils.Function = fh.inner
-    if insn.mnem not in ("call", "jmp"):
+    # example:
+    #
+    #    call dword [0x00473038]
+
+    if insn.mnem != "call":
         return
 
-    if insn.mnem == "jmp":
-        if f.vw.getFunctionMeta(f.va, "Thunk"):
-            return
-
     # traditional call via IAT
     if isinstance(insn.opers[0], envi.archs.i386.disasm.i386ImmMemOper):
         oper = insn.opers[0]
         target = oper.getOperAddr(insn)
 
         imports = get_imports(f.vw)
-        if target in imports:
-            dll, symbol = imports[target]
-            for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-                yield API(name), ih.address
+        if target in imports.keys():
+            for feature, va in capa.features.extractors.helpers.generate_api_features(imports[target], insn.va):
+                yield feature, va
 
     # call via thunk on x86,
     # see 9324d1a8ae37a36ae560c37448c9705a at 0x407985
     #
-    # this is also how calls to internal functions may be decoded on x32 and x64.
+    # this is also how calls to internal functions may be decoded on x64.
     # see Lab21-01.exe_:0x140001178
-    #
-    # follow chained thunks, e.g. in 82bf6347acf15e5d883715dc289d8a2b at 0x14005E0FF in
-    # 0x140059342 (viv) / 0x14005E0C0 (IDA)
-    # 14005E0FF call    j_ElfClearEventLogFileW (14005AAF8)
-    #   14005AAF8 jmp     ElfClearEventLogFileW (14005E196)
-    #     14005E196 jmp     cs:__imp_ElfClearEventLogFileW
-
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386PcRelOper):
-        imports = get_imports(f.vw)
-        target = capa.features.extractors.viv.helpers.get_coderef_from(f.vw, insn.va)
-        if not target:
+        target = insn.opers[0].getOperValue(insn)
+
+        try:
+            thunk = f.vw.getFunctionMeta(target, "Thunk")
+        except vivisect.exc.InvalidFunction:
             return
-
-        if viv_utils.flirt.is_library_function(f.vw, target):
-            name = viv_utils.get_function_name(f.vw, target)
-            yield API(name), ih.address
-            if name.startswith("_"):
-                # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                # extract features for both the mangled and un-mangled representations.
-                # e.g. `_fwrite` -> `fwrite`
-                # see: https://stackoverflow.com/a/2628384/87207
-                yield API(name[1:]), ih.address
-            return
-
-        for _ in range(THUNK_CHAIN_DEPTH_DELTA):
-            if target in imports:
-                dll, symbol = imports[target]
-                for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-                    yield API(name), ih.address
-
-            # if jump leads to an ENDBRANCH instruction, skip it
-            if f.vw.getByteDef(target)[1].startswith(b"\xf3\x0f\x1e"):
-                target += 4
-
-            target = capa.features.extractors.viv.helpers.get_coderef_from(f.vw, target)
-            if not target:
-                return
+        else:
+            if thunk:
+                for feature, va in capa.features.extractors.helpers.generate_api_features(thunk, insn.va):
+                    yield feature, va
 
     # call via import on x64
     # see Lab21-01.exe_:0x14000118C
@@ -141,10 +100,9 @@ def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterato
         target = op.getOperAddr(insn)
 
         imports = get_imports(f.vw)
-        if target in imports:
-            dll, symbol = imports[target]
-            for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-                yield API(name), ih.address
+        if target in imports.keys():
+            for feature, va in capa.features.extractors.helpers.generate_api_features(imports[target], insn.va):
+                yield feature, va
 
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper):
         try:
@@ -158,10 +116,40 @@ def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterato
             return
 
         imports = get_imports(f.vw)
-        if target in imports:
-            dll, symbol = imports[target]
-            for name in capa.features.extractors.helpers.generate_symbols(dll, symbol):
-                yield API(name), ih.address
+        if target in imports.keys():
+            for feature, va in capa.features.extractors.helpers.generate_api_features(imports[target], insn.va):
+                yield feature, va
+
+
+def extract_insn_number_features(f, bb, insn):
+    """parse number features from the given instruction."""
+    # example:
+    #
+    #     push    3136B0h         ; dwControlCode
+    for oper in insn.opers:
+        # this is for both x32 and x64
+        if not isinstance(oper, (envi.archs.i386.disasm.i386ImmOper, envi.archs.i386.disasm.i386ImmMemOper)):
+            continue
+
+        if isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
+            v = oper.getOperValue(oper)
+        else:
+            v = oper.getOperAddr(oper)
+
+        if f.vw.probeMemory(v, 1, envi.memory.MM_READ):
+            # this is a valid address
+            # assume its not also a constant.
+            continue
+
+        if insn.mnem == "add" and insn.opers[0].isReg() and insn.opers[0].reg == envi.archs.i386.disasm.REG_ESP:
+            # skip things like:
+            #
+            #    .text:00401140                 call    sub_407E2B
+            #    .text:00401145                 add     esp, 0Ch
+            return
+
+        yield Number(v), insn.va
+        yield Number(v, arch=get_arch(f.vw)), insn.va
 
 
 def derefs(vw, p):
@@ -196,7 +184,7 @@ def derefs(vw, p):
         p = next
 
 
-def read_memory(vw, va: int, size: int) -> bytes:
+def read_memory(vw, va, size):
     # as documented in #176, vivisect will not readMemory() when the section is not marked readable.
     #
     # but here, we don't care about permissions.
@@ -209,10 +197,10 @@ def read_memory(vw, va: int, size: int) -> bytes:
             mva, msize, mperms, mfname = mmap
             offset = va - mva
             return mbytes[offset : offset + size]
-    raise envi.exc.SegmentationViolation(va)
+    raise envi.SegmentationViolation(va)
 
 
-def read_bytes(vw, va: int) -> bytes:
+def read_bytes(vw, va):
     """
     read up to MAX_BYTES_FEATURE_SIZE from the given address.
 
@@ -221,7 +209,7 @@ def read_bytes(vw, va: int) -> bytes:
     """
     segm = vw.getSegment(va)
     if not segm:
-        raise envi.exc.SegmentationViolation(va)
+        raise envi.SegmentationViolation()
 
     segm_end = segm[0] + segm[1]
     try:
@@ -230,23 +218,20 @@ def read_bytes(vw, va: int) -> bytes:
             return read_memory(vw, va, segm_end - va)
         else:
             return read_memory(vw, va, MAX_BYTES_FEATURE_SIZE)
-    except envi.exc.SegmentationViolation:
+    except envi.SegmentationViolation:
         raise
 
 
-def extract_insn_bytes_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_bytes_features(f, bb, insn):
     """
     parse byte sequence features from the given instruction.
     example:
         #     push    offset iid_004118d4_IShellLinkA ; riid
     """
-    insn: envi.Opcode = ih.inner
-    f: viv_utils.Function = fh.inner
-
-    if insn.mnem == "call":
-        return
-
     for oper in insn.opers:
+        if insn.mnem == "call":
+            continue
+
         if isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
             v = oper.getOperValue(oper)
         elif isinstance(oper, envi.archs.i386.disasm.i386RegMemOper):
@@ -265,19 +250,19 @@ def extract_insn_bytes_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera
         for v in derefs(f.vw, v):
             try:
                 buf = read_bytes(f.vw, v)
-            except envi.exc.SegmentationViolation:
+            except envi.SegmentationViolation:
                 continue
 
             if capa.features.extractors.helpers.all_zeros(buf):
                 continue
 
-            yield Bytes(buf), ih.address
+            yield Bytes(buf), insn.va
 
 
-def read_string(vw, offset: int) -> str:
+def read_string(vw, offset):
     try:
         alen = vw.detectString(offset)
-    except envi.exc.SegmentationViolation:
+    except envi.SegmentationViolation:
         pass
     else:
         if alen > 0:
@@ -285,7 +270,7 @@ def read_string(vw, offset: int) -> str:
 
     try:
         ulen = vw.detectUnicode(offset)
-    except envi.exc.SegmentationViolation:
+    except envi.SegmentationViolation:
         pass
     except IndexError:
         # potential vivisect bug detecting Unicode at segment end
@@ -296,27 +281,76 @@ def read_string(vw, offset: int) -> str:
                 # vivisect seems to mis-detect the end unicode strings
                 # off by one, too short
                 ulen += 1
-            else:
-                # vivisect seems to mis-detect the end unicode strings
-                # off by two, too short
-                ulen += 2
             return read_memory(vw, offset, ulen).decode("utf-16")
 
     raise ValueError("not a string", offset)
 
 
-def is_security_cookie(f, bb, insn) -> bool:
+def extract_insn_string_features(f, bb, insn):
+    """parse string features from the given instruction."""
+    # example:
+    #
+    #     push    offset aAcr     ; "ACR  > "
+
+    for oper in insn.opers:
+        if isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
+            v = oper.getOperValue(oper)
+        elif isinstance(oper, envi.archs.i386.disasm.i386SibOper):
+            # like 0x401000 in `mov eax, 0x401000[2 * ebx]`
+            v = oper.imm
+        elif isinstance(oper, envi.archs.amd64.disasm.Amd64RipRelOper):
+            v = oper.getOperAddr(insn)
+        else:
+            continue
+
+        for v in derefs(f.vw, v):
+            try:
+                s = read_string(f.vw, v)
+            except ValueError:
+                continue
+            else:
+                yield String(s.rstrip("\x00")), insn.va
+
+
+def extract_insn_offset_features(f, bb, insn):
+    """parse structure offset features from the given instruction."""
+    # example:
+    #
+    #     .text:0040112F    cmp     [esi+4], ebx
+    for oper in insn.opers:
+        # this is for both x32 and x64
+        if not isinstance(oper, envi.archs.i386.disasm.i386RegMemOper):
+            continue
+
+        if oper.reg == envi.archs.i386.disasm.REG_ESP:
+            continue
+
+        if oper.reg == envi.archs.i386.disasm.REG_EBP:
+            continue
+
+        # TODO: do x64 support for real.
+        if oper.reg == envi.archs.amd64.disasm.REG_RBP:
+            continue
+
+        # viv already decodes offsets as signed
+        v = oper.disp
+
+        yield Offset(v), insn.va
+        yield Offset(v, arch=get_arch(f.vw)), insn.va
+
+
+def is_security_cookie(f, bb, insn):
     """
     check if an instruction is related to security cookie checks
     """
     # security cookie check should use SP or BP
     oper = insn.opers[1]
     if oper.isReg() and oper.reg not in [
-        envi.archs.i386.regs.REG_ESP,
-        envi.archs.i386.regs.REG_EBP,
+        envi.archs.i386.disasm.REG_ESP,
+        envi.archs.i386.disasm.REG_EBP,
         # TODO: do x64 support for real.
-        envi.archs.amd64.regs.REG_RBP,
-        envi.archs.amd64.regs.REG_RSP,
+        envi.archs.amd64.disasm.REG_RBP,
+        envi.archs.amd64.disasm.REG_RSP,
     ]:
         return False
 
@@ -333,18 +367,12 @@ def is_security_cookie(f, bb, insn) -> bool:
     return False
 
 
-def extract_insn_nzxor_characteristic_features(
-    fh: FunctionHandle, bbhandle: BBHandle, ih: InsnHandle
-) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_nzxor_characteristic_features(f, bb, insn):
     """
     parse non-zeroing XOR instruction from the given instruction.
     ignore expected non-zeroing XORs, e.g. security cookies.
     """
-    insn: envi.Opcode = ih.inner
-    bb: viv_utils.BasicBlock = bbhandle.inner
-    f: viv_utils.Function = fh.inner
-
-    if insn.mnem not in ("xor", "xorpd", "xorps", "pxor"):
+    if insn.mnem != "xor":
         return
 
     if insn.opers[0] == insn.opers[1]:
@@ -353,40 +381,19 @@ def extract_insn_nzxor_characteristic_features(
     if is_security_cookie(f, bb, insn):
         return
 
-    yield Characteristic("nzxor"), ih.address
+    yield Characteristic("nzxor"), insn.va
 
 
-def extract_insn_mnemonic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_mnemonic_features(f, bb, insn):
     """parse mnemonic features from the given instruction."""
-    yield Mnemonic(ih.inner.mnem), ih.address
+    yield Mnemonic(insn.mnem), insn.va
 
 
-def extract_insn_obfs_call_plus_5_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """
-    parse call $+5 instruction from the given instruction.
-    """
-    insn: envi.Opcode = ih.inner
-
-    if insn.mnem != "call":
-        return
-
-    if isinstance(insn.opers[0], envi.archs.i386.disasm.i386PcRelOper):
-        if insn.va + 5 == insn.opers[0].getOperValue(insn):
-            yield Characteristic("call $+5"), ih.address
-
-    if isinstance(insn.opers[0], envi.archs.i386.disasm.i386ImmMemOper) or isinstance(
-        insn.opers[0], envi.archs.amd64.disasm.Amd64RipRelOper
-    ):
-        if insn.va + 5 == insn.opers[0].getOperAddr(insn):
-            yield Characteristic("call $+5"), ih.address
-
-
-def extract_insn_peb_access_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_peb_access_characteristic_features(f, bb, insn):
     """
     parse peb access from the given function. fs:[0x30] on x86, gs:[0x60] on x64
     """
     # TODO handle where fs/gs are loaded into a register or onto the stack and used later
-    insn: envi.Opcode = ih.inner
 
     if insn.mnem not in ["push", "mov"]:
         return
@@ -405,7 +412,7 @@ def extract_insn_peb_access_characteristic_features(f, bb, ih: InsnHandle) -> It
             if (isinstance(oper, envi.archs.i386.disasm.i386RegMemOper) and oper.disp == 0x30) or (
                 isinstance(oper, envi.archs.i386.disasm.i386ImmMemOper) and oper.imm == 0x30
             ):
-                yield Characteristic("peb access"), ih.address
+                yield Characteristic("peb access"), insn.va
     elif "gs" in prefix:
         for oper in insn.opers:
             if (
@@ -413,25 +420,23 @@ def extract_insn_peb_access_characteristic_features(f, bb, ih: InsnHandle) -> It
                 or (isinstance(oper, envi.archs.amd64.disasm.i386SibOper) and oper.imm == 0x60)
                 or (isinstance(oper, envi.archs.amd64.disasm.i386ImmMemOper) and oper.imm == 0x60)
             ):
-                yield Characteristic("peb access"), ih.address
+                yield Characteristic("peb access"), insn.va
     else:
         pass
 
 
-def extract_insn_segment_access_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    """parse the instruction for access to fs or gs"""
-    insn: envi.Opcode = ih.inner
-
+def extract_insn_segment_access_features(f, bb, insn):
+    """ parse the instruction for access to fs or gs """
     prefix = insn.getPrefixName()
 
     if prefix == "fs":
-        yield Characteristic("fs access"), ih.address
+        yield Characteristic("fs access"), insn.va
 
     if prefix == "gs":
-        yield Characteristic("gs access"), ih.address
+        yield Characteristic("gs access"), insn.va
 
 
-def get_section(vw, va: int):
+def get_section(vw, va):
     for start, length, _, __ in vw.getMemoryMaps():
         if start <= va < start + length:
             return start
@@ -439,18 +444,11 @@ def get_section(vw, va: int):
     raise KeyError(va)
 
 
-def extract_insn_cross_section_cflow(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_insn_cross_section_cflow(f, bb, insn):
     """
     inspect the instruction for a CALL or JMP that crosses section boundaries.
     """
-    insn: envi.Opcode = ih.inner
-    f: viv_utils.Function = fh.inner
-
     for va, flags in insn.getBranches():
-        if va is None:
-            # va may be none for dynamic branches that haven't been resolved, such as `jmp eax`.
-            continue
-
         if flags & envi.BR_FALL:
             continue
 
@@ -472,7 +470,7 @@ def extract_insn_cross_section_cflow(fh: FunctionHandle, bb, ih: InsnHandle) ->
                     continue
 
             if get_section(f.vw, insn.va) != get_section(f.vw, va):
-                yield Characteristic("cross section flow"), ih.address
+                yield Characteristic("cross section flow"), insn.va
 
         except KeyError:
             continue
@@ -480,10 +478,7 @@ def extract_insn_cross_section_cflow(fh: FunctionHandle, bb, ih: InsnHandle) ->
 
 # this is a feature that's most relevant at the function scope,
 # however, its most efficient to extract at the instruction scope.
-def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    insn: envi.Opcode = ih.inner
-    f: viv_utils.Function = fh.inner
-
+def extract_function_calls_from(f, bb, insn):
     if insn.mnem != "call":
         return
 
@@ -493,7 +488,7 @@ def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera
     if isinstance(insn.opers[0], envi.archs.i386.disasm.i386ImmMemOper):
         oper = insn.opers[0]
         target = oper.getOperAddr(insn)
-        yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
+        yield Characteristic("calls from"), target
 
     # call via thunk on x86,
     # see 9324d1a8ae37a36ae560c37448c9705a at 0x407985
@@ -502,190 +497,43 @@ def extract_function_calls_from(fh: FunctionHandle, bb, ih: InsnHandle) -> Itera
     # see Lab21-01.exe_:0x140001178
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386PcRelOper):
         target = insn.opers[0].getOperValue(insn)
-        yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
+        yield Characteristic("calls from"), target
 
     # call via IAT, x64
     elif isinstance(insn.opers[0], envi.archs.amd64.disasm.Amd64RipRelOper):
         op = insn.opers[0]
         target = op.getOperAddr(insn)
-        yield Characteristic("calls from"), AbsoluteVirtualAddress(target)
+        yield Characteristic("calls from"), target
 
     if target and target == f.va:
         # if we found a jump target and it's the function address
         # mark as recursive
-        yield Characteristic("recursive call"), AbsoluteVirtualAddress(target)
+        yield Characteristic("recursive call"), target
 
 
 # this is a feature that's most relevant at the function or basic block scope,
 # however, its most efficient to extract at the instruction scope.
-def extract_function_indirect_call_characteristic_features(f, bb, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+def extract_function_indirect_call_characteristic_features(f, bb, insn):
     """
     extract indirect function call characteristic (e.g., call eax or call dword ptr [edx+4])
     does not include calls like => call ds:dword_ABD4974
     """
-    insn: envi.Opcode = ih.inner
-
     if insn.mnem != "call":
         return
 
     # Checks below work for x86 and x64
     if isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegOper):
         # call edx
-        yield Characteristic("indirect call"), ih.address
+        yield Characteristic("indirect call"), insn.va
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386RegMemOper):
         # call dword ptr [eax+50h]
-        yield Characteristic("indirect call"), ih.address
+        yield Characteristic("indirect call"), insn.va
     elif isinstance(insn.opers[0], envi.archs.i386.disasm.i386SibOper):
         # call qword ptr [rsp+78h]
-        yield Characteristic("indirect call"), ih.address
+        yield Characteristic("indirect call"), insn.va
 
 
-def extract_op_number_features(
-    fh: FunctionHandle, bb, ih: InsnHandle, i, oper: envi.Operand
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse number features from the given operand.
-
-    example:
-        push    3136B0h         ; dwControlCode
-    """
-    insn: envi.Opcode = ih.inner
-    f: viv_utils.Function = fh.inner
-
-    # this is for both x32 and x64
-    if not isinstance(oper, (envi.archs.i386.disasm.i386ImmOper, envi.archs.i386.disasm.i386ImmMemOper)):
-        return
-
-    if isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
-        v = oper.getOperValue(oper)
-    else:
-        v = oper.getOperAddr(oper)
-
-    if f.vw.probeMemory(v, 1, envi.memory.MM_READ):
-        # this is a valid address
-        # assume its not also a constant.
-        return
-
-    if insn.mnem == "add" and insn.opers[0].isReg() and insn.opers[0].reg == envi.archs.i386.regs.REG_ESP:
-        # skip things like:
-        #
-        #    .text:00401140                 call    sub_407E2B
-        #    .text:00401145                 add     esp, 0Ch
-        return
-
-    yield Number(v), ih.address
-    yield OperandNumber(i, v), ih.address
-
-    if insn.mnem == "add" and 0 < v < MAX_STRUCTURE_SIZE and isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
-        # for pattern like:
-        #
-        #     add eax, 0x10
-        #
-        # assume 0x10 is also an offset (imagine eax is a pointer).
-        yield Offset(v), ih.address
-        yield OperandOffset(i, v), ih.address
-
-
-def extract_op_offset_features(
-    fh: FunctionHandle, bb, ih: InsnHandle, i, oper: envi.Operand
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse structure offset features from the given operand."""
-    # example:
-    #
-    #     .text:0040112F    cmp     [esi+4], ebx
-    insn: envi.Opcode = ih.inner
-    f: viv_utils.Function = fh.inner
-
-    # this is for both x32 and x64
-    # like [esi + 4]
-    #       reg   ^
-    #             disp
-    if isinstance(oper, envi.archs.i386.disasm.i386RegMemOper):
-        if oper.reg == envi.archs.i386.regs.REG_ESP:
-            return
-
-        if oper.reg == envi.archs.i386.regs.REG_EBP:
-            return
-
-        # TODO: do x64 support for real.
-        if oper.reg == envi.archs.amd64.regs.REG_RBP:
-            return
-
-        # viv already decodes offsets as signed
-        v = oper.disp
-
-        yield Offset(v), ih.address
-        yield OperandOffset(i, v), ih.address
-
-        if insn.mnem == "lea" and i == 1 and not f.vw.probeMemory(v, 1, envi.memory.MM_READ):
-            # for pattern like:
-            #
-            #     lea eax, [ebx + 1]
-            #
-            # assume 1 is also an offset (imagine ebx is a zero register).
-            yield Number(v), ih.address
-            yield OperandNumber(i, v), ih.address
-
-    # like: [esi + ecx + 16384]
-    #        reg   ^     ^
-    #              index ^
-    #                    disp
-    elif isinstance(oper, envi.archs.i386.disasm.i386SibOper):
-        # viv already decodes offsets as signed
-        v = oper.disp
-
-        yield Offset(v), ih.address
-        yield OperandOffset(i, v), ih.address
-
-
-def extract_op_string_features(
-    fh: FunctionHandle, bb, ih: InsnHandle, i, oper: envi.Operand
-) -> Iterator[Tuple[Feature, Address]]:
-    """parse string features from the given operand."""
-    # example:
-    #
-    #     push    offset aAcr     ; "ACR  > "
-    insn: envi.Opcode = ih.inner
-    f: viv_utils.Function = fh.inner
-
-    if isinstance(oper, envi.archs.i386.disasm.i386ImmOper):
-        v = oper.getOperValue(oper)
-    elif isinstance(oper, envi.archs.i386.disasm.i386ImmMemOper):
-        # like 0x10056CB4 in `lea eax, dword [0x10056CB4]`
-        v = oper.imm
-    elif isinstance(oper, envi.archs.i386.disasm.i386SibOper):
-        # like 0x401000 in `mov eax, 0x401000[2 * ebx]`
-        v = oper.imm
-    elif isinstance(oper, envi.archs.amd64.disasm.Amd64RipRelOper):
-        v = oper.getOperAddr(insn)
-    else:
-        return
-
-    for v in derefs(f.vw, v):
-        try:
-            s = read_string(f.vw, v)
-        except ValueError:
-            continue
-        else:
-            yield String(s.rstrip("\x00")), ih.address
-
-
-def extract_operand_features(f: FunctionHandle, bb, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
-    for i, oper in enumerate(insn.inner.opers):
-        for op_handler in OPERAND_HANDLERS:
-            for feature, addr in op_handler(f, bb, insn, i, oper):
-                yield feature, addr
-
-
-OPERAND_HANDLERS: List[
-    Callable[[FunctionHandle, BBHandle, InsnHandle, int, envi.Operand], Iterator[Tuple[Feature, Address]]]
-] = [
-    extract_op_number_features,
-    extract_op_offset_features,
-    extract_op_string_features,
-]
-
-
-def extract_features(f, bb, insn) -> Iterator[Tuple[Feature, Address]]:
+def extract_features(f, bb, insn):
     """
     extract features from the given insn.
 
@@ -695,23 +543,24 @@ def extract_features(f, bb, insn) -> Iterator[Tuple[Feature, Address]]:
       insn (vivisect...Instruction): the instruction to process.
 
     yields:
-      Tuple[Feature, Address]: the features and their location found in this insn.
+      Feature, set[VA]: the features and their location found in this insn.
     """
     for insn_handler in INSTRUCTION_HANDLERS:
-        for feature, addr in insn_handler(f, bb, insn):
-            yield feature, addr
+        for feature, va in insn_handler(f, bb, insn):
+            yield feature, va
 
 
-INSTRUCTION_HANDLERS: List[Callable[[FunctionHandle, BBHandle, InsnHandle], Iterator[Tuple[Feature, Address]]]] = [
+INSTRUCTION_HANDLERS = (
     extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_string_features,
     extract_insn_bytes_features,
+    extract_insn_offset_features,
     extract_insn_nzxor_characteristic_features,
     extract_insn_mnemonic_features,
-    extract_insn_obfs_call_plus_5_characteristic_features,
     extract_insn_peb_access_characteristic_features,
     extract_insn_cross_section_cflow,
     extract_insn_segment_access_features,
     extract_function_calls_from,
     extract_function_indirect_call_characteristic_features,
-    extract_operand_features,
-]
+)

capa/features/file.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,33 +6,22 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from capa.features.common import Feature
+from capa.features import Feature
 
 
 class Export(Feature):
-    def __init__(self, value: str, description=None):
+    def __init__(self, value, description=None):
         # value is export name
         super(Export, self).__init__(value, description=description)
 
 
 class Import(Feature):
-    def __init__(self, value: str, description=None):
+    def __init__(self, value, description=None):
         # value is import name
         super(Import, self).__init__(value, description=description)
 
 
 class Section(Feature):
-    def __init__(self, value: str, description=None):
+    def __init__(self, value, description=None):
         # value is section name
         super(Section, self).__init__(value, description=description)
-
-
-class FunctionName(Feature):
-    """recognized name for statically linked function"""
-
-    def __init__(self, name: str, description=None):
-        # value is function name
-        super(FunctionName, self).__init__(name, description=description)
-        # override the name property set by `capa.features.Feature`
-        # that would be `functionname` (note missing dash)
-        self.name = "function-name"

capa/features/freeze.py

@@ -1,7 +1,46 @@
 """
 capa freeze file format: `| capa0000 | + zlib(utf-8(json(...)))`
 
-Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+json format:
+
+    {
+      'version': 1,
+      'functions': {
+        int(function va): {
+          'basic blocks': {
+            int(basic block va): {
+              'instructions': [instruction va, ...]
+            },
+            ...
+          },
+          ...
+        },
+        ...
+      },
+      'scopes': {
+        'file': [
+          (str(name), [any(arg), ...], int(va), ()),
+          ...
+        },
+        'function': [
+          (str(name), [any(arg), ...], int(va), (int(function va), )),
+          ...
+        ],
+        'basic block': [
+          (str(name), [any(arg), ...], int(va), (int(function va),
+                                                 int(basic block va))),
+          ...
+        ],
+        'instruction': [
+          (str(name), [any(arg), ...], int(va), (int(function va),
+                                                 int(basic block va),
+                                                 int(instruction va))),
+          ...
+        ],
+      }
+    }
+
+Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
 You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -9,351 +48,162 @@ Unless required by applicable law or agreed to in writing, software distributed
  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
+import json
 import zlib
 import logging
-import collections
-from enum import Enum
-from typing import Any, Set, Dict, List, Type, Tuple
 
-from pydantic import Field, BaseModel
-
-import capa.helpers
+import capa.features
 import capa.features.file
 import capa.features.insn
-import capa.features.common
-import capa.features.address
 import capa.features.basicblock
-import capa.features.extractors.base_extractor
-from capa.helpers import assert_never
+import capa.features.extractors
+from capa.helpers import hex
 
 logger = logging.getLogger(__name__)
 
 
-class HashableModel(BaseModel):
-    class Config:
-        frozen = True
+def serialize_feature(feature):
+    return feature.freeze_serialize()
 
 
-class AddressType(str, Enum):
-    ABSOLUTE = "absolute"
-    RELATIVE = "relative"
-    FILE = "file"
-    DN_TOKEN = "dn token"
-    DN_TOKEN_OFFSET = "dn token offset"
-    NO_ADDRESS = "no address"
+KNOWN_FEATURES = {F.__name__: F for F in capa.features.Feature.__subclasses__()}
 
 
-class Address(HashableModel):
-    type: AddressType
-    value: Any
-
-    @classmethod
-    def from_capa(cls, a: capa.features.address.Address) -> "Address":
-        if isinstance(a, capa.features.address.AbsoluteVirtualAddress):
-            return cls(type=AddressType.ABSOLUTE, value=int(a))
-
-        elif isinstance(a, capa.features.address.RelativeVirtualAddress):
-            return cls(type=AddressType.RELATIVE, value=int(a))
-
-        elif isinstance(a, capa.features.address.FileOffsetAddress):
-            return cls(type=AddressType.FILE, value=int(a))
-
-        elif isinstance(a, capa.features.address.DNTokenAddress):
-            # TODO: probably need serialization here
-            return cls(type=AddressType.DN_TOKEN, value=a.token)
-
-        elif isinstance(a, capa.features.address.DNTokenOffsetAddress):
-            # TODO: probably need serialization here
-            return cls(type=AddressType.DN_TOKEN_OFFSET, value=(a.token, a.offset))
-
-        elif a == capa.features.address.NO_ADDRESS or isinstance(a, capa.features.address._NoAddress):
-            return cls(type=AddressType.NO_ADDRESS, value=None)
-
-        elif isinstance(a, capa.features.address.Address) and not issubclass(type(a), capa.features.address.Address):
-            raise ValueError("don't use an Address instance directly")
-
-        elif isinstance(a, capa.features.address.Address):
-            raise ValueError("don't use an Address instance directly")
-
-        else:
-            assert_never(a)
-
-    def to_capa(self) -> capa.features.address.Address:
-        if self.type is AddressType.ABSOLUTE:
-            return capa.features.address.AbsoluteVirtualAddress(self.value)
-
-        elif self.type is AddressType.RELATIVE:
-            return capa.features.address.RelativeVirtualAddress(self.value)
-
-        elif self.type is AddressType.FILE:
-            return capa.features.address.FileOffsetAddress(self.value)
-
-        elif self.type is AddressType.DN_TOKEN:
-            return capa.features.address.DNTokenAddress(self.value)
-
-        elif self.type is AddressType.DN_TOKEN_OFFSET:
-            return capa.features.address.DNTokenOffsetAddress(*self.value)
-
-        elif self.type is AddressType.NO_ADDRESS:
-            return capa.features.address.NO_ADDRESS
-
-        else:
-            assert_never(self.type)
+def deserialize_feature(doc):
+    F = KNOWN_FEATURES[doc[0]]
+    return F.freeze_deserialize(doc[1])
 
 
-KNOWN_FEATURES: Dict[str, Type[capa.features.common.Feature]] = {
-    F.__name__: F for F in capa.features.common.Feature.__subclasses__()
-}
-KNOWN_FEATURES.update({F.__name__: F for F in capa.features.insn._Operand.__subclasses__()})  # type: ignore
-
-
-class Feature(HashableModel):
-    name: str
-    args: Tuple[Any, ...]
-
-    @classmethod
-    def from_capa(cls, f: capa.features.common.Feature) -> "Feature":
-        name, args = f.freeze_serialize()
-        return cls(name=name, args=tuple(args))
-
-    def to_capa(self) -> capa.features.common.Feature:
-        F = KNOWN_FEATURES[self.name]
-        return F.freeze_deserialize(self.args)
-
-
-class GlobalFeature(HashableModel):
-    feature: Feature
-
-
-class FileFeature(HashableModel):
-    address: Address
-    feature: Feature
-
-
-class FunctionFeature(HashableModel):
-    """
-    args:
-        function: the address of the function to which this feature belongs.
-        address: the address at which this feature is found.
-
-    function != address because, e.g., the feature may be found *within* the scope (function).
-    versus right at its starting address.
-    """
-
-    function: Address
-    address: Address
-    feature: Feature
-
-
-class BasicBlockFeature(HashableModel):
-    """
-    args:
-        basic_block: the address of the basic block to which this feature belongs.
-        address: the address at which this feature is found.
-
-    basic_block != address because, e.g., the feature may be found *within* the scope (basic block).
-    versus right at its starting address.
-    """
-
-    basic_block: Address
-    address: Address
-    feature: Feature
-
-
-class InstructionFeature(HashableModel):
-    """
-    args:
-        instruction: the address of the instruction to which this feature belongs.
-        address: the address at which this feature is found.
-
-    instruction != address because, e.g., the feature may be found *within* the scope (basic block),
-    versus right at its starting address.
-    """
-
-    instruction: Address
-    address: Address
-    feature: Feature
-
-
-class InstructionFeatures(BaseModel):
-    address: Address
-    features: List[InstructionFeature]
-
-
-class BasicBlockFeatures(BaseModel):
-    address: Address
-    features: List[BasicBlockFeature]
-    instructions: List[InstructionFeatures]
-
-
-class FunctionFeatures(BaseModel):
-    address: Address
-    features: List[FunctionFeature]
-    basic_blocks: List[BasicBlockFeatures] = Field(alias="basic block")
-
-    class Config:
-        allow_population_by_field_name = True
-
-
-class Features(BaseModel):
-    global_: List[GlobalFeature] = Field(alias="global")
-    file: List[FileFeature]
-    functions: List[FunctionFeatures]
-
-    class Config:
-        allow_population_by_field_name = True
-
-
-class Freeze(BaseModel):
-    version: int = 2
-    base_address: Address = Field(alias="base address")
-    features: Features
-
-    class Config:
-        allow_population_by_field_name = True
-
-
-def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> str:
+def dumps(extractor):
     """
     serialize the given extractor to a string
+
+    args:
+      extractor: capa.features.extractor.FeatureExtractor:
+
+    returns:
+      str: the serialized features.
     """
+    ret = {"version": 1, "functions": {}, "scopes": {"file": [], "function": [], "basic block": [], "instruction": [],}}
 
-    global_features: List[GlobalFeature] = []
-    for feature, _ in extractor.extract_global_features():
-        global_features.append(
-            GlobalFeature(
-                feature=Feature.from_capa(feature),
-            )
-        )
+    for feature, va in extractor.extract_file_features():
+        ret["scopes"]["file"].append(serialize_feature(feature) + (hex(va), ()))
 
-    file_features: List[FileFeature] = []
-    for feature, address in extractor.extract_file_features():
-        file_features.append(
-            FileFeature(
-                feature=Feature.from_capa(feature),
-                address=Address.from_capa(address),
-            )
-        )
-
-    function_features: List[FunctionFeatures] = []
     for f in extractor.get_functions():
-        faddr = Address.from_capa(f.address)
-        ffeatures = [
-            FunctionFeature(
-                function=faddr,
-                address=Address.from_capa(addr),
-                feature=Feature.from_capa(feature),
-            )
-            for feature, addr in extractor.extract_function_features(f)
-        ]
+        ret["functions"][hex(f)] = {}
+
+        for feature, va in extractor.extract_function_features(f):
+            ret["scopes"]["function"].append(serialize_feature(feature) + (hex(va), (hex(f),)))
 
-        basic_blocks = []
         for bb in extractor.get_basic_blocks(f):
-            bbaddr = Address.from_capa(bb.address)
-            bbfeatures = [
-                BasicBlockFeature(
-                    basic_block=bbaddr,
-                    address=Address.from_capa(addr),
-                    feature=Feature.from_capa(feature),
-                )
-                for feature, addr in extractor.extract_basic_block_features(f, bb)
-            ]
+            ret["functions"][hex(f)][hex(bb)] = []
 
-            instructions = []
-            for insn in extractor.get_instructions(f, bb):
-                iaddr = Address.from_capa(insn.address)
-                ifeatures = [
-                    InstructionFeature(
-                        instruction=iaddr,
-                        address=Address.from_capa(addr),
-                        feature=Feature.from_capa(feature),
+            for feature, va in extractor.extract_basic_block_features(f, bb):
+                ret["scopes"]["basic block"].append(serialize_feature(feature) + (hex(va), (hex(f), hex(bb),)))
+
+            for insnva, insn in sorted(
+                [(insn.__int__(), insn) for insn in extractor.get_instructions(f, bb)], key=lambda p: p[0]
+            ):
+                ret["functions"][hex(f)][hex(bb)].append(hex(insnva))
+
+                for feature, va in extractor.extract_insn_features(f, bb, insn):
+                    ret["scopes"]["instruction"].append(
+                        serialize_feature(feature) + (hex(va), (hex(f), hex(bb), hex(insnva),))
                     )
-                    for feature, addr in extractor.extract_insn_features(f, bb, insn)
-                ]
-                instructions.append(
-                    InstructionFeatures(
-                        address=iaddr,
-                        features=ifeatures,
-                    )
-                )
-
-            basic_blocks.append(
-                BasicBlockFeatures(
-                    address=bbaddr,
-                    features=bbfeatures,
-                    instructions=instructions,
-                )
-            )
-
-        function_features.append(
-            FunctionFeatures(
-                address=faddr,
-                features=ffeatures,
-                basic_blocks=basic_blocks,
-            )
-        )
-
-    features = Features(
-        global_=global_features,
-        file=file_features,
-        functions=function_features,
-    )
-
-    freeze = Freeze(
-        version=2,
-        base_address=Address.from_capa(extractor.get_base_address()),
-        features=features,
-    )
-
-    return freeze.json()
+    return json.dumps(ret)
 
 
-def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
+def loads(s):
     """deserialize a set of features (as a NullFeatureExtractor) from a string."""
-    import capa.features.extractors.null as null
+    doc = json.loads(s)
 
-    freeze = Freeze.parse_raw(s)
-    if freeze.version != 2:
-        raise ValueError("unsupported freeze format version: %d", freeze.version)
+    if doc.get("version") != 1:
+        raise ValueError("unsupported freeze format version: %d" % (doc.get("version")))
 
-    return null.NullFeatureExtractor(
-        base_address=freeze.base_address.to_capa(),
-        global_features=[f.feature.to_capa() for f in freeze.features.global_],
-        file_features=[(f.address.to_capa(), f.feature.to_capa()) for f in freeze.features.file],
-        functions={
-            f.address.to_capa(): null.FunctionFeatures(
-                features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in f.features],
-                basic_blocks={
-                    bb.address.to_capa(): null.BasicBlockFeatures(
-                        features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in bb.features],
-                        instructions={
-                            i.address.to_capa(): null.InstructionFeatures(
-                                features=[(fe.address.to_capa(), fe.feature.to_capa()) for fe in i.features]
-                            )
-                            for i in bb.instructions
-                        },
-                    )
-                    for bb in f.basic_blocks
-                },
-            )
-            for f in freeze.features.functions
-        },
-    )
+    features = {
+        "file features": [],
+        "functions": {},
+    }
+
+    for fva, function in doc.get("functions", {}).items():
+        fva = int(fva, 0x10)
+        features["functions"][fva] = {
+            "features": [],
+            "basic blocks": {},
+        }
+
+        for bbva, bb in function.items():
+            bbva = int(bbva, 0x10)
+            features["functions"][fva]["basic blocks"][bbva] = {
+                "features": [],
+                "instructions": {},
+            }
+
+            for insnva in bb:
+                insnva = int(insnva, 0x10)
+                features["functions"][fva]["basic blocks"][bbva]["instructions"][insnva] = {
+                    "features": [],
+                }
+
+    # in the following blocks, each entry looks like:
+    #
+    #     ('MatchedRule', ('foo', ), '0x401000', ('0x401000', ))
+    #      ^^^^^^^^^^^^^  ^^^^^^^^^  ^^^^^^^^^^  ^^^^^^^^^^^^^^
+    #      feature name   args       addr         func/bb/insn
+    for feature in doc.get("scopes", {}).get("file", []):
+        va, loc = feature[2:]
+        va = int(va, 0x10)
+        feature = deserialize_feature(feature[:2])
+        features["file features"].append((va, feature))
+
+    for feature in doc.get("scopes", {}).get("function", []):
+        # fetch the pair like:
+        #
+        #     ('0x401000', ('0x401000', ))
+        #      ^^^^^^^^^^  ^^^^^^^^^^^^^^
+        #      addr         func/bb/insn
+        va, loc = feature[2:]
+        va = int(va, 0x10)
+        loc = [int(lo, 0x10) for lo in loc]
+
+        # decode the feature from the pair like:
+        #
+        #     ('MatchedRule', ('foo', ))
+        #      ^^^^^^^^^^^^^  ^^^^^^^^^
+        #      feature name   args
+        feature = deserialize_feature(feature[:2])
+        features["functions"][loc[0]]["features"].append((va, feature))
+
+    for feature in doc.get("scopes", {}).get("basic block", []):
+        va, loc = feature[2:]
+        va = int(va, 0x10)
+        loc = [int(lo, 0x10) for lo in loc]
+        feature = deserialize_feature(feature[:2])
+        features["functions"][loc[0]]["basic blocks"][loc[1]]["features"].append((va, feature))
+
+    for feature in doc.get("scopes", {}).get("instruction", []):
+        va, loc = feature[2:]
+        va = int(va, 0x10)
+        loc = [int(lo, 0x10) for lo in loc]
+        feature = deserialize_feature(feature[:2])
+        features["functions"][loc[0]]["basic blocks"][loc[1]]["instructions"][loc[2]]["features"].append((va, feature))
+
+    return capa.features.extractors.NullFeatureExtractor(features)
 
 
 MAGIC = "capa0000".encode("ascii")
 
 
-def dump(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> bytes:
+def dump(extractor):
     """serialize the given extractor to a byte array."""
     return MAGIC + zlib.compress(dumps(extractor).encode("utf-8"))
 
 
-def is_freeze(buf: bytes) -> bool:
+def is_freeze(buf):
     return buf[: len(MAGIC)] == MAGIC
 
 
-def load(buf: bytes) -> capa.features.extractors.base_extractor.FeatureExtractor:
+def load(buf):
     """deserialize a set of features (as a NullFeatureExtractor) from a byte array."""
     if not is_freeze(buf):
         raise ValueError("missing magic header")
@@ -369,16 +219,35 @@ def main(argv=None):
     if argv is None:
         argv = sys.argv[1:]
 
+    formats = [
+        ("auto", "(default) detect file type automatically"),
+        ("pe", "Windows PE file"),
+        ("sc32", "32-bit shellcode"),
+        ("sc64", "64-bit shellcode"),
+    ]
+    format_help = ", ".join(["%s: %s" % (f[0], f[1]) for f in formats])
+
     parser = argparse.ArgumentParser(description="save capa features to a file")
-    capa.main.install_common_args(parser, {"sample", "format", "backend", "signatures"})
+    parser.add_argument("sample", type=str, help="Path to sample to analyze")
     parser.add_argument("output", type=str, help="Path to output file")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose output")
+    parser.add_argument("-q", "--quiet", action="store_true", help="Disable all output but errors")
+    parser.add_argument(
+        "-f", "--format", choices=[f[0] for f in formats], default="auto", help="Select sample format, %s" % format_help
+    )
     args = parser.parse_args(args=argv)
-    capa.main.handle_common_args(args)
 
-    sigpaths = capa.main.get_signatures(args.signatures)
-
-    extractor = capa.main.get_extractor(args.sample, args.format, args.backend, sigpaths, False)
+    if args.quiet:
+        logging.basicConfig(level=logging.ERROR)
+        logging.getLogger().setLevel(logging.ERROR)
+    elif args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+        logging.getLogger().setLevel(logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+        logging.getLogger().setLevel(logging.INFO)
 
+    extractor = capa.main.get_extractor(args.sample, args.format)
     with open(args.output, "wb") as f:
         f.write(dump(extractor))
 

capa/features/insn.py

@@ -1,99 +1,43 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import abc
-from typing import Union
 
-import capa.render.utils
-from capa.features.common import Feature
+from capa.features import Feature
 
 
 class API(Feature):
-    def __init__(self, name: str, description=None):
-        super(API, self).__init__(name, description=description)
+    def __init__(self, name, description=None):
+        # Downcase library name if given
+        if "." in name:
+            modname, impname = name.split(".")
+            name = modname.lower() + "." + impname
+
+        super(API, self).__init__(name, description)
 
 
 class Number(Feature):
-    def __init__(self, value: Union[int, float], description=None):
-        super(Number, self).__init__(value, description=description)
+    def __init__(self, value, arch=None, description=None):
+        super(Number, self).__init__(value, arch=arch, description=description)
 
     def get_value_str(self):
-        if isinstance(self.value, int):
-            return capa.render.utils.hex(self.value)
-        elif isinstance(self.value, float):
-            return str(self.value)
+        if self.value < 0:
+            return "-0x%X" % (-self.value)
         else:
-            raise ValueError("invalid value type")
-
-
-# max recognized structure size (and therefore, offset size)
-MAX_STRUCTURE_SIZE = 0x10000
+            return "0x%X" % self.value
 
 
 class Offset(Feature):
-    def __init__(self, value: int, description=None):
-        super(Offset, self).__init__(value, description=description)
+    def __init__(self, value, arch=None, description=None):
+        super(Offset, self).__init__(value, arch=arch, description=description)
 
     def get_value_str(self):
-        return capa.render.utils.hex(self.value)
+        return "0x%X" % self.value
 
 
 class Mnemonic(Feature):
-    def __init__(self, value: str, description=None):
+    def __init__(self, value, description=None):
         super(Mnemonic, self).__init__(value, description=description)
-
-
-# max number of operands to consider for a given instrucion.
-# since we only support Intel and .NET, we can assume this is 3
-# which covers cases up to e.g. "vinserti128 ymm0,ymm0,ymm5,1"
-MAX_OPERAND_COUNT = 4
-MAX_OPERAND_INDEX = MAX_OPERAND_COUNT - 1
-
-
-class _Operand(Feature, abc.ABC):
-    # superclass: don't use directly
-    # subclasses should set self.name and provide the value string formatter
-    def __init__(self, index: int, value: int, description=None):
-        super(_Operand, self).__init__(value, description=description)
-        self.index = index
-
-    def __hash__(self):
-        return hash((self.name, self.value))
-
-    def __eq__(self, other):
-        return super().__eq__(other) and self.index == other.index
-
-    def freeze_serialize(self):
-        return (self.__class__.__name__, (self.index, self.value))
-
-
-class OperandNumber(_Operand):
-    # cached names so we don't do extra string formatting every ctor
-    NAMES = ["operand[%d].number" % i for i in range(MAX_OPERAND_COUNT)]
-
-    # operand[i].number: 0x12
-    def __init__(self, index: int, value: int, description=None):
-        super(OperandNumber, self).__init__(index, value, description=description)
-        self.name = self.NAMES[index]
-
-    def get_value_str(self) -> str:
-        assert isinstance(self.value, int)
-        return capa.render.utils.hex(self.value)
-
-
-class OperandOffset(_Operand):
-    # cached names so we don't do extra string formatting every ctor
-    NAMES = ["operand[%d].offset" % i for i in range(MAX_OPERAND_COUNT)]
-
-    # operand[i].offset: 0x12
-    def __init__(self, index: int, value: int, description=None):
-        super(OperandOffset, self).__init__(index, value, description=description)
-        self.name = self.NAMES[index]
-
-    def get_value_str(self) -> str:
-        assert isinstance(self.value, int)
-        return capa.render.utils.hex(self.value)

capa/helpers.py

@@ -1,118 +1,36 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+
 import os
-import logging
-from typing import NoReturn
-
-from capa.exceptions import UnsupportedFormatError
-from capa.features.common import FORMAT_SC32, FORMAT_SC64, FORMAT_UNKNOWN
-
-EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32")
-EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64")
-
-
-logger = logging.getLogger("capa")
 
 _hex = hex
 
 
 def hex(i):
-    return _hex(int(i))
+    # under py2.7, long integers get formatted with a trailing `L`
+    # and this is not pretty. so strip it out.
+    return _hex(oint(i)).rstrip("L")
 
 
-def get_file_taste(sample_path: str) -> bytes:
+def oint(i):
+    # there seems to be some trouble with using `int(viv_utils.Function)`
+    # with the black magic we do with binding the `__int__()` routine.
+    # i haven't had a chance to debug this yet (and i have no hotel wifi).
+    # so in the meantime, detect this, and call the method directly.
+    try:
+        return int(i)
+    except TypeError:
+        return i.__int__()
+
+
+def get_file_taste(sample_path):
     if not os.path.exists(sample_path):
         raise IOError("sample path %s does not exist or cannot be accessed" % sample_path)
     with open(sample_path, "rb") as f:
         taste = f.read(8)
     return taste
-
-
-def is_runtime_ida():
-    try:
-        import idc
-    except ImportError:
-        return False
-    else:
-        return True
-
-
-def assert_never(value: NoReturn) -> NoReturn:
-    assert False, f"Unhandled value: {value} ({type(value).__name__})"
-
-
-def get_format_from_extension(sample: str) -> str:
-    if sample.endswith(EXTENSIONS_SHELLCODE_32):
-        return FORMAT_SC32
-    elif sample.endswith(EXTENSIONS_SHELLCODE_64):
-        return FORMAT_SC64
-    return FORMAT_UNKNOWN
-
-
-def get_auto_format(path: str) -> str:
-    format_ = get_format(path)
-    if format_ == FORMAT_UNKNOWN:
-        format_ = get_format_from_extension(path)
-    if format_ == FORMAT_UNKNOWN:
-        raise UnsupportedFormatError()
-    return format_
-
-
-def get_format(sample: str) -> str:
-    # imported locally to avoid import cycle
-    from capa.features.extractors.common import extract_format
-
-    with open(sample, "rb") as f:
-        buf = f.read()
-
-    for feature, _ in extract_format(buf):
-        assert isinstance(feature.value, str)
-        return feature.value
-
-    return FORMAT_UNKNOWN
-
-
-def log_unsupported_format_error():
-    logger.error("-" * 80)
-    logger.error(" Input file does not appear to be a PE or ELF file.")
-    logger.error(" ")
-    logger.error(
-        " capa currently only supports analyzing PE and ELF files (or shellcode, when using --format sc32|sc64)."
-    )
-    logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
-    logger.error("-" * 80)
-
-
-def log_unsupported_os_error():
-    logger.error("-" * 80)
-    logger.error(" Input file does not appear to target a supported OS.")
-    logger.error(" ")
-    logger.error(
-        " capa currently only supports analyzing executables for some operating systems (including Windows and Linux)."
-    )
-    logger.error("-" * 80)
-
-
-def log_unsupported_arch_error():
-    logger.error("-" * 80)
-    logger.error(" Input file does not appear to target a supported architecture.")
-    logger.error(" ")
-    logger.error(" capa currently only supports analyzing x86 (32- and 64-bit).")
-    logger.error("-" * 80)
-
-
-def log_unsupported_runtime_error():
-    logger.error("-" * 80)
-    logger.error(" Unsupported runtime or Python interpreter.")
-    logger.error(" ")
-    logger.error(" capa supports running under Python 3.7 and higher.")
-    logger.error(" ")
-    logger.error(
-        " If you're seeing this message on the command line, please ensure you're running a supported Python version."
-    )
-    logger.error("-" * 80)

capa/ida/explorer/item.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,6 +6,7 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+import sys
 import codecs
 
 import idc
@@ -16,9 +17,9 @@ import capa.ida.helpers
 
 
 def info_to_name(display):
-    """extract root value from display name
+    """ extract root value from display name
 
-    e.g. function(my_function) => my_function
+        e.g. function(my_function) => my_function
     """
     try:
         return display.split("(")[1].rstrip(")")
@@ -27,326 +28,247 @@ def info_to_name(display):
 
 
 def location_to_hex(location):
-    """convert location to hex for display"""
+    """ convert location to hex for display """
     return "%08X" % location
 
 
-class CapaExplorerDataItem:
-    """store data for CapaExplorerDataModel"""
+class CapaExplorerDataItem(object):
+    """ store data for CapaExplorerDataModel """
 
-    def __init__(self, parent, data, can_check=True):
-        """initialize item"""
+    def __init__(self, parent, data):
+        """ """
         self.pred = parent
         self._data = data
         self.children = []
         self._checked = False
-        self._can_check = can_check
 
-        # default state for item
-        self.flags = QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable
-
-        if self._can_check:
-            self.flags = self.flags | QtCore.Qt.ItemIsUserCheckable | QtCore.Qt.ItemIsTristate
+        self.flags = (
+            QtCore.Qt.ItemIsEnabled
+            | QtCore.Qt.ItemIsSelectable
+            | QtCore.Qt.ItemIsTristate
+            | QtCore.Qt.ItemIsUserCheckable
+        )
 
         if self.pred:
             self.pred.appendChild(self)
 
     def setIsEditable(self, isEditable=False):
-        """modify item editable flags
-
-        @param isEditable: True, can edit, False cannot edit
-        """
+        """ modify item flags to be editable or not """
         if isEditable:
             self.flags |= QtCore.Qt.ItemIsEditable
         else:
             self.flags &= ~QtCore.Qt.ItemIsEditable
 
     def setChecked(self, checked):
-        """set item as checked
-
-        @param checked: True, item checked, False item not checked
-        """
+        """ set item as checked """
         self._checked = checked
 
-    def canCheck(self):
-        """ """
-        return self._can_check
-
     def isChecked(self):
-        """get item is checked"""
+        """ get item is checked """
         return self._checked
 
     def appendChild(self, item):
-        """add a new child to specified item
+        """ add child item
 
-        @param item: CapaExplorerDataItem
+            @param item: CapaExplorerDataItem*
         """
         self.children.append(item)
 
     def child(self, row):
-        """get child row
+        """ get child row
 
-        @param row: row number
+            @param row: TODO
         """
         return self.children[row]
 
     def childCount(self):
-        """get child count"""
+        """ get child count """
         return len(self.children)
 
     def columnCount(self):
-        """get column count"""
+        """ get column count """
         return len(self._data)
 
     def data(self, column):
-        """get data at column
-
-        @param: column number
-        """
+        """ get data at column """
         try:
             return self._data[column]
         except IndexError:
             return None
 
     def parent(self):
-        """get parent"""
+        """ get parent """
         return self.pred
 
     def row(self):
-        """get row location"""
+        """ get row location """
         if self.pred:
             return self.pred.children.index(self)
         return 0
 
     def setData(self, column, value):
-        """set data in column
-
-        @param column: column number
-        @value: value to set (assume str)
-        """
+        """ set data in column """
         self._data[column] = value
 
     def children(self):
-        """yield children"""
+        """ yield children """
         for child in self.children:
             yield child
 
     def removeChildren(self):
-        """remove children"""
+        """ remove children from node """
         del self.children[:]
 
     def __str__(self):
-        """get string representation of columns
-
-        used for copy-n-paste operations
-        """
+        """ get string representation of columns """
         return " ".join([data for data in self._data if data])
 
     @property
     def info(self):
-        """return data stored in information column"""
+        """ return data stored in information column """
         return self._data[0]
 
     @property
     def location(self):
-        """return data stored in location column"""
+        """ return data stored in location column """
         try:
-            # address stored as str, convert to int before return
             return int(self._data[1], 16)
         except ValueError:
             return None
 
     @property
     def details(self):
-        """return data stored in details column"""
+        """ return data stored in details column """
         return self._data[2]
 
 
 class CapaExplorerRuleItem(CapaExplorerDataItem):
-    """store data for rule result"""
+    """ store data relevant to capa function result """
 
     fmt = "%s (%d matches)"
 
-    def __init__(self, parent, name, namespace, count, source, can_check=True):
-        """initialize item
-
-        @param parent: parent node
-        @param name: rule name
-        @param namespace: rule namespace
-        @param count: number of match for this rule
-        @param source: rule source (tooltip)
-        """
-        display = self.fmt % (name, count) if count > 1 else name
-        super(CapaExplorerRuleItem, self).__init__(parent, [display, "", namespace], can_check)
+    def __init__(self, parent, display, count, source):
+        """ """
+        display = self.fmt % (display, count) if count > 1 else display
+        super(CapaExplorerRuleItem, self).__init__(parent, [display, "", ""])
         self._source = source
 
     @property
     def source(self):
-        """return rule source to display (tooltip)"""
+        """ return rule contents for display """
         return self._source
 
 
 class CapaExplorerRuleMatchItem(CapaExplorerDataItem):
-    """store data for rule match"""
+    """ store data relevant to capa function match result """
 
     def __init__(self, parent, display, source=""):
-        """initialize item
-
-        @param parent: parent node
-        @param display: text to display in UI
-        @param source: rule match source to display (tooltip)
-        """
+        """ """
         super(CapaExplorerRuleMatchItem, self).__init__(parent, [display, "", ""])
         self._source = source
 
     @property
     def source(self):
-        """return rule contents for display"""
+        """ return rule contents for display """
         return self._source
 
 
 class CapaExplorerFunctionItem(CapaExplorerDataItem):
-    """store data for function match"""
+    """ store data relevant to capa function result """
 
     fmt = "function(%s)"
 
-    def __init__(self, parent, location, can_check=True):
-        """initialize item
-
-        @param parent: parent node
-        @param location: virtual address of function as seen by IDA
-        """
-        # location can be an Address now, so need to get the VA
-        location = int(location)
+    def __init__(self, parent, location):
+        """ """
         super(CapaExplorerFunctionItem, self).__init__(
-            parent, [self.fmt % idaapi.get_name(location), location_to_hex(location), ""], can_check
+            parent, [self.fmt % idaapi.get_name(location), location_to_hex(location), ""]
         )
 
     @property
     def info(self):
-        """return function name"""
+        """ """
         info = super(CapaExplorerFunctionItem, self).info
         display = info_to_name(info)
         return display if display else info
 
     @info.setter
     def info(self, display):
-        """set function name
-
-        called when user changes function name in plugin UI
-
-        @param display: new function name to display
-        """
+        """ """
         self._data[0] = self.fmt % display
 
 
 class CapaExplorerSubscopeItem(CapaExplorerDataItem):
-    """store data for subscope match"""
+    """ store data relevant to subscope """
 
     fmt = "subscope(%s)"
 
     def __init__(self, parent, scope):
-        """initialize item
-
-        @param parent: parent node
-        @param scope: subscope name
-        """
+        """ """
         super(CapaExplorerSubscopeItem, self).__init__(parent, [self.fmt % scope, "", ""])
 
 
 class CapaExplorerBlockItem(CapaExplorerDataItem):
-    """store data for basic block match"""
+    """ store data relevant to capa basic block result """
 
     fmt = "basic block(loc_%08X)"
 
     def __init__(self, parent, location):
-        """initialize item
-
-        @param parent: parent node
-        @param location: virtual address of basic block as seen by IDA
-        """
+        """ """
         super(CapaExplorerBlockItem, self).__init__(parent, [self.fmt % location, location_to_hex(location), ""])
 
 
 class CapaExplorerDefaultItem(CapaExplorerDataItem):
-    """store data for default match e.g. statement (and, or)"""
+    """ store data relevant to capa default result """
 
     def __init__(self, parent, display, details="", location=None):
-        """initialize item
-
-        @param parent: parent node
-        @param display: text to display in UI
-        @param details: text to display in details section of UI
-        @param location: virtual address as seen by IDA
-        """
+        """ """
         location = location_to_hex(location) if location else ""
         super(CapaExplorerDefaultItem, self).__init__(parent, [display, location, details])
 
 
 class CapaExplorerFeatureItem(CapaExplorerDataItem):
-    """store data for feature match"""
+    """ store data relevant to capa feature result """
 
     def __init__(self, parent, display, location="", details=""):
-        """initialize item
-
-        @param parent: parent node
-        @param display: text to display in UI
-        @param details: text to display in details section of UI
-        @param location: virtual address as seen by IDA
-        """
+        """ """
         location = location_to_hex(location) if location else ""
         super(CapaExplorerFeatureItem, self).__init__(parent, [display, location, details])
 
 
 class CapaExplorerInstructionViewItem(CapaExplorerFeatureItem):
-    """store data for instruction match"""
+    """ store data relevant to an instruction preview """
 
     def __init__(self, parent, display, location):
-        """initialize item
-
-        details section shows disassembly view for match
-
-        @param parent: parent node
-        @param display: text to display in UI
-        @param location: virtual address as seen by IDA
-        """
+        """ """
         details = capa.ida.helpers.get_disasm_line(location)
         super(CapaExplorerInstructionViewItem, self).__init__(parent, display, location=location, details=details)
         self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)
 
 
 class CapaExplorerByteViewItem(CapaExplorerFeatureItem):
-    """store data for byte match"""
+    """ store data relevant to byte preview """
 
     def __init__(self, parent, display, location):
-        """initialize item
-
-        details section shows byte preview for match
-
-        @param parent: parent node
-        @param display: text to display in UI
-        @param location: virtual address as seen by IDA
-        """
+        """ """
         byte_snap = idaapi.get_bytes(location, 32)
 
-        details = ""
         if byte_snap:
             byte_snap = codecs.encode(byte_snap, "hex").upper()
-            details = " ".join([byte_snap[i : i + 2].decode() for i in range(0, len(byte_snap), 2)])
+            if sys.version_info >= (3, 0):
+                details = " ".join([byte_snap[i : i + 2].decode() for i in range(0, len(byte_snap), 2)])
+            else:
+                details = " ".join([byte_snap[i : i + 2] for i in range(0, len(byte_snap), 2)])
+        else:
+            details = ""
 
         super(CapaExplorerByteViewItem, self).__init__(parent, display, location=location, details=details)
         self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)
 
 
 class CapaExplorerStringViewItem(CapaExplorerFeatureItem):
-    """store data for string match"""
+    """ store data relevant to string preview """
 
-    def __init__(self, parent, display, location, value):
-        """initialize item
-
-        @param parent: parent node
-        @param display: text to display in UI
-        @param location: virtual address as seen by IDA
-        """
-        super(CapaExplorerStringViewItem, self).__init__(parent, display, location=location, details=value)
+    def __init__(self, parent, display, location):
+        """ """
+        super(CapaExplorerStringViewItem, self).__init__(parent, display, location=location)
         self.ida_highlight = idc.get_color(location, idc.CIC_ITEM)

capa/ida/explorer/model.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at: [package root]/LICENSE.txt
@@ -6,18 +6,16 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
-from typing import List
 from collections import deque
 
 import idc
+import six
 import idaapi
-from PyQt5 import QtGui, QtCore
+from PyQt5 import Qt, QtGui, QtCore
 
-import capa.rules
 import capa.ida.helpers
 import capa.render.utils as rutils
-import capa.features.common
-from capa.ida.plugin.item import (
+from capa.ida.explorer.item import (
     CapaExplorerDataItem,
     CapaExplorerRuleItem,
     CapaExplorerBlockItem,
@@ -32,11 +30,11 @@ from capa.ida.plugin.item import (
 )
 
 # default highlight color used in IDA window
-DEFAULT_HIGHLIGHT = 0xE6C700
+DEFAULT_HIGHLIGHT = 0xD096FF
 
 
 class CapaExplorerDataModel(QtCore.QAbstractItemModel):
-    """model for displaying hierarchical results return by capa"""
+    """ """
 
     COLUMN_INDEX_RULE_INFORMATION = 0
     COLUMN_INDEX_VIRTUAL_ADDRESS = 1
@@ -45,16 +43,14 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
     COLUMN_COUNT = 3
 
     def __init__(self, parent=None):
-        """initialize model"""
+        """ """
         super(CapaExplorerDataModel, self).__init__(parent)
-        # root node does not have parent, contains header columns
         self.root_node = CapaExplorerDataItem(None, ["Rule Information", "Address", "Details"])
 
     def reset(self):
-        """reset UI elements (e.g. checkboxes, IDA color highlights)
-
-        called when view wants to reset UI display
-        """
+        """ """
+        # reset checkboxes and color highlights
+        # TODO: make less hacky
         for idx in range(self.root_node.childCount()):
             root_index = self.index(idx, 0, QtCore.QModelIndex())
             for model_index in self.iterateChildrenIndexFromRootIndex(root_index, ignore_root=False):
@@ -63,20 +59,17 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                 self.dataChanged.emit(model_index, model_index)
 
     def clear(self):
-        """clear model data
-
-        called when view wants to clear UI display
-        """
+        """ """
         self.beginResetModel()
         self.root_node.removeChildren()
         self.endResetModel()
 
     def columnCount(self, model_index):
-        """return number of columns for the children of the given parent
+        """ get the number of columns for the children of the given parent
 
-        @param model_index: QModelIndex
+            @param model_index: QModelIndex*
 
-        @retval column count
+            @retval column count
         """
         if model_index.isValid():
             return model_index.internalPointer().columnCount()
@@ -84,14 +77,12 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             return self.root_node.columnCount()
 
     def data(self, model_index, role):
-        """return data stored at given index by display role
+        """ get data stored under the given role for the item referred to by the index
 
-        this function is used to control UI elements (e.g. text font, color, etc.) based on column, item type, etc.
+            @param model_index: QModelIndex*
+            @param role: QtCore.Qt.*
 
-        @param model_index: QModelIndex
-        @param role: QtCore.Qt.*
-
-        @retval data to be displayed
+            @retval data to be displayed
         """
         if not model_index.isValid():
             return None
@@ -113,8 +104,6 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
         if role == QtCore.Qt.CheckStateRole and column == CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION:
             # inform view how to display content of checkbox - un/checked
-            if not item.canCheck():
-                return None
             return QtCore.Qt.Checked if item.isChecked() else QtCore.Qt.Unchecked
 
         if role == QtCore.Qt.FontRole and column in (
@@ -142,14 +131,14 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             )
             and column == CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION
         ):
-            # set bold font for important items
+            # set bold font for top-level rules
             font = QtGui.QFont()
             font.setBold(True)
             return font
 
         if role == QtCore.Qt.ForegroundRole and column == CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS:
             # set color for virtual address column
-            return QtGui.QColor(37, 147, 215)
+            return QtGui.QColor(88, 139, 174)
 
         if (
             role == QtCore.Qt.ForegroundRole
@@ -162,11 +151,11 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return None
 
     def flags(self, model_index):
-        """return item flags for given index
+        """ get item flags for given index
 
-        @param model_index: QModelIndex
+            @param model_index: QModelIndex*
 
-        @retval QtCore.Qt.ItemFlags
+            @retval QtCore.Qt.ItemFlags
         """
         if not model_index.isValid():
             return QtCore.Qt.NoItemFlags
@@ -174,13 +163,13 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return model_index.internalPointer().flags
 
     def headerData(self, section, orientation, role):
-        """return data for the given role and section in the header with the specified orientation
+        """ get data for the given role and section in the header with the specified orientation
 
-        @param section: int
-        @param orientation: QtCore.Qt.Orientation
-        @param role: QtCore.Qt.DisplayRole
+            @param section: int
+            @param orientation: QtCore.Qt.Orientation
+            @param role: QtCore.Qt.DisplayRole
 
-        @retval header data
+            @retval header data list()
         """
         if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
             return self.root_node.data(section)
@@ -188,13 +177,13 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return None
 
     def index(self, row, column, parent):
-        """return index of the item by row, column, and parent index
+        """ get index of the item in the model specified by the given row, column and parent index
 
-        @param row: item row
-        @param column: item column
-        @param parent: QModelIndex of parent
+            @param row: int
+            @param column: int
+            @param parent: QModelIndex*
 
-        @retval QModelIndex of item
+            @retval QModelIndex*
         """
         if not self.hasIndex(row, column, parent):
             return QtCore.QModelIndex()
@@ -212,13 +201,13 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             return QtCore.QModelIndex()
 
     def parent(self, model_index):
-        """return parent index by child index
+        """ get parent of the model item with the given index
 
-        if the item has no parent, an invalid QModelIndex is returned
+            if the item has no parent, an invalid QModelIndex* is returned
 
-        @param model_index: QModelIndex of child
+            @param model_index: QModelIndex*
 
-        @retval QModelIndex of parent
+            @retval QModelIndex*
         """
         if not model_index.isValid():
             return QtCore.QModelIndex()
@@ -232,12 +221,12 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return self.createIndex(parent.row(), 0, parent)
 
     def iterateChildrenIndexFromRootIndex(self, model_index, ignore_root=True):
-        """depth-first traversal of child nodes
+        """ depth-first traversal of child nodes
 
-        @param model_index: QModelIndex of starting item
-        @param ignore_root: True, do not yield root index, False yield root index
+            @param model_index: QModelIndex*
+            @param ignore_root: if set, do not return root index
 
-        @retval yield QModelIndex
+            @retval yield QModelIndex*
         """
         visited = set()
         stack = deque((model_index,))
@@ -259,10 +248,10 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                     stack.append(child_index.child(idx, 0))
 
     def reset_ida_highlighting(self, item, checked):
-        """reset IDA highlight for item
+        """ reset IDA highlight for an item
 
-        @param item: CapaExplorerDataItem
-        @param checked: True, item checked, False item not checked
+            @param item: capa explorer item
+            @param checked: indicates item is or not checked
         """
         if not isinstance(
             item, (CapaExplorerStringViewItem, CapaExplorerInstructionViewItem, CapaExplorerByteViewItem)
@@ -286,11 +275,13 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                 idc.set_color(item.location, idc.CIC_ITEM, item.ida_highlight)
 
     def setData(self, model_index, value, role):
-        """set data at index by role
+        """ set the role data for the item at index to value
 
-        @param model_index: QModelIndex of item
-        @param value: value to set
-        @param role: QtCore.Qt.EditRole
+            @param model_index: QModelIndex*
+            @param value: QVariant*
+            @param role: QtCore.Qt.EditRole
+
+            @retval True/False
         """
         if not model_index.isValid():
             return False
@@ -325,13 +316,14 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return False
 
     def rowCount(self, model_index):
-        """return number of rows under item by index
+        """ get the number of rows under the given parent
 
-        when the parent is valid it means that is returning the number of children of parent
+            when the parent is valid it means that is returning the number of
+            children of parent
 
-        @param model_index: QModelIndex
+            @param model_index: QModelIndex*
 
-        @retval row count
+            @retval row count
         """
         if model_index.column() > 0:
             return 0
@@ -344,26 +336,24 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return item.childCount()
 
     def render_capa_doc_statement_node(self, parent, statement, locations, doc):
-        """render capa statement read from doc
+        """ render capa statement read from doc
 
-        @param parent: parent to which new child is assigned
-        @param statement: statement read from doc
-        @param locations: locations of children (applies to range only?)
-        @param doc: result doc
+            @param parent: parent to which new child is assigned
+            @param statement: statement read from doc
+            @param locations: locations of children (applies to range only?)
+            @param doc: capa result doc
+
+            "statement": {
+                "type": "or"
+            },
         """
         if statement["type"] in ("and", "or", "optional"):
-            display = statement["type"]
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
-            return CapaExplorerDefaultItem(parent, display)
+            return CapaExplorerDefaultItem(parent, statement["type"])
         elif statement["type"] == "not":
             # TODO: do we display 'not'
             pass
         elif statement["type"] == "some":
-            display = "%d or more" % statement["count"]
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
-            return CapaExplorerDefaultItem(parent, display)
+            return CapaExplorerDefaultItem(parent, str(statement["count"]) + " or more")
         elif statement["type"] == "range":
             # `range` is a weird node, its almost a hybrid of statement + feature.
             # it is a specific feature repeated multiple times.
@@ -380,9 +370,6 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             else:
                 display += "between %d and %d" % (statement["min"], statement["max"])
 
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
-
             parent2 = CapaExplorerFeatureItem(parent, display=display)
 
             for location in locations:
@@ -391,19 +378,33 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
             return parent2
         elif statement["type"] == "subscope":
-            display = statement[statement["type"]]
-            if statement.get("description"):
-                display += " (%s)" % statement["description"]
-            return CapaExplorerSubscopeItem(parent, display)
+            return CapaExplorerSubscopeItem(parent, statement[statement["type"]])
         else:
             raise RuntimeError("unexpected match statement type: " + str(statement))
 
     def render_capa_doc_match(self, parent, match, doc):
-        """render capa match read from doc
+        """ render capa match read from doc
 
-        @param parent: parent node to which new child is assigned
-        @param match: match read from doc
-        @param doc: result doc
+            @param parent: parent node to which new child is assigned
+            @param match: match read from doc
+            @param doc: capa result doc
+
+            "matches": {
+                "0": {
+                    "children": [],
+                    "locations": [
+                        4317184
+                    ],
+                    "node": {
+                        "feature": {
+                            "section": ".rsrc",
+                            "type": "section"
+                        },
+                        "type": "feature"
+                    },
+                    "success": true
+                }
+            },
         """
         if not match["success"]:
             # TODO: display failed branches at some point? Help with debugging rules?
@@ -429,40 +430,16 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         for child in match.get("children", []):
             self.render_capa_doc_match(parent2, child, doc)
 
-    def render_capa_doc_by_function(self, doc):
-        """ """
-        matches_by_function = {}
-        for rule in rutils.capability_rules(doc):
-            for ea in rule["matches"].keys():
-                ea = capa.ida.helpers.get_func_start_ea(ea)
-                if ea is None:
-                    # file scope, skip rendering in this mode
-                    continue
-                if not matches_by_function.get(ea, ()):
-                    # new function root
-                    matches_by_function[ea] = (CapaExplorerFunctionItem(self.root_node, ea, can_check=False), [])
-                function_root, match_cache = matches_by_function[ea]
-                if rule["meta"]["name"] in match_cache:
-                    # rule match already rendered for this function root, skip it
-                    continue
-                match_cache.append(rule["meta"]["name"])
-                CapaExplorerRuleItem(
-                    function_root,
-                    rule["meta"]["name"],
-                    rule["meta"].get("namespace"),
-                    len(rule["matches"]),
-                    rule["source"],
-                    can_check=False,
-                )
+    def render_capa_doc(self, doc):
+        """ render capa features specified in doc
+
+            @param doc: capa result doc
+        """
+        # inform model that changes are about to occur
+        self.beginResetModel()
 
-    def render_capa_doc_by_program(self, doc):
-        """ """
         for rule in rutils.capability_rules(doc):
-            rule_name = rule["meta"]["name"]
-            rule_namespace = rule["meta"].get("namespace")
-            parent = CapaExplorerRuleItem(
-                self.root_node, rule_name, rule_namespace, len(rule["matches"]), rule["source"]
-            )
+            parent = CapaExplorerRuleItem(self.root_node, rule["meta"]["name"], len(rule["matches"]), rule["source"])
 
             for (location, match) in doc["rules"][rule["meta"]["name"]]["matches"].items():
                 if rule["meta"]["scope"] == capa.rules.FILE_SCOPE:
@@ -476,58 +453,51 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
                 self.render_capa_doc_match(parent2, match, doc)
 
-    def render_capa_doc(self, doc, by_function):
-        """render capa features specified in doc
-
-        @param doc: capa result doc
-        """
-        # inform model that changes are about to occur
-        self.beginResetModel()
-
-        if by_function:
-            self.render_capa_doc_by_function(doc)
-        else:
-            self.render_capa_doc_by_program(doc)
-
         # inform model changes have ended
         self.endResetModel()
 
     def capa_doc_feature_to_display(self, feature):
-        """convert capa doc feature type string to display string for ui
+        """ convert capa doc feature type string to display string for ui
 
-        @param feature: capa feature read from doc
+            @param feature: capa feature read from doc
+
+            Example:
+                "feature": {
+                    "bytes": "01 14 02 00 00 00 00 00 C0 00 00 00 00 00 00 46",
+                    "description": "CLSID_ShellLink",
+                    "type": "bytes"
+                }
+
+                bytes(01 14 02 00 00 00 00 00 C0 00 00 00 00 00 00 46 = CLSID_ShellLink)
         """
-        key = feature["type"]
-        value = feature[feature["type"]]
-        if value:
-            if key == "string":
-                value = '"%s"' % capa.features.common.escape_string(value)
+        if feature[feature["type"]]:
             if feature.get("description", ""):
-                return "%s(%s = %s)" % (key, value, feature["description"])
+                return "%s(%s = %s)" % (feature["type"], feature[feature["type"]], feature["description"])
             else:
-                return "%s(%s)" % (key, value)
+                return "%s(%s)" % (feature["type"], feature[feature["type"]])
         else:
-            return "%s" % key
+            return "%s" % feature["type"]
 
     def render_capa_doc_feature_node(self, parent, feature, locations, doc):
-        """process capa doc feature node
+        """ process capa doc feature node
 
-        @param parent: parent node to which child is assigned
-        @param feature: capa doc feature node
-        @param locations: locations identified for feature
-        @param doc: capa doc
+            @param parent: parent node to which child is assigned
+            @param feature: capa doc feature node
+            @param locations: locations identified for feature
+            @param doc: capa doc
+
+            Example:
+              "feature": {
+                "description": "FILE_WRITE_DATA",
+                "number": "0x2",
+                "type": "number"
+              }
         """
         display = self.capa_doc_feature_to_display(feature)
 
         if len(locations) == 1:
             # only one location for feature so no need to nest children
-            parent2 = self.render_capa_doc_feature(
-                parent,
-                feature,
-                next(iter(locations)),
-                doc,
-                display=display,
-            )
+            parent2 = self.render_capa_doc_feature(parent, feature, next(iter(locations)), doc, display=display,)
         else:
             # feature has multiple children, nest  under one parent feature node
             parent2 = CapaExplorerFeatureItem(parent, display)
@@ -538,22 +508,21 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
         return parent2
 
     def render_capa_doc_feature(self, parent, feature, location, doc, display="-"):
-        """render capa feature read from doc
+        """ render capa feature read from doc
 
-        @param parent: parent node to which new child is assigned
-        @param feature: feature read from doc
-        @param doc: capa feature doc
-        @param location: address of feature
-        @param display: text to display in plugin UI
+            @param parent: parent node to which new child is assigned
+            @param feature: feature read from doc
+            @param doc: capa feature doc
+            @param location: address of feature
+            @param display: text to display in plugin ui
+
+            Example:
+              "feature": {
+                "description": "FILE_WRITE_DATA",
+                "number": "0x2",
+                "type": "number"
+              }
         """
-
-        # convert to offset from locations: List[Address]
-        try:
-            location = int(location)
-        except TypeError:
-            # e.g. capa.features.address._NoAddress, global features
-            return
-
         # special handling for characteristic pending type
         if feature["type"] == "characteristic":
             if feature[feature["type"]] in ("embedded pe",):
@@ -571,15 +540,8 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
                 parent, display, source=doc["rules"].get(feature[feature["type"]], {}).get("source", "")
             )
 
-        if feature["type"] in ("regex", "substring"):
-            for s, locations in feature["matches"].items():
-                if location in locations:
-                    return CapaExplorerStringViewItem(
-                        parent, display, location, '"' + capa.features.common.escape_string(s) + '"'
-                    )
-
-            # programming error: the given location should always be found in the regex matches
-            raise ValueError("regex match at location not found")
+        if feature["type"] == "regex":
+            return CapaExplorerFeatureItem(parent, display, location, details=feature["match"])
 
         if feature["type"] == "basicblock":
             return CapaExplorerBlockItem(parent, location)
@@ -590,6 +552,10 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             "mnemonic",
             "number",
             "offset",
+            "number/x32",
+            "number/x64",
+            "offset/x32",
+            "offset/x64",
         ):
             # display instruction preview
             return CapaExplorerInstructionViewItem(parent, display, location)
@@ -600,26 +566,19 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
         if feature["type"] in ("string",):
             # display string preview
-            return CapaExplorerStringViewItem(
-                parent, display, location, '"%s"' % capa.features.common.escape_string(feature[feature["type"]])
-            )
+            return CapaExplorerStringViewItem(parent, display, location)
 
-        if feature["type"] in ("import", "export", "function-name"):
+        if feature["type"] in ("import", "export"):
             # display no preview
-            return CapaExplorerFeatureItem(parent, location=location, display=display)
-
-        if feature["type"] in ("arch", "os", "format"):
             return CapaExplorerFeatureItem(parent, display=display)
 
         raise RuntimeError("unexpected feature type: " + str(feature["type"]))
 
     def update_function_name(self, old_name, new_name):
-        """update all instances of old function name with new function name
+        """ update all instances of old function name with new function name
 
-        called when user updates function name using plugin UI
-
-        @param old_name: old function name
-        @param new_name: new function name
+            @param old_name: previous function name
+            @param new_name: new function name
         """
         # create empty root index for search
         root_index = self.index(0, 0, QtCore.QModelIndex())

capa/ida/explorer/proxy.py

@@ -0,0 +1,89 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+from PyQt5 import QtCore
+
+from capa.ida.explorer.model import CapaExplorerDataModel
+
+
+class CapaExplorerSortFilterProxyModel(QtCore.QSortFilterProxyModel):
+    def __init__(self, parent=None):
+        """ """
+        super(CapaExplorerSortFilterProxyModel, self).__init__(parent)
+
+    def lessThan(self, left, right):
+        """ true if the value of the left item is less than value of right item
+
+            @param left: QModelIndex*
+            @param right: QModelIndex*
+
+            @retval True/False
+        """
+        ldata = left.internalPointer().data(left.column())
+        rdata = right.internalPointer().data(right.column())
+
+        if (
+            ldata
+            and rdata
+            and left.column() == CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS
+            and left.column() == right.column()
+        ):
+            # convert virtual address before compare
+            return int(ldata, 16) < int(rdata, 16)
+        else:
+            # compare as lowercase
+            return ldata.lower() < rdata.lower()
+
+    def filterAcceptsRow(self, row, parent):
+        """ true if the item in the row indicated by the given row and parent
+            should be included in the model; otherwise returns false
+
+            @param row: int
+            @param parent: QModelIndex*
+
+            @retval True/False
+        """
+        if self.filter_accepts_row_self(row, parent):
+            return True
+
+        alpha = parent
+        while alpha.isValid():
+            if self.filter_accepts_row_self(alpha.row(), alpha.parent()):
+                return True
+            alpha = alpha.parent()
+
+        if self.index_has_accepted_children(row, parent):
+            return True
+
+        return False
+
+    def add_single_string_filter(self, column, string):
+        """ add fixed string filter
+
+            @param column: key column
+            @param string: string to sort
+        """
+        self.setFilterKeyColumn(column)
+        self.setFilterFixedString(string)
+
+    def index_has_accepted_children(self, row, parent):
+        """ """
+        model_index = self.sourceModel().index(row, 0, parent)
+
+        if model_index.isValid():
+            for idx in range(self.sourceModel().rowCount(model_index)):
+                if self.filter_accepts_row_self(idx, model_index):
+                    return True
+                if self.index_has_accepted_children(idx, model_index):
+                    return True
+
+        return False
+
+    def filter_accepts_row_self(self, row, parent):
+        """ """
+        return super(CapaExplorerSortFilterProxyModel, self).filterAcceptsRow(row, parent)

capa/ida/explorer/view.py

@@ -0,0 +1,262 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import idc
+import idaapi
+from PyQt5 import QtGui, QtCore, QtWidgets
+
+from capa.ida.explorer.item import CapaExplorerRuleItem, CapaExplorerFunctionItem
+from capa.ida.explorer.model import CapaExplorerDataModel
+
+
+class CapaExplorerQtreeView(QtWidgets.QTreeView):
+    """ capa explorer QTreeView implementation
+
+        view controls UI action responses and displays data from
+        CapaExplorerDataModel
+
+        view does not modify CapaExplorerDataModel directly - data
+        modifications should be implemented in CapaExplorerDataModel
+    """
+
+    def __init__(self, model, parent=None):
+        """ initialize CapaExplorerQTreeView """
+        super(CapaExplorerQtreeView, self).__init__(parent)
+
+        self.setModel(model)
+
+        self.model = model
+        self.parent = parent
+
+        # configure custom UI controls
+        self.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
+        self.setExpandsOnDoubleClick(False)
+        self.setSortingEnabled(True)
+        self.model.setDynamicSortFilter(False)
+
+        # configure view columns to auto-resize
+        for idx in range(CapaExplorerDataModel.COLUMN_COUNT):
+            self.header().setSectionResizeMode(idx, QtWidgets.QHeaderView.Interactive)
+
+        # connect slots to resize columns when expanded or collapsed
+        self.expanded.connect(self.resize_columns_to_content)
+        self.collapsed.connect(self.resize_columns_to_content)
+
+        # connect slots
+        self.customContextMenuRequested.connect(self.slot_custom_context_menu_requested)
+        self.doubleClicked.connect(self.slot_double_click)
+
+        self.setStyleSheet("QTreeView::item {padding-right: 15 px;padding-bottom: 2 px;}")
+
+    def reset(self):
+        """ reset user interface changes
+
+            called when view should reset any user interface changes
+            made since the last reset e.g. IDA window highlighting
+        """
+        self.collapseAll()
+        self.resize_columns_to_content()
+
+    def resize_columns_to_content(self):
+        """ reset view columns to contents """
+        self.header().resizeSections(QtWidgets.QHeaderView.ResizeToContents)
+
+    def map_index_to_source_item(self, model_index):
+        """ map proxy model index to source model item
+
+            @param model_index: QModelIndex*
+
+            @retval QObject*
+        """
+        return self.model.mapToSource(model_index).internalPointer()
+
+    def send_data_to_clipboard(self, data):
+        """ copy data to the clipboard
+
+            @param data: data to be copied
+        """
+        clip = QtWidgets.QApplication.clipboard()
+        clip.clear(mode=clip.Clipboard)
+        clip.setText(data, mode=clip.Clipboard)
+
+    def new_action(self, display, data, slot):
+        """ create action for context menu
+
+            @param display: text displayed to user in context menu
+            @param data: data passed to slot
+            @param slot: slot to connect
+
+            @retval QAction*
+        """
+        action = QtWidgets.QAction(display, self.parent)
+        action.setData(data)
+        action.triggered.connect(lambda checked: slot(action))
+
+        return action
+
+    def load_default_context_menu_actions(self, data):
+        """ yield actions specific to function custom context menu
+
+            @param data: tuple
+
+            @yield QAction*
+        """
+        default_actions = (
+            ("Copy column", data, self.slot_copy_column),
+            ("Copy row", data, self.slot_copy_row),
+        )
+
+        # add default actions
+        for action in default_actions:
+            yield self.new_action(*action)
+
+    def load_function_context_menu_actions(self, data):
+        """ yield actions specific to function custom context menu
+
+            @param data: tuple
+
+            @yield QAction*
+        """
+        function_actions = (("Rename function", data, self.slot_rename_function),)
+
+        # add function actions
+        for action in function_actions:
+            yield self.new_action(*action)
+
+        # add default actions
+        for action in self.load_default_context_menu_actions(data):
+            yield action
+
+    def load_default_context_menu(self, pos, item, model_index):
+        """ create default custom context menu
+
+            creates custom context menu containing default actions
+
+            @param pos: TODO
+            @param item: TODO
+            @param model_index: TODO
+
+            @retval QMenu*
+        """
+        menu = QtWidgets.QMenu()
+
+        for action in self.load_default_context_menu_actions((pos, item, model_index)):
+            menu.addAction(action)
+
+        return menu
+
+    def load_function_item_context_menu(self, pos, item, model_index):
+        """ create function custom context menu
+
+            creates custom context menu containing actions specific to functions
+            and the default actions
+
+            @param pos: TODO
+            @param item: TODO
+            @param model_index: TODO
+
+            @retval QMenu*
+        """
+        menu = QtWidgets.QMenu()
+
+        for action in self.load_function_context_menu_actions((pos, item, model_index)):
+            menu.addAction(action)
+
+        return menu
+
+    def show_custom_context_menu(self, menu, pos):
+        """ display custom context menu in view
+
+            @param menu: TODO
+            @param pos: TODO
+        """
+        if menu:
+            menu.exec_(self.viewport().mapToGlobal(pos))
+
+    def slot_copy_column(self, action):
+        """ slot connected to custom context menu
+
+            allows user to select a column and copy the data
+            to clipboard
+
+            @param action: QAction*
+        """
+        _, item, model_index = action.data()
+        self.send_data_to_clipboard(item.data(model_index.column()))
+
+    def slot_copy_row(self, action):
+        """ slot connected to custom context menu
+
+            allows user to select a row and copy the space-delimited
+            data to clipboard
+
+            @param action: QAction*
+        """
+        _, item, _ = action.data()
+        self.send_data_to_clipboard(str(item))
+
+    def slot_rename_function(self, action):
+        """ slot connected to custom context menu
+
+            allows user to select a edit a function name and push
+            changes to IDA
+
+            @param action: QAction*
+        """
+        _, item, model_index = action.data()
+
+        # make item temporary edit, reset after user is finished
+        item.setIsEditable(True)
+        self.edit(model_index)
+        item.setIsEditable(False)
+
+    def slot_custom_context_menu_requested(self, pos):
+        """ slot connected to custom context menu request
+
+            displays custom context menu to user containing action
+            relevant to the data item selected
+
+            @param pos: TODO
+        """
+        model_index = self.indexAt(pos)
+
+        if not model_index.isValid():
+            return
+
+        item = self.map_index_to_source_item(model_index)
+        column = model_index.column()
+        menu = None
+
+        if CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION == column and isinstance(item, CapaExplorerFunctionItem):
+            # user hovered function item
+            menu = self.load_function_item_context_menu(pos, item, model_index)
+        else:
+            # user hovered default item
+            menu = self.load_default_context_menu(pos, item, model_index)
+
+        # show custom context menu at view position
+        self.show_custom_context_menu(menu, pos)
+
+    def slot_double_click(self, model_index):
+        """ slot connected to double click event
+
+            @param model_index: QModelIndex*
+        """
+        if not model_index.isValid():
+            return
+
+        item = self.map_index_to_source_item(model_index)
+        column = model_index.column()
+
+        if CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS == column and item.location:
+            # user double-clicked virtual address column - navigate IDA to address
+            idc.jumpto(item.location)
+
+        if CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION == column:
+            # user double-clicked information column - un/expand
+            self.collapse(model_index) if self.isExpanded(model_index) else self.expand(model_index)

capa/ida/helpers.py

@@ -1,163 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import logging
-import datetime
-
-import idc
-import idaapi
-import idautils
-import ida_bytes
-import ida_loader
-
-import capa
-import capa.version
-import capa.features.common
-
-logger = logging.getLogger("capa")
-
-# file type as returned by idainfo.file_type
-SUPPORTED_FILE_TYPES = (
-    idaapi.f_PE,
-    idaapi.f_ELF,
-    idaapi.f_BIN,
-    # idaapi.f_MACHO,
-)
-
-# arch type as returned by idainfo.procname
-SUPPORTED_ARCH_TYPES = ("metapc",)
-
-
-def inform_user_ida_ui(message):
-    idaapi.info("%s. Please refer to IDA Output window for more information." % message)
-
-
-def is_supported_ida_version():
-    version = float(idaapi.get_kernel_version())
-    if version < 7.4 or version >= 8:
-        warning_msg = "This plugin does not support your IDA Pro version"
-        logger.warning(warning_msg)
-        logger.warning("Your IDA Pro version is: %s. Supported versions are: IDA >= 7.4 and IDA < 8.0." % version)
-        return False
-    return True
-
-
-def is_supported_file_type():
-    file_info = idaapi.get_inf_structure()
-    if file_info.filetype not in SUPPORTED_FILE_TYPES:
-        logger.error("-" * 80)
-        logger.error(" Input file does not appear to be a supported file type.")
-        logger.error(" ")
-        logger.error(
-            " capa currently only supports analyzing PE, ELF, or binary files containing x86 (32- and 64-bit) shellcode."
-        )
-        logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
-        logger.error("-" * 80)
-        return False
-    return True
-
-
-def is_supported_arch_type():
-    file_info = idaapi.get_inf_structure()
-    if file_info.procname not in SUPPORTED_ARCH_TYPES or not any((file_info.is_32bit(), file_info.is_64bit())):
-        logger.error("-" * 80)
-        logger.error(" Input file does not appear to target a supported architecture.")
-        logger.error(" ")
-        logger.error(" capa currently only supports analyzing x86 (32- and 64-bit).")
-        logger.error("-" * 80)
-        return False
-    return True
-
-
-def get_disasm_line(va):
-    """ """
-    return idc.generate_disasm_line(va, idc.GENDSM_FORCE_CODE)
-
-
-def is_func_start(ea):
-    """check if function stat exists at virtual address"""
-    f = idaapi.get_func(ea)
-    return f and f.start_ea == ea
-
-
-def get_func_start_ea(ea):
-    """ """
-    f = idaapi.get_func(ea)
-    return f if f is None else f.start_ea
-
-
-def get_file_md5():
-    """ """
-    md5 = idautils.GetInputFileMD5()
-    if not isinstance(md5, str):
-        md5 = capa.features.common.bytes_to_str(md5)
-    return md5
-
-
-def get_file_sha256():
-    """ """
-    sha256 = idaapi.retrieve_input_file_sha256()
-    if not isinstance(sha256, str):
-        sha256 = capa.features.common.bytes_to_str(sha256)
-    return sha256
-
-
-def collect_metadata():
-    """ """
-    md5 = get_file_md5()
-    sha256 = get_file_sha256()
-
-    return {
-        "timestamp": datetime.datetime.now().isoformat(),
-        # "argv" is not relevant here
-        "sample": {
-            "md5": md5,
-            "sha1": "",  # not easily accessible
-            "sha256": sha256,
-            "path": idaapi.get_input_file_path(),
-        },
-        "analysis": {
-            "format": idaapi.get_file_type_name(),
-            "extractor": "ida",
-            "base_address": idaapi.get_imagebase(),
-            "layout": {
-                # this is updated after capabilities have been collected.
-                # will look like:
-                #
-                # "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
-            },
-        },
-        "version": capa.version.__version__,
-    }
-
-
-class IDAIO:
-    """
-    An object that acts as a file-like object,
-    using bytes from the current IDB workspace.
-    """
-
-    def __init__(self):
-        super(IDAIO, self).__init__()
-        self.offset = 0
-
-    def seek(self, offset, whence=0):
-        assert whence == 0
-        self.offset = offset
-
-    def read(self, size):
-        ea = ida_loader.get_fileregion_ea(self.offset)
-        if ea == idc.BADADDR:
-            # best guess, such as if file is mapped at address 0x0.
-            ea = self.offset
-
-        logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, ea)
-        return ida_bytes.get_bytes(ea, size)
-
-    def close(self):
-        return

capa/ida/helpers/__init__.py

@@ -0,0 +1,107 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+import datetime
+
+import idc
+import six
+import idaapi
+import idautils
+
+import capa
+
+logger = logging.getLogger("capa")
+
+SUPPORTED_IDA_VERSIONS = [
+    "7.1",
+    "7.2",
+    "7.3",
+    "7.4",
+    "7.5",
+]
+
+# file type names as returned by idaapi.get_file_type_name()
+SUPPORTED_FILE_TYPES = [
+    "Portable executable for 80386 (PE)",
+    "Portable executable for AMD64 (PE)",
+    "Binary file",  # x86/AMD64 shellcode support
+]
+
+
+def inform_user_ida_ui(message):
+    idaapi.info("%s. Please refer to IDA Output window for more information." % message)
+
+
+def is_supported_ida_version():
+    version = idaapi.get_kernel_version()
+    if version not in SUPPORTED_IDA_VERSIONS:
+        warning_msg = "This plugin does not support your IDA Pro version"
+        logger.warning(warning_msg)
+        logger.warning(
+            "Your IDA Pro version is: %s. Supported versions are: %s." % (version, ", ".join(SUPPORTED_IDA_VERSIONS))
+        )
+        capa.ida.helpers.inform_user_ida_ui(warning_msg)
+        return False
+    return True
+
+
+def is_supported_file_type():
+    file_type = idaapi.get_file_type_name()
+    if file_type not in SUPPORTED_FILE_TYPES:
+        logger.error("-" * 80)
+        logger.error(" Input file does not appear to be a PE file.")
+        logger.error(" ")
+        logger.error(
+            " capa currently only supports analyzing PE files (or binary files containing x86/AMD64 shellcode) with IDA."
+        )
+        logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
+        logger.error("-" * 80)
+        inform_user_ida_ui("capa does not support the format of this file")
+        return False
+    return True
+
+
+def get_disasm_line(va):
+    """ """
+    return idc.generate_disasm_line(va, idc.GENDSM_FORCE_CODE)
+
+
+def is_func_start(ea):
+    """ check if function stat exists at virtual address """
+    f = idaapi.get_func(ea)
+    return f and f.start_ea == ea
+
+
+def get_func_start_ea(ea):
+    """ """
+    f = idaapi.get_func(ea)
+    return f if f is None else f.start_ea
+
+
+def collect_metadata():
+    md5 = idautils.GetInputFileMD5()
+    if not isinstance(md5, six.string_types):
+        md5 = capa.features.bytes_to_str(md5)
+
+    sha256 = idaapi.retrieve_input_file_sha256()
+    if not isinstance(sha256, six.string_types):
+        sha256 = capa.features.bytes_to_str(sha256)
+
+    return {
+        "timestamp": datetime.datetime.now().isoformat(),
+        # "argv" is not relevant here
+        "sample": {
+            "md5": md5,
+            "sha1": "",  # not easily accessible
+            "sha256": sha256,
+            "path": idaapi.get_input_file_path(),
+        },
+        "analysis": {"format": idaapi.get_file_type_name(), "extractor": "ida",},
+        "version": capa.version.__version__,
+    }

capa/ida/ida_capa_explorer.py

@@ -0,0 +1,573 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import os
+import json
+import logging
+import collections
+
+import idaapi
+from PyQt5 import QtGui, QtCore, QtWidgets
+
+import capa.main
+import capa.rules
+import capa.ida.helpers
+import capa.render.utils as rutils
+import capa.features.extractors.ida
+from capa.ida.explorer.view import CapaExplorerQtreeView
+from capa.ida.explorer.model import CapaExplorerDataModel
+from capa.ida.explorer.proxy import CapaExplorerSortFilterProxyModel
+
+PLUGIN_NAME = "capa explorer"
+
+logger = logging.getLogger("capa")
+
+
+class CapaExplorerIdaHooks(idaapi.UI_Hooks):
+    def __init__(self, screen_ea_changed_hook, action_hooks):
+        """ facilitate IDA UI hooks
+
+            @param screen_ea_changed_hook: function hook for IDA screen ea changed
+            @param action_hooks: dict of IDA action handles
+        """
+        super(CapaExplorerIdaHooks, self).__init__()
+
+        self.screen_ea_changed_hook = screen_ea_changed_hook
+        self.process_action_hooks = action_hooks
+        self.process_action_handle = None
+        self.process_action_meta = {}
+
+    def preprocess_action(self, name):
+        """ called prior to action completed
+
+            @param name: name of action defined by idagui.cfg
+
+            @retval must be 0
+        """
+        self.process_action_handle = self.process_action_hooks.get(name, None)
+
+        if self.process_action_handle:
+            self.process_action_handle(self.process_action_meta)
+
+        # must return 0 for IDA
+        return 0
+
+    def postprocess_action(self):
+        """ called after action completed """
+        if not self.process_action_handle:
+            return
+
+        self.process_action_handle(self.process_action_meta, post=True)
+        self.reset()
+
+    def screen_ea_changed(self, curr_ea, prev_ea):
+        """ called after screen location is changed
+
+            @param curr_ea: current location
+            @param prev_ea: prev location
+        """
+        self.screen_ea_changed_hook(idaapi.get_current_widget(), curr_ea, prev_ea)
+
+    def reset(self):
+        """ reset internal state """
+        self.process_action_handle = None
+        self.process_action_meta.clear()
+
+
+class CapaExplorerForm(idaapi.PluginForm):
+    def __init__(self):
+        """ """
+        super(CapaExplorerForm, self).__init__()
+
+        self.form_title = PLUGIN_NAME
+        self.file_loc = __file__
+
+        self.parent = None
+        self.ida_hooks = None
+        self.doc = None
+
+        # models
+        self.model_data = None
+        self.model_proxy = None
+
+        # user interface elements
+        self.view_limit_results_by_function = None
+        self.view_tree = None
+        self.view_summary = None
+        self.view_attack = None
+        self.view_tabs = None
+        self.view_menu_bar = None
+
+    def OnCreate(self, form):
+        """ """
+        self.parent = self.FormToPyQtWidget(form)
+        self.load_interface()
+        self.load_capa_results()
+        self.load_ida_hooks()
+
+        self.view_tree.reset()
+
+        logger.info("form created.")
+
+    def Show(self):
+        """ """
+        return idaapi.PluginForm.Show(
+            self, self.form_title, options=(idaapi.PluginForm.WOPN_TAB | idaapi.PluginForm.WCLS_CLOSE_LATER)
+        )
+
+    def OnClose(self, form):
+        """ form is closed """
+        self.unload_ida_hooks()
+        self.ida_reset()
+
+        logger.info("form closed.")
+
+    def load_interface(self):
+        """ load user interface """
+        # load models
+        self.model_data = CapaExplorerDataModel()
+        self.model_proxy = CapaExplorerSortFilterProxyModel()
+        self.model_proxy.setSourceModel(self.model_data)
+
+        # load tree
+        self.view_tree = CapaExplorerQtreeView(self.model_proxy, self.parent)
+
+        # load summary table
+        self.load_view_summary()
+        self.load_view_attack()
+
+        # load parent tab and children tab views
+        self.load_view_tabs()
+        self.load_view_checkbox_limit_by()
+        self.load_view_summary_tab()
+        self.load_view_attack_tab()
+        self.load_view_tree_tab()
+
+        # load menu bar and sub menus
+        self.load_view_menu_bar()
+        self.load_file_menu()
+
+        # load parent view
+        self.load_view_parent()
+
+    def load_view_tabs(self):
+        """ load tabs """
+        tabs = QtWidgets.QTabWidget()
+        self.view_tabs = tabs
+
+    def load_view_menu_bar(self):
+        """ load menu bar """
+        bar = QtWidgets.QMenuBar()
+        self.view_menu_bar = bar
+
+    def load_view_summary(self):
+        """ load capa summary table """
+        table_headers = [
+            "Capability",
+            "Namespace",
+        ]
+
+        table = QtWidgets.QTableWidget()
+
+        table.setColumnCount(len(table_headers))
+        table.verticalHeader().setVisible(False)
+        table.setSortingEnabled(False)
+        table.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
+        table.setFocusPolicy(QtCore.Qt.NoFocus)
+        table.setSelectionMode(QtWidgets.QAbstractItemView.NoSelection)
+        table.setHorizontalHeaderLabels(table_headers)
+        table.horizontalHeader().setDefaultAlignment(QtCore.Qt.AlignLeft)
+        table.setShowGrid(False)
+        table.setStyleSheet("QTableWidget::item { padding: 25px; }")
+
+        self.view_summary = table
+
+    def load_view_attack(self):
+        """ load MITRE ATT&CK table """
+        table_headers = [
+            "ATT&CK Tactic",
+            "ATT&CK Technique ",
+        ]
+
+        table = QtWidgets.QTableWidget()
+
+        table.setColumnCount(len(table_headers))
+        table.verticalHeader().setVisible(False)
+        table.setSortingEnabled(False)
+        table.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
+        table.setFocusPolicy(QtCore.Qt.NoFocus)
+        table.setSelectionMode(QtWidgets.QAbstractItemView.NoSelection)
+        table.setHorizontalHeaderLabels(table_headers)
+        table.horizontalHeader().setDefaultAlignment(QtCore.Qt.AlignLeft)
+        table.setShowGrid(False)
+        table.setStyleSheet("QTableWidget::item { padding: 25px; }")
+
+        self.view_attack = table
+
+    def load_view_checkbox_limit_by(self):
+        """ load limit results by function checkbox """
+        check = QtWidgets.QCheckBox("Limit results to current function")
+        check.setChecked(False)
+        check.stateChanged.connect(self.slot_checkbox_limit_by_changed)
+
+        self.view_limit_results_by_function = check
+
+    def load_view_parent(self):
+        """ load view parent """
+        layout = QtWidgets.QVBoxLayout()
+
+        layout.addWidget(self.view_tabs)
+        layout.setMenuBar(self.view_menu_bar)
+
+        self.parent.setLayout(layout)
+
+    def load_view_tree_tab(self):
+        """ load capa tree tab view """
+        layout = QtWidgets.QVBoxLayout()
+        layout.addWidget(self.view_limit_results_by_function)
+        layout.addWidget(self.view_tree)
+
+        tab = QtWidgets.QWidget()
+        tab.setLayout(layout)
+
+        self.view_tabs.addTab(tab, "Tree View")
+
+    def load_view_summary_tab(self):
+        """ load capa summary tab view """
+        layout = QtWidgets.QVBoxLayout()
+        layout.addWidget(self.view_summary)
+
+        tab = QtWidgets.QWidget()
+        tab.setLayout(layout)
+
+        self.view_tabs.addTab(tab, "Summary")
+
+    def load_view_attack_tab(self):
+        """ load MITRE ATT&CK tab view """
+        layout = QtWidgets.QVBoxLayout()
+        layout.addWidget(self.view_attack)
+
+        tab = QtWidgets.QWidget()
+        tab.setLayout(layout)
+
+        self.view_tabs.addTab(tab, "MITRE")
+
+    def load_file_menu(self):
+        """ load file menu actions """
+        actions = (
+            ("Reset view", "Reset plugin view", self.reset),
+            ("Run analysis", "Run capa analysis on current database", self.reload),
+            ("Export results...", "Export capa results as JSON file", self.export_json),
+        )
+
+        menu = self.view_menu_bar.addMenu("File")
+        for (name, _, handle) in actions:
+            action = QtWidgets.QAction(name, self.parent)
+            action.triggered.connect(handle)
+            menu.addAction(action)
+
+    def export_json(self):
+        """ export capa results as JSON file """
+        if not self.doc:
+            idaapi.info("No capa results to export.")
+            return
+        path = idaapi.ask_file(True, "*.json", "Choose file")
+        if os.path.exists(path) and 1 != idaapi.ask_yn(1, "File already exists. Overwrite?"):
+            return
+        with open(path, "wb") as export_file:
+            export_file.write(
+                json.dumps(self.doc, sort_keys=True, cls=capa.render.CapaJsonObjectEncoder).encode("utf-8")
+            )
+
+    def load_ida_hooks(self):
+        """ load IDA Pro UI hooks """
+        action_hooks = {
+            "MakeName": self.ida_hook_rename,
+            "EditFunction": self.ida_hook_rename,
+        }
+
+        self.ida_hooks = CapaExplorerIdaHooks(self.ida_hook_screen_ea_changed, action_hooks)
+        self.ida_hooks.hook()
+
+    def unload_ida_hooks(self):
+        """ unload IDA Pro UI hooks """
+        if self.ida_hooks:
+            self.ida_hooks.unhook()
+
+    def ida_hook_rename(self, meta, post=False):
+        """ hook for IDA rename action
+
+            called twice, once before action and once after
+            action completes
+
+            @param meta: metadata cache
+            @param post: indicates pre or post action
+        """
+        location = idaapi.get_screen_ea()
+        if not location or not capa.ida.helpers.is_func_start(location):
+            return
+
+        curr_name = idaapi.get_name(location)
+
+        if post:
+            # post action update data model w/ current name
+            self.model_data.update_function_name(meta.get("prev_name", ""), curr_name)
+        else:
+            # pre action so save current name for replacement later
+            meta["prev_name"] = curr_name
+
+    def ida_hook_screen_ea_changed(self, widget, new_ea, old_ea):
+        """ hook for IDA screen ea changed
+
+            @param widget: IDA widget type
+            @param new_ea: destination ea
+            @param old_ea: source ea
+         """
+        if not self.view_limit_results_by_function.isChecked():
+            # ignore if checkbox not selected
+            return
+
+        if idaapi.get_widget_type(widget) != idaapi.BWN_DISASM:
+            # ignore views other than asm
+            return
+
+        # attempt to map virtual addresses to function start addresses
+        new_func_start = capa.ida.helpers.get_func_start_ea(new_ea)
+        old_func_start = capa.ida.helpers.get_func_start_ea(old_ea)
+
+        if new_func_start and new_func_start == old_func_start:
+            # navigated within the same function - do nothing
+            return
+
+        if new_func_start:
+            # navigated to new function - filter for function start virtual address
+            match = capa.ida.explorer.item.location_to_hex(new_func_start)
+        else:
+            # navigated to virtual address not in valid function - clear filter
+            match = ""
+
+        # filter on virtual address to avoid updating filter string if function name is changed
+        self.model_proxy.add_single_string_filter(CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS, match)
+        self.view_tree.resize_columns_to_content()
+
+    def load_capa_results(self):
+        """ run capa analysis and render results in UI """
+        logger.info("-" * 80)
+        logger.info(" Using default embedded rules.")
+        logger.info(" ")
+        logger.info(" You can see the current default rule set here:")
+        logger.info("     https://github.com/fireeye/capa-rules")
+        logger.info("-" * 80)
+
+        rules_path = os.path.join(os.path.dirname(self.file_loc), "../..", "rules")
+        rules = capa.main.get_rules(rules_path)
+        rules = capa.rules.RuleSet(rules)
+
+        meta = capa.ida.helpers.collect_metadata()
+
+        capabilities, counts = capa.main.find_capabilities(
+            rules, capa.features.extractors.ida.IdaFeatureExtractor(), True
+        )
+        meta["analysis"].update(counts)
+
+        # support binary files specifically for x86/AMD64 shellcode
+        # warn user binary file is loaded but still allow capa to process it
+        # TODO: check specific architecture of binary files based on how user configured IDA processors
+        if idaapi.get_file_type_name() == "Binary file":
+            logger.warning("-" * 80)
+            logger.warning(" Input file appears to be a binary file.")
+            logger.warning(" ")
+            logger.warning(
+                " capa currently only supports analyzing binary files containing x86/AMD64 shellcode with IDA."
+            )
+            logger.warning(
+                " This means the results may be misleading or incomplete if the binary file loaded in IDA is not x86/AMD64."
+            )
+            logger.warning(" If you don't know the input file type, you can try using the `file` utility to guess it.")
+            logger.warning("-" * 80)
+
+            capa.ida.helpers.inform_user_ida_ui("capa encountered warnings during analysis")
+
+        if capa.main.has_file_limitation(rules, capabilities, is_standalone=False):
+            capa.ida.helpers.inform_user_ida_ui("capa encountered warnings during analysis")
+
+        logger.info("analysis completed.")
+
+        self.doc = capa.render.convert_capabilities_to_result_document(meta, rules, capabilities)
+
+        self.model_data.render_capa_doc(self.doc)
+        self.render_capa_doc_summary()
+        self.render_capa_doc_mitre_summary()
+
+        self.set_view_tree_default_sort_order()
+
+        logger.info("render views completed.")
+
+    def set_view_tree_default_sort_order(self):
+        """ set capa tree view default sort order """
+        self.view_tree.sortByColumn(CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION, QtCore.Qt.AscendingOrder)
+
+    def render_capa_doc_summary(self):
+        """ render capa summary results """
+        for (row, rule) in enumerate(rutils.capability_rules(self.doc)):
+            count = len(rule["matches"])
+
+            if count == 1:
+                capability = rule["meta"]["name"]
+            else:
+                capability = "%s (%d matches)" % (rule["meta"]["name"], count)
+
+            self.view_summary.setRowCount(row + 1)
+
+            self.view_summary.setItem(row, 0, self.render_new_table_header_item(capability))
+            self.view_summary.setItem(row, 1, QtWidgets.QTableWidgetItem(rule["meta"]["namespace"]))
+
+        # resize columns to content
+        self.view_summary.resizeColumnsToContents()
+
+    def render_capa_doc_mitre_summary(self):
+        """ render capa MITRE ATT&CK results """
+        tactics = collections.defaultdict(set)
+
+        for rule in rutils.capability_rules(self.doc):
+            if not rule["meta"].get("att&ck"):
+                continue
+
+            for attack in rule["meta"]["att&ck"]:
+                tactic, _, rest = attack.partition("::")
+                if "::" in rest:
+                    technique, _, rest = rest.partition("::")
+                    subtechnique, _, id = rest.rpartition(" ")
+                    tactics[tactic].add((technique, subtechnique, id))
+                else:
+                    technique, _, id = rest.rpartition(" ")
+                    tactics[tactic].add((technique, id))
+
+        column_one = []
+        column_two = []
+
+        for (tactic, techniques) in sorted(tactics.items()):
+            column_one.append(tactic.upper())
+            # add extra space when more than one technique
+            column_one.extend(["" for i in range(len(techniques) - 1)])
+
+            for spec in sorted(techniques):
+                if len(spec) == 2:
+                    technique, id = spec
+                    column_two.append("%s %s" % (technique, id))
+                elif len(spec) == 3:
+                    technique, subtechnique, id = spec
+                    column_two.append("%s::%s %s" % (technique, subtechnique, id))
+                else:
+                    raise RuntimeError("unexpected ATT&CK spec format")
+
+        self.view_attack.setRowCount(max(len(column_one), len(column_two)))
+
+        for row, value in enumerate(column_one):
+            self.view_attack.setItem(row, 0, self.render_new_table_header_item(value))
+
+        for row, value in enumerate(column_two):
+            self.view_attack.setItem(row, 1, QtWidgets.QTableWidgetItem(value))
+
+        # resize columns to content
+        self.view_attack.resizeColumnsToContents()
+
+    def render_new_table_header_item(self, text):
+        """ create new table header item with default style """
+        item = QtWidgets.QTableWidgetItem(text)
+        item.setForeground(QtGui.QColor(88, 139, 174))
+
+        font = QtGui.QFont()
+        font.setBold(True)
+
+        item.setFont(font)
+
+        return item
+
+    def ida_reset(self):
+        """ reset IDA UI """
+        self.model_data.reset()
+        self.view_tree.reset()
+        self.view_limit_results_by_function.setChecked(False)
+        self.set_view_tree_default_sort_order()
+
+    def reload(self):
+        """ reload views and re-run capa analysis """
+        self.ida_reset()
+        self.model_proxy.invalidate()
+        self.model_data.clear()
+        self.view_summary.setRowCount(0)
+        self.load_capa_results()
+
+        logger.info("reload complete.")
+        idaapi.info("%s reload completed." % PLUGIN_NAME)
+
+    def reset(self):
+        """ reset UI elements
+
+            e.g. checkboxes and IDA highlighting
+        """
+        self.ida_reset()
+
+        logger.info("reset completed.")
+        idaapi.info("%s reset completed." % PLUGIN_NAME)
+
+    def slot_menu_bar_hovered(self, action):
+        """ display menu action tooltip
+
+            @param action: QtWidgets.QAction*
+
+            @reference: https://stackoverflow.com/questions/21725119/why-wont-qtooltips-appear-on-qactions-within-a-qmenu
+        """
+        QtWidgets.QToolTip.showText(
+            QtGui.QCursor.pos(), action.toolTip(), self.view_menu_bar, self.view_menu_bar.actionGeometry(action)
+        )
+
+    def slot_checkbox_limit_by_changed(self):
+        """ slot activated if checkbox clicked
+
+            if checked, configure function filter if screen location is located
+            in function, otherwise clear filter
+        """
+        match = ""
+        if self.view_limit_results_by_function.isChecked():
+            location = capa.ida.helpers.get_func_start_ea(idaapi.get_screen_ea())
+            if location:
+                match = capa.ida.explorer.item.location_to_hex(location)
+
+        self.model_proxy.add_single_string_filter(CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS, match)
+
+        self.view_tree.resize_columns_to_content()
+
+
+def main():
+    """ TODO: move to idaapi.plugin_t class """
+    logging.basicConfig(level=logging.INFO)
+
+    if not capa.ida.helpers.is_supported_ida_version():
+        return -1
+
+    if not capa.ida.helpers.is_supported_file_type():
+        return -1
+
+    global CAPA_EXPLORER_FORM
+
+    try:
+        # there is an instance, reload it
+        CAPA_EXPLORER_FORM
+        CAPA_EXPLORER_FORM.Close()
+        CAPA_EXPLORER_FORM = CapaExplorerForm()
+    except Exception:
+        # there is no instance yet
+        CAPA_EXPLORER_FORM = CapaExplorerForm()
+
+    CAPA_EXPLORER_FORM.Show()
+
+
+if __name__ == "__main__":
+    main()

capa/ida/plugin/README.md

@@ -1,139 +0,0 @@
-![capa explorer](../../../.github/capa-explorer-logo.png)
-
-capa explorer is an IDAPython plugin that integrates the FLARE team's open-source framework, capa, with IDA Pro. capa is a framework that uses a well-defined collection of rules to 
-identify capabilities in a program. You can run capa against a PE file or shellcode and it tells you what it thinks the program can do. For example, it might suggest that 
-the program is a backdoor, can install services, or relies on HTTP to communicate. capa explorer runs capa directly against your IDA Pro database (IDB) without requiring access
-to the original binary file. Once a database has been analyzed, capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted from your IDB.
-
-We love using capa explorer during malware analysis because it teaches us what parts of a program suggest a behavior. As we click on rows, capa explorer jumps directly 
-to important addresses in the IDB and highlights key features in the Disassembly view so they stand out visually. To illustrate, we use capa explorer to 
-analyze Lab 14-02 from [Practical Malware Analysis](https://nostarch.com/malware) (PMA) available [here](https://practicalmalwareanalysis.com/labs/). Our goal is to understand 
-the program's functionality.
-
-After loading Lab 14-02 into IDA and analyzing the database with capa explorer, we see that capa detected a rule match for `self delete via COMSPEC environment variable`:
-
-![](../../../doc/img/explorer_condensed.png)
-
-We can use capa explorer to navigate our Disassembly view directly to the suspect function and get an assembly-level breakdown of why capa matched `self delete via COMSPEC environment variable`.
-
-![](../../../doc/img/explorer_expanded.png)
-
-Using the `Rule Information` and `Details` columns capa explorer shows us that the suspect function matched `self delete via COMSPEC environment variable` because it contains capa rule matches for `create process`, `get COMSPEC environment variable`,
-and `query environment variable`, references to the strings `COMSPEC`, ` > nul`, and `/c del `, and calls to the Windows API functions `GetEnvironmentVariableA` and `ShellExecuteEx`.
-
-capa explorer also helps you build new capa rules. To start select the `Rule Generator` tab, navigate to a function in your Disassembly view,
-and click `Analyze`. capa explorer will extract features from the function and display them in the `Features` pane. You can add features listed in this pane to the `Editor` pane
-by either double-clicking a feature or using multi-select + right-click to add multiple features at once. The `Preview` and `Editor` panes help edit your rule. Use the `Preview` pane
-to modify the rule text directly and the `Editor` pane to construct and rearrange your hierarchy of statements and features. When you finish a rule you can save it directly to a file by clicking `Save`.
-
-![](../../../doc/img/rulegen_expanded.png)
-
-For more information on the FLARE team's open-source framework, capa, check out the overview in our first [blog](https://www.fireeye.com/blog/threat-research/2020/07/capa-automatically-identify-malware-capabilities.html).
-
-## Getting Started
-
-### Requirements
-
-capa explorer supports Python versions >= 3.7.x and the following IDA Pro versions:
-
-* IDA 7.4
-* IDA 7.5
-* IDA 7.6 (caveat below)
-* IDA 7.7
-
-capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x). Based on our testing the following matrix shows the Python versions supported
-by each supported IDA version:
-
-| | IDA 7.4 | IDA 7.5 | IDA 7.6 |
-| --- | --- | --- | --- |
-| Python 3.7.x | Yes | Yes | Yes |
-| Python 3.8.x | Partial (see below) | Yes | Yes |
-| Python 3.9.x | No | Partial (see below) | Yes |
-
-To use capa explorer with IDA 7.4 and Python 3.8.x you must follow the instructions provided by hex-rays [here](https://hex-rays.com/blog/ida-7-4-and-python-3-8/).
-
-To use capa explorer with IDA 7.5 and Python 3.9.x you must follow the instructions provided by hex-rays [here](https://hex-rays.com/blog/python-3-9-support-for-ida-7-5/).
-
-If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues).
-
-#### IDA 7.6 caveat: IDA 7.6sp1 or patch required
-
-As described [here](https://www.hex-rays.com/blog/ida-7-6-empty-qtreeview-qtreewidget/):
-
-> A rather nasty issue evaded our testing and found its way into IDA 7.6: using the PyQt5 modules that are shipped with IDA, QTreeView (or QTreeWidget) instances will always fail to display contents.
-
-Therefore, in order to use capa under IDA 7.6 you need the [Service Pack 1 for IDA 7.6](https://www.hex-rays.com/products/ida/news/7_6sp1). Alternatively, you can download and install the fix corresponding to your IDA installation, replacing the original QtWidgets DLL with the one contained in the .zip file (links to Hex-Rays):
-
-
-  - Windows: [pyqt5_qtwidgets_win](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_win.zip)
-  - Linux: [pyqt5_qtwidgets_linux](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_linux.zip)
-  - MacOS (Intel): [pyqt5_qtwidgets_mac_x64](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_mac_x64.zip)
-  - MacOS (AppleSilicon): [pyqt5_qtwidgets_mac_arm](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_mac_arm.zip)
-
-
-### Supported File Types
-
-capa explorer is limited to the file types supported by capa, which include:
-
-* Windows x86 (32- and 64-bit) PE and ELF files
-* Windows x86 (32- and 64-bit) shellcode
-
-### Installation
-
-You can install capa explorer using the following steps:
-
-1. Install capa and its dependencies from PyPI for the Python interpreter used by your IDA installation:
-    ```
-    $ pip install flare-capa
-    ```
-3. Download the [standard collection of capa rules](https://github.com/mandiant/capa-rules) (capa explorer needs capa rules to analyze a database)
-4. Copy [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory
-
-### Usage
-
-1. Open IDA and analyze a supported file type (select the `Manual Load` and `Load Resources` options in IDA for best results)
-2. Open capa explorer in IDA by navigating to `Edit > Plugins > FLARE capa explorer` or using the keyboard shortcut `Alt+F5`
-   You can also use `ida_loader.load_and_run_plugin("capa_explorer", arg)`. `arg` is a bitflag for which setting the LSB enables automatic analysis. See `capa.ida.plugin.form.Options` for more details.
-3. Select the `Program Analysis` tab
-4. Click the `Analyze` button
-
-When running capa explorer for the first time you are prompted to select a file directory containing capa rules. The plugin conveniently
-remembers your selection for future runs; you can change this selection and other default settings by clicking `Settings`. We recommend 
-downloading and using the [standard collection of capa rules](https://github.com/mandiant/capa-rules) when getting started with the plugin.
-
-#### Tips for Program Analysis
-
-* Start analysis by clicking the `Analyze` button
-* Reset the plugin user interface and remove highlighting from your Disassembly view by clicking the `Reset` button
-* Change your capa rules directory and other default settings by clicking `Settings`
-* Hover your cursor over a rule match to view the source content of the rule
-* Double-click the `Address` column to navigate your Disassembly view to the address of the associated feature
-* Double-click a result in the `Rule Information` column to expand its children
-* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in your Dissasembly view
-
-#### Tips for Rule Generator
-
-* Navigate to a function in your Disassembly view and click`Analyze` to get started
-* Double-click or use multi-select + right-click to add features from the `Features` pane to the `Editor` pane
-* Right-click features in the `Editor` pane to make context-specific modifications
-* Drag-and-drop (single click + multi-select support) features in the `Editor` pane to construct your hierarchy of statements and features
-* Right-click anywhere in the `Editor` pane not on a feature to remove all features
-* Add descriptions or comments to a feature by editing the corresponding column in the `Editor` pane
-* Directly edit rule text and metadata fields using the `Preview` pane
-* Change the default rule author and default rule scope displayed in the `Preview` pane by clicking `Settings`
-
-## Development
-
-capa explorer is packaged with capa so you will need to install capa locally for development. You can install capa locally by following the steps outlined in `Method 3: Inspecting the capa source code` of the [capa 
-installation guide](https://github.com/mandiant/capa/blob/master/doc/installation.md#method-3-inspecting-the-capa-source-code). Once installed, copy [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) 
-to your plugins directory to install capa explorer in IDA.
-
-### Components
-
-capa explorer consists of two main components:
-
-* An [feature extractor](https://github.com/mandiant/capa/tree/master/capa/features/extractors/ida) built on top of IDA's binary analysis engine
-  * This component uses IDAPython to extract [capa features](https://github.com/mandiant/capa-rules/blob/master/doc/format.md#extracted-features) from your IDBs such as strings, 
-disassembly, and control flow; these extracted features are used by capa to find feature combinations that result in a rule match
-* An [interactive user interface](https://github.com/mandiant/capa/tree/master/capa/ida/plugin) for displaying and exploring capa rule matches
-  * This component integrates the feature extractor and capa, providing an interactive user interface to dissect rule matches found by capa using features extracted directly from your IDBs

capa/ida/plugin/__init__.py

@@ -1,125 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-import logging
-
-import idaapi
-import ida_kernwin
-
-from capa.ida.plugin.form import CapaExplorerForm
-from capa.ida.plugin.icon import ICON
-
-logger = logging.getLogger(__name__)
-
-
-class CapaExplorerPlugin(idaapi.plugin_t):
-
-    # Mandatory definitions
-    PLUGIN_NAME = "FLARE capa explorer"
-    PLUGIN_VERSION = "1.0.0"
-    PLUGIN_AUTHORS = "michael.hunhoff@mandiant.com, william.ballenthin@mandiant.com, moritz.raabe@mandiant.com"
-
-    wanted_name = PLUGIN_NAME
-    wanted_hotkey = "ALT-F5"
-    comment = "IDA Pro plugin for the FLARE team's capa tool to identify capabilities in executable files."
-    website = "https://github.com/mandiant/capa"
-    help = "See https://github.com/mandiant/capa/blob/master/doc/usage.md"
-    version = ""
-    flags = 0
-
-    def __init__(self):
-        """initialize plugin"""
-        self.form = None
-
-    def init(self):
-        """called when IDA is loading the plugin"""
-        logging.basicConfig(level=logging.INFO)
-
-        import capa.ida.helpers
-
-        # do not load plugin if IDA version/file type not supported
-        if not capa.ida.helpers.is_supported_ida_version():
-            return idaapi.PLUGIN_SKIP
-        if not capa.ida.helpers.is_supported_file_type():
-            return idaapi.PLUGIN_SKIP
-        if not capa.ida.helpers.is_supported_arch_type():
-            return idaapi.PLUGIN_SKIP
-        return idaapi.PLUGIN_OK
-
-    def term(self):
-        """called when IDA is unloading the plugin"""
-        pass
-
-    def run(self, arg):
-        """
-        called when IDA is running the plugin as a script
-
-        args:
-          arg (int): bitflag. Setting LSB enables automatic analysis upon
-          loading. The other bits are currently undefined. See `form.Options`.
-        """
-        self.form = CapaExplorerForm(self.PLUGIN_NAME, arg)
-        return True
-
-
-# set the capa plugin icon.
-#
-# TL;DR: temporarily install a UI hook set the icon.
-#
-# Long form:
-#
-# in the IDAPython `plugin_t` life cycle,
-#   - `init` decides if a plugin should be registered
-#   - `run` executes the main logic (shows the window)
-#   - `term` cleans this up
-#
-# we want to associate an icon with the plugin action - which is created by IDA.
-# however, this action is created by IDA *after* `init` is called.
-# so, we can't do this in `plugin_t.init`.
-# we also can't spawn a thread and do it after a delay,
-#  since `ida_kernwin.update_action_icon` must be called from the main thread.
-# so we need to register a callback that's invoked from the main thread after the plugin is registered.
-#
-# after a lot of guess-and-check, we can use `UI_Hooks.updated_actions` to
-#  receive notications after IDA has created an action for each plugin.
-# so, create this hook, wait for capa plugin to load, set the icon, and unhook.
-
-
-class OnUpdatedActionsHook(ida_kernwin.UI_Hooks):
-    """register a callback to be invoked each time the UI actions are updated"""
-
-    def __init__(self, cb):
-        super(OnUpdatedActionsHook, self).__init__()
-        self.cb = cb
-
-    def updated_actions(self):
-        if self.cb():
-            # uninstall the callback once its run successfully
-            self.unhook()
-
-
-def install_icon():
-    plugin_name = CapaExplorerPlugin.PLUGIN_NAME
-    action_name = "Edit/Plugins/" + plugin_name
-
-    if action_name not in ida_kernwin.get_registered_actions():
-        # keep the hook registered
-        return False
-
-    # resource leak here. need to call `ida_kernwin.free_custom_icon`?
-    # however, since we're not cycling this icon a lot, its probably ok.
-    # expect to leak exactly one icon per application load.
-    icon = ida_kernwin.load_custom_icon(data=ICON)
-
-    ida_kernwin.update_action_icon(action_name, icon)
-
-    # uninstall the hook
-    return True
-
-
-h = OnUpdatedActionsHook(install_icon)
-h.hook()

capa/ida/plugin/capa_explorer.py

@@ -1,17 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-from capa.ida.plugin import CapaExplorerPlugin
-
-
-def PLUGIN_ENTRY():
-    """mandatory entry point for IDAPython plugins
-
-    copy this script to your IDA plugins directory and start the plugin by navigating to Edit > Plugins in IDA Pro
-    """
-    return CapaExplorerPlugin()

capa/ida/plugin/hooks.py

@@ -1,60 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-
-import idaapi
-
-
-class CapaExplorerIdaHooks(idaapi.UI_Hooks):
-    def __init__(self, screen_ea_changed_hook, action_hooks):
-        """facilitate IDA UI hooks
-
-        @param screen_ea_changed_hook: function hook for IDA screen ea changed
-        @param action_hooks: dict of IDA action handles
-        """
-        super(CapaExplorerIdaHooks, self).__init__()
-
-        self.screen_ea_changed_hook = screen_ea_changed_hook
-        self.process_action_hooks = action_hooks
-        self.process_action_handle = None
-        self.process_action_meta = {}
-
-    def preprocess_action(self, name):
-        """called prior to action completed
-
-        @param name: name of action defined by idagui.cfg
-
-        @retval must be 0
-        """
-        self.process_action_handle = self.process_action_hooks.get(name, None)
-
-        if self.process_action_handle:
-            self.process_action_handle(self.process_action_meta)
-
-        # must return 0 for IDA
-        return 0
-
-    def postprocess_action(self):
-        """called after action completed"""
-        if not self.process_action_handle:
-            return
-
-        self.process_action_handle(self.process_action_meta, post=True)
-        self.reset()
-
-    def screen_ea_changed(self, curr_ea, prev_ea):
-        """called after screen location is changed
-
-        @param curr_ea: current location
-        @param prev_ea: prev location
-        """
-        self.screen_ea_changed_hook(idaapi.get_current_widget(), curr_ea, prev_ea)
-
-    def reset(self):
-        """reset internal state"""
-        self.process_action_handle = None
-        self.process_action_meta.clear()

capa/ida/plugin/proxy.py

@@ -1,225 +0,0 @@
-# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at: [package root]/LICENSE.txt
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and limitations under the License.
-from PyQt5 import QtCore
-from PyQt5.QtCore import Qt
-
-from capa.ida.plugin.model import CapaExplorerDataModel
-
-
-class CapaExplorerRangeProxyModel(QtCore.QSortFilterProxyModel):
-    """filter results based on virtual address range as seen by IDA
-
-    implements filtering for "limit results by current function" checkbox in plugin UI
-
-    minimum and maximum virtual addresses are used to filter results to a specific address range. this allows
-    basic blocks to be included when limiting results to a specific function
-    """
-
-    def __init__(self, parent=None):
-        """initialize proxy filter"""
-        super(CapaExplorerRangeProxyModel, self).__init__(parent)
-        self.min_ea = None
-        self.max_ea = None
-
-    def lessThan(self, left, right):
-        """return True if left item is less than right item, else False
-
-        @param left: QModelIndex of left
-        @param right: QModelIndex of right
-        """
-        ldata = left.internalPointer().data(left.column())
-        rdata = right.internalPointer().data(right.column())
-
-        if (
-            ldata
-            and rdata
-            and left.column() == CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS
-            and left.column() == right.column()
-        ):
-            # convert virtual address before compare
-            return int(ldata, 16) < int(rdata, 16)
-        else:
-            # compare as lowercase
-            return ldata.lower() < rdata.lower()
-
-    def filterAcceptsRow(self, row, parent):
-        """return true if the item in the row indicated by the given row and parent should be included in the model;
-        otherwise return false
-
-        @param row: row number
-        @param parent: QModelIndex of parent
-        """
-        if self.filter_accepts_row_self(row, parent):
-            return True
-
-        alpha = parent
-        while alpha.isValid():
-            if self.filter_accepts_row_self(alpha.row(), alpha.parent()):
-                return True
-            alpha = alpha.parent()
-
-        if self.index_has_accepted_children(row, parent):
-            return True
-
-        return False
-
-    def index_has_accepted_children(self, row, parent):
-        """return True if parent has one or more children that match filter, else False
-
-        @param row: row number
-        @param parent: QModelIndex of parent
-        """
-        model_index = self.sourceModel().index(row, 0, parent)
-
-        if model_index.isValid():
-            for idx in range(self.sourceModel().rowCount(model_index)):
-                if self.filter_accepts_row_self(idx, model_index):
-                    return True
-                if self.index_has_accepted_children(idx, model_index):
-                    return True
-
-        return False
-
-    def filter_accepts_row_self(self, row, parent):
-        """return True if filter accepts row, else False
-
-        @param row: row number
-        @param parent: QModelIndex of parent
-        """
-        # filter not set
-        if self.min_ea is None and self.max_ea is None:
-            return True
-
-        index = self.sourceModel().index(row, 0, parent)
-        data = index.internalPointer().data(CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS)
-
-        # virtual address may be empty
-        if not data:
-            return False
-
-        # convert virtual address str to int
-        ea = int(data, 16)
-
-        if self.min_ea <= ea and ea < self.max_ea:
-            return True
-
-        return False
-
-    def add_address_range_filter(self, min_ea, max_ea):
-        """add new address range filter
-
-        called when user checks "limit results by current function" in plugin UI
-
-        @param min_ea: minimum virtual address as seen by IDA
-        @param max_ea: maximum virtual address as seen by IDA
-        """
-        self.min_ea = min_ea
-        self.max_ea = max_ea
-
-        self.setFilterKeyColumn(CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS)
-        self.invalidateFilter()
-
-    def reset_address_range_filter(self):
-        """remove address range filter (accept all results)
-
-        called when user un-checks "limit results by current function" in plugin UI
-        """
-        self.min_ea = None
-        self.max_ea = None
-        self.invalidateFilter()
-
-
-class CapaExplorerSearchProxyModel(QtCore.QSortFilterProxyModel):
-    """A SortFilterProxyModel that accepts rows with a substring match for a configurable query.
-
-    Looks for matches in the text of all rows.
-    Displays the entire tree row if any of the tree branches,
-     that is, you can filter by rule name, or also
-     filter by "characteristic(nzxor)" to filter matches with some feature.
-    """
-
-    def __init__(self, parent=None):
-        """ """
-        super(CapaExplorerSearchProxyModel, self).__init__(parent)
-        self.query = ""
-        self.setFilterKeyColumn(-1)  # all columns
-
-    def filterAcceptsRow(self, row, parent):
-        """true if the item in the row indicated by the given row and parent
-        should be included in the model; otherwise returns false
-
-        @param row: int
-        @param parent: QModelIndex*
-
-        @retval True/False
-        """
-        # this row matches, accept it
-        if self.filter_accepts_row_self(row, parent):
-            return True
-
-        # the parent of this row matches, accept it
-        alpha = parent
-        while alpha.isValid():
-            if self.filter_accepts_row_self(alpha.row(), alpha.parent()):
-                return True
-            alpha = alpha.parent()
-
-        # this row is a parent, and a child matches, accept it
-        if self.index_has_accepted_children(row, parent):
-            return True
-
-        return False
-
-    def index_has_accepted_children(self, row, parent):
-        """returns True if the given row or its children should be accepted"""
-        source_model = self.sourceModel()
-        model_index = source_model.index(row, 0, parent)
-
-        if model_index.isValid():
-            for idx in range(source_model.rowCount(model_index)):
-                if self.filter_accepts_row_self(idx, model_index):
-                    return True
-                if self.index_has_accepted_children(idx, model_index):
-                    return True
-
-        return False
-
-    def filter_accepts_row_self(self, row, parent):
-        """returns True if the given row should be accepted"""
-        if self.query == "":
-            return True
-
-        source_model = self.sourceModel()
-
-        for column in (
-            CapaExplorerDataModel.COLUMN_INDEX_RULE_INFORMATION,
-            CapaExplorerDataModel.COLUMN_INDEX_VIRTUAL_ADDRESS,
-            CapaExplorerDataModel.COLUMN_INDEX_DETAILS,
-        ):
-            index = source_model.index(row, column, parent)
-            data = source_model.data(index, Qt.DisplayRole)
-
-            if not data:
-                continue
-
-            if not isinstance(data, str):
-                # sanity check: should already be a string, but double check
-                continue
-
-            # case in-sensitive matching
-            if self.query.lower() in data.lower():
-                return True
-
-        return False
-
-    def set_query(self, query):
-        self.query = query
-        self.invalidateFilter()
-
-    def reset_query(self):
-        self.set_query("")

capa/ida/plugin_helpers.py

@@ -0,0 +1,99 @@
+# Copyright (C) 2020 FireEye, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import os
+import logging
+
+import idc
+import idaapi
+from PyQt5.QtCore import Qt
+from PyQt5.QtWidgets import QTreeWidgetItem, QTreeWidgetItemIterator
+
+CAPA_EXTENSION = ".capas"
+
+
+logger = logging.getLogger("capa_ida")
+
+
+def get_input_file(freeze=True):
+    """
+    get input file path
+
+        freeze (bool): if True, get freeze file if it exists
+    """
+    # try original file in same directory as idb/i64 without idb/i64 file extension
+    input_file = idc.get_idb_path()[:-4]
+
+    if freeze:
+        # use frozen file if it exists
+        freeze_file_cand = "%s%s" % (input_file, CAPA_EXTENSION)
+        if os.path.isfile(freeze_file_cand):
+            return freeze_file_cand
+
+    if not os.path.isfile(input_file):
+        # TM naming
+        input_file = "%s.mal_" % idc.get_idb_path()[:-4]
+        if not os.path.isfile(input_file):
+            input_file = idaapi.ask_file(0, "*.*", "Please specify input file.")
+    if not input_file:
+        raise ValueError("could not find input file")
+    return input_file
+
+
+def get_orig_color_feature_vas(vas):
+    orig_colors = {}
+    for va in vas:
+        orig_colors[va] = idc.get_color(va, idc.CIC_ITEM)
+    return orig_colors
+
+
+def reset_colors(orig_colors):
+    if orig_colors:
+        for va, color in orig_colors.iteritems():
+            idc.set_color(va, idc.CIC_ITEM, orig_colors[va])
+
+
+def reset_selection(tree):
+    iterator = QTreeWidgetItemIterator(tree, QTreeWidgetItemIterator.Checked)
+    while iterator.value():
+        item = iterator.value()
+        item.setCheckState(0, Qt.Unchecked)  # column, state
+        iterator += 1
+
+
+def get_disasm_line(va):
+    return idc.generate_disasm_line(va, idc.GENDSM_FORCE_CODE)
+
+
+def get_selected_items(tree, skip_level_1=False):
+    selected = []
+    iterator = QTreeWidgetItemIterator(tree, QTreeWidgetItemIterator.Checked)
+    while iterator.value():
+        item = iterator.value()
+        if skip_level_1:
+            # hacky way to check if item is at level 1, if so, skip
+            # alternative, check if text in disasm column
+            if item.parent() and item.parent().parent() is None:
+                iterator += 1
+                continue
+        if item.text(1):
+            # logger.debug('selected %s, %s', item.text(0), item.text(1))
+            selected.append(int(item.text(1), 0x10))
+        iterator += 1
+    return selected
+
+
+def add_child_item(parent, values, feature=None):
+    child = QTreeWidgetItem(parent)
+    child.setFlags(child.flags() | Qt.ItemIsTristate | Qt.ItemIsUserCheckable)
+    for i, v in enumerate(values):
+        child.setText(i, v)
+        if feature:
+            child.setData(0, 0x100, feature)
+        child.setCheckState(0, Qt.Unchecked)
+    return child

capa/optimizer.py

@@ -1,70 +0,0 @@
-import logging
-
-import capa.engine as ceng
-import capa.features.common
-
-logger = logging.getLogger(__name__)
-
-
-def get_node_cost(node):
-    if isinstance(node, (capa.features.common.OS, capa.features.common.Arch, capa.features.common.Format)):
-        # we assume these are the most restrictive features:
-        # authors commonly use them at the start of rules to restrict the category of samples to inspect
-        return 0
-
-    # elif "everything else":
-    #   return 1
-    #
-    # this should be all hash-lookup features.
-    # see below.
-
-    elif isinstance(node, (capa.features.common.Substring, capa.features.common.Regex, capa.features.common.Bytes)):
-        # substring and regex features require a full scan of each string
-        # which we anticipate is more expensive then a hash lookup feature (e.g. mnemonic or count).
-        #
-        # TODO: compute the average cost of these feature relative to hash feature
-        # and adjust the factor accordingly.
-        return 2
-
-    elif isinstance(node, (ceng.Not, ceng.Range)):
-        # the cost of these nodes are defined by the complexity of their single child.
-        return 1 + get_node_cost(node.child)
-
-    elif isinstance(node, (ceng.And, ceng.Or, ceng.Some)):
-        # the cost of these nodes is the full cost of their children
-        # as this is the worst-case scenario.
-        return 1 + sum(map(get_node_cost, node.children))
-
-    else:
-        # this should be all hash-lookup features.
-        # we give this a arbitrary weight of 1.
-        # the only thing more "important" than this is checking OS/Arch/Format.
-        return 1
-
-
-def optimize_statement(statement):
-    # this routine operates in-place
-
-    if isinstance(statement, (ceng.And, ceng.Or, ceng.Some)):
-        # has .children
-        statement.children = sorted(statement.children, key=lambda n: get_node_cost(n))
-        return
-    elif isinstance(statement, (ceng.Not, ceng.Range)):
-        # has .child
-        optimize_statement(statement.child)
-        return
-    else:
-        # appears to be "simple"
-        return
-
-
-def optimize_rule(rule):
-    # this routine operates in-place
-    optimize_statement(rule.statement)
-
-
-def optimize_rules(rules):
-    logger.debug("optimizing %d rules", len(rules))
-    for rule in rules:
-        optimize_rule(rule)
-    return rules

capa/perf.py

@@ -1,10 +0,0 @@
-import collections
-from typing import Dict
-
-# this structure is unstable and may change before the next major release.
-counters: Dict[str, int] = collections.Counter()
-
-
-def reset():
-    global counters
-    counters = collections.Counter()