diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6b8916d3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Explicitly declare text files you want to always be normalized and converted +# to native line endings on checkout. +*.py text +*.yml text +*.md text +*.txt text diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md index f7f029a8..9cd16c0d 100644 --- a/.github/CODE_OF_CONDUCT.md +++ b/.github/CODE_OF_CONDUCT.md @@ -1,46 +1,46 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version] - -[homepage]: https://contributor-covenant.org -[version]: https://contributor-covenant.org/version/1/4/ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version] + +[homepage]: https://contributor-covenant.org +[version]: https://contributor-covenant.org/version/1/4/ diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5f1bb7ef..707d0932 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,197 +1,197 @@ -# Contributing to Capa - -First off, thanks for taking the time to contribute! - -The following is a set of guidelines for contributing to capa and its packages, which are hosted in the [FireEye Organization](https://github.com/fireeye) on GitHub. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. - -#### Table Of Contents - -[Code of Conduct](#code-of-conduct) - -[What should I know before I get started?](#what-should-i-know-before-i-get-started) - * [Capa and its Repositories](#capa-and-its-repositories) - * [Capa Design Decisions](#design-decisions) - -[How Can I Contribute?](#how-can-i-contribute) - * [Reporting Bugs](#reporting-bugs) - * [Suggesting Enhancements](#suggesting-enhancements) - * [Your First Code Contribution](#your-first-code-contribution) - * [Pull Requests](#pull-requests) - -[Styleguides](#styleguides) - * [Git Commit Messages](#git-commit-messages) - * [Python Styleguide](#python-styleguide) - * [Rules Styleguide](#rules-styleguide) - -## Code of Conduct - -This project and everyone participating in it is governed by the [Capa Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers. - -## What should I know before I get started? - -### Capa and its repositories - -We host the capa project as three Github repositories: - - [capa](https://github.com/fireeye/capa) - - [capa-rules](https://github.com/fireeye/capa-rules) - - [capa-testfiles](https://github.com/fireeye/capa-testfiles) - -The command line tools, logic engine, and other Python source code are found in the `capa` repository. -This is the repository to fork when you want to enhance the features, performance, or user interface of capa. -Do *not* push rules directly to this repository, instead... - -The standard rules contributed by the community are found in the `capa-rules` repository. -When you have an idea for a new rule, you should open a PR against `capa-rules`. -We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and discussions should happen. -If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`. -Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way. - -Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository. -These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites; - however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate, - a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth. - -### Design Decisions - -When we make a significant decision in how we maintain the project and what we can or cannot support, - we will document it in the [capa issues tracker](https://github.com/fireeye/capa/issues). -This is the best place review our discussions about what/how/why we do things in the project. -If you have a question, check to see if it is documented there. -If it is *not* documented there, or you can't find an answer, please open a issue. -We'll link to existing issues when appropriate to keep discussions in one place. - -## How Can I Contribute? - -### Reporting Bugs - -This section guides you through submitting a bug report for capa. -Following these guidelines helps maintainers and the community understand your report, reproduce the behavior, and find related reports. - -Before creating bug reports, please check [this list](#before-submitting-a-bug-report) - as you might find out that you don't need to create one. -When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report). -Fill out [the required template](./ISSUE_TEMPLATE/bug_report.md), - the information it asks for helps us resolve issues faster. - -> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one. - -#### Before Submitting A Bug Report - -* **Determine [which repository the problem should be reported in](#capa-and-its-repositories)**. -* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the problem has already been reported. If it has **and the issue is still open**, add a comment to the existing issue instead of opening a new one. - -#### How Do I Submit A (Good) Bug Report? - -Bugs are tracked as [GitHub issues](https://guides.github.com/features/issues/). -After you've determined [which repository](#capa-and-its-repositories) your bug is related to, - create an issue on that repository and provide the following information by filling in - [the template](./ISSUE_TEMPLATE/bug_report.md). - -Explain the problem and include additional details to help maintainers reproduce the problem: - -* **Use a clear and descriptive title** for the issue to identify the problem. -* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started capa, e.g. which command exactly you used in the terminal, or how you started capa otherwise. -* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). -* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior. -* **Explain which behavior you expected to see instead and why.** -* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. -* **If you're reporting that capa crashed**, include the stack trace from the terminal. Include the stack trace in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist. -* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below. - -Provide more context by answering these questions: - -* **Did the problem start happening recently** (e.g. after updating to a new version of capa) or was this always a problem? -* If the problem started happening recently, **can you reproduce the problem in an older version of capa?** What's the most recent version in which the problem doesn't happen? You can download older versions of capa from [the releases page](https://github.com/fireeye/capa/releases). -* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens. -* If the problem is related to working with files (e.g. opening and editing files), **does the problem happen for all files and projects or only some?** Does the problem happen only when working with local or remote files (e.g. on network drives), with files of a specific type (e.g. only JavaScript or Python files), with large files or files with very long lines, or with files in a specific encoding? Is there anything else special about the files you are using? - -Include details about your configuration and environment: - -* **Which version of capa are you using?** You can get the exact version by running `capa --version` in your terminal. -* **What's the name and version of the OS you're using**? - -### Suggesting Enhancements - -This section guides you through submitting an enhancement suggestion for capa, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion and find related suggestions. - -Before creating enhancement suggestions, please check [this list](#before-submitting-an-enhancement-suggestion) as you might find out that you don't need to create one. When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](./ISSUE_TEMPLATE/feature_request.md), including the steps that you imagine you would take if the feature you're requesting existed. - -#### Before Submitting An Enhancement Suggestion - -* **Determine [which repository the enhancement should be suggested in](#capa-and-its-repositories).** -* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one. - -#### How Do I Submit A (Good) Enhancement Suggestion? - -Enhancement suggestions are tracked as [GitHub issues](https://guides.github.com/features/issues/). After you've determined [which repository](#capa-and-its-repositories) your enhancement suggestion is related to, create an issue on that repository and provide the following information: - -* **Use a clear and descriptive title** for the issue to identify the suggestion. -* **Provide a step-by-step description of the suggested enhancement** in as many details as possible. -* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). -* **Describe the current behavior** and **explain which behavior you expected to see instead** and why. -* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of capa which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. -* **Explain why this enhancement would be useful** to most capa users and isn't something that can or should be implemented as an external tool that uses capa as a library. -* **Specify which version of capa you're using.** You can get the exact version by running `capa --version` in your terminal. -* **Specify the name and version of the OS you're using.** - -### Your First Code Contribution - -Unsure where to begin contributing to capa? You can start by looking through these `good-first-issue` and `rule-idea` issues: - -* [good-first-issue](https://github.com/fireeye/capa/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code, and a test or two. -* [rule-idea](https://github.com/fireeye/capa-rules/issues?q=is%3Aissue+is%3Aopen+label%3A%22rule+idea%22) - issues that describe potential new rule ideas. - -Both issue lists are sorted by total number of comments. While not perfect, number of comments is a reasonable proxy for impact a given change will have. - -#### Local development - -capa and all its resources can be developed locally. -For instructions on how to do this, see the "Method 3" section of the [installation guide](https://github.com/fireeye/capa/blob/master/doc/installation.md). - -### Pull Requests - -The process described here has several goals: - -- Maintain capa's quality -- Fix problems that are important to users -- Engage the community in working toward the best possible capa -- Enable a sustainable system for capa's maintainers to review contributions - -Please follow these steps to have your contribution considered by the maintainers: - -1. Follow all instructions in [the template](PULL_REQUEST_TEMPLATE.md) -2. Follow the [styleguides](#styleguides) -3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing
What if the status checks are failing? If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.
- -While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted. - -## Styleguides - -### Git Commit Messages - -* Use the present tense ("Add feature" not "Added feature") -* Use the imperative mood ("Move cursor to..." not "Moves cursor to...") -* Prefix the first line with the component in question ("rules: ..." or "render: ...") -* Reference issues and pull requests liberally after the first line - -### Python Styleguide - -All Python code must adhere to the style guide used by capa: - - 1. [PEP8](https://www.python.org/dev/peps/pep-0008/), with clarifications from - 2. [Willi's style guide](https://docs.google.com/document/d/1iRpeg-w4DtibwytUyC_dDT7IGhNGBP25-nQfuBa-Fyk/edit?usp=sharing), formatted with - 3. [isort](https://pypi.org/project/isort/) (with line width 120 and ordered by line length), and formatted with - 4. [black](https://github.com/psf/black) (with line width 120), and formatted with - 5. [dos2unix](https://linux.die.net/man/1/dos2unix) - -Our CI pipeline will reformat and enforce the Python styleguide. - -### Rules Styleguide - -All (non-nursery) capa rules must: - - 1. pass the [linter](https://github.com/fireeye/capa/blob/master/scripts/lint.py), and - 2. be formatted with [capafmt](https://github.com/fireeye/capa/blob/master/scripts/capafmt.py) - -This ensures that all rules meet the same minimum level of quality and are structured in a consistent way. -Our CI pipeline will reformat and enforce the capa rules styleguide. +# Contributing to Capa + +First off, thanks for taking the time to contribute! + +The following is a set of guidelines for contributing to capa and its packages, which are hosted in the [FireEye Organization](https://github.com/fireeye) on GitHub. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. + +#### Table Of Contents + +[Code of Conduct](#code-of-conduct) + +[What should I know before I get started?](#what-should-i-know-before-i-get-started) + * [Capa and its Repositories](#capa-and-its-repositories) + * [Capa Design Decisions](#design-decisions) + +[How Can I Contribute?](#how-can-i-contribute) + * [Reporting Bugs](#reporting-bugs) + * [Suggesting Enhancements](#suggesting-enhancements) + * [Your First Code Contribution](#your-first-code-contribution) + * [Pull Requests](#pull-requests) + +[Styleguides](#styleguides) + * [Git Commit Messages](#git-commit-messages) + * [Python Styleguide](#python-styleguide) + * [Rules Styleguide](#rules-styleguide) + +## Code of Conduct + +This project and everyone participating in it is governed by the [Capa Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers. + +## What should I know before I get started? + +### Capa and its repositories + +We host the capa project as three Github repositories: + - [capa](https://github.com/fireeye/capa) + - [capa-rules](https://github.com/fireeye/capa-rules) + - [capa-testfiles](https://github.com/fireeye/capa-testfiles) + +The command line tools, logic engine, and other Python source code are found in the `capa` repository. +This is the repository to fork when you want to enhance the features, performance, or user interface of capa. +Do *not* push rules directly to this repository, instead... + +The standard rules contributed by the community are found in the `capa-rules` repository. +When you have an idea for a new rule, you should open a PR against `capa-rules`. +We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and discussions should happen. +If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`. +Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way. + +Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository. +These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites; + however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate, + a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth. + +### Design Decisions + +When we make a significant decision in how we maintain the project and what we can or cannot support, + we will document it in the [capa issues tracker](https://github.com/fireeye/capa/issues). +This is the best place review our discussions about what/how/why we do things in the project. +If you have a question, check to see if it is documented there. +If it is *not* documented there, or you can't find an answer, please open a issue. +We'll link to existing issues when appropriate to keep discussions in one place. + +## How Can I Contribute? + +### Reporting Bugs + +This section guides you through submitting a bug report for capa. +Following these guidelines helps maintainers and the community understand your report, reproduce the behavior, and find related reports. + +Before creating bug reports, please check [this list](#before-submitting-a-bug-report) + as you might find out that you don't need to create one. +When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report). +Fill out [the required template](./ISSUE_TEMPLATE/bug_report.md), + the information it asks for helps us resolve issues faster. + +> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one. + +#### Before Submitting A Bug Report + +* **Determine [which repository the problem should be reported in](#capa-and-its-repositories)**. +* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the problem has already been reported. If it has **and the issue is still open**, add a comment to the existing issue instead of opening a new one. + +#### How Do I Submit A (Good) Bug Report? + +Bugs are tracked as [GitHub issues](https://guides.github.com/features/issues/). +After you've determined [which repository](#capa-and-its-repositories) your bug is related to, + create an issue on that repository and provide the following information by filling in + [the template](./ISSUE_TEMPLATE/bug_report.md). + +Explain the problem and include additional details to help maintainers reproduce the problem: + +* **Use a clear and descriptive title** for the issue to identify the problem. +* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started capa, e.g. which command exactly you used in the terminal, or how you started capa otherwise. +* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). +* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior. +* **Explain which behavior you expected to see instead and why.** +* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. +* **If you're reporting that capa crashed**, include the stack trace from the terminal. Include the stack trace in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist. +* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below. + +Provide more context by answering these questions: + +* **Did the problem start happening recently** (e.g. after updating to a new version of capa) or was this always a problem? +* If the problem started happening recently, **can you reproduce the problem in an older version of capa?** What's the most recent version in which the problem doesn't happen? You can download older versions of capa from [the releases page](https://github.com/fireeye/capa/releases). +* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens. +* If the problem is related to working with files (e.g. opening and editing files), **does the problem happen for all files and projects or only some?** Does the problem happen only when working with local or remote files (e.g. on network drives), with files of a specific type (e.g. only JavaScript or Python files), with large files or files with very long lines, or with files in a specific encoding? Is there anything else special about the files you are using? + +Include details about your configuration and environment: + +* **Which version of capa are you using?** You can get the exact version by running `capa --version` in your terminal. +* **What's the name and version of the OS you're using**? + +### Suggesting Enhancements + +This section guides you through submitting an enhancement suggestion for capa, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion and find related suggestions. + +Before creating enhancement suggestions, please check [this list](#before-submitting-an-enhancement-suggestion) as you might find out that you don't need to create one. When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](./ISSUE_TEMPLATE/feature_request.md), including the steps that you imagine you would take if the feature you're requesting existed. + +#### Before Submitting An Enhancement Suggestion + +* **Determine [which repository the enhancement should be suggested in](#capa-and-its-repositories).** +* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one. + +#### How Do I Submit A (Good) Enhancement Suggestion? + +Enhancement suggestions are tracked as [GitHub issues](https://guides.github.com/features/issues/). After you've determined [which repository](#capa-and-its-repositories) your enhancement suggestion is related to, create an issue on that repository and provide the following information: + +* **Use a clear and descriptive title** for the issue to identify the suggestion. +* **Provide a step-by-step description of the suggested enhancement** in as many details as possible. +* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). +* **Describe the current behavior** and **explain which behavior you expected to see instead** and why. +* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of capa which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. +* **Explain why this enhancement would be useful** to most capa users and isn't something that can or should be implemented as an external tool that uses capa as a library. +* **Specify which version of capa you're using.** You can get the exact version by running `capa --version` in your terminal. +* **Specify the name and version of the OS you're using.** + +### Your First Code Contribution + +Unsure where to begin contributing to capa? You can start by looking through these `good-first-issue` and `rule-idea` issues: + +* [good-first-issue](https://github.com/fireeye/capa/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code, and a test or two. +* [rule-idea](https://github.com/fireeye/capa-rules/issues?q=is%3Aissue+is%3Aopen+label%3A%22rule+idea%22) - issues that describe potential new rule ideas. + +Both issue lists are sorted by total number of comments. While not perfect, number of comments is a reasonable proxy for impact a given change will have. + +#### Local development + +capa and all its resources can be developed locally. +For instructions on how to do this, see the "Method 3" section of the [installation guide](https://github.com/fireeye/capa/blob/master/doc/installation.md). + +### Pull Requests + +The process described here has several goals: + +- Maintain capa's quality +- Fix problems that are important to users +- Engage the community in working toward the best possible capa +- Enable a sustainable system for capa's maintainers to review contributions + +Please follow these steps to have your contribution considered by the maintainers: + +1. Follow all instructions in [the template](PULL_REQUEST_TEMPLATE.md) +2. Follow the [styleguides](#styleguides) +3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing
What if the status checks are failing? If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.
+ +While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted. + +## Styleguides + +### Git Commit Messages + +* Use the present tense ("Add feature" not "Added feature") +* Use the imperative mood ("Move cursor to..." not "Moves cursor to...") +* Prefix the first line with the component in question ("rules: ..." or "render: ...") +* Reference issues and pull requests liberally after the first line + +### Python Styleguide + +All Python code must adhere to the style guide used by capa: + + 1. [PEP8](https://www.python.org/dev/peps/pep-0008/), with clarifications from + 2. [Willi's style guide](https://docs.google.com/document/d/1iRpeg-w4DtibwytUyC_dDT7IGhNGBP25-nQfuBa-Fyk/edit?usp=sharing), formatted with + 3. [isort](https://pypi.org/project/isort/) (with line width 120 and ordered by line length), and formatted with + 4. [black](https://github.com/psf/black) (with line width 120), and formatted with + 5. [dos2unix](https://linux.die.net/man/1/dos2unix) + +Our CI pipeline will reformat and enforce the Python styleguide. + +### Rules Styleguide + +All (non-nursery) capa rules must: + + 1. pass the [linter](https://github.com/fireeye/capa/blob/master/scripts/lint.py), and + 2. be formatted with [capafmt](https://github.com/fireeye/capa/blob/master/scripts/capafmt.py) + +This ensures that all rules meet the same minimum level of quality and are structured in a consistent way. +Our CI pipeline will reformat and enforce the capa rules styleguide. diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 720726aa..0579ad35 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,47 +1,47 @@ ---- -name: Bug report -about: Create a report to help us improve - ---- - - -### Description - - - -### Steps to Reproduce - - - - - -**Expected behavior:** - - - -**Actual behavior:** - - - -### Versions - - - -### Additional Information - - - +--- +name: Bug report +about: Create a report to help us improve + +--- + + +### Description + + + +### Steps to Reproduce + + + + + +**Expected behavior:** + + + +**Actual behavior:** + + + +### Versions + + + +### Additional Information + + + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 55b1bb5b..e366f1ea 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,35 +1,35 @@ ---- -name: Feature request -about: Suggest an idea for capa - ---- - - -### Summary - - - -### Motivation - - - -### Describe alternatives you've considered - - - -## Additional context - - - +--- +name: Feature request +about: Suggest an idea for capa + +--- + + +### Summary + + + +### Motivation + + + +### Describe alternatives you've considered + + + +## Additional context + + + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..f22f4fe9 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,31 @@ + + +### Description + + + +closes # (issue) + +### Documentation + +- [ ] I have updated the [CHANGELOG.md](/CHANGELOG.md), this is required for: + - Bug fixes (non-breaking change which fixes an issue) + - New features (non-breaking change which adds functionality) + - Breaking changes (fix or feature that would cause existing functionality to not work as expected) +- [ ] This change requires a documentation update + - [ ] I have made the corresponding changes to the documentation + +### Tests + +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] No new tests needed diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4e12cabf..df0e81bc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,81 +1,196 @@ -name: build - -on: - # TODO: remove upon merge - push: - branches: ["function-id-flirt"] - - release: - types: [edited, published] - -jobs: - build: - name: PyInstaller for ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-16.04 - # use old linux so that the shared library versioning is more portable - artifact_name: capa - asset_name: linux - - os: windows-latest - artifact_name: capa.exe - asset_name: windows - - os: macos-latest - artifact_name: capa - asset_name: macos - steps: - - name: Checkout capa - uses: actions/checkout@v2 - with: - submodules: true - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - if: matrix.os == 'ubuntu-latest' - run: sudo apt-get install -y libyaml-dev - - name: Install PyInstaller - run: pip install 'pyinstaller==4.2' - - name: Install capa - run: pip install -e . - - name: Build standalone executable - run: pyinstaller .github/pyinstaller/pyinstaller.spec - - name: Does it run? - run: dist/capa -d "tests/data/Practical Malware Analysis Lab 01-01.dll_" - - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.asset_name }} - path: dist/${{ matrix.artifact_name }} - - zip: - name: zip ${{ matrix.asset_name }} - runs-on: ubuntu-latest - needs: build - strategy: - matrix: - include: - - asset_name: linux - artifact_name: capa - - asset_name: windows - artifact_name: capa.exe - - asset_name: macos - artifact_name: capa - steps: - - name: Download ${{ matrix.asset_name }} - uses: actions/download-artifact@v2 - with: - name: ${{ matrix.asset_name }} - - name: Set executable flag - run: chmod +x ${{ matrix.artifact_name }} - - name: Set zip name - run: echo "zip_name=capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV - - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }} - run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }} - - name: Upload ${{ env.zip_name }} to GH Release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN}} - file: ${{ env.zip_name }} - tag: ${{ github.ref }} +<<<<<<< HEAD +name: build + +on: + # TODO: remove upon merge + push: + branches: ["function-id-flirt"] + + release: + types: [edited, published] + +jobs: + build: + name: PyInstaller for ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-16.04 + # use old linux so that the shared library versioning is more portable + artifact_name: capa + asset_name: linux + - os: windows-latest + artifact_name: capa.exe + asset_name: windows + - os: macos-latest + artifact_name: capa + asset_name: macos + steps: + - name: Checkout capa + uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - if: matrix.os == 'ubuntu-latest' + run: sudo apt-get install -y libyaml-dev + - name: Install PyInstaller + run: pip install 'pyinstaller==4.2' + - name: Install capa + run: pip install -e . + - name: Build standalone executable + run: pyinstaller .github/pyinstaller/pyinstaller.spec + - name: Does it run? + run: dist/capa -d "tests/data/Practical Malware Analysis Lab 01-01.dll_" + - uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.asset_name }} + path: dist/${{ matrix.artifact_name }} + + zip: + name: zip ${{ matrix.asset_name }} + runs-on: ubuntu-latest + needs: build + strategy: + matrix: + include: + - asset_name: linux + artifact_name: capa + - asset_name: windows + artifact_name: capa.exe + - asset_name: macos + artifact_name: capa + steps: + - name: Download ${{ matrix.asset_name }} + uses: actions/download-artifact@v2 + with: + name: ${{ matrix.asset_name }} + - name: Set executable flag + run: chmod +x ${{ matrix.artifact_name }} + - name: Set zip name + run: echo "zip_name=capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV + - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }} + run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }} + - name: Upload ${{ env.zip_name }} to GH Release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN}} + file: ${{ env.zip_name }} + tag: ${{ github.ref }} +======= +name: build + +on: + push: + branches: [master] + release: + types: [edited, published] + +jobs: + build: + name: PyInstaller for ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-16.04 + # use old linux so that the shared library versioning is more portable + artifact_name: capa + asset_name: linux + - os: windows-2019 + artifact_name: capa.exe + asset_name: windows + - os: macos-10.15 + artifact_name: capa + asset_name: macos + steps: + - name: Checkout capa + uses: actions/checkout@v2 + with: + submodules: true + # using Python 3.8 to support running across multiple operating systems including Windows 7 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - if: matrix.os == 'ubuntu-16.04' + run: sudo apt-get install -y libyaml-dev + - name: Install PyInstaller + run: pip install 'pyinstaller==4.2' + - name: Install capa + run: pip install -e . + - name: Build standalone executable + run: pyinstaller .github/pyinstaller/pyinstaller.spec + - name: Does it run? + run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_" + - uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.asset_name }} + path: dist/${{ matrix.artifact_name }} + + test_run: + # test that binaries run on push to master + if: github.event_name == 'push' + name: Test run on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: [build] + strategy: + matrix: + include: + # OSs not already tested above + - os: ubuntu-18.04 + artifact_name: capa + asset_name: linux + - os: ubuntu-20.04 + artifact_name: capa + asset_name: linux + - os: windows-2016 + artifact_name: capa.exe + asset_name: windows + steps: + - name: Download ${{ matrix.asset_name }} + uses: actions/download-artifact@v2 + with: + name: ${{ matrix.asset_name }} + - name: Set executable flag + if: matrix.os != 'windows-2016' + run: chmod +x ${{ matrix.artifact_name }} + - name: Run capa + run: ./${{ matrix.artifact_name }} -h + + zip_and_upload: + # upload zipped binaries to Release page + if: github.event_name == 'release' + name: zip and upload ${{ matrix.asset_name }} + runs-on: ubuntu-20.04 + needs: [build] + strategy: + matrix: + include: + - asset_name: linux + artifact_name: capa + - asset_name: windows + artifact_name: capa.exe + - asset_name: macos + artifact_name: capa + steps: + - name: Download ${{ matrix.asset_name }} + uses: actions/download-artifact@v2 + with: + name: ${{ matrix.asset_name }} + - name: Set executable flag + run: chmod +x ${{ matrix.artifact_name }} + - name: Set zip name + run: echo "zip_name=capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV + - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }} + run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }} + - name: Upload ${{ env.zip_name }} to GH Release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN}} + file: ${{ env.zip_name }} + tag: ${{ github.ref }} +>>>>>>> master diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2258de3b..a5119e4c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,29 +1,30 @@ -# This workflows will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -name: publish to pypi - -on: - release: - types: [published] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '2.7' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload --skip-existing dist/* \ No newline at end of file +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: publish to pypi + +on: + release: + types: [published] + +jobs: + deploy: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.6' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload --skip-existing dist/* + diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml new file mode 100644 index 00000000..a2d94315 --- /dev/null +++ b/.github/workflows/tag.yml @@ -0,0 +1,29 @@ +name: tag + +on: + release: + types: [published] + +jobs: + tag: + name: Tag capa rules + runs-on: ubuntu-20.04 + steps: + - name: Checkout capa-rules + uses: actions/checkout@v2 + with: + repository: fireeye/capa-rules + token: ${{ secrets.CAPA_TOKEN }} + - name: Tag capa-rules + run: | + # user information is needed to create annotated tags (with a message) + git config user.email 'capa-dev@fireeye.com' + git config user.name 'Capa Bot' + name=${{ github.event.release.tag_name }} + git tag $name -m "https://github.com/fireeye/capa/releases/$name" + - name: Push tag to capa-rules + uses: ad-m/github-push-action@master + with: + repository: fireeye/capa-rules + github_token: ${{ secrets.CAPA_TOKEN }} + tags: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c3faac83..03888c8e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ on: jobs: code_style: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout capa uses: actions/checkout@v2 @@ -17,14 +17,14 @@ jobs: with: python-version: 3.8 - name: Install dependencies - run: pip install 'isort==5.*' black + run: pip install -e .[dev] - name: Lint with isort run: isort --profile black --length-sort --line-width 120 -c . - name: Lint with black run: black -l 120 --check . rule_linter: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout capa with rules submodule uses: actions/checkout@v2 @@ -34,37 +34,43 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 - # We don't need vivisect, so we can install capa using Python3 - name: Install capa run: pip install -e . - name: Run rule linter run: python scripts/lint.py rules/ tests: - name: Tests in ${{ matrix.python }} - runs-on: ubuntu-latest + name: Tests in ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} needs: [code_style, rule_linter] strategy: fail-fast: false matrix: + os: [ubuntu-20.04, windows-2019, macos-10.15] + # across all operating systems + python-version: [3.6, 3.9] include: - - python: 2.7 - - python: 3.7 - - python: 3.8 - - python: 3.9.1 + # on Ubuntu run these as well + - os: ubuntu-20.04 + python-version: 3.7 + - os: ubuntu-20.04 + python-version: 3.8 steps: - name: Checkout capa with submodules uses: actions/checkout@v2 with: submodules: true - - name: Set up Python ${{ matrix.python }} + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python }} + python-version: ${{ matrix.python-version }} - name: Install pyyaml + if: matrix.os == 'ubuntu-20.04' run: sudo apt-get install -y libyaml-dev + - name: Install Microsoft Visual C++ 9.0 + if: matrix.os == 'windows-2019' && matrix.python-version == '2.7' + run: choco install vcpython27 - name: Install capa run: pip install -e .[dev] - name: Run tests run: pytest tests/ - diff --git a/CHANGELOG.md b/CHANGELOG.md index 42cfd75f..aeeb8bb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,484 +1,683 @@ -# Change Log - -## v1.5.0 (2021-02-05) - -This release brings support for running capa under Python 3 via [SMDA](https://github.com/danielplohmann/smda), more thorough CI testing and linting, better extraction of strings and byte features, and 50 (!) new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: - - - @johnk3r - - @doomedraven - - @stvemillertime - - @itreallynick - - @0x534a - -@dzbeck also added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for many rules. - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - - -### New Features - - - py3 support via SMDA #355 @danielplohmann @jcrussell - - scripts: example of using capa as a library #372, #380 @doomedraven - - ci: enable dependabot #373 @mr-tz - - ci: lint rules @mr-tz - - ci: lint rule format #401 @mr-tz - - freeze: add base address #391 @mr-tz - - json: meta: add base address #412 @mr-tz - -### New Rules (50) - - - 64-bit execution via heavens gate @recvfrom - - contain anti-disasm techniques @mr-tz - - check for microsoft office emulation @re-fox - - check for windows sandbox via device @re-fox - - check for windows sandbox via dns suffix @re-fox - - check for windows sandbox via genuine state @re-fox - - check for windows sandbox via process name @re-fox - - check for windows sandbox via registry @re-fox - - capture microphone audio @re-fox - - capture public ip @re-fox - - get domain trust relationships @johnk3r - - check HTTP status code @mr-tz - - compiled with perl2exe @re-fox - - compiled with ps2exe @re-fox - - compiled with pyarmor @stvemillertime, @itreallynick - - validate payment card number using luhn algorithm @re-fox - - hash data using fnv @re-fox @mr-tz - - generate random numbers via WinAPI @mike-hunhoff @johnk3r - - enumerate files recursively @re-fox - - get file system object information @mike-hunhoff - - read virtual disk @re-fox - - register minifilter driver @mike-hunhoff - - start minifilter driver @mike-hunhoff - - enumerate gui resources @johnk3r - - simulate CTRL ALT DEL @mike-hunhoff - - hijack thread execution @0x534a - - inject dll @0x534a - - inject pe @0x534a - - create or open registry key @mike-hunhoff - - delete registry value @mike-hunhoff - - query or enumerate registry key @mike-hunhoff - - query or enumerate registry value @mike-hunhoff - - resume thread @0x534a - - suspend thread @0x534a - - allocate memory @0x534a - - allocate RW memory @0x534a - - contain pusha popa sequence @mr-tz - - create or open file @mike-hunhoff - - open process @0x534a - - open thread @0x534a - - get kernel32 base address @mr-tz - - get ntdll base address @mr-tz - - encrypt or decrypt data via BCrypt @mike-hunhoff - - generate random numbers using the Delphi LCG @williballenthin - - hash data via BCrypt @mike-hunhoff - - migrate process to active window station @williballenthin - - patch process command line @williballenthin - - resolve function by hash @williballenthin - - persist via Winlogon Helper DLL registry key @0x534a - - schedule task via command line @0x534a - -### Bug Fixes - - - doc: pyinstaller build process @mr-tz - - ida: better bytes extraction #409 @mike-hunhoff - - viv: better unicode string extraction #364 @mike-hunhoff - - viv: better unicode string extraction #378 @mr-tz - - viv: more xor instructions #379 @mr-tz - - viv: decrease logging verbosity #381 @mr-tz - - rules: fix api description syntax #403 @mike-hunhoff - - main: disable progress background thread #410 @mike-hunhoff - -### Changes - - - rules: return lib rules for scopes #398 @mr-tz - -### Raw diffs - - - [capa v1.4.1...v1.5.0](https://github.com/fireeye/capa/compare/v1.4.1...v1.5.0) - - [capa-rules v1.4.0...v1.5.0](https://github.com/fireeye/capa-rules/compare/v1.4.0...v1.5.0) - -## v1.4.1 (2020-10-23) - -This release fixes an issue building capa on our CI server, which prevented us from building standalone binaries for v1.4.1. - -### Bug Fixes - - - install VC dependencies for Python 2.7 during Windows build - -### Raw diffs - - - [capa v1.4.0...v1.4.1](https://github.com/fireeye/capa/compare/v1.4.0...v1.4.1) - - [capa-rules v1.4.0...v1.4.1](https://github.com/fireeye/capa-rules/compare/v1.4.0...v1.4.1) - -## v1.4.0 (2020-10-23) - -This capa release includes changes to the rule parsing, enhanced feature extraction, various bug fixes, and improved capa scripts. Everyone should benefit from the improved functionality and performance. The community helped to add 69 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: - - - @mwilliams31 - - @yt0ng - -@dzbeck added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for 86 rules. - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - -### New features - - - script that demonstrates bulk processing @williballenthin #307 - - main: render MBC table @mr-tz #332 - - ida backend: improve detection of APIs called via two or more chained thunks @mike-hunhoff #340 - - viv backend: improve detection of APIs called via two or more chained thunks @mr-tz #341 - - features: extract APIs called via jmp instruction @mr-tz #337 - -### New rules - - - clear the Windows event log @mike-hunhoff - - crash the Windows event logging service @mike-hunhoff - - packed with kkrunchy @re-fox - - packed with nspack @re-fox - - packed with pebundle @re-fox - - packed with pelocknt @re-fox - - packed with peshield @re-fox - - packed with petite @re-fox - - packed with rlpack @re-fox - - packed with upack @re-fox - - packed with y0da crypter @re-fox - - compiled with rust @re-fox - - compute adler32 checksum @mwilliams31 - - encrypt-data-using-hc-128 @recvfrom - - manipulate console @williballenthin - - references logon banner @re-fox - - terminate process via fastfail @re-fox - - delete volume shadow copies @mr-tz - - authenticate HMAC @mr-tz - - compiled from EPL @williballenthin - - compiled with Go @williballenthin - - create Restart Manager session @mike-hunhoff - - decode data using Base64 via WinAPI @mike-hunhoff - - empty recycle bin quietly @mwilliams31 - - enumerate network shares @mike-hunhoff - - hook routines via microsoft detours @williballenthin - - hooked by API Override @williballenthin - - impersonate user @mike-hunhoff - - the @williballenthin packer detection package, thanks to Hexacorn for the data, see https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/ - - packed with CCG - - packed with Crunch - - packed with Dragon Armor - - packed with enigma - - packed with Epack - - packed with MaskPE - - packed with MEW - - packed with Mpress - - packed with Neolite - - packed with PECompact - - packed with Pepack - - packed with Perplex - - packed with ProCrypt - - packed with RPCrypt - - packed with SeauSFX - - packed with Shrinker - - packed with Simple Pack - - packed with StarForce - - packed with SVKP - - packed with Themida - - packed with TSULoader - - packed with VProtect - - packed with WWPACK - - rebuilt by ImpRec - - packaged as a Pintool - - packaged as a CreateInstall installer - - packaged as a WinZip self-extracting archive - - reference 114DNS DNS server @williballenthin - - reference AliDNS DNS server @williballenthin - - reference Cloudflare DNS server @williballenthin - - reference Comodo Secure DNS server @williballenthin - - reference Google Public DNS server @williballenthin - - reference Hurricane Electric DNS server @williballenthin - - reference kornet DNS server @williballenthin - - reference L3 DNS server @williballenthin - - reference OpenDNS DNS server @williballenthin - - reference Quad9 DNS server @williballenthin - - reference Verisign DNS server @williballenthin - - run as service @mike-hunhoff - - schedule task via ITaskService @mike-hunhoff - - references DNS over HTTPS endpoints @yt0ng - -### Bug fixes - - - ida plugin: fix tree-view exception @mike-hunhoff #315 - - ida plugin: fix feature count @mike-hunhoff - - main: fix reported total rule count @williballenthin #325 - - features: fix handling of API names with multiple periods @mike-hunhoff #329 - - ida backend: find all byte sequences instead of only first @mike-hunhoff #335 - - features: display 0 value @mr-tz #338 - - ida backend: extract ordinal and name imports @mr-tz #343 - - show-features: improvements and support within IDA @mr-tz #342 - - main: sanity check MBC rendering @williballenthin - - main: handle sample path that contains non-ASCII characters @mr-tz #328 - -### Changes - - - rules: use yaml.CLoader for better performance @williballenthin #306 - - rules: parse descriptions for statements @mr-tz #312 - -### Raw diffs - - - [capa v1.3.0...v1.4.0](https://github.com/fireeye/capa/compare/v1.3.0...v1.4.0) - - [capa-rules v1.3.0...v1.4.0](https://github.com/fireeye/capa-rules/compare/v1.3.0...v1.4.0) - -## v1.3.0 (2020-09-14) - -This release brings newly updated mappings to the [Malware Behavior Catalog version 2.0](https://github.com/MBCProject/mbc-markdown), many enhancements to the IDA Pro plugin, [flare-capa on PyPI](https://pypi.org/project/flare-capa/), a bunch of bug fixes to improve feature extraction, and four new rules. We received contributions from ten reverse engineers, including seven new ones: - - - @dzbeck - - @recvfrom - - @toomanybananas - - @cclauss - - @adamprescott91 - - @weslambert - - @stevemk14ebr - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - -### Key changes to IDA Plugin - -The IDA Pro integration is now distributed as a real plugin, instead of a script. This enables a few things: - - - keyboard shortcuts and file menu integration - - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%` - - generally doing thing the "right way" - -How to get this new version? Its easy: download [capa_explorer.py](https://raw.githubusercontent.com/fireeye/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. - -Please refer to the plugin [readme](https://github.com/fireeye/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin. - -Please open an issue in this repository if you notice anything weird. - -### New features - - - ida plugin: now a real plugin, not a script @mike-hunhoff - - core: distributed via PyPI as [flare-capa](https://pypi.org/project/flare-capa/) @williballenthin - - features: enable automatic A/W handling for imports @williballenthin @Ana06 #246 - - ida plugin: persist rules directory setting via [ida-settings](https://github.com/williballenthin/ida-settings) @williballenthin #268 - - ida plugin: add search bar to results view @williballenthin #285 - - ida plugin: add `Analyze` and `Reset` buttons to tree view @mike-hunhoff #304 - - ida plugin: add status label to tree view @mike-hunhoff - - ida plugin: add progress indicator @mike-hunhoff, @mr-tz - -### New rules - - - compiled with py2exe @re-fox - - resolve path using msvcrt @re-fox - - decompress data using QuickLZ @edeca - - encrypt data using sosemanuk @recvfrom - -### Bug fixes - - - rule: reduce FP in DNS resolution @toomanybananas - - engine: report correct strings matched via regex @williballenthin #262 - - formatter: correctly format descriptions in two-line syntax @williballenthin @recvfrom #263 - - viv: better extract offsets from SibOper operands @williballenthin @edeca #276 - - import-to-ida: fix import error @cclauss - - viv: don't write settings to ~/.viv/viv.json @williballenthin @rakuy0 @weslambert #244 - - ida plugin: remove dependency loop that resulted in unnecessary overhead @mike-hunhoff #303 - - ida plugin: correctly highlight regex matches in IDA Disassembly view @mike-hunhoff #305 - - ida plugin: better handle rule directory prompt and failure case @stevemk14ebr @mike-hunhoff #309 - -### Changes - - - rules: update meta mapping to MBC 2.0! @dzbeck - - render: don't display rules that are also matched by other rules @williballenthin @Ana06 #224 - - ida plugin: simplify tabs, removing summary and adding detail to results view @williballenthin #286 - - ida plugin: analysis is no longer automatically started when plugin is first opened @mike-hunhoff #304 - - ida plugin: user must manually select a capa rules directory before analysis can be performed @mike-hunhoff - - ida plugin: user interface controls are disabled until analysis is performed @mike-hunhoff #304 - -### Raw diffs - - - [capa v1.2.0...v1.3.0](https://github.com/fireeye/capa/compare/v1.2.0...v1.3.0) - - [capa-rules v1.2.0...v1.3.0](https://github.com/fireeye/capa-rules/compare/v1.2.0...v1.3.0) - -## v1.2.0 (2020-08-31) - -This release brings UI enhancements, especially for the IDA Pro plugin, -investment towards py3 support, -fixes some bugs identified by the community, -and 46 (!) new rules. -We received contributions from ten reverse engineers, including five new ones: - - - @agithubuserlol - - @recvfrom - - @D4nch3n - - @edeca - - @winniepe - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). -Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) -and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - -### New features - - - ida plugin: display arch flavors @mike-hunhoff - - ida plugin: display block descriptions @mike-hunhoff - - ida backend: extract features from nested pointers @mike-hunhoff - - main: show more progress output @williballenthin - - core: pin dependency versions #258 @recvfrom - -### New rules - - bypass UAC via AppInfo ALPC @agithubuserlol - - bypass UAC via token manipulation @agithubuserlol - - check for sandbox and av modules @re-fox - - check for sandbox username @re-fox - - check if process is running under wine @re-fox - - validate credit card number using luhn algorithm @re-fox - - validate credit card number using luhn algorithm with no lookup table @re-fox - - hash data using FNV @edeca @mr-tz - - link many functions at runtime @mr-tz - - reference public RSA key @mr-tz - - packed with ASPack @williballenthin - - delete internet cache @mike-hunhoff - - enumerate internet cache @mike-hunhoff - - send ICMP echo request @mike-hunhoff - - check for debugger via API @mike-hunhoff - - check for hardware breakpoints @mike-hunhoff - - check for kernel debugger via shared user data structure @mike-hunhoff - - check for protected handle exception @mike-hunhoff - - check for software breakpoints @mike-hunhoff - - check for trap flag exception @mike-hunhoff - - check for unexpected memory writes @mike-hunhoff - - check process job object @mike-hunhoff - - reference anti-VM strings targeting Parallels @mike-hunhoff - - reference anti-VM strings targeting Qemu @mike-hunhoff - - reference anti-VM strings targeting VirtualBox @mike-hunhoff - - reference anti-VM strings targeting VirtualPC @mike-hunhoff - - reference anti-VM strings targeting VMWare @mike-hunhoff - - reference anti-VM strings targeting Xen @mike-hunhoff - - reference analysis tools strings @mike-hunhoff - - reference WMI statements @mike-hunhoff - - get number of processor cores @mike-hunhoff - - get number of processors @mike-hunhoff - - enumerate disk properties @mike-hunhoff - - get disk size @mike-hunhoff - - get process heap flags @mike-hunhoff - - get process heap force flags @mike-hunhoff - - get Explorer PID @mike-hunhoff - - delay execution @mike-hunhoff - - check for process debug object @mike-hunhoff - - check license value @mike-hunhoff - - check ProcessDebugFlags @mike-hunhoff - - check ProcessDebugPort @mike-hunhoff - - check SystemKernelDebuggerInformation @mike-hunhoff - - check thread yield allowed @mike-hunhoff - - enumerate system firmware tables @mike-hunhoff - - get system firmware table @mike-hunhoff - - hide thread from debugger @mike-hunhoff - -### Bug fixes - - - ida backend: extract unmapped immediate number features @mike-hunhoff - - ida backend: fix stack cookie check #257 @mike-hunhoff - - viv backend: better extract gs segment access @williballenthin - - core: enable counting of string features #241 @D4nch3n @williballenthin - - core: enable descriptions on feature with arch flavors @mike-hunhoff - - core: update git links for non-SSH access #259 @recvfrom - -### Changes - - - ida plugin: better default display showing first level nesting @winniepe - - remove unused `characteristic(switch)` feature @ana06 - - prepare testing infrastructure for multiple backends/py3 @williballenthin - - ci: zip build artifacts @ana06 - - ci: build all supported python versions @ana06 - - code style and formatting @mr-tz - -### Raw diffs - - - [capa v1.1.0...v1.2.0](https://github.com/fireeye/capa/compare/v1.1.0...v1.2.0) - - [capa-rules v1.1.0...v1.2.0](https://github.com/fireeye/capa-rules/compare/v1.1.0...v1.2.0) - -## v1.1.0 (2020-08-05) - -This release brings new rule format updates, such as adding `offset/x32` and negative offsets, -fixes some bugs identified by the community, and 28 (!) new rules. -We received contributions from eight reverse engineers, including four new ones: - - - @re-fox - - @psifertex - - @bitsofbinary - - @threathive - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - -### New features - - - import: add Binary Ninja import script #205 #207 @psifertex - - rules: offsets can be negative #197 #208 @williballenthin - - rules: enable descriptions for statement nodes #194 #209 @Ana06 - - rules: add arch flavors to number and offset features #210 #216 @williballenthin - - render: show SHA1/SHA256 in default report #164 @threathive - - tests: add tests for IDA Pro backend #202 @williballenthin - -### New rules - - - check for unmoving mouse cursor @BitsOfBinary - - check mutex and exit @re-fox - - parse credit card information @re-fox - - read ini file @re-fox - - validate credit card number with luhn algorithm @re-fox - - change the wallpaper @re-fox - - acquire debug privileges @williballenthin - - import public key @williballenthin - - terminate process by name @williballenthin - - encrypt data using DES @re-fox - - encrypt data using DES via WinAPI @re-fox - - hash data using sha1 via x86 extensions @re-fox - - hash data using sha256 via x86 extensions @re-fox - - capture network configuration via ipconfig @re-fox - - hash data via WinCrypt @mike-hunhoff - - get file attributes @mike-hunhoff - - allocate thread local storage @mike-hunhoff - - get thread local storage value @mike-hunhoff - - set thread local storage @mike-hunhoff - - get session integrity level @mike-hunhoff - - add file to cabinet file @mike-hunhoff - - flush cabinet file @mike-hunhoff - - open cabinet file @mike-hunhoff - - gather firefox profile information @re-fox - - encrypt data using skipjack @re-fox - - encrypt data using camellia @re-fox - - hash data using tiger @re-fox - - encrypt data using blowfish @re-fox - - encrypt data using twofish @re-fox - -### Bug fixes - - - linter: fix exception when examples is `None` @Ana06 - - linter: fix suggested recommendations via templating @williballenthin - - render: fix exception when rendering counts @williballenthin - - render: fix render of negative offsets @williballenthin - - extractor: fix segmentation violation from vivisect @williballenthin - - main: fix crash when .viv cannot be saved #168 @secshoggoth @williballenthin - - main: fix shellcode .viv save path @williballenthin - -### Changes - - - doc: explain how to bypass gatekeeper on macOS @psifertex - - doc: explain supported linux distributions @Ana06 - - doc: explain submodule update with --init @psifertex - - main: improve program help output @mr-tz - - main: disable progress when run in quiet mode @mr-tz - - main: assert supported IDA versions @mr-tz - - extractor: better identify nested pointers to strings @williballenthin - - setup: specify vivisect download url @Ana06 - - setup: pin vivisect version @williballenthin - - setup: bump vivisect dependency version @williballenthin - - setup: set Python project name to `flare-capa` @williballenthin - - ci: run tests and linter via Github Actions @Ana06 - - hooks: run style checkers and hide stashed output @Ana06 - - linter: ignore period in rule filename @williballenthin - - linter: warn on nursery rule with no changes needed @williballenthin - -### Raw diffs - - - [capa v1.0.0...v1.1.0](https://github.com/fireeye/capa/compare/v1.0.0...v1.1.0) - - [capa-rules v1.0.0...v1.1.0](https://github.com/fireeye/capa-rules/compare/v1.0.0...v1.1.0) +# Change Log + +## master (unreleased) + +The first Python 3 ONLY capa version. + +### New Features + +- main: auto detect shellcode based on file extension #516 @mr-tz + +### New Rules + +### Bug Fixes + +- build: use Python 3.8 for PyInstaller to support consistently running across multiple operating systems including Windows 7 #505 @mr-tz + +### Changes + +- py3: drop Python 2 support #480 @Ana06 +- deps: bump ruamel yaml parser to 0.17.4 #519 @williballenthin +- explorer: explain how to install IDA 7.6 patch to enable the plugin #528 @williballenthin + +### Development + +- ci: add capa release link to capa-rules tag #517 @Ana06 + +### Raw diffs + + +- [capa v1.6.1...master](https://github.com/fireeye/capa/compare/v1.6.1...master) +- [capa-rules v1.6.1...master](https://github.com/fireeye/capa-rules/compare/v1.6.1...master) + + +## v1.6.2 (2021-04-13) + +This release backports a fix to capa 1.6: The Windows binary was built with Python 3.9 which doesn't support Windows 7. + +### Bug Fixes + +- build: use Python 3.8 for PyInstaller to support consistently running across multiple operating systems including Windows 7 @mr-tz @Ana06 + +### Raw diffs + + - [capa v1.6.1...v1.6.2](https://github.com/fireeye/capa/compare/v1.6.1...v1.6.2) + - [capa-rules v1.6.1...v1.6.2](https://github.com/fireeye/capa-rules/compare/v1.6.1...v1.6.2) + +## v1.6.1 (2021-04-07) + +This release includes several bug fixes, such as a vivisect issue that prevented capa from working on Windows with Python 3. It also adds 17 new rules and a bunch of improvements in the rules and IDA rule generator. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. + +### Upcoming changes + +**This is the very last capa release that supports Python 2.** The next release will be v2.0 and will have breaking changes, including the removal of Python 2 support. + +### New features + +- explorer: add support for multi-line tab and SHIFT + Tab #474 @mike-hunhoff + +![multi-line tab in rule generator](doc/img/changelog/tab.gif) + +### New Rules (17) + +- encrypt data using RC4 with custom key via WinAPI @MalwareMechanic +- encrypt data using Curve25519 @dandonov +- packaged as an IExpress self-extracting archive @recvfrom +- create registry key via offline registry library @johnk3r +- open registry key via offline registry library @johnk3r +- query registry key via offline registry library @johnk3r +- set registry key via offline registry library @johnk3r +- delete registry key via offline registry library @johnk3r +- enumerate PE sections @Ana06 +- inject DLL reflectively @Ana06 +- inspect section memory permissions @Ana06 +- parse PE exports @Ana06 +- rebuild import table @Ana06 +- compare security identifiers @mike-hunhoff +- get user security identifier @mike-hunhoff +- listen for remote procedure calls @mike-hunhoff +- query remote server for available data @mike-hunhoff + +### Bug Fixes + +- vivisect: update to v1.0.1 which includes bug fix for #459 (capa failed in Windows with Python 3 and vivisect) #512 @williballenthin +- explorer: fix initialize rules directory #464 @mike-hunhoff +- explorer: support subscope rules #493 @mike-hunhoff +- explorer: add checks to validate matched data when searching #500 @mike-hunhoff +- features, explorer: add support for string features with special characters e.g. '\n' #468 @mike-hunhoff + +### Changes + +- vivisect: raises `IncompatibleVivVersion` instead of `UnicodeDecodeError` when using incompatible Python 2 `.viv` files with Python3 #479 @Ana06 +- explorer: improve settings modification #465 @mike-hunhoff +- rules: improvements @mr-tz, @re-fox, @mike-hunhoff +- rules, lint: enforce string with double quotes formatting in rules #468 @mike-hunhoff +- lint: ensure LF end of line #485 #486 @mr-tz +- setup: pin dependencies #513 #504 @Ana06 @mr-tz + +### Development + +- ci: test on Windows, Ubuntu, macOS across Python versions #470 @mr-tz @Ana06 +- ci: pin OS versions #491 @williballenthin +- ci: tag capa-rules on release #476 @Ana06 +- doc: document release process #476 @Ana06 +- doc: Improve README badges #477 #478 @ana06 @mr-tz +- doc: update capa explorer documentation #503 @mike-hunhoff +- doc: add PR template #495 @mr-tz +- changelog: document incompatibility of viv files #475 @Ana06 +- rule loading: ignore files starting with .git #492 @mr-tz + +### Raw diffs + + - [capa v1.6.0...v1.6.1](https://github.com/fireeye/capa/compare/v1.6.0...v1.6.1) + - [capa-rules v1.6.0...v1.6.1](https://github.com/fireeye/capa-rules/compare/v1.6.0...v1.6.1) + + +## v1.6.0 (2021-03-09) + +This release adds the capa explorer rule generator plugin for IDA Pro, vivisect support for Python 3 and 12 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. Thank you also to the vivisect development team (@rakuy0, @atlas0fd00m) for the Python 3 support (`vivisect==1.0.0`) and the fixes for Python 2 (`vivisect==0.2.1`). + +### Rule Generator IDA Plugin + +The capa explorer IDA plugin now helps you quickly build new capa rules using features extracted directly from your IDA database. Without leaving the plugin interface you can use the features extracted by capa explorer to develop and test new rules and save your work directly to your capa rules directory. To get started select the new `Rule Generator` tab, navigate to a function in the IDA `Disassembly` view, and click `Analyze`. For more information check out the capa explorer [readme](https://github.com/fireeye/capa/blob/master/capa/ida/plugin/README.md). + +![](doc/img/rulegen_expanded.png) + +### Python 2/3 vivisect workspace compatibility + +This version of capa adds Python 3 support in vivisect. Note that `.viv` files (generated by vivisect) are not compatible between Python 2 and Python 3. When updating to Python 3 you need to delete all the `.viv` files for capa to work. + +If you get the following error (or a similar one), you most likely need to delete `.viv` files: +``` +UnicodeDecodeError: 'ascii' codec can't decode byte 0x90 in position 2: ordinal not in range(128) +``` + +### Upcoming changes + +**This is the last capa release that supports Python 2.** The next release will be v2.0 and will have breaking changes, including the removal of Python 2 support. + +If you have workflows that rely on the Python 2 version and need future maintenance, please reach out. We may be able to supply limited backports of key fixes and features. + +### New features + +- explorer: Add capa explorer rule generator plugin for IDA Pro. Now capa explorer helps you build new capa rules! #426, #438, #439 @mike-hunhoff +- python: Python 3 support in vivisect #421 @Ana06 +- main: Add backend option in Python 3 to select the backend to be used (either SMDA or vivisect) #421 @Ana06 +- python: Python 3 support in IDA #429, #437 @mike-hunhoff +- ci: test pyinstaller CI #452 @williballenthin +- scripts: enable multiple backends in `show-features.py` #429 @mike-hunhoff +- scripts: add `scripts/vivisect-py2-vs-py3.sh` to compare vivisect Python 2 vs 3 (can easily be modified to test run times and compare different versions) #421 @Ana06 + +### New Rules (12) + +- patch process command line @re-fox @williballenthin (graduated from nursery) +- compiled with dmd @re-fox +- compiled with exe4j @johnk3r +- compiled from Visual Basic @williballenthin +- capture screenshot in Go @TcM1911 +- compiled with Nim @mike-hunhoff +- linked against Go process enumeration library @TcM1911 +- linked against Go registry library @TcM1911 +- linked against Go WMI library @TcM1911 +- linked against Go static asset library @TcM1911 +- inspect load icon resource @mike-hunhoff +- linked against XZip @mr-tz + +### Bug Fixes + +- ida: check for unmapped addresses when resolving data references #436 @mike-hunhoff + +### Changes + +- setup: vivisect v1.0.0 is the default backend for Python3 (it was SMDA before) #421 @Ana06 +- setup: bump vivisect to 0.2.1 #454 @mr-tz +- linter: adding ntoskrnl, ntdll overlap lint #428 @mike-hunhoff +- ci: use py3.9 and pyinstaller 4.2 to build standalone binaries #452 @williballenthin +- scripts: remove old migration script #450 @williballenthin + +### Development + +- main: factor out common cli argument handling #450 @williballenthin + +### Raw diffs + + - [capa v1.5.1...v1.6.0](https://github.com/fireeye/capa/compare/v1.5.1...v1.6.0) + - [capa-rules v1.5.1...v1.6.0](https://github.com/fireeye/capa-rules/compare/v1.5.1...v1.6.0) + + +## v1.5.1 (2021-02-09) + +This release fixes the version number that we forgot to update for v1.5.0 (therefore, v1.5.0 was not published to pypi). It also includes 1 new rule and some rule improvements. + +### New Rules (1) + +- encrypt data using vest @re-fox + +### Raw diffs + + - [capa v1.5.0...v1.5.1](https://github.com/fireeye/capa/compare/v1.5.1...v1.6.0) + - [capa-rules v1.5.0...v1.5.1](https://github.com/fireeye/capa-rules/compare/v1.5.1...v1.6.0) + + +## v1.5.0 (2021-02-05) + +This release brings support for running capa under Python 3 via [SMDA](https://github.com/danielplohmann/smda), more thorough CI testing and linting, better extraction of strings and byte features, and 50 (!) new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: + + - @johnk3r + - @doomedraven + - @stvemillertime + - @itreallynick + - @0x534a + +@dzbeck also added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for many rules. + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + + +### New Features + + - py3 support via SMDA #355 @danielplohmann @jcrussell + - scripts: example of using capa as a library #372, #380 @doomedraven + - ci: enable dependabot #373 @mr-tz + - ci: lint rules @mr-tz + - ci: lint rule format #401 @mr-tz + - freeze: add base address #391 @mr-tz + - json: meta: add base address #412 @mr-tz + +### New Rules (50) + + - 64-bit execution via heavens gate @recvfrom + - contain anti-disasm techniques @mr-tz + - check for microsoft office emulation @re-fox + - check for windows sandbox via device @re-fox + - check for windows sandbox via dns suffix @re-fox + - check for windows sandbox via genuine state @re-fox + - check for windows sandbox via process name @re-fox + - check for windows sandbox via registry @re-fox + - capture microphone audio @re-fox + - capture public ip @re-fox + - get domain trust relationships @johnk3r + - check HTTP status code @mr-tz + - compiled with perl2exe @re-fox + - compiled with ps2exe @re-fox + - compiled with pyarmor @stvemillertime, @itreallynick + - validate payment card number using luhn algorithm @re-fox + - hash data using fnv @re-fox @mr-tz + - generate random numbers via WinAPI @mike-hunhoff @johnk3r + - enumerate files recursively @re-fox + - get file system object information @mike-hunhoff + - read virtual disk @re-fox + - register minifilter driver @mike-hunhoff + - start minifilter driver @mike-hunhoff + - enumerate gui resources @johnk3r + - simulate CTRL ALT DEL @mike-hunhoff + - hijack thread execution @0x534a + - inject dll @0x534a + - inject pe @0x534a + - create or open registry key @mike-hunhoff + - delete registry value @mike-hunhoff + - query or enumerate registry key @mike-hunhoff + - query or enumerate registry value @mike-hunhoff + - resume thread @0x534a + - suspend thread @0x534a + - allocate memory @0x534a + - allocate RW memory @0x534a + - contain pusha popa sequence @mr-tz + - create or open file @mike-hunhoff + - open process @0x534a + - open thread @0x534a + - get kernel32 base address @mr-tz + - get ntdll base address @mr-tz + - encrypt or decrypt data via BCrypt @mike-hunhoff + - generate random numbers using the Delphi LCG @williballenthin + - hash data via BCrypt @mike-hunhoff + - migrate process to active window station @williballenthin + - patch process command line @williballenthin + - resolve function by hash @williballenthin + - persist via Winlogon Helper DLL registry key @0x534a + - schedule task via command line @0x534a + +### Bug Fixes + + - doc: pyinstaller build process @mr-tz + - ida: better bytes extraction #409 @mike-hunhoff + - viv: better unicode string extraction #364 @mike-hunhoff + - viv: better unicode string extraction #378 @mr-tz + - viv: more xor instructions #379 @mr-tz + - viv: decrease logging verbosity #381 @mr-tz + - rules: fix api description syntax #403 @mike-hunhoff + - main: disable progress background thread #410 @mike-hunhoff + +### Changes + + - rules: return lib rules for scopes #398 @mr-tz + +### Raw diffs + + - [capa v1.4.1...v1.5.0](https://github.com/fireeye/capa/compare/v1.4.1...v1.5.0) + - [capa-rules v1.4.0...v1.5.0](https://github.com/fireeye/capa-rules/compare/v1.4.0...v1.5.0) + +## v1.4.1 (2020-10-23) + +This release fixes an issue building capa on our CI server, which prevented us from building standalone binaries for v1.4.1. + +### Bug Fixes + + - install VC dependencies for Python 2.7 during Windows build + +### Raw diffs + + - [capa v1.4.0...v1.4.1](https://github.com/fireeye/capa/compare/v1.4.0...v1.4.1) + - [capa-rules v1.4.0...v1.4.1](https://github.com/fireeye/capa-rules/compare/v1.4.0...v1.4.1) + +## v1.4.0 (2020-10-23) + +This capa release includes changes to the rule parsing, enhanced feature extraction, various bug fixes, and improved capa scripts. Everyone should benefit from the improved functionality and performance. The community helped to add 69 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: + + - @mwilliams31 + - @yt0ng + +@dzbeck added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for 86 rules. + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + +### New features + + - script that demonstrates bulk processing @williballenthin #307 + - main: render MBC table @mr-tz #332 + - ida backend: improve detection of APIs called via two or more chained thunks @mike-hunhoff #340 + - viv backend: improve detection of APIs called via two or more chained thunks @mr-tz #341 + - features: extract APIs called via jmp instruction @mr-tz #337 + +### New rules + + - clear the Windows event log @mike-hunhoff + - crash the Windows event logging service @mike-hunhoff + - packed with kkrunchy @re-fox + - packed with nspack @re-fox + - packed with pebundle @re-fox + - packed with pelocknt @re-fox + - packed with peshield @re-fox + - packed with petite @re-fox + - packed with rlpack @re-fox + - packed with upack @re-fox + - packed with y0da crypter @re-fox + - compiled with rust @re-fox + - compute adler32 checksum @mwilliams31 + - encrypt-data-using-hc-128 @recvfrom + - manipulate console @williballenthin + - references logon banner @re-fox + - terminate process via fastfail @re-fox + - delete volume shadow copies @mr-tz + - authenticate HMAC @mr-tz + - compiled from EPL @williballenthin + - compiled with Go @williballenthin + - create Restart Manager session @mike-hunhoff + - decode data using Base64 via WinAPI @mike-hunhoff + - empty recycle bin quietly @mwilliams31 + - enumerate network shares @mike-hunhoff + - hook routines via microsoft detours @williballenthin + - hooked by API Override @williballenthin + - impersonate user @mike-hunhoff + - the @williballenthin packer detection package, thanks to Hexacorn for the data, see https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/ + - packed with CCG + - packed with Crunch + - packed with Dragon Armor + - packed with enigma + - packed with Epack + - packed with MaskPE + - packed with MEW + - packed with Mpress + - packed with Neolite + - packed with PECompact + - packed with Pepack + - packed with Perplex + - packed with ProCrypt + - packed with RPCrypt + - packed with SeauSFX + - packed with Shrinker + - packed with Simple Pack + - packed with StarForce + - packed with SVKP + - packed with Themida + - packed with TSULoader + - packed with VProtect + - packed with WWPACK + - rebuilt by ImpRec + - packaged as a Pintool + - packaged as a CreateInstall installer + - packaged as a WinZip self-extracting archive + - reference 114DNS DNS server @williballenthin + - reference AliDNS DNS server @williballenthin + - reference Cloudflare DNS server @williballenthin + - reference Comodo Secure DNS server @williballenthin + - reference Google Public DNS server @williballenthin + - reference Hurricane Electric DNS server @williballenthin + - reference kornet DNS server @williballenthin + - reference L3 DNS server @williballenthin + - reference OpenDNS DNS server @williballenthin + - reference Quad9 DNS server @williballenthin + - reference Verisign DNS server @williballenthin + - run as service @mike-hunhoff + - schedule task via ITaskService @mike-hunhoff + - references DNS over HTTPS endpoints @yt0ng + +### Bug fixes + + - ida plugin: fix tree-view exception @mike-hunhoff #315 + - ida plugin: fix feature count @mike-hunhoff + - main: fix reported total rule count @williballenthin #325 + - features: fix handling of API names with multiple periods @mike-hunhoff #329 + - ida backend: find all byte sequences instead of only first @mike-hunhoff #335 + - features: display 0 value @mr-tz #338 + - ida backend: extract ordinal and name imports @mr-tz #343 + - show-features: improvements and support within IDA @mr-tz #342 + - main: sanity check MBC rendering @williballenthin + - main: handle sample path that contains non-ASCII characters @mr-tz #328 + +### Changes + + - rules: use yaml.CLoader for better performance @williballenthin #306 + - rules: parse descriptions for statements @mr-tz #312 + +### Raw diffs + + - [capa v1.3.0...v1.4.0](https://github.com/fireeye/capa/compare/v1.3.0...v1.4.0) + - [capa-rules v1.3.0...v1.4.0](https://github.com/fireeye/capa-rules/compare/v1.3.0...v1.4.0) + +## v1.3.0 (2020-09-14) + +This release brings newly updated mappings to the [Malware Behavior Catalog version 2.0](https://github.com/MBCProject/mbc-markdown), many enhancements to the IDA Pro plugin, [flare-capa on PyPI](https://pypi.org/project/flare-capa/), a bunch of bug fixes to improve feature extraction, and four new rules. We received contributions from ten reverse engineers, including seven new ones: + + - @dzbeck + - @recvfrom + - @toomanybananas + - @cclauss + - @adamprescott91 + - @weslambert + - @stevemk14ebr + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + +### Key changes to IDA Plugin + +The IDA Pro integration is now distributed as a real plugin, instead of a script. This enables a few things: + + - keyboard shortcuts and file menu integration + - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%` + - generally doing thing the "right way" + +How to get this new version? Its easy: download [capa_explorer.py](https://raw.githubusercontent.com/fireeye/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. + +Please refer to the plugin [readme](https://github.com/fireeye/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin. + +Please open an issue in this repository if you notice anything weird. + +### New features + + - ida plugin: now a real plugin, not a script @mike-hunhoff + - core: distributed via PyPI as [flare-capa](https://pypi.org/project/flare-capa/) @williballenthin + - features: enable automatic A/W handling for imports @williballenthin @Ana06 #246 + - ida plugin: persist rules directory setting via [ida-settings](https://github.com/williballenthin/ida-settings) @williballenthin #268 + - ida plugin: add search bar to results view @williballenthin #285 + - ida plugin: add `Analyze` and `Reset` buttons to tree view @mike-hunhoff #304 + - ida plugin: add status label to tree view @mike-hunhoff + - ida plugin: add progress indicator @mike-hunhoff, @mr-tz + +### New rules + + - compiled with py2exe @re-fox + - resolve path using msvcrt @re-fox + - decompress data using QuickLZ @edeca + - encrypt data using sosemanuk @recvfrom + +### Bug fixes + + - rule: reduce FP in DNS resolution @toomanybananas + - engine: report correct strings matched via regex @williballenthin #262 + - formatter: correctly format descriptions in two-line syntax @williballenthin @recvfrom #263 + - viv: better extract offsets from SibOper operands @williballenthin @edeca #276 + - import-to-ida: fix import error @cclauss + - viv: don't write settings to ~/.viv/viv.json @williballenthin @rakuy0 @weslambert #244 + - ida plugin: remove dependency loop that resulted in unnecessary overhead @mike-hunhoff #303 + - ida plugin: correctly highlight regex matches in IDA Disassembly view @mike-hunhoff #305 + - ida plugin: better handle rule directory prompt and failure case @stevemk14ebr @mike-hunhoff #309 + +### Changes + + - rules: update meta mapping to MBC 2.0! @dzbeck + - render: don't display rules that are also matched by other rules @williballenthin @Ana06 #224 + - ida plugin: simplify tabs, removing summary and adding detail to results view @williballenthin #286 + - ida plugin: analysis is no longer automatically started when plugin is first opened @mike-hunhoff #304 + - ida plugin: user must manually select a capa rules directory before analysis can be performed @mike-hunhoff + - ida plugin: user interface controls are disabled until analysis is performed @mike-hunhoff #304 + +### Raw diffs + + - [capa v1.2.0...v1.3.0](https://github.com/fireeye/capa/compare/v1.2.0...v1.3.0) + - [capa-rules v1.2.0...v1.3.0](https://github.com/fireeye/capa-rules/compare/v1.2.0...v1.3.0) + +## v1.2.0 (2020-08-31) + +This release brings UI enhancements, especially for the IDA Pro plugin, +investment towards py3 support, +fixes some bugs identified by the community, +and 46 (!) new rules. +We received contributions from ten reverse engineers, including five new ones: + + - @agithubuserlol + - @recvfrom + - @D4nch3n + - @edeca + - @winniepe + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). +Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) +and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + +### New features + + - ida plugin: display arch flavors @mike-hunhoff + - ida plugin: display block descriptions @mike-hunhoff + - ida backend: extract features from nested pointers @mike-hunhoff + - main: show more progress output @williballenthin + - core: pin dependency versions #258 @recvfrom + +### New rules + - bypass UAC via AppInfo ALPC @agithubuserlol + - bypass UAC via token manipulation @agithubuserlol + - check for sandbox and av modules @re-fox + - check for sandbox username @re-fox + - check if process is running under wine @re-fox + - validate credit card number using luhn algorithm @re-fox + - validate credit card number using luhn algorithm with no lookup table @re-fox + - hash data using FNV @edeca @mr-tz + - link many functions at runtime @mr-tz + - reference public RSA key @mr-tz + - packed with ASPack @williballenthin + - delete internet cache @mike-hunhoff + - enumerate internet cache @mike-hunhoff + - send ICMP echo request @mike-hunhoff + - check for debugger via API @mike-hunhoff + - check for hardware breakpoints @mike-hunhoff + - check for kernel debugger via shared user data structure @mike-hunhoff + - check for protected handle exception @mike-hunhoff + - check for software breakpoints @mike-hunhoff + - check for trap flag exception @mike-hunhoff + - check for unexpected memory writes @mike-hunhoff + - check process job object @mike-hunhoff + - reference anti-VM strings targeting Parallels @mike-hunhoff + - reference anti-VM strings targeting Qemu @mike-hunhoff + - reference anti-VM strings targeting VirtualBox @mike-hunhoff + - reference anti-VM strings targeting VirtualPC @mike-hunhoff + - reference anti-VM strings targeting VMWare @mike-hunhoff + - reference anti-VM strings targeting Xen @mike-hunhoff + - reference analysis tools strings @mike-hunhoff + - reference WMI statements @mike-hunhoff + - get number of processor cores @mike-hunhoff + - get number of processors @mike-hunhoff + - enumerate disk properties @mike-hunhoff + - get disk size @mike-hunhoff + - get process heap flags @mike-hunhoff + - get process heap force flags @mike-hunhoff + - get Explorer PID @mike-hunhoff + - delay execution @mike-hunhoff + - check for process debug object @mike-hunhoff + - check license value @mike-hunhoff + - check ProcessDebugFlags @mike-hunhoff + - check ProcessDebugPort @mike-hunhoff + - check SystemKernelDebuggerInformation @mike-hunhoff + - check thread yield allowed @mike-hunhoff + - enumerate system firmware tables @mike-hunhoff + - get system firmware table @mike-hunhoff + - hide thread from debugger @mike-hunhoff + +### Bug fixes + + - ida backend: extract unmapped immediate number features @mike-hunhoff + - ida backend: fix stack cookie check #257 @mike-hunhoff + - viv backend: better extract gs segment access @williballenthin + - core: enable counting of string features #241 @D4nch3n @williballenthin + - core: enable descriptions on feature with arch flavors @mike-hunhoff + - core: update git links for non-SSH access #259 @recvfrom + +### Changes + + - ida plugin: better default display showing first level nesting @winniepe + - remove unused `characteristic(switch)` feature @ana06 + - prepare testing infrastructure for multiple backends/py3 @williballenthin + - ci: zip build artifacts @ana06 + - ci: build all supported python versions @ana06 + - code style and formatting @mr-tz + +### Raw diffs + + - [capa v1.1.0...v1.2.0](https://github.com/fireeye/capa/compare/v1.1.0...v1.2.0) + - [capa-rules v1.1.0...v1.2.0](https://github.com/fireeye/capa-rules/compare/v1.1.0...v1.2.0) + +## v1.1.0 (2020-08-05) + +This release brings new rule format updates, such as adding `offset/x32` and negative offsets, +fixes some bugs identified by the community, and 28 (!) new rules. +We received contributions from eight reverse engineers, including four new ones: + + - @re-fox + - @psifertex + - @bitsofbinary + - @threathive + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + +### New features + + - import: add Binary Ninja import script #205 #207 @psifertex + - rules: offsets can be negative #197 #208 @williballenthin + - rules: enable descriptions for statement nodes #194 #209 @Ana06 + - rules: add arch flavors to number and offset features #210 #216 @williballenthin + - render: show SHA1/SHA256 in default report #164 @threathive + - tests: add tests for IDA Pro backend #202 @williballenthin + +### New rules + + - check for unmoving mouse cursor @BitsOfBinary + - check mutex and exit @re-fox + - parse credit card information @re-fox + - read ini file @re-fox + - validate credit card number with luhn algorithm @re-fox + - change the wallpaper @re-fox + - acquire debug privileges @williballenthin + - import public key @williballenthin + - terminate process by name @williballenthin + - encrypt data using DES @re-fox + - encrypt data using DES via WinAPI @re-fox + - hash data using sha1 via x86 extensions @re-fox + - hash data using sha256 via x86 extensions @re-fox + - capture network configuration via ipconfig @re-fox + - hash data via WinCrypt @mike-hunhoff + - get file attributes @mike-hunhoff + - allocate thread local storage @mike-hunhoff + - get thread local storage value @mike-hunhoff + - set thread local storage @mike-hunhoff + - get session integrity level @mike-hunhoff + - add file to cabinet file @mike-hunhoff + - flush cabinet file @mike-hunhoff + - open cabinet file @mike-hunhoff + - gather firefox profile information @re-fox + - encrypt data using skipjack @re-fox + - encrypt data using camellia @re-fox + - hash data using tiger @re-fox + - encrypt data using blowfish @re-fox + - encrypt data using twofish @re-fox + +### Bug fixes + + - linter: fix exception when examples is `None` @Ana06 + - linter: fix suggested recommendations via templating @williballenthin + - render: fix exception when rendering counts @williballenthin + - render: fix render of negative offsets @williballenthin + - extractor: fix segmentation violation from vivisect @williballenthin + - main: fix crash when .viv cannot be saved #168 @secshoggoth @williballenthin + - main: fix shellcode .viv save path @williballenthin + +### Changes + + - doc: explain how to bypass gatekeeper on macOS @psifertex + - doc: explain supported linux distributions @Ana06 + - doc: explain submodule update with --init @psifertex + - main: improve program help output @mr-tz + - main: disable progress when run in quiet mode @mr-tz + - main: assert supported IDA versions @mr-tz + - extractor: better identify nested pointers to strings @williballenthin + - setup: specify vivisect download url @Ana06 + - setup: pin vivisect version @williballenthin + - setup: bump vivisect dependency version @williballenthin + - setup: set Python project name to `flare-capa` @williballenthin + - ci: run tests and linter via Github Actions @Ana06 + - hooks: run style checkers and hide stashed output @Ana06 + - linter: ignore period in rule filename @williballenthin + - linter: warn on nursery rule with no changes needed @williballenthin + +### Raw diffs + + - [capa v1.0.0...v1.1.0](https://github.com/fireeye/capa/compare/v1.0.0...v1.1.0) + - [capa-rules v1.0.0...v1.1.0](https://github.com/fireeye/capa-rules/compare/v1.0.0...v1.1.0) diff --git a/README.md b/README.md index fbabc401..f66f5e02 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ ![capa](.github/logo.png) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) +[![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/releases) +[![Number of rules](https://img.shields.io/badge/rules-488-blue.svg)](https://github.com/fireeye/capa-rules) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) -[![Number of rules](https://img.shields.io/badge/rules-469-blue.svg)](https://github.com/fireeye/capa-rules) +[![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) capa detects capabilities in executable files. @@ -146,8 +149,8 @@ rule: The [github.com/fireeye/capa-rules](https://github.com/fireeye/capa-rules) repository contains hundreds of standard library rules that are distributed with capa. Please learn to write rules and contribute new entries as you find interesting techniques in malware. -If you use IDA Pro, then you can use the [capa explorer plugin](capa/ida/plugin/). -capa explorer lets you quickly identify and navigate to interesting areas of a program and help you build new capa rules out of the features extracted directly from your IDB. +If you use IDA Pro, then you can use the [capa explorer](capa/ida/plugin/) plugin. +capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted directly from your IDA Pro database. ![capa + IDA Pro integration](doc/img/explorer_expanded.png) diff --git a/capa/features/__init__.py b/capa/features/__init__.py index 8bd517f2..c88cd153 100644 --- a/capa/features/__init__.py +++ b/capa/features/__init__.py @@ -27,17 +27,28 @@ VALID_ARCH = (ARCH_X32, ARCH_X64) def bytes_to_str(b): - if sys.version_info[0] >= 3: - return str(codecs.encode(b, "hex").decode("utf-8")) - else: - return codecs.encode(b, "hex") + return str(codecs.encode(b, "hex").decode("utf-8")) def hex_string(h): - """ render hex string e.g. "0a40b1" as "0A 40 B1" """ + """render hex string e.g. "0a40b1" as "0A 40 B1" """ return " ".join(h[i : i + 2] for i in range(0, len(h), 2)).upper() +def escape_string(s): + """escape special characters""" + s = repr(s) + if not s.startswith(('"', "'")): + # u'hello\r\nworld' -> hello\\r\\nworld + s = s[2:-1] + else: + # 'hello\r\nworld' -> hello\\r\\nworld + s = s[1:-1] + s = s.replace("\\'", "'") # repr() may escape "'" in some edge cases, remove + s = s.replace('"', '\\"') # repr() does not escape '"', add + return s + + class Feature(object): def __init__(self, value, arch=None, description=None): """ diff --git a/capa/features/extractors/helpers.py b/capa/features/extractors/helpers.py index e78be96e..8dc39bea 100644 --- a/capa/features/extractors/helpers.py +++ b/capa/features/extractors/helpers.py @@ -16,10 +16,7 @@ MIN_STACKSTRING_LEN = 8 def xor_static(data, i): - if sys.version_info >= (3, 0): - return bytes(c ^ i for c in data) - else: - return "".join(chr(ord(c) ^ i) for c in data) + return bytes(c ^ i for c in data) def is_aw_function(symbol): diff --git a/capa/features/extractors/ida/__init__.py b/capa/features/extractors/ida/__init__.py index 362b0da7..689975b9 100644 --- a/capa/features/extractors/ida/__init__.py +++ b/capa/features/extractors/ida/__init__.py @@ -34,10 +34,7 @@ def add_ea_int_cast(o): this bit of skullduggery lets use cast viv-utils objects as ints. the correct way of doing this is to update viv-utils (or subclass the objects here). """ - if sys.version_info[0] >= 3: - setattr(o, "__int__", types.MethodType(get_ea, o)) - else: - setattr(o, "__int__", types.MethodType(get_ea, o, type(o))) + setattr(o, "__int__", types.MethodType(get_ea, o)) return o diff --git a/capa/features/extractors/ida/basicblock.py b/capa/features/extractors/ida/basicblock.py index f878ab1a..0a0e08f0 100644 --- a/capa/features/extractors/ida/basicblock.py +++ b/capa/features/extractors/ida/basicblock.py @@ -39,18 +39,11 @@ def get_printable_len(op): raise ValueError("Unhandled operand data type 0x%x." % op.dtype) def is_printable_ascii(chars): - if sys.version_info[0] >= 3: - return all(c < 127 and chr(c) in string.printable for c in chars) - else: - return all(ord(c) < 127 and c in string.printable for c in chars) + return all(c < 127 and chr(c) in string.printable for c in chars) def is_printable_utf16le(chars): - if sys.version_info[0] >= 3: - if all(c == 0x00 for c in chars[1::2]): - return is_printable_ascii(chars[::2]) - else: - if all(c == "\x00" for c in chars[1::2]): - return is_printable_ascii(chars[::2]) + if all(c == 0x00 for c in chars[1::2]): + return is_printable_ascii(chars[::2]) if is_printable_ascii(chars): return idaapi.get_dtype_size(op.dtype) diff --git a/capa/features/extractors/ida/file.py b/capa/features/extractors/ida/file.py index 5ca2370d..6f55ed45 100644 --- a/capa/features/extractors/ida/file.py +++ b/capa/features/extractors/ida/file.py @@ -78,7 +78,7 @@ def extract_file_embedded_pe(): def extract_file_export_names(): - """ extract function exports """ + """extract function exports""" for (_, _, ea, name) in idautils.Entries(): yield Export(name), ea @@ -144,7 +144,7 @@ def extract_file_strings(): def extract_features(): - """ extract file features """ + """extract file features""" for file_handler in FILE_HANDLERS: for feature, va in file_handler(): yield feature, va diff --git a/capa/features/extractors/ida/helpers.py b/capa/features/extractors/ida/helpers.py index 38fa3775..4b7dcdeb 100644 --- a/capa/features/extractors/ida/helpers.py +++ b/capa/features/extractors/ida/helpers.py @@ -23,11 +23,7 @@ def find_byte_sequence(start, end, seq): end: max virtual address seq: bytes to search e.g. b"\x01\x03" """ - if sys.version_info[0] >= 3: - seq = " ".join(["%02x" % b for b in seq]) - else: - seq = " ".join(["%02x" % ord(b) for b in seq]) - + seq = " ".join(["%02x" % b for b in seq]) while True: ea = idaapi.find_binary(start, end, seq, 0, idaapi.SEARCH_DOWN) if ea == idaapi.BADADDR: @@ -83,7 +79,7 @@ def get_segment_buffer(seg): def get_file_imports(): - """ get file imports """ + """get file imports""" imports = {} for idx in range(idaapi.get_import_module_qty()): @@ -120,7 +116,7 @@ def get_instructions_in_range(start, end): def is_operand_equal(op1, op2): - """ compare two IDA op_t """ + """compare two IDA op_t""" if op1.flags != op2.flags: return False @@ -146,7 +142,7 @@ def is_operand_equal(op1, op2): def is_basic_block_equal(bb1, bb2): - """ compare two IDA BasicBlock """ + """compare two IDA BasicBlock""" if bb1.start_ea != bb2.start_ea: return False @@ -160,7 +156,7 @@ def is_basic_block_equal(bb1, bb2): def basic_block_size(bb): - """ calculate size of basic block """ + """calculate size of basic block""" return bb.end_ea - bb.start_ea @@ -178,7 +174,7 @@ def read_bytes_at(ea, count): def find_string_at(ea, min=4): - """ check if ASCII string exists at a given virtual address """ + """check if ASCII string exists at a given virtual address""" found = idaapi.get_strlit_contents(ea, -1, idaapi.STRTYPE_C) if found and len(found) > min: try: @@ -232,23 +228,23 @@ def get_op_phrase_info(op): def is_op_write(insn, op): - """ Check if an operand is written to (destination operand) """ + """Check if an operand is written to (destination operand)""" return idaapi.has_cf_chg(insn.get_canon_feature(), op.n) def is_op_read(insn, op): - """ Check if an operand is read from (source operand) """ + """Check if an operand is read from (source operand)""" return idaapi.has_cf_use(insn.get_canon_feature(), op.n) def is_op_offset(insn, op): - """ Check is an operand has been marked as an offset (by auto-analysis or manually) """ + """Check is an operand has been marked as an offset (by auto-analysis or manually)""" flags = idaapi.get_flags(insn.ea) return ida_bytes.is_off(flags, op.n) def is_sp_modified(insn): - """ determine if instruction modifies SP, ESP, RSP """ + """determine if instruction modifies SP, ESP, RSP""" for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)): if op.reg == idautils.procregs.sp.reg and is_op_write(insn, op): # register is stack and written @@ -257,7 +253,7 @@ def is_sp_modified(insn): def is_bp_modified(insn): - """ check if instruction modifies BP, EBP, RBP """ + """check if instruction modifies BP, EBP, RBP""" for op in get_insn_ops(insn, target_ops=(idaapi.o_reg,)): if op.reg == idautils.procregs.bp.reg and is_op_write(insn, op): # register is base and written @@ -266,12 +262,12 @@ def is_bp_modified(insn): def is_frame_register(reg): - """ check if register is sp or bp """ + """check if register is sp or bp""" return reg in (idautils.procregs.sp.reg, idautils.procregs.bp.reg) def get_insn_ops(insn, target_ops=()): - """ yield op_t for instruction, filter on type if specified """ + """yield op_t for instruction, filter on type if specified""" for op in insn.ops: if op.type == idaapi.o_void: # avoid looping all 6 ops if only subset exists @@ -282,7 +278,7 @@ def get_insn_ops(insn, target_ops=()): def is_op_stack_var(ea, index): - """ check if operand is a stack variable """ + """check if operand is a stack variable""" return idaapi.is_stkvar(idaapi.get_flags(ea), index) @@ -336,7 +332,7 @@ def is_basic_block_tight_loop(bb): def find_data_reference_from_insn(insn, max_depth=10): - """ search for data reference from instruction, return address of instruction if no reference exists """ + """search for data reference from instruction, return address of instruction if no reference exists""" depth = 0 ea = insn.ea @@ -379,5 +375,5 @@ def get_function_blocks(f): def is_basic_block_return(bb): - """ check if basic block is return block """ + """check if basic block is return block""" return bb.type == idaapi.fcb_ret diff --git a/capa/features/extractors/ida/insn.py b/capa/features/extractors/ida/insn.py index 3f5aef52..ec2f8070 100644 --- a/capa/features/extractors/ida/insn.py +++ b/capa/features/extractors/ida/insn.py @@ -53,7 +53,7 @@ def get_imports(ctx): def check_for_api_call(ctx, insn): - """ check instruction for API call """ + """check instruction for API call""" if not insn.get_canon_mnem() in ("call", "jmp"): return @@ -256,7 +256,7 @@ def bb_stack_cookie_registers(bb): def is_nzxor_stack_cookie_delta(f, bb, insn): - """ check if nzxor exists within stack cookie delta """ + """check if nzxor exists within stack cookie delta""" # security cookie check should use SP or BP if not capa.features.extractors.ida.helpers.is_frame_register(insn.Op2.reg): return False @@ -279,7 +279,7 @@ def is_nzxor_stack_cookie_delta(f, bb, insn): def is_nzxor_stack_cookie(f, bb, insn): - """ check if nzxor is related to stack cookie """ + """check if nzxor is related to stack cookie""" if contains_stack_cookie_keywords(idaapi.get_cmt(insn.ea, False)): # Example: # xor ecx, ebp ; StackCookie diff --git a/capa/features/extractors/smda/__init__.py b/capa/features/extractors/smda/__init__.py index b7ea3ec1..9d58a7de 100644 --- a/capa/features/extractors/smda/__init__.py +++ b/capa/features/extractors/smda/__init__.py @@ -15,8 +15,6 @@ from capa.features.extractors import FeatureExtractor class SmdaFeatureExtractor(FeatureExtractor): def __init__(self, smda_report: SmdaReport, path): super(SmdaFeatureExtractor, self).__init__() - if sys.version_info < (3, 0): - raise UnsupportedRuntimeError("SMDA should only be used with Python 3.") self.smda_report = smda_report self.path = path diff --git a/capa/features/extractors/smda/basicblock.py b/capa/features/extractors/smda/basicblock.py index 07477139..98b91ec3 100644 --- a/capa/features/extractors/smda/basicblock.py +++ b/capa/features/extractors/smda/basicblock.py @@ -15,7 +15,7 @@ def _bb_has_tight_loop(f, bb): def extract_bb_tight_loop(f, bb): - """ check basic block for tight loop indicators """ + """check basic block for tight loop indicators""" if _bb_has_tight_loop(f, bb): yield Characteristic("tight loop"), bb.offset @@ -39,7 +39,7 @@ def get_operands(smda_ins): def extract_stackstring(f, bb): - """ check basic block for stackstring indicators """ + """check basic block for stackstring indicators""" if _bb_has_stackstring(f, bb): yield Characteristic("stack string"), bb.offset diff --git a/capa/features/extractors/smda/insn.py b/capa/features/extractors/smda/insn.py index bf23f607..48ef4c1c 100644 --- a/capa/features/extractors/smda/insn.py +++ b/capa/features/extractors/smda/insn.py @@ -293,7 +293,7 @@ def extract_insn_peb_access_characteristic_features(f, bb, insn): def extract_insn_segment_access_features(f, bb, insn): - """ parse the instruction for access to fs or gs """ + """parse the instruction for access to fs or gs""" operands = [o.strip() for o in insn.operands.split(",")] for operand in operands: if "fs:" in operand: diff --git a/capa/features/extractors/viv/basicblock.py b/capa/features/extractors/viv/basicblock.py index 76ffefe6..a76aa179 100644 --- a/capa/features/extractors/viv/basicblock.py +++ b/capa/features/extractors/viv/basicblock.py @@ -45,7 +45,7 @@ def _bb_has_tight_loop(f, bb): def extract_bb_tight_loop(f, bb): - """ check basic block for tight loop indicators """ + """check basic block for tight loop indicators""" if _bb_has_tight_loop(f, bb): yield Characteristic("tight loop"), bb.va @@ -68,7 +68,7 @@ def _bb_has_stackstring(f, bb): def extract_stackstring(f, bb): - """ check basic block for stackstring indicators """ + """check basic block for stackstring indicators""" if _bb_has_stackstring(f, bb): yield Characteristic("stack string"), bb.va diff --git a/capa/features/extractors/viv/insn.py b/capa/features/extractors/viv/insn.py index 4e78a3da..19162e61 100644 --- a/capa/features/extractors/viv/insn.py +++ b/capa/features/extractors/viv/insn.py @@ -486,7 +486,7 @@ def extract_insn_peb_access_characteristic_features(f, bb, insn): def extract_insn_segment_access_features(f, bb, insn): - """ parse the instruction for access to fs or gs """ + """parse the instruction for access to fs or gs""" prefix = insn.getPrefixName() if prefix == "fs": diff --git a/capa/features/freeze.py b/capa/features/freeze.py index fb2f4db0..b97f1244 100644 --- a/capa/features/freeze.py +++ b/capa/features/freeze.py @@ -264,15 +264,14 @@ def main(argv=None): parser.add_argument( "-f", "--format", choices=[f[0] for f in formats], default="auto", help="Select sample format, %s" % format_help ) - if sys.version_info >= (3, 0): - parser.add_argument( - "-b", - "--backend", - type=str, - help="select the backend to use", - choices=(capa.main.BACKEND_VIV, capa.main.BACKEND_SMDA), - default=capa.main.BACKEND_VIV, - ) + parser.add_argument( + "-b", + "--backend", + type=str, + help="select the backend to use", + choices=(capa.main.BACKEND_VIV, capa.main.BACKEND_SMDA), + default=capa.main.BACKEND_VIV, + ) parser.add_argument( "--signature", action="append", @@ -293,8 +292,7 @@ def main(argv=None): logging.basicConfig(level=logging.INFO) logging.getLogger().setLevel(logging.INFO) - backend = args.backend if sys.version_info > (3, 0) else capa.main.BACKEND_VIV - extractor = capa.main.get_extractor(args.sample, args.format, backend, sigpaths=args.signatures) + extractor = capa.main.get_extractor(args.sample, args.format, args.backend, sigpaths=args.signatures) with open(args.output, "wb") as f: f.write(dump(extractor)) diff --git a/capa/helpers.py b/capa/helpers.py index 317bf1db..f5e1e5b0 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -12,9 +12,7 @@ _hex = hex def hex(i): - # under py2.7, long integers get formatted with a trailing `L` - # and this is not pretty. so strip it out. - return _hex(oint(i)).rstrip("L") + return _hex(oint(i)) def oint(i): diff --git a/capa/ida/helpers/__init__.py b/capa/ida/helpers/__init__.py index 020483f9..41e4b463 100644 --- a/capa/ida/helpers/__init__.py +++ b/capa/ida/helpers/__init__.py @@ -10,7 +10,6 @@ import logging import datetime import idc -import six import idaapi import idautils @@ -24,6 +23,7 @@ SUPPORTED_IDA_VERSIONS = [ "7.3", "7.4", "7.5", + "7.6", ] # file type names as returned by idaapi.get_file_type_name() @@ -71,7 +71,7 @@ def get_disasm_line(va): def is_func_start(ea): - """ check if function stat exists at virtual address """ + """check if function stat exists at virtual address""" f = idaapi.get_func(ea) return f and f.start_ea == ea @@ -85,7 +85,7 @@ def get_func_start_ea(ea): def get_file_md5(): """ """ md5 = idautils.GetInputFileMD5() - if not isinstance(md5, six.string_types): + if not isinstance(md5, str): md5 = capa.features.bytes_to_str(md5) return md5 @@ -93,7 +93,7 @@ def get_file_md5(): def get_file_sha256(): """ """ sha256 = idaapi.retrieve_input_file_sha256() - if not isinstance(sha256, six.string_types): + if not isinstance(sha256, str): sha256 = capa.features.bytes_to_str(sha256) return sha256 diff --git a/capa/ida/plugin/README.md b/capa/ida/plugin/README.md index 54306b89..13f14010 100644 --- a/capa/ida/plugin/README.md +++ b/capa/ida/plugin/README.md @@ -1,13 +1,12 @@ ![capa explorer](../../../.github/capa-explorer-logo.png) -capa explorer is an IDA Pro plugin written in Python that integrates the FLARE team's open-source framework, capa, with IDA. capa is a framework that uses a well-defined collection of rules to +capa explorer is an IDAPython plugin that integrates the FLARE team's open-source framework, capa, with IDA Pro. capa is a framework that uses a well-defined collection of rules to identify capabilities in a program. You can run capa against a PE file or shellcode and it tells you what it thinks the program can do. For example, it might suggest that -the program is a backdoor, can install services, or relies on HTTP to communicate. You can use capa explorer to run capa directly on an IDA database without requiring access -to or execution of the source binary. Once a database has been analyzed, capa explorer can be used to quickly identify and navigate to interesting areas of a program and manually build new capa rules out -of the features extracted directly from your IDB. +the program is a backdoor, can install services, or relies on HTTP to communicate. capa explorer runs capa directly against your IDA Pro database (IDB) without requiring access +to the original binary file. Once a database has been analyzed, capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted from your IDB. We love using capa explorer during malware analysis because it teaches us what parts of a program suggest a behavior. As we click on rows, capa explorer jumps directly -to important addresses in the IDA Pro database and highlights key features in the Disassembly view so they stand out visually. To illustrate, we use capa explorer to +to important addresses in the IDB and highlights key features in the Disassembly view so they stand out visually. To illustrate, we use capa explorer to analyze Lab 14-02 from [Practical Malware Analysis](https://nostarch.com/malware) (PMA) available [here](https://practicalmalwareanalysis.com/labs/). Our goal is to understand the program's functionality. @@ -15,16 +14,15 @@ After loading Lab 14-02 into IDA and analyzing the database with capa explorer, ![](../../../doc/img/explorer_condensed.png) -We can use capa explorer to navigate the IDA Disassembly view directly to the suspect function and get an assembly-level breakdown of why capa matched `self delete via COMSPEC environment variable` -for this particular function. +We can use capa explorer to navigate our Disassembly view directly to the suspect function and get an assembly-level breakdown of why capa matched `self delete via COMSPEC environment variable`. ![](../../../doc/img/explorer_expanded.png) Using the `Rule Information` and `Details` columns capa explorer shows us that the suspect function matched `self delete via COMSPEC environment variable` because it contains capa rule matches for `create process`, `get COMSPEC environment variable`, -and `query environment variable`, references to the strings `COMSPEC`, ` > nul`, and `/c del`, and calls to the Windows API functions `GetEnvironmentVariableA` and `ShellExecuteEx`. +and `query environment variable`, references to the strings `COMSPEC`, ` > nul`, and `/c del `, and calls to the Windows API functions `GetEnvironmentVariableA` and `ShellExecuteEx`. -capa explorer also helps you build new capa rules. To start select the `Rule Generator` tab, navigate to a function in the IDA `Disassembly` view, -and click `Analyze`. capa explorer will extract features from this function and display them in the `Function Features` pane. You can add features listed in this pane to the `Editor` pane +capa explorer also helps you build new capa rules. To start select the `Rule Generator` tab, navigate to a function in your Disassembly view, +and click `Analyze`. capa explorer will extract features from the function and display them in the `Features` pane. You can add features listed in this pane to the `Editor` pane by either double-clicking a feature or using multi-select + right-click to add multiple features at once. The `Preview` and `Editor` panes help edit your rule. Use the `Preview` pane to modify the rule text directly and the `Editor` pane to construct and rearrange your hierarchy of statements and features. When you finish a rule you can save it directly to a file by clicking `Save`. @@ -36,15 +34,32 @@ For more information on the FLARE team's open-source framework, capa, check out ### Requirements -capa explorer supports the following IDA setups: +capa explorer supports Python >= 3.6 and the following IDA Pro versions: -* IDA Pro 7.4+ with Python 2.7 or Python 3. +* IDA 7.4 +* IDA 7.5 +* IDA 7.6 (caveat below) If you encounter issues with your specific setup, please open a new [Issue](https://github.com/fireeye/capa/issues). +#### IDA 7.6 caveat: IDA needs a patch + +As described [here](https://www.hex-rays.com/blog/ida-7-6-empty-qtreeview-qtreewidget/): + +> A rather nasty issue evaded our testing and found its way into IDA 7.6: using the PyQt5 modules that are shipped with IDA, QTreeView (or QTreeWidget) instances will always fail to display contents. + +Therefore, in order to use capa under IDA 7.6, you need to download and install the fix corresponding to your IDA installation, replacing the original QtWidgets DLL with the one contained in the .zip file (links to Hex-Rays): + + + - Windows: [pyqt5_qtwidgets_win](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_win.zip) + - Linux: [pyqt5_qtwidgets_linux](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_linux.zip) + - MacOS (Intel): [pyqt5_qtwidgets_mac_x64](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_mac_x64.zip) + - MacOS (AppleSilicon): [pyqt5_qtwidgets_mac_arm](https://www.hex-rays.com/wp-content/uploads/2021/04/pyqt5_qtwidgets_mac_arm.zip) + + ### Supported File Types -capa explorer is limited to the file types supported by capa, which includes: +capa explorer is limited to the file types supported by capa, which include: * Windows 32-bit and 64-bit PE files * Windows 32-bit and 64-bit shellcode @@ -62,50 +77,48 @@ You can install capa explorer using the following steps: ### Usage -1. Run IDA and analyze a supported file type (select the `Manual Load` and `Load Resources` options in IDA for best results) +1. Open IDA and analyze a supported file type (select the `Manual Load` and `Load Resources` options in IDA for best results) 2. Open capa explorer in IDA by navigating to `Edit > Plugins > FLARE capa explorer` or using the keyboard shortcut `Alt+F5` 3. Select the `Program Analysis` tab 4. Click the `Analyze` button When running capa explorer for the first time you are prompted to select a file directory containing capa rules. The plugin conveniently -remembers your selection for future runs; you can change this selection by navigating to `Settings > Change default rules directory...`. We recommend +remembers your selection for future runs; you can change this selection and other default settings by clicking `Settings`. We recommend downloading and using the [standard collection of capa rules](https://github.com/fireeye/capa-rules) when getting started with the plugin. #### Tips for Program Analysis * Start analysis by clicking the `Analyze` button -* Reset the plugin user interface and remove highlighting from IDA disassembly view by clicking the `Reset` button -* Change your capa rules directory by navigating to `Settings > Change default rules directory...` from the plugin menu +* Reset the plugin user interface and remove highlighting from your Disassembly view by clicking the `Reset` button +* Change your capa rules directory and other default settings by clicking `Settings` * Hover your cursor over a rule match to view the source content of the rule -* Double-click the `Address` column to navigate the IDA Disassembly view to the associated feature +* Double-click the `Address` column to navigate your Disassembly view to the address of the associated feature * Double-click a result in the `Rule Information` column to expand its children -* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in the IDA Dissasembly view +* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in your Dissasembly view #### Tips for Rule Generator -* Navigate to a function in the `Disassembly` view and click`Analyze` to get started -* Double-click or multi-select + right-click in the `Function Features` pane to add features to the `Editor` pane -* Right-click features in the `Editor` pane to make modifications -* Drag-and-drop (single click + multi-select support) features in the `Editor` pane to quickly build a hierarchy of statements and features -* Right-click anywhere in the `Editor` pane not on a feature to quickly remove all features -* Add descriptions/comments by placing editing the appropriate column in the `Editor` pane -* Directly edit rule text, including rule metadata fields using the `Preview` pane -* Change the default rule author and default scope displayed in the `Preview` pane by navigating to `Settings` +* Navigate to a function in your Disassembly view and click`Analyze` to get started +* Double-click or use multi-select + right-click to add features from the `Features` pane to the `Editor` pane +* Right-click features in the `Editor` pane to make context-specific modifications +* Drag-and-drop (single click + multi-select support) features in the `Editor` pane to construct your hierarchy of statements and features +* Right-click anywhere in the `Editor` pane not on a feature to remove all features +* Add descriptions or comments to a feature by editing the corresponding column in the `Editor` pane +* Directly edit rule text and metadata fields using the `Preview` pane +* Change the default rule author and default rule scope displayed in the `Preview` pane by clicking `Settings` ## Development -Because capa explorer is packaged with capa you will need to install capa locally for development. - -You can install capa locally by following the steps outlined in `Method 3: Inspecting the capa source code` of the [capa +capa explorer is packaged with capa so you will need to install capa locally for development. You can install capa locally by following the steps outlined in `Method 3: Inspecting the capa source code` of the [capa installation guide](https://github.com/fireeye/capa/blob/master/doc/installation.md#method-3-inspecting-the-capa-source-code). Once installed, copy [capa_explorer.py](https://raw.githubusercontent.com/fireeye/capa/master/capa/ida/plugin/capa_explorer.py) -to your IDA plugins directory to run the plugin in IDA. +to your plugins directory to install capa explorer in IDA. ### Components capa explorer consists of two main components: -* An IDA [feature extractor](https://github.com/fireeye/capa/tree/master/capa/features/extractors/ida) built on top of IDA's binary analysis engine - * This component uses IDAPython to extract [capa features](https://github.com/fireeye/capa-rules/blob/master/doc/format.md#extracted-features) from the IDA database such as strings, +* An [feature extractor](https://github.com/fireeye/capa/tree/master/capa/features/extractors/ida) built on top of IDA's binary analysis engine + * This component uses IDAPython to extract [capa features](https://github.com/fireeye/capa-rules/blob/master/doc/format.md#extracted-features) from your IDBs such as strings, disassembly, and control flow; these extracted features are used by capa to find feature combinations that result in a rule match * An [interactive user interface](https://github.com/fireeye/capa/tree/master/capa/ida/plugin) for displaying and exploring capa rule matches - * This component integrates the IDA feature extractor and capa, providing an interactive user interface to dissect rule matches found by capa using features extracted by the IDA feature extractor + * This component integrates the feature extractor and capa, providing an interactive user interface to dissect rule matches found by capa using features extracted directly from your IDBs diff --git a/capa/ida/plugin/form.py b/capa/ida/plugin/form.py index c2ed292e..f16f5abe 100644 --- a/capa/ida/plugin/form.py +++ b/capa/ida/plugin/form.py @@ -37,6 +37,10 @@ from capa.ida.plugin.proxy import CapaExplorerRangeProxyModel, CapaExplorerSearc logger = logging.getLogger(__name__) settings = ida_settings.IDASettings("capa") +CAPA_SETTINGS_RULE_PATH = "rule_path" +CAPA_SETTINGS_RULEGEN_AUTHOR = "rulegen_author" +CAPA_SETTINGS_RULEGEN_SCOPE = "rulegen_scope" + def write_file(path, data): """ """ @@ -166,6 +170,60 @@ class CapaExplorerFeatureExtractor(capa.features.extractors.ida.IdaFeatureExtrac return super(CapaExplorerFeatureExtractor, self).extract_function_features(f) +class QLineEditClicked(QtWidgets.QLineEdit): + def __init__(self, content, parent=None): + """ """ + super(QLineEditClicked, self).__init__(content, parent) + + def mouseReleaseEvent(self, e): + """ """ + old = self.text() + new = str( + QtWidgets.QFileDialog.getExistingDirectory( + self.parent(), "Please select a capa rules directory", settings.user.get(CAPA_SETTINGS_RULE_PATH, "") + ) + ) + if new: + self.setText(new) + else: + self.setText(old) + + +class CapaSettingsInputDialog(QtWidgets.QDialog): + def __init__(self, title, parent=None): + """ """ + super(CapaSettingsInputDialog, self).__init__(parent) + + self.setWindowTitle(title) + self.setMinimumWidth(500) + self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowContextHelpButtonHint) + + self.edit_rule_path = QLineEditClicked(settings.user.get(CAPA_SETTINGS_RULE_PATH, "")) + self.edit_rule_author = QtWidgets.QLineEdit(settings.user.get(CAPA_SETTINGS_RULEGEN_AUTHOR, "")) + self.edit_rule_scope = QtWidgets.QComboBox() + + scopes = ("file", "function", "basic block") + + self.edit_rule_scope.addItems(scopes) + self.edit_rule_scope.setCurrentIndex(scopes.index(settings.user.get(CAPA_SETTINGS_RULEGEN_SCOPE, "function"))) + + buttons = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Cancel, self) + + layout = QtWidgets.QFormLayout(self) + layout.addRow("capa rules path", self.edit_rule_path) + layout.addRow("Default rule author", self.edit_rule_author) + layout.addRow("Default rule scope", self.edit_rule_scope) + + layout.addWidget(buttons) + + buttons.accepted.connect(self.accept) + buttons.rejected.connect(self.reject) + + def get_values(self): + """ """ + return self.edit_rule_path.text(), self.edit_rule_author.text(), self.edit_rule_scope.currentText() + + class CapaExplorerForm(idaapi.PluginForm): """form element for plugin interface""" @@ -197,11 +255,11 @@ class CapaExplorerForm(idaapi.PluginForm): self.view_rulegen = None self.view_tabs = None self.view_tab_rulegen = None - self.view_menu_bar = None self.view_status_label = None self.view_buttons = None self.view_analyze_button = None self.view_reset_button = None + self.view_settings_button = None self.view_save_button = None self.view_rulegen_preview = None @@ -273,10 +331,6 @@ class CapaExplorerForm(idaapi.PluginForm): self.load_view_status_label() self.load_view_buttons() - # load menu bar and sub menus - self.load_view_menu_bar() - self.load_configure_menu() - # load parent view self.load_view_parent() @@ -285,11 +339,6 @@ class CapaExplorerForm(idaapi.PluginForm): tabs = QtWidgets.QTabWidget() self.view_tabs = tabs - def load_view_menu_bar(self): - """load menu bar""" - bar = QtWidgets.QMenuBar() - self.view_menu_bar = bar - def load_view_checkbox_limit_by(self): """load limit results by function checkbox""" check = QtWidgets.QCheckBox("Limit results to current function") @@ -319,19 +368,23 @@ class CapaExplorerForm(idaapi.PluginForm): analyze_button = QtWidgets.QPushButton("Analyze") reset_button = QtWidgets.QPushButton("Reset") save_button = QtWidgets.QPushButton("Save") + settings_button = QtWidgets.QPushButton("Settings") analyze_button.clicked.connect(self.slot_analyze) reset_button.clicked.connect(self.slot_reset) save_button.clicked.connect(self.slot_save) + settings_button.clicked.connect(self.slot_settings) layout = QtWidgets.QHBoxLayout() layout.addWidget(analyze_button) layout.addWidget(reset_button) - layout.addStretch(2) + layout.addWidget(settings_button) + layout.addStretch(3) layout.addWidget(save_button, alignment=QtCore.Qt.AlignRight) self.view_analyze_button = analyze_button self.view_reset_button = reset_button + self.view_settings_button = settings_button self.view_save_button = save_button self.view_buttons = layout @@ -350,7 +403,6 @@ class CapaExplorerForm(idaapi.PluginForm): layout.addWidget(self.view_tabs) layout.addLayout(self.view_buttons) layout.addWidget(self.view_status_label) - layout.setMenuBar(self.view_menu_bar) self.parent.setLayout(layout) @@ -450,27 +502,6 @@ class CapaExplorerForm(idaapi.PluginForm): self.view_tabs.addTab(tab, "Rule Generator") - def load_configure_menu(self): - """ """ - actions = ( - ("Change default rules directory...", "Set default rules directory", self.slot_change_rules_dir), - ("Change default rule author...", "Set default rule author", self.slot_change_rule_author), - ("Change default rule scope...", "Set default rule scope", self.slot_change_rule_scope), - ) - self.load_menu("Settings", actions) - - def load_menu(self, title, actions): - """load menu actions - - @param title: menu name displayed in UI - @param actions: tuple of tuples containing action name, tooltip, and slot function - """ - menu = self.view_menu_bar.addMenu(title) - for (name, _, slot) in actions: - action = QtWidgets.QAction(name, self.parent) - action.triggered.connect(slot) - menu.addAction(action) - def load_ida_hooks(self): """load IDA UI hooks""" # map named action (defined in idagui.cfg) to Python function @@ -567,7 +598,7 @@ class CapaExplorerForm(idaapi.PluginForm): try: # resolve rules directory - check self and settings first, then ask user - if not os.path.exists(settings.user.get("rule_path", "")): + if not os.path.exists(settings.user.get(CAPA_SETTINGS_RULE_PATH, "")): idaapi.info("Please select a file directory containing capa rules.") path = self.ask_user_directory() if not path: @@ -575,7 +606,7 @@ class CapaExplorerForm(idaapi.PluginForm): "You must select a file directory containing capa rules before analysis can be run. The standard collection of capa rules can be downloaded from https://github.com/fireeye/capa-rules." ) return False - settings.user["rule_path"] = path + settings.user[CAPA_SETTINGS_RULE_PATH] = path except Exception as e: logger.error("Failed to load capa rules (error: %s).", e) return False @@ -584,8 +615,9 @@ class CapaExplorerForm(idaapi.PluginForm): logger.info("User cancelled analysis.") return False - rule_path = settings.user["rule_path"] + rule_path = settings.user[CAPA_SETTINGS_RULE_PATH] try: + # TODO refactor: this first part is identical to capa.main.get_rules if not os.path.exists(rule_path): raise IOError("rule path %s does not exist or cannot be accessed" % rule_path) @@ -601,8 +633,8 @@ class CapaExplorerForm(idaapi.PluginForm): continue for file in files: if not file.endswith(".yml"): - if not (file.endswith(".md") or file.endswith(".git") or file.endswith(".txt")): - # expect to see readme.md, format.md, and maybe a .git directory + if not (file.startswith(".git") or file.endswith((".git", ".md", ".txt"))): + # expect to see .git* files, readme.md, format.md, and maybe a .git directory # other things maybe are rules, but are mis-named. logger.warning("skipping non-.yml file: %s", file) continue @@ -613,7 +645,8 @@ class CapaExplorerForm(idaapi.PluginForm): total_paths = len(rule_paths) for (i, rule_path) in enumerate(rule_paths): update_wait_box( - "loading capa rules from %s (%d of %d)" % (settings.user["rule_path"], i + 1, total_paths) + "loading capa rules from %s (%d of %d)" + % (settings.user[CAPA_SETTINGS_RULE_PATH], i + 1, total_paths) ) if ida_kernwin.user_cancelled(): raise UserCancelledError("user cancelled") @@ -632,12 +665,14 @@ class CapaExplorerForm(idaapi.PluginForm): logger.info("User cancelled analysis.") return False except Exception as e: - capa.ida.helpers.inform_user_ida_ui("Failed to load capa rules from %s" % settings.user["rule_path"]) - logger.error("Failed to load rules from %s (error: %s).", settings.user["rule_path"], e) + capa.ida.helpers.inform_user_ida_ui( + "Failed to load capa rules from %s" % settings.user[CAPA_SETTINGS_RULE_PATH] + ) + logger.error("Failed to load rules from %s (error: %s).", settings.user[CAPA_SETTINGS_RULE_PATH], e) logger.error( "Make sure your file directory contains properly formatted capa rules. You can download the standard collection of capa rules from https://github.com/fireeye/capa-rules." ) - settings.user["rule_path"] = "" + settings.user[CAPA_SETTINGS_RULE_PATH] = "" return False self.ruleset_cache = ruleset @@ -743,7 +778,7 @@ class CapaExplorerForm(idaapi.PluginForm): try: self.model_data.render_capa_doc(self.doc, self.view_show_results_by_function.isChecked()) self.set_view_status_label( - "capa rules directory: %s (%d rules)" % (settings.user["rule_path"], len(self.rules_cache)) + "capa rules directory: %s (%d rules)" % (settings.user[CAPA_SETTINGS_RULE_PATH], len(self.rules_cache)) ) except Exception as e: logger.error("Failed to render results (error: %s)", e) @@ -881,14 +916,14 @@ class CapaExplorerForm(idaapi.PluginForm): # load preview and feature tree self.view_rulegen_preview.load_preview_meta( f.start_ea if f else None, - settings.user.get("rulegen_author", ""), - settings.user.get("rulegen_scope", "function"), + settings.user.get(CAPA_SETTINGS_RULEGEN_AUTHOR, ""), + settings.user.get(CAPA_SETTINGS_RULEGEN_SCOPE, "function"), ) self.view_rulegen_features.load_features(file_features, func_features) # self.view_rulegen_header_label.setText("Function Features (%s)" % trim_function_name(f)) self.set_view_status_label( - "capa rules directory: %s (%d rules)" % (settings.user["rule_path"], len(self.rules_cache)) + "capa rules directory: %s (%d rules)" % (settings.user[CAPA_SETTINGS_RULE_PATH], len(self.rules_cache)) ) except Exception as e: logger.error("Failed to render views (error: %s)" % e) @@ -985,6 +1020,12 @@ class CapaExplorerForm(idaapi.PluginForm): # create deep copy of current rules, add our new rule rules = copy.copy(self.rules_cache) + + # ensure subscope rules are included + for sub in rule.extract_subscope_rules(): + rules.append(sub) + + # include our new rule in the list rules.append(rule) try: @@ -1066,6 +1107,16 @@ class CapaExplorerForm(idaapi.PluginForm): elif self.view_tabs.currentIndex() == 1: self.save_function_analysis() + def slot_settings(self): + """ """ + dialog = CapaSettingsInputDialog("capa explorer settings", parent=self.parent) + if dialog.exec_(): + ( + settings.user[CAPA_SETTINGS_RULE_PATH], + settings.user[CAPA_SETTINGS_RULEGEN_AUTHOR], + settings.user[CAPA_SETTINGS_RULEGEN_SCOPE], + ) = dialog.get_values() + def save_program_analysis(self): """ """ if not self.doc: @@ -1143,42 +1194,16 @@ class CapaExplorerForm(idaapi.PluginForm): """create Qt dialog to ask user for a directory""" return str( QtWidgets.QFileDialog.getExistingDirectory( - self.parent, "Please select a capa rules directory", settings.user["rule_path"] + self.parent, "Please select a capa rules directory", settings.user.get(CAPA_SETTINGS_RULE_PATH, "") ) ) def ask_user_capa_rule_file(self): """ """ return QtWidgets.QFileDialog.getSaveFileName( - None, "Please select a capa rule to edit", settings.user["rule_path"], "*.yml" + None, "Please select a capa rule to edit", settings.user.get(CAPA_SETTINGS_RULE_PATH, ""), "*.yml" )[0] - def slot_change_rule_scope(self): - """ """ - scope = idaapi.ask_str(str(settings.user.get("rulegen_scope", "function")), 0, "Enter default rule scope") - if scope: - settings.user["rulegen_scope"] = scope - idaapi.info("Run analysis again for your changes to take effect.") - - def slot_change_rule_author(self): - """ """ - author = idaapi.ask_str(str(settings.user.get("rulegen_author", "")), 0, "Enter default rule author") - if author: - settings.user["rulegen_author"] = author - idaapi.info("Run analysis again for your changes to take effect.") - - def slot_change_rules_dir(self): - """allow user to change rules directory - - user selection stored in settings for future runs - """ - path = self.ask_user_directory() - if path: - settings.user["rule_path"] = path - self.rules_cache = None - self.ruleset_cache = None - idaapi.info("Run analysis again for your changes to take effect.") - def set_view_status_label(self, text): """update status label control diff --git a/capa/ida/plugin/item.py b/capa/ida/plugin/item.py index a1854cdd..77db0c32 100644 --- a/capa/ida/plugin/item.py +++ b/capa/ida/plugin/item.py @@ -202,7 +202,7 @@ class CapaExplorerRuleMatchItem(CapaExplorerDataItem): @property def source(self): - """ return rule contents for display """ + """return rule contents for display""" return self._source @@ -328,14 +328,10 @@ class CapaExplorerByteViewItem(CapaExplorerFeatureItem): """ byte_snap = idaapi.get_bytes(location, 32) + details = "" if byte_snap: byte_snap = codecs.encode(byte_snap, "hex").upper() - if sys.version_info >= (3, 0): - details = " ".join([byte_snap[i : i + 2].decode() for i in range(0, len(byte_snap), 2)]) - else: - details = " ".join([byte_snap[i : i + 2] for i in range(0, len(byte_snap), 2)]) - else: - details = "" + details = " ".join([byte_snap[i : i + 2].decode() for i in range(0, len(byte_snap), 2)]) super(CapaExplorerByteViewItem, self).__init__(parent, display, location=location, details=details) self.ida_highlight = idc.get_color(location, idc.CIC_ITEM) diff --git a/capa/ida/plugin/model.py b/capa/ida/plugin/model.py index 08573fb8..6dcd191c 100644 --- a/capa/ida/plugin/model.py +++ b/capa/ida/plugin/model.py @@ -488,13 +488,17 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel): @param feature: capa feature read from doc """ - if feature[feature["type"]]: + key = feature["type"] + value = feature[feature["type"]] + if value: + if key == "string": + value = '"%s"' % capa.features.escape_string(value) if feature.get("description", ""): - return "%s(%s = %s)" % (feature["type"], feature[feature["type"]], feature["description"]) + return "%s(%s = %s)" % (key, value, feature["description"]) else: - return "%s(%s)" % (feature["type"], feature[feature["type"]]) + return "%s(%s)" % (key, value) else: - return "%s" % feature["type"] + return "%s" % key def render_capa_doc_feature_node(self, parent, feature, locations, doc): """process capa doc feature node @@ -551,7 +555,9 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel): ) if feature["type"] == "regex": - return CapaExplorerStringViewItem(parent, display, location, feature["match"]) + return CapaExplorerStringViewItem( + parent, display, location, '"%s"' % capa.features.escape_string(feature["match"]) + ) if feature["type"] == "basicblock": return CapaExplorerBlockItem(parent, location) @@ -576,7 +582,9 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel): if feature["type"] in ("string",): # display string preview - return CapaExplorerStringViewItem(parent, display, location, feature[feature["type"]]) + return CapaExplorerStringViewItem( + parent, display, location, '"%s"' % capa.features.escape_string(feature[feature["type"]]) + ) if feature["type"] in ("import", "export"): # display no preview diff --git a/capa/ida/plugin/proxy.py b/capa/ida/plugin/proxy.py index 7511cc64..2ce48134 100644 --- a/capa/ida/plugin/proxy.py +++ b/capa/ida/plugin/proxy.py @@ -5,7 +5,6 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import six from PyQt5 import QtCore from PyQt5.QtCore import Qt @@ -208,7 +207,7 @@ class CapaExplorerSearchProxyModel(QtCore.QSortFilterProxyModel): if not data: continue - if not isinstance(data, six.string_types): + if not isinstance(data, str): # sanity check: should already be a string, but double check continue diff --git a/capa/ida/plugin/view.py b/capa/ida/plugin/view.py index 38d681b8..52a29916 100644 --- a/capa/ida/plugin/view.py +++ b/capa/ida/plugin/view.py @@ -178,6 +178,9 @@ def build_context_menu(o, actions): class CapaExplorerRulgenPreview(QtWidgets.QTextEdit): + + INDENT = " " * 2 + def __init__(self, parent=None): """ """ super(CapaExplorerRulgenPreview, self).__init__(parent) @@ -210,12 +213,99 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit): self.setText("\n".join(metadata_default)) def keyPressEvent(self, e): - """ """ - if e.key() == QtCore.Qt.Key_Tab: - self.insertPlainText(" " * 2) + """intercept key press events""" + if e.key() in (QtCore.Qt.Key_Tab, QtCore.Qt.Key_Backtab): + # apparently it's not easy to implement tabs as spaces, or multi-line tab or SHIFT + Tab + # so we need to implement it ourselves so we can retain properly formatted capa rules + # when a user uses the Tab key + if self.textCursor().selection().isEmpty(): + # single line, only worry about Tab + if e.key() == QtCore.Qt.Key_Tab: + self.insertPlainText(self.INDENT) + else: + # multi-line tab or SHIFT + Tab + cur = self.textCursor() + select_start_ppos = cur.selectionStart() + select_end_ppos = cur.selectionEnd() + + scroll_ppos = self.verticalScrollBar().sliderPosition() + + # determine lineno for first selected line, and column + cur.setPosition(select_start_ppos) + start_lineno = self.count_previous_lines_from_block(cur.block()) + start_lineco = cur.columnNumber() + + # determine lineno for last selected line + cur.setPosition(select_end_ppos) + end_lineno = self.count_previous_lines_from_block(cur.block()) + + # now we need to indent or dedent the selected lines. for now, we read the text, modify + # the lines between start_lineno and end_lineno accordingly, and then reset the view + # this might not be the best solution, but it avoids messing around with cursor positions + # to determine the beginning of lines + + plain = self.toPlainText().splitlines() + + if e.key() == QtCore.Qt.Key_Tab: + # user Tab, indent selected lines + lines_modified = end_lineno - start_lineno + first_modified = True + change = [self.INDENT + line for line in plain[start_lineno : end_lineno + 1]] + else: + # user SHIFT + Tab, dedent selected lines + lines_modified = 0 + first_modified = False + change = [] + for (lineno, line) in enumerate(plain[start_lineno : end_lineno + 1]): + if line.startswith(self.INDENT): + if lineno == 0: + # keep track if first line is modified, so we can properly display + # the text selection later + first_modified = True + lines_modified += 1 + line = line[len(self.INDENT) :] + change.append(line) + + # apply modifications, and reset view + plain[start_lineno : end_lineno + 1] = change + self.setPlainText("\n".join(plain) + "\n") + + # now we need to properly adjust the selection positions, so users don't have to + # re-select when indenting or dedenting the same lines repeatedly + if e.key() == QtCore.Qt.Key_Tab: + # user Tab, increase increment selection positions + select_start_ppos += len(self.INDENT) + select_end_ppos += (lines_modified * len(self.INDENT)) + len(self.INDENT) + elif lines_modified: + # user SHIFT + Tab, decrease selection positions + if start_lineco not in (0, 1) and first_modified: + # only decrease start position if not in first column + select_start_ppos -= len(self.INDENT) + select_end_ppos -= lines_modified * len(self.INDENT) + + # apply updated selection and restore previous scroll position + self.set_selection(select_start_ppos, select_end_ppos, len(self.toPlainText())) + self.verticalScrollBar().setSliderPosition(scroll_ppos) else: super(CapaExplorerRulgenPreview, self).keyPressEvent(e) + def count_previous_lines_from_block(self, block): + """calculate number of lines preceding block""" + count = 0 + while True: + block = block.previous() + if not block.isValid(): + break + count += block.lineCount() + return count + + def set_selection(self, start, end, max): + """set text selection""" + cursor = self.textCursor() + cursor.setPosition(start) + cursor.setPosition(end if end < max else max, QtGui.QTextCursor.KeepAnchor) + self.setTextCursor(cursor) + class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget): @@ -325,6 +415,11 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget): # create a new parent under root node, by default; new node added last position in tree new_parent = self.new_expression_node(self.root, (action.data()[0], "")) + if "basic block" in action.data()[0]: + # add default child expression when nesting under basic block + new_parent.setExpanded(True) + new_parent = self.new_expression_node(new_parent, ("- or:", "")) + for o in self.get_features(selected=True): # take child from its parent by index, add to new parent new_parent.addChild(o.parent().takeChild(o.parent().indexOfChild(o))) @@ -335,6 +430,15 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget): def slot_edit_expression(self, action): """ """ expression, o = action.data() + if "basic block" in expression and "basic block" not in o.text( + CapaExplorerRulgenEditor.get_column_feature_index() + ): + # current expression is "basic block", and not changing to "basic block" expression + children = o.takeChildren() + new_parent = self.new_expression_node(o, ("- or:", "")) + for child in children: + new_parent.addChild(child) + new_parent.setExpanded(True) o.setText(CapaExplorerRulgenEditor.get_column_feature_index(), expression) def slot_clear_all(self, action): @@ -520,11 +624,23 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget): # single features for (k, v) in filter(lambda t: t[1] == 1, counted): - self.new_feature_node(self.root, ("- %s: %s" % (k.name.lower(), k.get_value_str()), "")) + if isinstance(k, (capa.features.String,)): + value = '"%s"' % capa.features.escape_string(k.get_value_str()) + else: + value = k.get_value_str() + self.new_feature_node(self.root, ("- %s: %s" % (k.name.lower(), value), "")) # n > 1 features for (k, v) in filter(lambda t: t[1] > 1, counted): - self.new_feature_node(self.root, ("- count(%s): %d" % (str(k), v), "")) + if k.value: + if isinstance(k, (capa.features.String,)): + value = '"%s"' % capa.features.escape_string(k.get_value_str()) + else: + value = k.get_value_str() + display = "- count(%s(%s)): %d" % (k.name.lower(), value, v) + else: + display = "- count(%s): %d" % (k.name.lower(), v) + self.new_feature_node(self.root, (display, "")) self.expandAll() self.update_preview() @@ -699,9 +815,11 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget): if text: for o in iterate_tree(self): data = o.data(0, 0x100) - if data and text.lower() not in data.get_value_str().lower(): - o.setHidden(True) - continue + if data: + to_match = data.get_value_str() + if not to_match or text.lower() not in to_match.lower(): + o.setHidden(True) + continue o.setHidden(False) o.setExpanded(True) else: @@ -776,16 +894,20 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget): def format_address(e): return "%X" % e if e else "" + def format_feature(feature): + """ """ + name = feature.name.lower() + value = feature.get_value_str() + if isinstance(feature, (capa.features.String,)): + value = '"%s"' % capa.features.escape_string(value) + return "%s(%s)" % (name, value) + for (feature, eas) in sorted(features.items(), key=lambda k: sorted(k[1])): if isinstance(feature, capa.features.basicblock.BasicBlock): # filter basic blocks for now, we may want to add these back in some time # in the future continue - if isinstance(feature, capa.features.String): - # strip string for display - feature.value = feature.value.strip() - # level 0 if type(feature) not in self.parent_items: self.parent_items[type(feature)] = self.new_parent_node(parent, (feature.name.lower(),)) @@ -794,20 +916,22 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget): if feature not in self.parent_items: if len(eas) > 1: self.parent_items[feature] = self.new_parent_node( - self.parent_items[type(feature)], (str(feature),), feature=feature + self.parent_items[type(feature)], (format_feature(feature),), feature=feature ) else: self.parent_items[feature] = self.new_leaf_node( - self.parent_items[type(feature)], (str(feature),), feature=feature + self.parent_items[type(feature)], (format_feature(feature),), feature=feature ) # level n > 1 if len(eas) > 1: for ea in sorted(eas): - self.new_leaf_node(self.parent_items[feature], (str(feature), format_address(ea)), feature=feature) + self.new_leaf_node( + self.parent_items[feature], (format_feature(feature), format_address(ea)), feature=feature + ) else: ea = eas.pop() - for (i, v) in enumerate((str(feature), format_address(ea))): + for (i, v) in enumerate((format_feature(feature), format_address(ea))): self.parent_items[feature].setText(i, v) self.parent_items[feature].setData(0, 0x100, feature) diff --git a/capa/main.py b/capa/main.py index 84310005..fd0d1381 100644 --- a/capa/main.py +++ b/capa/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 """ Copyright (C) 2020 FireEye, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); @@ -38,6 +38,8 @@ RULES_PATH_DEFAULT_STRING = "(embedded rules)" SUPPORTED_FILE_MAGIC = set([b"MZ"]) BACKEND_VIV = "vivisect" BACKEND_SMDA = "smda" +EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32") +EXTENSIONS_SHELLCODE_64 = ("sc64", "raw64") logger = logging.getLogger("capa") @@ -389,26 +391,15 @@ def get_workspace(path, format, sigpaths): return vw -def get_extractor_py2(path, format, sigpaths, disable_progress=False): - import capa.features.extractors.viv - - with halo.Halo(text="analyzing program", spinner="simpleDots", stream=sys.stderr, enabled=not disable_progress): - vw = get_workspace(path, format, sigpaths) - - try: - vw.saveWorkspace() - except IOError: - # see #168 for discussion around how to handle non-writable directories - logger.info("source directory is not writable, won't save intermediate workspace") - - return capa.features.extractors.viv.VivisectFeatureExtractor(vw, path) - - class UnsupportedRuntimeError(RuntimeError): pass -def get_extractor_py3(path, format, backend, sigpaths, disable_progress=False): +def get_extractor(path, format, backend, sigpaths, disable_progress=False): + """ + raises: + UnsupportedFormatError: + """ if backend == "smda": from smda.SmdaConfig import SmdaConfig from smda.Disassembler import Disassembler @@ -427,7 +418,11 @@ def get_extractor_py3(path, format, backend, sigpaths, disable_progress=False): import capa.features.extractors.viv with halo.Halo(text="analyzing program", spinner="simpleDots", stream=sys.stderr, enabled=not disable_progress): - vw = get_workspace(path, format, sigpaths) + if format == "auto" and path.endswith(EXTENSIONS_SHELLCODE_32): + format = "sc32" + elif format == "auto" and path.endswith(EXTENSIONS_SHELLCODE_64): + format = "sc64" + vw = get_workspace(path, format, sigpaths, should_save=False) try: vw.saveWorkspace() @@ -438,22 +433,6 @@ def get_extractor_py3(path, format, backend, sigpaths, disable_progress=False): return capa.features.extractors.viv.VivisectFeatureExtractor(vw, path) -def get_extractor(path, format, backend, sigpaths, disable_progress=False): - """ - args: - path (str): file system path to file to analyze. - format (str): "auto" for autodetection or one of "pe", "sc32" or "sc64" to override. - sigpaths (List[str]): file system paths to .sig/.pat files to identify functions. - - raises: - UnsupportedFormatError: - """ - if sys.version_info >= (3, 0): - return get_extractor_py3(path, format, backend, sigpaths, disable_progress=disable_progress) - else: - return get_extractor_py2(path, format, sigpaths, disable_progress=disable_progress) - - def is_nursery_rule_path(path): """ The nursery is a spot for rules that have not yet been fully polished. @@ -485,8 +464,8 @@ def get_rules(rule_path, disable_progress=False): for file in files: if not file.endswith(".yml"): - if not (file.endswith(".md") or file.endswith(".git") or file.endswith(".txt")): - # expect to see readme.md, format.md, and maybe a .git directory + if not (file.startswith(".git") or file.endswith((".git", ".md", ".txt"))): + # expect to see .git* files, readme.md, format.md, and maybe a .git directory # other things maybe are rules, but are mis-named. logger.warning("skipping non-.yml file: %s", file) continue @@ -566,7 +545,7 @@ def install_common_args(parser, wanted=None): wanted (Set[str]): collection of arguments to opt-into, including: - "sample": required positional argument to input file. - "format": flag to override file format. - - "backend": flag to override analysis backend under py3. + - "backend": flag to override analysis backend. - "rules": flag to override path to capa rules. - "tag": flag to override/specify which rules to match. """ @@ -604,22 +583,11 @@ def install_common_args(parser, wanted=None): # if "sample" in wanted: - if sys.version_info >= (3, 0): - parser.add_argument( - # Python 3 str handles non-ASCII arguments correctly - "sample", - type=str, - help="path to sample to analyze", - ) - else: - parser.add_argument( - # in #328 we noticed that the sample path is not handled correctly if it contains non-ASCII characters - # https://stackoverflow.com/a/22947334/ offers a solution and decoding using getfilesystemencoding works - # in our testing, however other sources suggest `sys.stdin.encoding` (https://stackoverflow.com/q/4012571/) - "sample", - type=lambda s: s.decode(sys.getfilesystemencoding()), - help="path to sample to analyze", - ) + parser.add_argument( + "sample", + type=str, + help="path to sample to analyze", + ) if "format" in wanted: formats = [ @@ -638,15 +606,15 @@ def install_common_args(parser, wanted=None): help="select sample format, %s" % format_help, ) - if "backend" in wanted and sys.version_info >= (3, 0): - parser.add_argument( - "-b", - "--backend", - type=str, - help="select the backend to use", - choices=(BACKEND_VIV, BACKEND_SMDA), - default=BACKEND_VIV, - ) + if "backend" in wanted: + parser.add_argument( + "-b", + "--backend", + type=str, + help="select the backend to use", + choices=(BACKEND_VIV, BACKEND_SMDA), + default=BACKEND_VIV, + ) if "rules" in wanted: parser.add_argument( @@ -694,10 +662,9 @@ def handle_common_args(args): # disable vivisect-related logging, it's verbose and not relevant for capa users set_vivisect_log_level(logging.CRITICAL) - # py2 doesn't know about cp65001, which is a variant of utf-8 on windows - # tqdm bails when trying to render the progress bar in this setup. - # because cp65001 is utf-8, we just map that codepage to the utf-8 codec. - # see #380 and: https://stackoverflow.com/a/3259271/87207 + # Since Python 3.8 cp65001 is an alias to utf_8, but not for Pyhton < 3.8 + # TODO: remove this code when only supporting Python 3.8+ + # https://stackoverflow.com/a/3259271/87207 import codecs codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None) @@ -717,6 +684,9 @@ def handle_common_args(args): def main(argv=None): + if sys.version_info < (3, 6): + raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.6+") + if argv is None: argv = sys.argv[1:] @@ -821,8 +791,7 @@ def main(argv=None): else: format = args.format try: - backend = args.backend if sys.version_info > (3, 0) else BACKEND_VIV - extractor = get_extractor(args.sample, args.format, backend, args.signatures, disable_progress=args.quiet) + extractor = get_extractor(args.sample, format, args.backend, args.signatures, disable_progress=args.quiet) except UnsupportedFormatError: logger.error("-" * 80) logger.error(" Input file does not appear to be a PE file.") @@ -833,16 +802,6 @@ def main(argv=None): logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.") logger.error("-" * 80) return -1 - except UnsupportedRuntimeError: - logger.error("-" * 80) - logger.error(" Unsupported runtime or Python interpreter.") - logger.error(" ") - logger.error(" capa supports running under Python 2.7 using Vivisect for binary analysis.") - logger.error(" It can also run within IDA Pro, using either Python 2.7 or 3.5+.") - logger.error(" ") - logger.error(" If you're seeing this message on the command line, please ensure you're running Python 2.7.") - logger.error("-" * 80) - return -1 meta = collect_metadata(argv, args.sample, args.rules, format, extractor) diff --git a/capa/render/__init__.py b/capa/render/__init__.py index 9de20bd5..7deb9b17 100644 --- a/capa/render/__init__.py +++ b/capa/render/__init__.py @@ -8,8 +8,6 @@ import json -import six - import capa.rules import capa.engine @@ -249,7 +247,7 @@ class CapaJsonObjectEncoder(json.JSONEncoder): """JSON encoder that emits Python sets as sorted lists""" def default(self, obj): - if isinstance(obj, (list, dict, int, float, bool, type(None))) or isinstance(obj, six.string_types): + if isinstance(obj, (list, dict, int, float, bool, type(None))) or isinstance(obj, str): return json.JSONEncoder.default(self, obj) elif isinstance(obj, set): return list(sorted(obj)) diff --git a/capa/render/default.py b/capa/render/default.py index 7ebad0da..52488011 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -8,7 +8,6 @@ import collections -import six import tabulate import capa.render.utils as rutils diff --git a/capa/render/utils.py b/capa/render/utils.py index a484dbb0..001fb4dc 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -6,7 +6,8 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import six +import io + import termcolor @@ -49,7 +50,7 @@ def capability_rules(doc): yield rule -class StringIO(six.StringIO): +class StringIO(io.StringIO): def writeln(self, s): self.write(s) self.write("\n") diff --git a/capa/render/vverbose.py b/capa/render/vverbose.py index 72425246..1627d09d 100644 --- a/capa/render/vverbose.py +++ b/capa/render/vverbose.py @@ -56,7 +56,11 @@ def render_statement(ostream, match, statement, indent=0): child = statement["child"] if child[child["type"]]: - value = rutils.bold2(child[child["type"]]) + if child["type"] == "string": + value = '"%s"' % capa.features.escape_string(child[child["type"]]) + else: + value = child[child["type"]] + value = rutils.bold2(value) if child.get("description"): ostream.write("count(%s(%s = %s)): " % (child["type"], value, child["description"])) else: @@ -90,6 +94,9 @@ def render_feature(ostream, match, feature, indent=0): key = "string" # render string for regex to mirror the rule source value = feature["match"] # the match provides more information than the value for regex + if key == "string": + value = '"%s"' % capa.features.escape_string(value) + ostream.write(key) ostream.write(": ") diff --git a/capa/rules.py b/capa/rules.py index b9a42056..de3bf4b9 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -18,7 +18,8 @@ try: except ImportError: from backports.functools_lru_cache import lru_cache -import six +import io + import yaml import ruamel.yaml @@ -244,7 +245,7 @@ def parse_description(s, value_type, description=None): """ s can be an int or a string """ - if value_type != "string" and isinstance(s, six.string_types) and DESCRIPTION_SEPARATOR in s: + if value_type != "string" and isinstance(s, str) and DESCRIPTION_SEPARATOR in s: if description: raise InvalidRule( 'unexpected value: "%s", only one description allowed (inline description with `%s`)' @@ -256,12 +257,11 @@ def parse_description(s, value_type, description=None): else: value = s - if isinstance(value, six.string_types): + if isinstance(value, str): if value_type == "bytes": try: value = codecs.decode(value.replace(" ", ""), "hex") - # TODO: Remove TypeError when Python2 is not used anymore - except (TypeError, binascii.Error): + except binascii.Error: raise InvalidRule('unexpected bytes value: "%s", must be a valid hex sequence' % value) if len(value) > MAX_BYTES_FEATURE_SIZE: @@ -406,7 +406,7 @@ def build_statements(d, scope): return Range(feature, min=min, max=max, description=description) else: raise InvalidRule("unexpected range: %s" % (count)) - elif key == "string" and not isinstance(d[key], six.string_types): + elif key == "string" and not isinstance(d[key], str): raise InvalidRule("ambiguous string value %s, must be defined as explicit string" % d[key]) else: Feature = parse_feature(key) @@ -699,7 +699,7 @@ class Rule(object): for key in hidden_meta.keys(): del meta[key] - ostream = six.BytesIO() + ostream = io.BytesIO() self._get_ruamel_yaml_parser().dump(definition, ostream) for key, value in hidden_meta.items(): @@ -736,6 +736,8 @@ class Rule(object): # the below regex makes these adjustments and while ugly, we don't have to explore the ruamel.yaml insides doc = re.sub(r"!!int '0x-([0-9a-fA-F]+)'", r"-0x\1", doc) + # normalize CRLF to LF + doc = doc.replace("\r\n", "\n") return doc @@ -936,7 +938,7 @@ class RuleSet(object): rules_filtered = set([]) for rule in rules: for k, v in rule.meta.items(): - if isinstance(v, six.string_types) and tag in v: + if isinstance(v, str) and tag in v: logger.debug('using rule "%s" and dependencies, found tag in meta.%s: %s', rule.name, k, v) rules_filtered.update(set(capa.rules.get_rules_and_dependencies(rules, rule.name))) break diff --git a/capa/version.py b/capa/version.py index 0f228f25..f49459c7 100644 --- a/capa/version.py +++ b/capa/version.py @@ -1 +1 @@ -__version__ = "1.5.1" +__version__ = "1.6.1" diff --git a/doc/img/changelog/tab.gif b/doc/img/changelog/tab.gif new file mode 100644 index 00000000..834fc954 Binary files /dev/null and b/doc/img/changelog/tab.gif differ diff --git a/doc/img/explorer_condensed.png b/doc/img/explorer_condensed.png index bd3cdcb4..8372cdd0 100644 Binary files a/doc/img/explorer_condensed.png and b/doc/img/explorer_condensed.png differ diff --git a/doc/img/explorer_expanded.png b/doc/img/explorer_expanded.png index a2f82e45..26cbbad7 100644 Binary files a/doc/img/explorer_expanded.png and b/doc/img/explorer_expanded.png differ diff --git a/doc/img/rulegen_expanded.png b/doc/img/rulegen_expanded.png index c3c4d0ee..5b3e5816 100644 Binary files a/doc/img/rulegen_expanded.png and b/doc/img/rulegen_expanded.png differ diff --git a/doc/installation.md b/doc/installation.md index 6bc7ab55..8ee74a82 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -42,6 +42,7 @@ If you'd like to review and modify the capa source code, you'll need to check it Next, clone the capa git repository. We use submodules to separate [code](https://github.com/fireeye/capa), [rules](https://github.com/fireeye/capa-rules), and [test data](https://github.com/fireeye/capa-testfiles). To clone everything use the `--recurse-submodules` option: +- CAUTION: The capa testfiles repository contains many malware samples. If you pull down everything using this method, you may want to install to a directory that won't trigger your anti-virus software. - `$ git clone --recurse-submodules https://github.com/fireeye/capa.git /local/path/to/src` (HTTPS) - `$ git clone --recurse-submodules git@github.com:fireeye/capa.git /local/path/to/src` (SSH) @@ -59,6 +60,25 @@ Use `pip` to install the source code in "editable" mode. This means that Python You'll find that the `capa.exe` (Windows) or `capa` (Linux/MacOS) executables in your path now invoke the capa binary from this directory. +#### Development + +##### venv [optional] + +For development, we recommend to use [venv](https://docs.python.org/3/tutorial/venv.html). It allows you to create a virtual environment: a self-contained directory tree that contains a Python installation for a particular version of Python, plus a number of additional packages. This approach avoids conflicts between the requirements of different applications on your computer. It also ensures that you don't overlook to add a new requirement to `setup.up` using a library already installed on your system. + +To create an environment (in the parent directory, to avoid commiting it by accident or messing with the linters), run: +`$ python3 -m venv ../capa-env` + +To activate `capa-env` in Linux or MacOS, run: +`$ source ../capa-env/bin/activate` + +To activate `capa-env` in Windows, run: +`$ ..\capa-env\Scripts\activate.bat` + +For more details about creating and using virtual environments, check out the [venv documentation](https://docs.python.org/3/tutorial/venv.html). + +##### Install development dependencies + We use the following tools to ensure consistent code style and formatting: - [black](https://github.com/psf/black) code formatter, with `-l 120` - [isort 5](https://pypi.org/project/isort/) code formatter, with `--profile black --length-sort --line-width 120` @@ -69,30 +89,28 @@ To install these development dependencies, run: `$ pip install -e /local/path/to/src[dev]` -Note that some development dependencies (including the black code formatter) require Python 3. - To check the code style, formatting and run the tests you can run the script `scripts/ci.sh`. You can run it with the argument `no_tests` to skip the tests and only run the code style and formatting: `scripts/ci.sh no_tests` +##### Setup hooks [optional] + +If you plan to contribute to capa, you may want to setup the hooks. +Run `scripts/setup-hooks.sh` to set the following hooks up: +- The `pre-commit` hook runs checks before every `git commit`. + It runs `scripts/ci.sh no_tests` aborting the commit if there are code style or rule linter offenses you need to fix. +- The `pre-push` hook runs checks before every `git push`. + It runs `scripts/ci.sh` aborting the push if there are code style or rule linter offenses or if the tests fail. + This way you can ensure everything is alright before sending a pull request. + +You can skip the checks by using the `--no-verify` git option. + ### 3. Compile binary using PyInstaller We compile capa standalone binaries using PyInstaller. To reproduce the build process check out the source code as described above and follow these steps. #### Install PyInstaller: -For Python 2.7: `$ pip install 'pyinstaller==3.*'` (PyInstaller 4 doesn't support Python 2.7) - -For Python 3: `$ pip install 'pyinstaller` +`$ pip install pyinstaller` (Python 3) #### Run Pyinstaller `$ pyinstaller .github/pyinstaller/pyinstaller.spec` You can find the compiled binary in the created directory `dist/`. - -### 4. Setup hooks [optional] -If you plan to contribute to capa, you may want to setup the hooks. -Run `scripts/setup-hooks.sh` to set the following hooks up: -- The `pre-commit` hook runs checks before every `git commit`. - It runs `scripts/ci.sh no_tests` aborting the commit if there are code style or rule linter offenses you need to fix. - You can skip this check by using the `--no-verify` git option. -- The `pre-push` hook runs checks before every `git push`. - It runs `scripts/ci.sh` aborting the push if there are code style or rule linter offenses or if the tests fail. - This way you can ensure everything is alright before sending a pull request. diff --git a/doc/release.md b/doc/release.md new file mode 100644 index 00000000..b1030472 --- /dev/null +++ b/doc/release.md @@ -0,0 +1,44 @@ +# Release checklist + +- [ ] Ensure all [milestoned issues/PRs](https://github.com/fireeye/capa/milestones) are addressed, or reassign to a new milestone. +- [ ] Add the `dont merge` label to all PRs that are close to be ready to merge (or merge them if they are ready) in [capa](https://github.com/fireeye/capa/pulls) and [capa-rules](https://github.com/fireeye/capa-rules/pulls). +- [ ] Ensure the [CI workflow succeeds in master](https://github.com/fireeye/capa/actions/workflows/tests.yml?query=branch%3Amaster). +- [ ] Ensure that `python scripts/lint.py rules/ --thorough` succeeds (only `missing examples` offenses are allowed in the nursery). +- [ ] Review changes + - capa https://github.com/fireeye/capa/compare/\...master + - capa-rules https://github.com/fireeye/capa-rules/compare/\\...master +- [ ] Update [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md) + - Do not forget to add a nice introduction thanking contributors + - Remember that we need a major release if we introduce breaking changes + - Sections + - New Features + - New Rules + - Bug Fixes + - Changes + - Development + - Raw diffs + - Update `Raw diffs` links + - Create placeholder for `master (unreleased)` section + ``` + ## master (unreleased) + + ### New Features + + ### New Rules + + ### Bug Fixes + + ### Changes + + ### Development + + ### Raw diffs + - [capa ...master](https://github.com/fireeye/capa/compare/...master) + - [capa-rules ...master](https://github.com/fireeye/capa-rules/compare/...master) + ``` +- [ ] Update [capa/version.py](https://github.com/fireeye/capa/blob/master/capa/version.py) +- [ ] Create a PR with the updated [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md) and [capa/version.py](https://github.com/fireeye/capa/blob/master/capa/version.py). Copy this checklist in the PR description. +- [ ] After PR review, merge the PR and [create the release in GH](https://github.com/fireeye/capa/releases/new) using text from the [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md). +- [ ] Verify GH actions [upload artifacts](https://github.com/fireeye/capa/releases), [publish to PyPI](https://pypi.org/project/flare-capa) and [create a tag in capa rules](https://github.com/fireeye/capa-rules/tags) upon completion. +- [ ] [Spread the word](https://twitter.com) + diff --git a/rules b/rules index 74f37214..f3ccebcd 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 74f372149f6fbe3275fba2ae12703b2caf274991 +Subproject commit f3ccebcd91e1c4ec9c2fec0aa7ae5e1a9382d1f8 diff --git a/scripts/bulk-process.py b/scripts/bulk-process.py index 8559a428..14f1d8ac 100644 --- a/scripts/bulk-process.py +++ b/scripts/bulk-process.py @@ -1,220 +1,220 @@ -#!/usr/bin/env python -""" -bulk-process - -Invoke capa recursively against a directory of samples -and emit a JSON document mapping the file paths to their results. - -By default, this will use subprocesses for parallelism. -Use `-n/--parallelism` to change the subprocess count from - the default of current CPU count. -Use `--no-mp` to use threads instead of processes, - which is probably not useful unless you set `--parallelism=1`. - -example: - - $ python scripts/bulk-process /tmp/suspicious - { - "/tmp/suspicious/suspicious.dll_": { - "rules": { - "encode data using XOR": { - "matches": { - "268440358": { - [...] - "/tmp/suspicious/1.dll_": { ... } - "/tmp/suspicious/2.dll_": { ... } - } - - -usage: - - usage: bulk-process.py [-h] [-r RULES] [-d] [-q] [-n PARALLELISM] [--no-mp] - input - - detect capabilities in programs. - - positional arguments: - input Path to directory of files to recursively analyze - - optional arguments: - -h, --help show this help message and exit - -r RULES, --rules RULES - Path to rule file or directory, use embedded rules by - default - -d, --debug Enable debugging output on STDERR - -q, --quiet Disable all output but errors - -n PARALLELISM, --parallelism PARALLELISM - parallelism factor - --no-mp disable subprocesses - -Copyright (C) 2020 FireEye, Inc. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. -You may obtain a copy of the License at: [package root]/LICENSE.txt -Unless required by applicable law or agreed to in writing, software distributed under the License - is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. -""" -import sys -import json -import logging -import os.path -import argparse -import multiprocessing -import multiprocessing.pool - -import capa -import capa.main -import capa.rules -import capa.render - -logger = logging.getLogger("capa") - - -def get_capa_results(args): - """ - run capa against the file at the given path, using the given rules. - - args is a tuple, containing: - rules (capa.rules.RuleSet): the rules to match - format (str): the name of the sample file format - path (str): the file system path to the sample to process - - args is a tuple because i'm not quite sure how to unpack multiple arguments using `map`. - - returns an dict with two required keys: - path (str): the file system path of the sample to process - status (str): either "error" or "ok" - - when status == "error", then a human readable message is found in property "error". - when status == "ok", then the capa results are found in the property "ok". - - the capa results are a dictionary with the following keys: - meta (dict): the meta analysis results - capabilities (dict): the matched capabilities and their result objects - """ - rules, format, path = args - logger.info("computing capa results for: %s", path) - try: - extractor = capa.main.get_extractor(path, format, capa.main.BACKEND_VIV, args.signatures, disable_progress=True) - except capa.main.UnsupportedFormatError: - # i'm 100% sure if multiprocessing will reliably raise exceptions across process boundaries. - # so instead, return an object with explicit success/failure status. - # - # if success, then status=ok, and results found in property "ok" - # if error, then status=error, and human readable message in property "error" - return { - "path": path, - "status": "error", - "error": "input file does not appear to be a PE file: %s" % path, - } - except capa.main.UnsupportedRuntimeError: - return { - "path": path, - "status": "error", - "error": "unsupported runtime or Python interpreter", - } - except Exception as e: - return { - "path": path, - "status": "error", - "error": "unexpected error: %s" % (e), - } - - meta = capa.main.collect_metadata("", path, "", format, extractor) - capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True) - meta["analysis"].update(counts) - - return { - "path": path, - "status": "ok", - "ok": { - "meta": meta, - "capabilities": capabilities, - }, - } - - -def main(argv=None): - if argv is None: - argv = sys.argv[1:] - - parser = argparse.ArgumentParser(description="detect capabilities in programs.") - capa.main.install_common_args(parser, wanted={"rules", "signatures"}) - parser.add_argument("input", type=str, help="Path to directory of files to recursively analyze") - parser.add_argument( - "-n", "--parallelism", type=int, default=multiprocessing.cpu_count(), help="parallelism factor" - ) - parser.add_argument("--no-mp", action="store_true", help="disable subprocesses") - args = parser.parse_args(args=argv) - capa.main.handle_common_args(args) - - if args.rules == "(embedded rules)": - logger.info("using default embedded rules") - logger.debug("detected running from source") - args.rules = os.path.join(os.path.dirname(__file__), "..", "rules") - logger.debug("default rule path (source method): %s", args.rules) - else: - logger.info("using rules path: %s", args.rules) - - try: - rules = capa.main.get_rules(args.rules) - rules = capa.rules.RuleSet(rules) - logger.info("successfully loaded %s rules", len(rules)) - except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e: - logger.error("%s", str(e)) - return -1 - - samples = [] - for (base, directories, files) in os.walk(args.input): - for file in files: - samples.append(os.path.join(base, file)) - - def pmap(f, args, parallelism=multiprocessing.cpu_count()): - """apply the given function f to the given args using subprocesses""" - return multiprocessing.Pool(parallelism).imap(f, args) - - def tmap(f, args, parallelism=multiprocessing.cpu_count()): - """apply the given function f to the given args using threads""" - return multiprocessing.pool.ThreadPool(parallelism).imap(f, args) - - def map(f, args, parallelism=None): - """apply the given function f to the given args in the current thread""" - for arg in args: - yield f(arg) - - if args.no_mp: - if args.parallelism == 1: - logger.debug("using current thread mapper") - mapper = map - else: - logger.debug("using threading mapper") - mapper = tmap - else: - logger.debug("using process mapper") - mapper = pmap - - results = {} - for result in mapper( - get_capa_results, [(rules, "pe", sample) for sample in samples], parallelism=args.parallelism - ): - if result["status"] == "error": - logger.warning(result["error"]) - elif result["status"] == "ok": - meta = result["ok"]["meta"] - capabilities = result["ok"]["capabilities"] - # our renderer expects to emit a json document for a single sample - # so we deserialize the json document, store it in a larger dict, and we'll subsequently re-encode. - results[result["path"]] = json.loads(capa.render.render_json(meta, rules, capabilities)) - else: - raise ValueError("unexpected status: %s" % (result["status"])) - - print(json.dumps(results)) - - logger.info("done.") - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +#!/usr/bin/env python +""" +bulk-process + +Invoke capa recursively against a directory of samples +and emit a JSON document mapping the file paths to their results. + +By default, this will use subprocesses for parallelism. +Use `-n/--parallelism` to change the subprocess count from + the default of current CPU count. +Use `--no-mp` to use threads instead of processes, + which is probably not useful unless you set `--parallelism=1`. + +example: + + $ python scripts/bulk-process /tmp/suspicious + { + "/tmp/suspicious/suspicious.dll_": { + "rules": { + "encode data using XOR": { + "matches": { + "268440358": { + [...] + "/tmp/suspicious/1.dll_": { ... } + "/tmp/suspicious/2.dll_": { ... } + } + + +usage: + + usage: bulk-process.py [-h] [-r RULES] [-d] [-q] [-n PARALLELISM] [--no-mp] + input + + detect capabilities in programs. + + positional arguments: + input Path to directory of files to recursively analyze + + optional arguments: + -h, --help show this help message and exit + -r RULES, --rules RULES + Path to rule file or directory, use embedded rules by + default + -d, --debug Enable debugging output on STDERR + -q, --quiet Disable all output but errors + -n PARALLELISM, --parallelism PARALLELISM + parallelism factor + --no-mp disable subprocesses + +Copyright (C) 2020 FireEye, Inc. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. +You may obtain a copy of the License at: [package root]/LICENSE.txt +Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. +""" +import sys +import json +import logging +import os.path +import argparse +import multiprocessing +import multiprocessing.pool + +import capa +import capa.main +import capa.rules +import capa.render + +logger = logging.getLogger("capa") + + +def get_capa_results(args): + """ + run capa against the file at the given path, using the given rules. + + args is a tuple, containing: + rules (capa.rules.RuleSet): the rules to match + format (str): the name of the sample file format + path (str): the file system path to the sample to process + + args is a tuple because i'm not quite sure how to unpack multiple arguments using `map`. + + returns an dict with two required keys: + path (str): the file system path of the sample to process + status (str): either "error" or "ok" + + when status == "error", then a human readable message is found in property "error". + when status == "ok", then the capa results are found in the property "ok". + + the capa results are a dictionary with the following keys: + meta (dict): the meta analysis results + capabilities (dict): the matched capabilities and their result objects + """ + rules, format, path = args + logger.info("computing capa results for: %s", path) + try: + extractor = capa.main.get_extractor(path, format, capa.main.BACKEND_VIV, args.signatures, disable_progress=True) + except capa.main.UnsupportedFormatError: + # i'm 100% sure if multiprocessing will reliably raise exceptions across process boundaries. + # so instead, return an object with explicit success/failure status. + # + # if success, then status=ok, and results found in property "ok" + # if error, then status=error, and human readable message in property "error" + return { + "path": path, + "status": "error", + "error": "input file does not appear to be a PE file: %s" % path, + } + except capa.main.UnsupportedRuntimeError: + return { + "path": path, + "status": "error", + "error": "unsupported runtime or Python interpreter", + } + except Exception as e: + return { + "path": path, + "status": "error", + "error": "unexpected error: %s" % (e), + } + + meta = capa.main.collect_metadata("", path, "", format, extractor) + capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True) + meta["analysis"].update(counts) + + return { + "path": path, + "status": "ok", + "ok": { + "meta": meta, + "capabilities": capabilities, + }, + } + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + + parser = argparse.ArgumentParser(description="detect capabilities in programs.") + capa.main.install_common_args(parser, wanted={"rules", "signatures"}) + parser.add_argument("input", type=str, help="Path to directory of files to recursively analyze") + parser.add_argument( + "-n", "--parallelism", type=int, default=multiprocessing.cpu_count(), help="parallelism factor" + ) + parser.add_argument("--no-mp", action="store_true", help="disable subprocesses") + args = parser.parse_args(args=argv) + capa.main.handle_common_args(args) + + if args.rules == "(embedded rules)": + logger.info("using default embedded rules") + logger.debug("detected running from source") + args.rules = os.path.join(os.path.dirname(__file__), "..", "rules") + logger.debug("default rule path (source method): %s", args.rules) + else: + logger.info("using rules path: %s", args.rules) + + try: + rules = capa.main.get_rules(args.rules) + rules = capa.rules.RuleSet(rules) + logger.info("successfully loaded %s rules", len(rules)) + except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e: + logger.error("%s", str(e)) + return -1 + + samples = [] + for (base, directories, files) in os.walk(args.input): + for file in files: + samples.append(os.path.join(base, file)) + + def pmap(f, args, parallelism=multiprocessing.cpu_count()): + """apply the given function f to the given args using subprocesses""" + return multiprocessing.Pool(parallelism).imap(f, args) + + def tmap(f, args, parallelism=multiprocessing.cpu_count()): + """apply the given function f to the given args using threads""" + return multiprocessing.pool.ThreadPool(parallelism).imap(f, args) + + def map(f, args, parallelism=None): + """apply the given function f to the given args in the current thread""" + for arg in args: + yield f(arg) + + if args.no_mp: + if args.parallelism == 1: + logger.debug("using current thread mapper") + mapper = map + else: + logger.debug("using threading mapper") + mapper = tmap + else: + logger.debug("using process mapper") + mapper = pmap + + results = {} + for result in mapper( + get_capa_results, [(rules, "pe", sample) for sample in samples], parallelism=args.parallelism + ): + if result["status"] == "error": + logger.warning(result["error"]) + elif result["status"] == "ok": + meta = result["ok"]["meta"] + capabilities = result["ok"]["capabilities"] + # our renderer expects to emit a json document for a single sample + # so we deserialize the json document, store it in a larger dict, and we'll subsequently re-encode. + results[result["path"]] = json.loads(capa.render.render_json(meta, rules, capabilities)) + else: + raise ValueError("unexpected status: %s" % (result["status"])) + + print(json.dumps(results)) + + logger.info("done.") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/capafmt.py b/scripts/capafmt.py index a0b2a7c6..1f110074 100644 --- a/scripts/capafmt.py +++ b/scripts/capafmt.py @@ -65,6 +65,8 @@ def main(argv=None): return 0 else: logger.info("rule requires reformatting (%s)", rule.name) + if "\r\n" in rule.definition: + logger.info("please make sure that the file uses LF (\\n) line endings only") return 1 if args.in_place: diff --git a/scripts/ci.sh b/scripts/ci.sh index f9b0bde1..204bf266 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -9,6 +9,7 @@ # See the License for the specific language governing permissions and limitations under the License. # Use a console with emojis support for a better experience +# Use venv to ensure that `python` calls the correct python version # Stash uncommited changes MSG="pre-push-$(date +%s)"; @@ -25,17 +26,8 @@ restore_stashed() { fi } -python_3() { - case "$(uname -s)" in - CYGWIN*|MINGW32*|MSYS*|MINGW*) - py -3 -m $1 > $2 2>&1;; - *) - python3 -m $1 > $2 2>&1;; - esac -} - # Run isort and print state -python_3 'isort --profile black --length-sort --line-width 120 -c .' 'isort-output.log'; +python -m isort --profile black --length-sort --line-width 120 -c . > isort-output.log 2>&1; if [ $? == 0 ]; then echo 'isort succeeded!! 💖'; else @@ -46,7 +38,7 @@ else fi # Run black and print state -python_3 'black -l 120 --check .' 'black-output.log'; +python -m black -l 120 --check . > black-output.log 2>&1; if [ $? == 0 ]; then echo 'black succeeded!! 💝'; else @@ -70,7 +62,7 @@ fi # Run tests except if first argument is no_tests if [ "$1" != 'no_tests' ]; then echo 'Running tests, please wait ⌛'; - pytest tests/ --maxfail=1; + python -m pytest tests/ --maxfail=1; if [ $? == 0 ]; then echo 'Tests succeed!! 🎉'; else diff --git a/scripts/lint.py b/scripts/lint.py index 59fd2d8a..74050ec9 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -25,13 +25,15 @@ import argparse import itertools import posixpath +import ruamel.yaml + import capa.main import capa.rules import capa.engine import capa.features import capa.features.insn -logger = logging.getLogger("capa.lint") +logger = logging.getLogger("lint") class Lint(object): @@ -303,6 +305,16 @@ class FeatureNtdllNtoskrnlApi(Lint): return False +class FormatLineFeedEOL(Lint): + name = "line(s) end with CRLF (\\r\\n)" + recommendation = "convert line endings to LF (\\n) for example using dos2unix" + + def check_rule(self, ctx, rule): + if len(rule.definition.split("\r\n")) > 0: + return False + return True + + class FormatSingleEmptyLineEOF(Lint): name = "EOF format" recommendation = "end file with a single empty line" @@ -323,12 +335,43 @@ class FormatIncorrect(Lint): if actual != expected: diff = difflib.ndiff(actual.splitlines(1), expected.splitlines(True)) - self.recommendation = self.recommendation_template.format("".join(diff)) + recommendation_template = self.recommendation_template + if "\r\n" in actual: + recommendation_template = ( + self.recommendation_template + "\nplease make sure that the file uses LF (\\n) line endings only" + ) + self.recommendation = recommendation_template.format("".join(diff)) return True return False +class FormatStringQuotesIncorrect(Lint): + name = "rule string quotes incorrect" + + def check_rule(self, ctx, rule): + events = capa.rules.Rule._get_ruamel_yaml_parser().parse(rule.definition) + for key in events: + if not (isinstance(key, ruamel.yaml.ScalarEvent) and key.value == "string"): + continue + value = next(events) # assume value is next event + if not isinstance(value, ruamel.yaml.ScalarEvent): + # ignore non-scalar + continue + if value.value.startswith("/") and value.value.endswith(("/", "/i")): + # ignore regex for now + continue + if value.style is None: + # no quotes + self.recommendation = 'add double quotes to "%s"' % value.value + return True + if value.style == "'": + # single quote + self.recommendation = 'change single quotes to double quotes for "%s"' % value.value + return True + return False + + def run_lints(lints, ctx, rule): for lint in lints: if lint.check_rule(ctx, rule): @@ -387,7 +430,9 @@ def lint_features(ctx, rule): FORMAT_LINTS = ( + FormatLineFeedEOL(), FormatSingleEmptyLineEOF(), + FormatStringQuotesIncorrect(), FormatIncorrect(), ) @@ -556,7 +601,7 @@ def main(argv=None): samples_path = os.path.join(os.path.dirname(__file__), "..", "tests", "data") - parser = argparse.ArgumentParser(description="A program.") + parser = argparse.ArgumentParser(description="Lint capa rules.") capa.main.install_common_args(parser, wanted={"tag"}) parser.add_argument("rules", type=str, help="Path to rules") parser.add_argument("--samples", type=str, default=samples_path, help="Path to samples") @@ -568,8 +613,12 @@ def main(argv=None): args = parser.parse_args(args=argv) capa.main.handle_common_args(args) - logging.getLogger("capa").setLevel(logging.CRITICAL) - logging.getLogger("viv_utils").setLevel(logging.CRITICAL) + if args.debug: + logging.getLogger("capa").setLevel(logging.DEBUG) + logging.getLogger("viv_utils").setLevel(logging.DEBUG) + else: + logging.getLogger("capa").setLevel(logging.ERROR) + logging.getLogger("viv_utils").setLevel(logging.ERROR) time0 = time.time() diff --git a/setup.py b/setup.py index 289139ec..4c825d4b 100644 --- a/setup.py +++ b/setup.py @@ -7,41 +7,25 @@ # See the License for the specific language governing permissions and limitations under the License. import os -import sys import setuptools requirements = [ - "six", - "tqdm", - "pyyaml", - "tabulate", - "colorama", - "termcolor", - "ruamel.yaml", - "wcwidth", + "tqdm==4.60.0", + "pyyaml==5.4.1", + "tabulate==0.8.9", + "colorama==0.4.4", + "termcolor==1.1.0", + "wcwidth==0.2.5", "ida-settings==2.1.0", + "viv-utils==0.6.0", + "halo==0.0.31", + "networkx==2.5.1", + "ruamel.yaml==0.17.4", + "vivisect==1.0.1", + "smda==1.5.14", ] -if sys.version_info >= (3, 0): - # py3 - requirements.append("halo") - requirements.append("networkx") - requirements.append("vivisect==1.0.0") - requirements.append("viv-utils[flirt]==0.5.0") - requirements.append("smda==1.5.13") -else: - # py2 - requirements.append("enum34==1.1.6") # v1.1.6 is needed by halo 0.0.30 / spinners 0.0.24 - requirements.append("halo==0.0.30") # halo==0.0.30 is the last version to support py2.7 - requirements.append("vivisect==0.1.0") - # TODO: issue here with flirt not being supported. - # but we're also no longer supporting py2 - # so this whole block goes away. - requirements.append("viv-utils==0.5.0") - requirements.append("networkx==2.2") # v2.2 is last version supported by Python 2.7 - requirements.append("backports.functools-lru-cache") - # this sets __version__ # via: http://stackoverflow.com/a/7071358/87207 # and: http://stackoverflow.com/a/2073599/87207 @@ -80,13 +64,13 @@ setuptools.setup( install_requires=requirements, extras_require={ "dev": [ - "pytest", - "pytest-sugar", - "pytest-instafail", - "pytest-cov", - "pycodestyle", - "black ; python_version>'3.0'", - "isort", + "pytest==6.2.3", + "pytest-sugar==0.9.4", + "pytest-instafail==0.4.2", + "pytest-cov==2.11.1", + "pycodestyle==2.7.0", + "black==21.4b0", + "isort==5.8.0", ] }, zip_safe=False, @@ -97,8 +81,8 @@ setuptools.setup( "Intended Audience :: Information Technology", "License :: OSI Approved :: Apache Software License", "Natural Language :: English", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Topic :: Security", ], + python_requires=">=3.6", ) diff --git a/tests/data b/tests/data index d1c9bfe2..a74770f4 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit d1c9bfe2e762fa75215e4a08694dafb216de1bff +Subproject commit a74770f417ec20a33047a57fd1c9e4667398d9e3 diff --git a/tests/fixtures.py b/tests/fixtures.py index de486b4c..a84a4bca 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -8,7 +8,6 @@ # See the License for the specific language governing permissions and limitations under the License. import os -import sys import os.path import binascii import contextlib diff --git a/tests/test_freeze.py b/tests/test_freeze.py index 11c5b956..4e27d5f8 100644 --- a/tests/test_freeze.py +++ b/tests/test_freeze.py @@ -5,7 +5,6 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import sys import textwrap import pytest @@ -174,7 +173,6 @@ def test_serialize_features(): roundtrip_feature(capa.features.file.Import("#11")) -@pytest.mark.xfail(sys.version_info >= (3, 0), reason="vivsect only works on py2") def test_freeze_sample(tmpdir, z9324d_extractor): # tmpdir fixture handles cleanup o = tmpdir.mkdir("capa").join("test.frz").strpath @@ -182,7 +180,6 @@ def test_freeze_sample(tmpdir, z9324d_extractor): assert capa.features.freeze.main([path, o, "-v"]) == 0 -@pytest.mark.xfail(sys.version_info >= (3, 0), reason="vivsect only works on py2") def test_freeze_load_sample(tmpdir, z9324d_extractor): o = tmpdir.mkdir("capa").join("test.frz") diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 7c9403be..9267164e 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -12,8 +12,6 @@ from capa.features.extractors import helpers def test_all_zeros(): - # Python 2: - # Python 3: a = b"\x00\x00\x00\x00" b = codecs.decode("00000000", "hex") c = b"\x01\x00\x00\x00" diff --git a/tests/test_ida_features.py b/tests/test_ida_features.py index 51de2139..4539a1ad 100644 --- a/tests/test_ida_features.py +++ b/tests/test_ida_features.py @@ -1,104 +1,104 @@ -# run this script from within IDA with ./tests/data/mimikatz.exe open -import sys -import logging -import os.path -import binascii -import traceback - -import pytest - -try: - sys.path.append(os.path.dirname(__file__)) - from fixtures import * -finally: - sys.path.pop() - - -logger = logging.getLogger("test_ida_features") - - -def check_input_file(wanted): - import idautils - - # some versions (7.4) of IDA return a truncated version of the MD5. - # https://github.com/idapython/bin/issues/11 - try: - found = idautils.GetInputFileMD5()[:31].decode("ascii").lower() - except UnicodeDecodeError: - # in IDA 7.5 or so, GetInputFileMD5 started returning raw binary - # rather than the hex digest - found = binascii.hexlify(idautils.GetInputFileMD5()[:15]).decode("ascii").lower() - - if not wanted.startswith(found): - raise RuntimeError("please run the tests against sample with MD5: `%s`" % (wanted)) - - -def get_ida_extractor(_path): - check_input_file("5f66b82558ca92e54e77f216ef4c066c") - - # have to import import this inline so pytest doesn't bail outside of IDA - import capa.features.extractors.ida - - return capa.features.extractors.ida.IdaFeatureExtractor() - - -@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") -def test_ida_features(): - for (sample, scope, feature, expected) in FEATURE_PRESENCE_TESTS + FEATURE_PRESENCE_TESTS_IDA: - id = make_test_id((sample, scope, feature, expected)) - - try: - check_input_file(get_sample_md5_by_name(sample)) - except RuntimeError: - print("SKIP %s" % (id)) - continue - - scope = resolve_scope(scope) - sample = resolve_sample(sample) - - try: - do_test_feature_presence(get_ida_extractor, sample, scope, feature, expected) - except Exception as e: - print("FAIL %s" % (id)) - traceback.print_exc() - else: - print("OK %s" % (id)) - - -@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") -def test_ida_feature_counts(): - for (sample, scope, feature, expected) in FEATURE_COUNT_TESTS: - id = make_test_id((sample, scope, feature, expected)) - - try: - check_input_file(get_sample_md5_by_name(sample)) - except RuntimeError: - print("SKIP %s" % (id)) - continue - - scope = resolve_scope(scope) - sample = resolve_sample(sample) - - try: - do_test_feature_count(get_ida_extractor, sample, scope, feature, expected) - except Exception as e: - print("FAIL %s" % (id)) - traceback.print_exc() - else: - print("OK %s" % (id)) - - -if __name__ == "__main__": - print("-" * 80) - - # invoke all functions in this module that start with `test_` - for name in dir(sys.modules[__name__]): - if not name.startswith("test_"): - continue - - test = getattr(sys.modules[__name__], name) - logger.debug("invoking test: %s", name) - sys.stderr.flush() - test() - - print("DONE") +# run this script from within IDA with ./tests/data/mimikatz.exe open +import sys +import logging +import os.path +import binascii +import traceback + +import pytest + +try: + sys.path.append(os.path.dirname(__file__)) + from fixtures import * +finally: + sys.path.pop() + + +logger = logging.getLogger("test_ida_features") + + +def check_input_file(wanted): + import idautils + + # some versions (7.4) of IDA return a truncated version of the MD5. + # https://github.com/idapython/bin/issues/11 + try: + found = idautils.GetInputFileMD5()[:31].decode("ascii").lower() + except UnicodeDecodeError: + # in IDA 7.5 or so, GetInputFileMD5 started returning raw binary + # rather than the hex digest + found = binascii.hexlify(idautils.GetInputFileMD5()[:15]).decode("ascii").lower() + + if not wanted.startswith(found): + raise RuntimeError("please run the tests against sample with MD5: `%s`" % (wanted)) + + +def get_ida_extractor(_path): + check_input_file("5f66b82558ca92e54e77f216ef4c066c") + + # have to import import this inline so pytest doesn't bail outside of IDA + import capa.features.extractors.ida + + return capa.features.extractors.ida.IdaFeatureExtractor() + + +@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") +def test_ida_features(): + for (sample, scope, feature, expected) in FEATURE_PRESENCE_TESTS + FEATURE_PRESENCE_TESTS_IDA: + id = make_test_id((sample, scope, feature, expected)) + + try: + check_input_file(get_sample_md5_by_name(sample)) + except RuntimeError: + print("SKIP %s" % (id)) + continue + + scope = resolve_scope(scope) + sample = resolve_sample(sample) + + try: + do_test_feature_presence(get_ida_extractor, sample, scope, feature, expected) + except Exception as e: + print("FAIL %s" % (id)) + traceback.print_exc() + else: + print("OK %s" % (id)) + + +@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") +def test_ida_feature_counts(): + for (sample, scope, feature, expected) in FEATURE_COUNT_TESTS: + id = make_test_id((sample, scope, feature, expected)) + + try: + check_input_file(get_sample_md5_by_name(sample)) + except RuntimeError: + print("SKIP %s" % (id)) + continue + + scope = resolve_scope(scope) + sample = resolve_sample(sample) + + try: + do_test_feature_count(get_ida_extractor, sample, scope, feature, expected) + except Exception as e: + print("FAIL %s" % (id)) + traceback.print_exc() + else: + print("OK %s" % (id)) + + +if __name__ == "__main__": + print("-" * 80) + + # invoke all functions in this module that start with `test_` + for name in dir(sys.modules[__name__]): + if not name.startswith("test_"): + continue + + test = getattr(sys.modules[__name__], name) + logger.debug("invoking test: %s", name) + sys.stderr.flush() + test() + + print("DONE") diff --git a/tests/test_main.py b/tests/test_main.py index 6732de2d..cde291d1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -6,7 +6,6 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import sys import json import textwrap @@ -58,10 +57,6 @@ def test_main_single_rule(z9324d_extractor, tmpdir): def test_main_non_ascii_filename(pingtaest_extractor, tmpdir, capsys): - # on py2.7, need to be careful about str (which can hold bytes) - # vs unicode (which is only unicode characters). - # on py3, this should not be needed. - # # here we print a string with unicode characters in it # (specifically, a byte string with utf-8 bytes in it, see file encoding) assert capa.main.main(["-q", pingtaest_extractor.path]) == 0 @@ -69,20 +64,14 @@ def test_main_non_ascii_filename(pingtaest_extractor, tmpdir, capsys): std = capsys.readouterr() # but here, we have to use a unicode instance, # because capsys has decoded the output for us. - if sys.version_info >= (3, 0): - assert pingtaest_extractor.path in std.out - else: - assert pingtaest_extractor.path.decode("utf-8") in std.out + assert pingtaest_extractor.path in std.out def test_main_non_ascii_filename_nonexistent(tmpdir, caplog): NON_ASCII_FILENAME = "täst_not_there.exe" assert capa.main.main(["-q", NON_ASCII_FILENAME]) == -1 - if sys.version_info >= (3, 0): - assert NON_ASCII_FILENAME in caplog.text - else: - assert NON_ASCII_FILENAME.decode("utf-8") in caplog.text + assert NON_ASCII_FILENAME in caplog.text def test_main_shellcode(z499c2_extractor): @@ -91,6 +80,8 @@ def test_main_shellcode(z499c2_extractor): assert capa.main.main([path, "-v", "-f", "sc32"]) == 0 assert capa.main.main([path, "-j", "-f", "sc32"]) == 0 assert capa.main.main([path, "-f", "sc32"]) == 0 + # auto detect shellcode based on file extension + assert capa.main.main([path]) == 0 def test_ruleset(): @@ -370,16 +361,15 @@ def test_not_render_rules_also_matched(z9324d_extractor, capsys): # It tests main works with different backends def test_backend_option(capsys): - if sys.version_info > (3, 0): - path = get_data_path_by_name("pma16-01") - assert capa.main.main([path, "-j", "-b", capa.main.BACKEND_VIV]) == 0 - std = capsys.readouterr() - std_json = json.loads(std.out) - assert std_json["meta"]["analysis"]["extractor"] == "VivisectFeatureExtractor" - assert len(std_json["rules"]) > 0 + path = get_data_path_by_name("pma16-01") + assert capa.main.main([path, "-j", "-b", capa.main.BACKEND_VIV]) == 0 + std = capsys.readouterr() + std_json = json.loads(std.out) + assert std_json["meta"]["analysis"]["extractor"] == "VivisectFeatureExtractor" + assert len(std_json["rules"]) > 0 - assert capa.main.main([path, "-j", "-b", capa.main.BACKEND_SMDA]) == 0 - std = capsys.readouterr() - std_json = json.loads(std.out) - assert std_json["meta"]["analysis"]["extractor"] == "SmdaFeatureExtractor" - assert len(std_json["rules"]) > 0 + assert capa.main.main([path, "-j", "-b", capa.main.BACKEND_SMDA]) == 0 + std = capsys.readouterr() + std_json = json.loads(std.out) + assert std_json["meta"]["analysis"]["extractor"] == "SmdaFeatureExtractor" + assert len(std_json["rules"]) > 0 diff --git a/tests/test_rules.py b/tests/test_rules.py index c08d7212..8bc42e8c 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -681,6 +681,25 @@ def test_explicit_string_values_int(): assert (String("0x123") in children) == True +def test_string_values_special_characters(): + rule = textwrap.dedent( + """ + rule: + meta: + name: test rule + features: + - or: + - string: "hello\\r\\nworld" + - string: "bye\\nbye" + description: "test description" + """ + ) + r = capa.rules.Rule.from_yaml(rule) + children = list(r.statement.get_children()) + assert (String("hello\r\nworld") in children) == True + assert (String("bye\nbye") in children) == True + + def test_regex_values_always_string(): rules = [ capa.rules.Rule.from_yaml( diff --git a/tests/test_smda_features.py b/tests/test_smda_features.py index 68790f8d..dae6185d 100644 --- a/tests/test_smda_features.py +++ b/tests/test_smda_features.py @@ -15,9 +15,9 @@ from fixtures import * FEATURE_PRESENCE_TESTS, indirect=["sample", "scope"], ) +@pytest.mark.xfail(sys.platform == "win32", reason="SMDA bug: https://github.com/danielplohmann/smda/issues/20") def test_smda_features(sample, scope, feature, expected): - with xfail(sys.version_info < (3, 0), reason="SMDA only works on py3"): - do_test_feature_presence(get_smda_extractor, sample, scope, feature, expected) + do_test_feature_presence(get_smda_extractor, sample, scope, feature, expected) @parametrize( @@ -26,5 +26,4 @@ def test_smda_features(sample, scope, feature, expected): indirect=["sample", "scope"], ) def test_smda_feature_counts(sample, scope, feature, expected): - with xfail(sys.version_info < (3, 0), reason="SMDA only works on py3"): - do_test_feature_count(get_smda_extractor, sample, scope, feature, expected) + do_test_feature_count(get_smda_extractor, sample, scope, feature, expected)