From 3d2b4dcc2684516844168de6596d751bb7cf8e36 Mon Sep 17 00:00:00 2001 From: Michael Hunhoff Date: Thu, 11 Mar 2021 17:09:14 -0700 Subject: [PATCH 01/18] adding support for multi-line tab and SHIFT + Tab --- capa/ida/plugin/view.py | 96 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/capa/ida/plugin/view.py b/capa/ida/plugin/view.py index 38d681b8..87eb5d8a 100644 --- a/capa/ida/plugin/view.py +++ b/capa/ida/plugin/view.py @@ -178,6 +178,9 @@ def build_context_menu(o, actions): class CapaExplorerRulgenPreview(QtWidgets.QTextEdit): + + INDENT = " " * 2 + def __init__(self, parent=None): """ """ super(CapaExplorerRulgenPreview, self).__init__(parent) @@ -210,12 +213,99 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit): self.setText("\n".join(metadata_default)) def keyPressEvent(self, e): - """ """ - if e.key() == QtCore.Qt.Key_Tab: - self.insertPlainText(" " * 2) + """intercept key press events""" + if e.key() in (QtCore.Qt.Key_Tab, QtCore.Qt.Key_Backtab): + # apparently it's not easy to implement tabs as spaces, or multi-line tab or SHIFT + Tab + # so we need to implement it ourselves so we can retain properly formatted capa rules + # when a user uses the Tab key + if self.textCursor().selection().isEmpty(): + # single line, only worry about Tab + if e.key() == QtCore.Qt.Key_Tab: + self.insertPlainText(self.INDENT) + else: + # multi-line tab or SHIFT + Tab + cur = self.textCursor() + select_start_ppos = cur.selectionStart() + select_end_ppos = cur.selectionEnd() + + scroll_ppos = self.verticalScrollBar().sliderPosition() + + # determine lineno for first selected line, and column + cur.setPosition(select_start_ppos) + start_lineno = self.count_previous_lines_from_block(cur.block()) + start_lineco = cur.columnNumber() + + # determine lineno for last selected line + cur.setPosition(select_end_ppos) + end_lineno = self.count_previous_lines_from_block(cur.block()) + + # now we need to indent or dedent the selected lines. for now, we read the text, modify + # the lines between start_lineno and end_lineno accordingly, and then reset the view + # this might not be the best solution, but it avoids messing around with cursor positions + # to determine the beginning of lines + + plain = self.toPlainText().splitlines() + + if e.key() == QtCore.Qt.Key_Tab: + # user Tab, indent selected lines + lines_modified = end_lineno - start_lineno + first_modified = True + change = [self.INDENT + line for line in plain[start_lineno : end_lineno + 1]] + else: + # user SHIFT + Tab, dedent selected lines + lines_modified = 0 + first_modified = False + change = [] + for (lineno, line) in enumerate(plain[start_lineno : end_lineno + 1]): + if line.startswith(self.INDENT): + if lineno == 0: + # keep track if first line is modified, so we can properly display + # the text selection later + first_modified = True + lines_modified += 1 + line = line[len(self.INDENT) :] + change.append(line) + + # apply modifications, and reset view + plain[start_lineno : end_lineno + 1] = change + self.setPlainText("\n".join(plain) + "\n") + + # now we need to properly adjust the selection positions, so users don't have to + # re-select when indenting or dedenting the same lines repeatedly + if e.key() == QtCore.Qt.Key_Tab: + # user Tab, increase increment selection positions + select_start_ppos += len(self.INDENT) + select_end_ppos += (lines_modified * len(self.INDENT)) + len(self.INDENT) + elif lines_modified: + # user SHIFT + Tab, decrease selection positions + if start_lineco not in (0, 1) and first_modified: + # only decrease start position if not in first column + select_start_ppos -= len(self.INDENT) + select_end_ppos -= lines_modified * len(self.INDENT) + + # apply updated selection and restore previous scroll position + self.set_selection(select_start_ppos, select_end_ppos, len(self.toPlainText())) + self.verticalScrollBar().setSliderPosition(scroll_ppos) else: super(CapaExplorerRulgenPreview, self).keyPressEvent(e) + def count_previous_lines_from_block(self, block): + """calculate number of lines preceding block""" + count = 0 + while True: + block = block.previous() + if not block.isValid(): + break + count += block.lineCount() + return count + + def set_selection(self, start, end, max): + """set text selection""" + cursor = self.textCursor() + cursor.setPosition(start) + cursor.setPosition(end if end < max else max, QtGui.QTextCursor.KeepAnchor) + self.setTextCursor(cursor) + class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget): From a31c616a21500735e3d3ddbfa7ce3df6d3b55001 Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Fri, 12 Mar 2021 10:43:58 +0100 Subject: [PATCH 02/18] changelog: document incompatibility of viv files `.viv` files (generated by vivisect) are not compatible between Python 2 and Python 3. This causes capa to raise an `UnicodeDecodeError` exception and should be documented better. I'll add this change to the release notes after the review. Related to https://github.com/fireeye/capa/issues/469 --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e24e811..d77d419c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ The capa explorer IDA plugin now helps you quickly build new capa rules using fe ![](doc/img/rulegen_expanded.png) +### Python 2/3 vivisect workspace compatibility + +This version of capa adds Python 3 support in vivisect. Note that `.viv` files (generated by vivisect) are not compatible between Python 2 and Python 3. When updating to Python 3 you need to delete all the `.viv` files for capa to work. + +If you get the following error (or a similar one), you most likely need to delete `.viv` files: +``` +UnicodeDecodeError: 'ascii' codec can't decode byte 0x90 in position 2: ordinal not in range(128) +``` + ### Upcoming changes **This is the last capa release that supports Python 2.** The next release will be v2.0 and will have breaking changes, including the removal of Python 2 support. From cff7d4bad49425fdde900967eec27cb76879b2a1 Mon Sep 17 00:00:00 2001 From: Moritz Date: Mon, 15 Mar 2021 11:54:11 +0100 Subject: [PATCH 03/18] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fbabc401..1d5235e5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ ![capa](.github/logo.png) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Number of rules](https://img.shields.io/badge/rules-469-blue.svg)](https://github.com/fireeye/capa-rules) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) From b2ab8ab54cf9bc5dce9cfc223441f38760882ba1 Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Mon, 15 Mar 2021 16:47:15 +0100 Subject: [PATCH 04/18] doc: Improve README badges - Add a link to the `PyPI - Python Version` badge. Otherwise it opens the image when clicking on it, which is inconsistent with the other labels. I arrived too late to point this out in: https://github.com/fireeye/capa/pull/477 - Add release badge with last release version. This may help users to realize that a new version has been released. - Add downloads badge. - Order labels by color. Closes https://github.com/fireeye/capa/issues/196 --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1d5235e5..955a2494 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ ![capa](.github/logo.png) -![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa) -[![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) +[![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/release) [![Number of rules](https://img.shields.io/badge/rules-469-blue.svg)](https://github.com/fireeye/capa-rules) +[![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) +[![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/release) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) capa detects capabilities in executable files. From cd01a01894f460301620e2c1eec8957534fc5aca Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Tue, 16 Mar 2021 10:51:50 +0100 Subject: [PATCH 05/18] setup: bump viv-utils to 0.5.0 In viv-utils `getWorkspace` raises `IncompatibleVivVersion` on Python 3 when `vw.loadWorkspace(viv_file)` raises `UnicodeDecodeError`. Fixes https://github.com/fireeye/capa/issues/469 As we use the same version in py2 and py3, define the viv-utils requirement once. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d8aafdc8..d564f74c 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ requirements = [ "ruamel.yaml", "wcwidth", "ida-settings==2.1.0", + "viv-utils==0.5.0", ] if sys.version_info >= (3, 0): @@ -28,14 +29,12 @@ if sys.version_info >= (3, 0): requirements.append("halo") requirements.append("networkx") requirements.append("vivisect==1.0.0") - requirements.append("viv-utils==0.3.19") requirements.append("smda==1.5.13") else: # py2 requirements.append("enum34==1.1.6") # v1.1.6 is needed by halo 0.0.30 / spinners 0.0.24 requirements.append("halo==0.0.30") # halo==0.0.30 is the last version to support py2.7 requirements.append("vivisect==0.2.1") - requirements.append("viv-utils==0.3.19") requirements.append("networkx==2.2") # v2.2 is last version supported by Python 2.7 requirements.append("backports.functools-lru-cache") From 30a83fa382c809d45a611288f803a82ed4e0fdff Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Tue, 16 Mar 2021 16:37:33 +0100 Subject: [PATCH 06/18] doc: Fix broken link in README Introduced in https://github.com/fireeye/capa/pull/478 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 955a2494..c1cc8554 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ ![capa](.github/logo.png) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) -[![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/release) +[![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/releases) [![Number of rules](https://img.shields.io/badge/rules-469-blue.svg)](https://github.com/fireeye/capa-rules) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) -[![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/release) +[![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) capa detects capabilities in executable files. From 153c6a7b01b72c65fdd71df6311bb9dab1e69fb5 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 18 Mar 2021 18:04:33 +0000 Subject: [PATCH 07/18] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index d1c9bfe2..21501a5a 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit d1c9bfe2e762fa75215e4a08694dafb216de1bff +Subproject commit 21501a5a4b1f8b92518559b27e6ad33a012f0893 From 1bfe0e0874ab458745d69dffaf3311e7047bdee1 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Thu, 18 Mar 2021 19:43:15 +0100 Subject: [PATCH 08/18] ensure LF end of line --- scripts/lint.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/lint.py b/scripts/lint.py index 80ccf2af..26702c3f 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -301,6 +301,16 @@ class FeatureNtdllNtoskrnlApi(Lint): return False +class FormatLineFeedEOL(Lint): + name = "line(s) end with CRLF (\\r\\n)" + recommendation = "convert line endings to LF (\\n) for example using dos2unix" + + def check_rule(self, ctx, rule): + if len(rule.definition.split("\r\n")) > 0: + return False + return True + + class FormatSingleEmptyLineEOF(Lint): name = "EOF format" recommendation = "end file with a single empty line" @@ -385,6 +395,7 @@ def lint_features(ctx, rule): FORMAT_LINTS = ( + FormatLineFeedEOL(), FormatSingleEmptyLineEOF(), FormatIncorrect(), ) @@ -554,7 +565,7 @@ def main(argv=None): samples_path = os.path.join(os.path.dirname(__file__), "..", "tests", "data") - parser = argparse.ArgumentParser(description="A program.") + parser = argparse.ArgumentParser(description="Lint capa rules.") capa.main.install_common_args(parser, wanted={"tag"}) parser.add_argument("rules", type=str, help="Path to rules") parser.add_argument("--samples", type=str, default=samples_path, help="Path to samples") @@ -566,8 +577,12 @@ def main(argv=None): args = parser.parse_args(args=argv) capa.main.handle_common_args(args) - logging.getLogger("capa").setLevel(logging.CRITICAL) - logging.getLogger("viv_utils").setLevel(logging.CRITICAL) + if args.debug: + logging.getLogger("capa").setLevel(logging.DEBUG) + logging.getLogger("viv_utils").setLevel(logging.DEBUG) + else: + logging.getLogger("capa").setLevel(logging.ERROR) + logging.getLogger("viv_utils").setLevel(logging.ERROR) time0 = time.time() From c945eaf80474cee50832c213ebee979c77f4ca09 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 18 Mar 2021 20:41:05 +0000 Subject: [PATCH 09/18] Sync capa rules submodule --- README.md | 2 +- rules | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c1cc8554..498a111d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-469-blue.svg)](https://github.com/fireeye/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-470-blue.svg)](https://github.com/fireeye/capa-rules) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index 6746530f..5bc15f9c 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 6746530f603962236ff3114fbd93418c57db88f5 +Subproject commit 5bc15f9ccd0d341f25def181e30c5523ad3657e4 From c7798b32543181757e2d2d6cc741dedb39a39b31 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Thu, 18 Mar 2021 19:43:15 +0100 Subject: [PATCH 10/18] ensure LF end of line --- scripts/lint.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/lint.py b/scripts/lint.py index 80ccf2af..26702c3f 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -301,6 +301,16 @@ class FeatureNtdllNtoskrnlApi(Lint): return False +class FormatLineFeedEOL(Lint): + name = "line(s) end with CRLF (\\r\\n)" + recommendation = "convert line endings to LF (\\n) for example using dos2unix" + + def check_rule(self, ctx, rule): + if len(rule.definition.split("\r\n")) > 0: + return False + return True + + class FormatSingleEmptyLineEOF(Lint): name = "EOF format" recommendation = "end file with a single empty line" @@ -385,6 +395,7 @@ def lint_features(ctx, rule): FORMAT_LINTS = ( + FormatLineFeedEOL(), FormatSingleEmptyLineEOF(), FormatIncorrect(), ) @@ -554,7 +565,7 @@ def main(argv=None): samples_path = os.path.join(os.path.dirname(__file__), "..", "tests", "data") - parser = argparse.ArgumentParser(description="A program.") + parser = argparse.ArgumentParser(description="Lint capa rules.") capa.main.install_common_args(parser, wanted={"tag"}) parser.add_argument("rules", type=str, help="Path to rules") parser.add_argument("--samples", type=str, default=samples_path, help="Path to samples") @@ -566,8 +577,12 @@ def main(argv=None): args = parser.parse_args(args=argv) capa.main.handle_common_args(args) - logging.getLogger("capa").setLevel(logging.CRITICAL) - logging.getLogger("viv_utils").setLevel(logging.CRITICAL) + if args.debug: + logging.getLogger("capa").setLevel(logging.DEBUG) + logging.getLogger("viv_utils").setLevel(logging.DEBUG) + else: + logging.getLogger("capa").setLevel(logging.ERROR) + logging.getLogger("viv_utils").setLevel(logging.ERROR) time0 = time.time() From 7e0b5236af319edbf0ad4661440432a016f5055e Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Fri, 19 Mar 2021 09:19:50 +0100 Subject: [PATCH 11/18] better deal with CRLF/LF issues --- capa/rules.py | 2 ++ scripts/capafmt.py | 2 ++ scripts/lint.py | 7 ++++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/capa/rules.py b/capa/rules.py index b9a42056..aeb2754a 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -736,6 +736,8 @@ class Rule(object): # the below regex makes these adjustments and while ugly, we don't have to explore the ruamel.yaml insides doc = re.sub(r"!!int '0x-([0-9a-fA-F]+)'", r"-0x\1", doc) + # normalize CRLF to LF + doc = doc.replace("\r\n", "\n") return doc diff --git a/scripts/capafmt.py b/scripts/capafmt.py index a0b2a7c6..1f110074 100644 --- a/scripts/capafmt.py +++ b/scripts/capafmt.py @@ -65,6 +65,8 @@ def main(argv=None): return 0 else: logger.info("rule requires reformatting (%s)", rule.name) + if "\r\n" in rule.definition: + logger.info("please make sure that the file uses LF (\\n) line endings only") return 1 if args.in_place: diff --git a/scripts/lint.py b/scripts/lint.py index 26702c3f..0c2c8087 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -331,7 +331,12 @@ class FormatIncorrect(Lint): if actual != expected: diff = difflib.ndiff(actual.splitlines(1), expected.splitlines(True)) - self.recommendation = self.recommendation_template.format("".join(diff)) + recommendation_template = self.recommendation_template + if "\r\n" in actual: + recommendation_template = ( + self.recommendation_template + "\nplease make sure that the file uses LF (\\n) line endings only" + ) + self.recommendation = recommendation_template.format("".join(diff)) return True return False From 8719a23de42d8eea212e215922e52adc070c63af Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Fri, 19 Mar 2021 09:25:28 +0100 Subject: [PATCH 12/18] dos2unix --- .github/CODE_OF_CONDUCT.md | 92 +- .github/CONTRIBUTING.md | 394 +++---- .github/ISSUE_TEMPLATE/bug_report.md | 94 +- .github/ISSUE_TEMPLATE/feature_request.md | 70 +- .github/workflows/build.yml | 154 +-- .github/workflows/publish.yml | 56 +- CHANGELOG.md | 1140 ++++++++++----------- scripts/bulk-process.py | 440 ++++---- tests/test_ida_features.py | 208 ++-- 9 files changed, 1324 insertions(+), 1324 deletions(-) diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md index f7f029a8..9cd16c0d 100644 --- a/.github/CODE_OF_CONDUCT.md +++ b/.github/CODE_OF_CONDUCT.md @@ -1,46 +1,46 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version] - -[homepage]: https://contributor-covenant.org -[version]: https://contributor-covenant.org/version/1/4/ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version] + +[homepage]: https://contributor-covenant.org +[version]: https://contributor-covenant.org/version/1/4/ diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5f1bb7ef..707d0932 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,197 +1,197 @@ -# Contributing to Capa - -First off, thanks for taking the time to contribute! - -The following is a set of guidelines for contributing to capa and its packages, which are hosted in the [FireEye Organization](https://github.com/fireeye) on GitHub. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. - -#### Table Of Contents - -[Code of Conduct](#code-of-conduct) - -[What should I know before I get started?](#what-should-i-know-before-i-get-started) - * [Capa and its Repositories](#capa-and-its-repositories) - * [Capa Design Decisions](#design-decisions) - -[How Can I Contribute?](#how-can-i-contribute) - * [Reporting Bugs](#reporting-bugs) - * [Suggesting Enhancements](#suggesting-enhancements) - * [Your First Code Contribution](#your-first-code-contribution) - * [Pull Requests](#pull-requests) - -[Styleguides](#styleguides) - * [Git Commit Messages](#git-commit-messages) - * [Python Styleguide](#python-styleguide) - * [Rules Styleguide](#rules-styleguide) - -## Code of Conduct - -This project and everyone participating in it is governed by the [Capa Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers. - -## What should I know before I get started? - -### Capa and its repositories - -We host the capa project as three Github repositories: - - [capa](https://github.com/fireeye/capa) - - [capa-rules](https://github.com/fireeye/capa-rules) - - [capa-testfiles](https://github.com/fireeye/capa-testfiles) - -The command line tools, logic engine, and other Python source code are found in the `capa` repository. -This is the repository to fork when you want to enhance the features, performance, or user interface of capa. -Do *not* push rules directly to this repository, instead... - -The standard rules contributed by the community are found in the `capa-rules` repository. -When you have an idea for a new rule, you should open a PR against `capa-rules`. -We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and discussions should happen. -If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`. -Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way. - -Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository. -These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites; - however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate, - a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth. - -### Design Decisions - -When we make a significant decision in how we maintain the project and what we can or cannot support, - we will document it in the [capa issues tracker](https://github.com/fireeye/capa/issues). -This is the best place review our discussions about what/how/why we do things in the project. -If you have a question, check to see if it is documented there. -If it is *not* documented there, or you can't find an answer, please open a issue. -We'll link to existing issues when appropriate to keep discussions in one place. - -## How Can I Contribute? - -### Reporting Bugs - -This section guides you through submitting a bug report for capa. -Following these guidelines helps maintainers and the community understand your report, reproduce the behavior, and find related reports. - -Before creating bug reports, please check [this list](#before-submitting-a-bug-report) - as you might find out that you don't need to create one. -When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report). -Fill out [the required template](./ISSUE_TEMPLATE/bug_report.md), - the information it asks for helps us resolve issues faster. - -> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one. - -#### Before Submitting A Bug Report - -* **Determine [which repository the problem should be reported in](#capa-and-its-repositories)**. -* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the problem has already been reported. If it has **and the issue is still open**, add a comment to the existing issue instead of opening a new one. - -#### How Do I Submit A (Good) Bug Report? - -Bugs are tracked as [GitHub issues](https://guides.github.com/features/issues/). -After you've determined [which repository](#capa-and-its-repositories) your bug is related to, - create an issue on that repository and provide the following information by filling in - [the template](./ISSUE_TEMPLATE/bug_report.md). - -Explain the problem and include additional details to help maintainers reproduce the problem: - -* **Use a clear and descriptive title** for the issue to identify the problem. -* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started capa, e.g. which command exactly you used in the terminal, or how you started capa otherwise. -* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). -* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior. -* **Explain which behavior you expected to see instead and why.** -* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. -* **If you're reporting that capa crashed**, include the stack trace from the terminal. Include the stack trace in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist. -* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below. - -Provide more context by answering these questions: - -* **Did the problem start happening recently** (e.g. after updating to a new version of capa) or was this always a problem? -* If the problem started happening recently, **can you reproduce the problem in an older version of capa?** What's the most recent version in which the problem doesn't happen? You can download older versions of capa from [the releases page](https://github.com/fireeye/capa/releases). -* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens. -* If the problem is related to working with files (e.g. opening and editing files), **does the problem happen for all files and projects or only some?** Does the problem happen only when working with local or remote files (e.g. on network drives), with files of a specific type (e.g. only JavaScript or Python files), with large files or files with very long lines, or with files in a specific encoding? Is there anything else special about the files you are using? - -Include details about your configuration and environment: - -* **Which version of capa are you using?** You can get the exact version by running `capa --version` in your terminal. -* **What's the name and version of the OS you're using**? - -### Suggesting Enhancements - -This section guides you through submitting an enhancement suggestion for capa, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion and find related suggestions. - -Before creating enhancement suggestions, please check [this list](#before-submitting-an-enhancement-suggestion) as you might find out that you don't need to create one. When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](./ISSUE_TEMPLATE/feature_request.md), including the steps that you imagine you would take if the feature you're requesting existed. - -#### Before Submitting An Enhancement Suggestion - -* **Determine [which repository the enhancement should be suggested in](#capa-and-its-repositories).** -* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one. - -#### How Do I Submit A (Good) Enhancement Suggestion? - -Enhancement suggestions are tracked as [GitHub issues](https://guides.github.com/features/issues/). After you've determined [which repository](#capa-and-its-repositories) your enhancement suggestion is related to, create an issue on that repository and provide the following information: - -* **Use a clear and descriptive title** for the issue to identify the suggestion. -* **Provide a step-by-step description of the suggested enhancement** in as many details as possible. -* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). -* **Describe the current behavior** and **explain which behavior you expected to see instead** and why. -* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of capa which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. -* **Explain why this enhancement would be useful** to most capa users and isn't something that can or should be implemented as an external tool that uses capa as a library. -* **Specify which version of capa you're using.** You can get the exact version by running `capa --version` in your terminal. -* **Specify the name and version of the OS you're using.** - -### Your First Code Contribution - -Unsure where to begin contributing to capa? You can start by looking through these `good-first-issue` and `rule-idea` issues: - -* [good-first-issue](https://github.com/fireeye/capa/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code, and a test or two. -* [rule-idea](https://github.com/fireeye/capa-rules/issues?q=is%3Aissue+is%3Aopen+label%3A%22rule+idea%22) - issues that describe potential new rule ideas. - -Both issue lists are sorted by total number of comments. While not perfect, number of comments is a reasonable proxy for impact a given change will have. - -#### Local development - -capa and all its resources can be developed locally. -For instructions on how to do this, see the "Method 3" section of the [installation guide](https://github.com/fireeye/capa/blob/master/doc/installation.md). - -### Pull Requests - -The process described here has several goals: - -- Maintain capa's quality -- Fix problems that are important to users -- Engage the community in working toward the best possible capa -- Enable a sustainable system for capa's maintainers to review contributions - -Please follow these steps to have your contribution considered by the maintainers: - -1. Follow all instructions in [the template](PULL_REQUEST_TEMPLATE.md) -2. Follow the [styleguides](#styleguides) -3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing
What if the status checks are failing? If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.
- -While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted. - -## Styleguides - -### Git Commit Messages - -* Use the present tense ("Add feature" not "Added feature") -* Use the imperative mood ("Move cursor to..." not "Moves cursor to...") -* Prefix the first line with the component in question ("rules: ..." or "render: ...") -* Reference issues and pull requests liberally after the first line - -### Python Styleguide - -All Python code must adhere to the style guide used by capa: - - 1. [PEP8](https://www.python.org/dev/peps/pep-0008/), with clarifications from - 2. [Willi's style guide](https://docs.google.com/document/d/1iRpeg-w4DtibwytUyC_dDT7IGhNGBP25-nQfuBa-Fyk/edit?usp=sharing), formatted with - 3. [isort](https://pypi.org/project/isort/) (with line width 120 and ordered by line length), and formatted with - 4. [black](https://github.com/psf/black) (with line width 120), and formatted with - 5. [dos2unix](https://linux.die.net/man/1/dos2unix) - -Our CI pipeline will reformat and enforce the Python styleguide. - -### Rules Styleguide - -All (non-nursery) capa rules must: - - 1. pass the [linter](https://github.com/fireeye/capa/blob/master/scripts/lint.py), and - 2. be formatted with [capafmt](https://github.com/fireeye/capa/blob/master/scripts/capafmt.py) - -This ensures that all rules meet the same minimum level of quality and are structured in a consistent way. -Our CI pipeline will reformat and enforce the capa rules styleguide. +# Contributing to Capa + +First off, thanks for taking the time to contribute! + +The following is a set of guidelines for contributing to capa and its packages, which are hosted in the [FireEye Organization](https://github.com/fireeye) on GitHub. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. + +#### Table Of Contents + +[Code of Conduct](#code-of-conduct) + +[What should I know before I get started?](#what-should-i-know-before-i-get-started) + * [Capa and its Repositories](#capa-and-its-repositories) + * [Capa Design Decisions](#design-decisions) + +[How Can I Contribute?](#how-can-i-contribute) + * [Reporting Bugs](#reporting-bugs) + * [Suggesting Enhancements](#suggesting-enhancements) + * [Your First Code Contribution](#your-first-code-contribution) + * [Pull Requests](#pull-requests) + +[Styleguides](#styleguides) + * [Git Commit Messages](#git-commit-messages) + * [Python Styleguide](#python-styleguide) + * [Rules Styleguide](#rules-styleguide) + +## Code of Conduct + +This project and everyone participating in it is governed by the [Capa Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers. + +## What should I know before I get started? + +### Capa and its repositories + +We host the capa project as three Github repositories: + - [capa](https://github.com/fireeye/capa) + - [capa-rules](https://github.com/fireeye/capa-rules) + - [capa-testfiles](https://github.com/fireeye/capa-testfiles) + +The command line tools, logic engine, and other Python source code are found in the `capa` repository. +This is the repository to fork when you want to enhance the features, performance, or user interface of capa. +Do *not* push rules directly to this repository, instead... + +The standard rules contributed by the community are found in the `capa-rules` repository. +When you have an idea for a new rule, you should open a PR against `capa-rules`. +We keep `capa` and `capa-rules` separate to distinguish where ideas, bugs, and discussions should happen. +If you're writing yaml it probably goes in `capa-rules` and if you're writing Python it probably goes in `capa`. +Also, we encourage users to develop their own rule repositories, so we treat our default set of rules in the same way. + +Test fixtures, such as malware samples and analysis workspaces, are found in the `capa-testfiles` repository. +These are files you'll need in order to run the linter (in `--thorough` mode) and full test suites; + however, they take up a lot of space (1GB+), so by keeping `capa-testfiles` separate, + a shallow checkout of `capa` and `capa-rules` doesn't take much bandwidth. + +### Design Decisions + +When we make a significant decision in how we maintain the project and what we can or cannot support, + we will document it in the [capa issues tracker](https://github.com/fireeye/capa/issues). +This is the best place review our discussions about what/how/why we do things in the project. +If you have a question, check to see if it is documented there. +If it is *not* documented there, or you can't find an answer, please open a issue. +We'll link to existing issues when appropriate to keep discussions in one place. + +## How Can I Contribute? + +### Reporting Bugs + +This section guides you through submitting a bug report for capa. +Following these guidelines helps maintainers and the community understand your report, reproduce the behavior, and find related reports. + +Before creating bug reports, please check [this list](#before-submitting-a-bug-report) + as you might find out that you don't need to create one. +When you are creating a bug report, please [include as many details as possible](#how-do-i-submit-a-good-bug-report). +Fill out [the required template](./ISSUE_TEMPLATE/bug_report.md), + the information it asks for helps us resolve issues faster. + +> **Note:** If you find a **Closed** issue that seems like it is the same thing that you're experiencing, open a new issue and include a link to the original issue in the body of your new one. + +#### Before Submitting A Bug Report + +* **Determine [which repository the problem should be reported in](#capa-and-its-repositories)**. +* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the problem has already been reported. If it has **and the issue is still open**, add a comment to the existing issue instead of opening a new one. + +#### How Do I Submit A (Good) Bug Report? + +Bugs are tracked as [GitHub issues](https://guides.github.com/features/issues/). +After you've determined [which repository](#capa-and-its-repositories) your bug is related to, + create an issue on that repository and provide the following information by filling in + [the template](./ISSUE_TEMPLATE/bug_report.md). + +Explain the problem and include additional details to help maintainers reproduce the problem: + +* **Use a clear and descriptive title** for the issue to identify the problem. +* **Describe the exact steps which reproduce the problem** in as many details as possible. For example, start by explaining how you started capa, e.g. which command exactly you used in the terminal, or how you started capa otherwise. +* **Provide specific examples to demonstrate the steps**. Include links to files or GitHub projects, or copy/pasteable snippets, which you use in those examples. If you're providing snippets in the issue, use [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). +* **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior. +* **Explain which behavior you expected to see instead and why.** +* **Include screenshots and animated GIFs** which show you following the described steps and clearly demonstrate the problem. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. +* **If you're reporting that capa crashed**, include the stack trace from the terminal. Include the stack trace in the issue in a [code block](https://help.github.com/articles/markdown-basics/#multiple-lines), a [file attachment](https://help.github.com/articles/file-attachments-on-issues-and-pull-requests/), or put it in a [gist](https://gist.github.com/) and provide link to that gist. +* **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened and share more information using the guidelines below. + +Provide more context by answering these questions: + +* **Did the problem start happening recently** (e.g. after updating to a new version of capa) or was this always a problem? +* If the problem started happening recently, **can you reproduce the problem in an older version of capa?** What's the most recent version in which the problem doesn't happen? You can download older versions of capa from [the releases page](https://github.com/fireeye/capa/releases). +* **Can you reliably reproduce the issue?** If not, provide details about how often the problem happens and under which conditions it normally happens. +* If the problem is related to working with files (e.g. opening and editing files), **does the problem happen for all files and projects or only some?** Does the problem happen only when working with local or remote files (e.g. on network drives), with files of a specific type (e.g. only JavaScript or Python files), with large files or files with very long lines, or with files in a specific encoding? Is there anything else special about the files you are using? + +Include details about your configuration and environment: + +* **Which version of capa are you using?** You can get the exact version by running `capa --version` in your terminal. +* **What's the name and version of the OS you're using**? + +### Suggesting Enhancements + +This section guides you through submitting an enhancement suggestion for capa, including completely new features and minor improvements to existing functionality. Following these guidelines helps maintainers and the community understand your suggestion and find related suggestions. + +Before creating enhancement suggestions, please check [this list](#before-submitting-an-enhancement-suggestion) as you might find out that you don't need to create one. When you are creating an enhancement suggestion, please [include as many details as possible](#how-do-i-submit-a-good-enhancement-suggestion). Fill in [the template](./ISSUE_TEMPLATE/feature_request.md), including the steps that you imagine you would take if the feature you're requesting existed. + +#### Before Submitting An Enhancement Suggestion + +* **Determine [which repository the enhancement should be suggested in](#capa-and-its-repositories).** +* **Perform a [cursory search](https://github.com/fireeye/capa/issues?q=is%3Aissue)** to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one. + +#### How Do I Submit A (Good) Enhancement Suggestion? + +Enhancement suggestions are tracked as [GitHub issues](https://guides.github.com/features/issues/). After you've determined [which repository](#capa-and-its-repositories) your enhancement suggestion is related to, create an issue on that repository and provide the following information: + +* **Use a clear and descriptive title** for the issue to identify the suggestion. +* **Provide a step-by-step description of the suggested enhancement** in as many details as possible. +* **Provide specific examples to demonstrate the steps**. Include copy/pasteable snippets which you use in those examples, as [Markdown code blocks](https://help.github.com/articles/markdown-basics/#multiple-lines). +* **Describe the current behavior** and **explain which behavior you expected to see instead** and why. +* **Include screenshots and animated GIFs** which help you demonstrate the steps or point out the part of capa which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. +* **Explain why this enhancement would be useful** to most capa users and isn't something that can or should be implemented as an external tool that uses capa as a library. +* **Specify which version of capa you're using.** You can get the exact version by running `capa --version` in your terminal. +* **Specify the name and version of the OS you're using.** + +### Your First Code Contribution + +Unsure where to begin contributing to capa? You can start by looking through these `good-first-issue` and `rule-idea` issues: + +* [good-first-issue](https://github.com/fireeye/capa/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code, and a test or two. +* [rule-idea](https://github.com/fireeye/capa-rules/issues?q=is%3Aissue+is%3Aopen+label%3A%22rule+idea%22) - issues that describe potential new rule ideas. + +Both issue lists are sorted by total number of comments. While not perfect, number of comments is a reasonable proxy for impact a given change will have. + +#### Local development + +capa and all its resources can be developed locally. +For instructions on how to do this, see the "Method 3" section of the [installation guide](https://github.com/fireeye/capa/blob/master/doc/installation.md). + +### Pull Requests + +The process described here has several goals: + +- Maintain capa's quality +- Fix problems that are important to users +- Engage the community in working toward the best possible capa +- Enable a sustainable system for capa's maintainers to review contributions + +Please follow these steps to have your contribution considered by the maintainers: + +1. Follow all instructions in [the template](PULL_REQUEST_TEMPLATE.md) +2. Follow the [styleguides](#styleguides) +3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing
What if the status checks are failing? If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.
+ +While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted. + +## Styleguides + +### Git Commit Messages + +* Use the present tense ("Add feature" not "Added feature") +* Use the imperative mood ("Move cursor to..." not "Moves cursor to...") +* Prefix the first line with the component in question ("rules: ..." or "render: ...") +* Reference issues and pull requests liberally after the first line + +### Python Styleguide + +All Python code must adhere to the style guide used by capa: + + 1. [PEP8](https://www.python.org/dev/peps/pep-0008/), with clarifications from + 2. [Willi's style guide](https://docs.google.com/document/d/1iRpeg-w4DtibwytUyC_dDT7IGhNGBP25-nQfuBa-Fyk/edit?usp=sharing), formatted with + 3. [isort](https://pypi.org/project/isort/) (with line width 120 and ordered by line length), and formatted with + 4. [black](https://github.com/psf/black) (with line width 120), and formatted with + 5. [dos2unix](https://linux.die.net/man/1/dos2unix) + +Our CI pipeline will reformat and enforce the Python styleguide. + +### Rules Styleguide + +All (non-nursery) capa rules must: + + 1. pass the [linter](https://github.com/fireeye/capa/blob/master/scripts/lint.py), and + 2. be formatted with [capafmt](https://github.com/fireeye/capa/blob/master/scripts/capafmt.py) + +This ensures that all rules meet the same minimum level of quality and are structured in a consistent way. +Our CI pipeline will reformat and enforce the capa rules styleguide. diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 720726aa..e33e7f8b 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,47 +1,47 @@ ---- -name: Bug report -about: Create a report to help us improve - ---- - - -### Description - - - -### Steps to Reproduce - - - - - -**Expected behavior:** - - - -**Actual behavior:** - - - -### Versions - - - -### Additional Information - - - +--- +name: Bug report +about: Create a report to help us improve + +--- + + +### Description + + + +### Steps to Reproduce + + + + + +**Expected behavior:** + + + +**Actual behavior:** + + + +### Versions + + + +### Additional Information + + + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 55b1bb5b..22e2fe09 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,35 +1,35 @@ ---- -name: Feature request -about: Suggest an idea for capa - ---- - - -### Summary - - - -### Motivation - - - -### Describe alternatives you've considered - - - -## Additional context - - - +--- +name: Feature request +about: Suggest an idea for capa + +--- + + +### Summary + + + +### Motivation + + + +### Describe alternatives you've considered + + + +## Additional context + + + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62d8ae56..5469889a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,77 +1,77 @@ -name: build - -on: - release: - types: [edited, published] - -jobs: - build: - name: PyInstaller for ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: ubuntu-16.04 - # use old linux so that the shared library versioning is more portable - artifact_name: capa - asset_name: linux - - os: windows-latest - artifact_name: capa.exe - asset_name: windows - - os: macos-latest - artifact_name: capa - asset_name: macos - steps: - - name: Checkout capa - uses: actions/checkout@v2 - with: - submodules: true - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - if: matrix.os == 'ubuntu-latest' - run: sudo apt-get install -y libyaml-dev - - name: Install PyInstaller - run: pip install 'pyinstaller==4.2' - - name: Install capa - run: pip install -e . - - name: Build standalone executable - run: pyinstaller .github/pyinstaller/pyinstaller.spec - - name: Does it run? - run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_" - - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.asset_name }} - path: dist/${{ matrix.artifact_name }} - - zip: - name: zip ${{ matrix.asset_name }} - runs-on: ubuntu-latest - needs: build - strategy: - matrix: - include: - - asset_name: linux - artifact_name: capa - - asset_name: windows - artifact_name: capa.exe - - asset_name: macos - artifact_name: capa - steps: - - name: Download ${{ matrix.asset_name }} - uses: actions/download-artifact@v2 - with: - name: ${{ matrix.asset_name }} - - name: Set executable flag - run: chmod +x ${{ matrix.artifact_name }} - - name: Set zip name - run: echo "zip_name=capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV - - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }} - run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }} - - name: Upload ${{ env.zip_name }} to GH Release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN}} - file: ${{ env.zip_name }} - tag: ${{ github.ref }} +name: build + +on: + release: + types: [edited, published] + +jobs: + build: + name: PyInstaller for ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-16.04 + # use old linux so that the shared library versioning is more portable + artifact_name: capa + asset_name: linux + - os: windows-latest + artifact_name: capa.exe + asset_name: windows + - os: macos-latest + artifact_name: capa + asset_name: macos + steps: + - name: Checkout capa + uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - if: matrix.os == 'ubuntu-latest' + run: sudo apt-get install -y libyaml-dev + - name: Install PyInstaller + run: pip install 'pyinstaller==4.2' + - name: Install capa + run: pip install -e . + - name: Build standalone executable + run: pyinstaller .github/pyinstaller/pyinstaller.spec + - name: Does it run? + run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_" + - uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.asset_name }} + path: dist/${{ matrix.artifact_name }} + + zip: + name: zip ${{ matrix.asset_name }} + runs-on: ubuntu-latest + needs: build + strategy: + matrix: + include: + - asset_name: linux + artifact_name: capa + - asset_name: windows + artifact_name: capa.exe + - asset_name: macos + artifact_name: capa + steps: + - name: Download ${{ matrix.asset_name }} + uses: actions/download-artifact@v2 + with: + name: ${{ matrix.asset_name }} + - name: Set executable flag + run: chmod +x ${{ matrix.artifact_name }} + - name: Set zip name + run: echo "zip_name=capa-${GITHUB_REF#refs/tags/}-${{ matrix.asset_name }}.zip" >> $GITHUB_ENV + - name: Zip ${{ matrix.artifact_name }} into ${{ env.zip_name }} + run: zip ${{ env.zip_name }} ${{ matrix.artifact_name }} + - name: Upload ${{ env.zip_name }} to GH Release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN}} + file: ${{ env.zip_name }} + tag: ${{ github.ref }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2258de3b..53768b91 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,29 +1,29 @@ -# This workflows will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -name: publish to pypi - -on: - release: - types: [published] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '2.7' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: publish to pypi + +on: + release: + types: [published] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '2.7' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel twine upload --skip-existing dist/* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index d77d419c..0cbdbdc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,570 +1,570 @@ -# Change Log - -## v1.6.0 (2021-03-09) - -This release adds the capa explorer rule generator plugin for IDA Pro, vivisect support for Python 3 and 12 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. Thank you also to the vivisect development team (@rakuy0, @atlas0fd00m) for the Python 3 support (`vivisect==1.0.0`) and the fixes for Python 2 (`vivisect==0.2.1`). - -### Rule Generator IDA Plugin - -The capa explorer IDA plugin now helps you quickly build new capa rules using features extracted directly from your IDA database. Without leaving the plugin interface you can use the features extracted by capa explorer to develop and test new rules and save your work directly to your capa rules directory. To get started select the new `Rule Generator` tab, navigate to a function in the IDA `Disassembly` view, and click `Analyze`. For more information check out the capa explorer [readme](https://github.com/fireeye/capa/blob/master/capa/ida/plugin/README.md). - -![](doc/img/rulegen_expanded.png) - -### Python 2/3 vivisect workspace compatibility - -This version of capa adds Python 3 support in vivisect. Note that `.viv` files (generated by vivisect) are not compatible between Python 2 and Python 3. When updating to Python 3 you need to delete all the `.viv` files for capa to work. - -If you get the following error (or a similar one), you most likely need to delete `.viv` files: -``` -UnicodeDecodeError: 'ascii' codec can't decode byte 0x90 in position 2: ordinal not in range(128) -``` - -### Upcoming changes - -**This is the last capa release that supports Python 2.** The next release will be v2.0 and will have breaking changes, including the removal of Python 2 support. - -If you have workflows that rely on the Python 2 version and need future maintenance, please reach out. We may be able to supply limited backports of key fixes and features. - -### New features - -- explorer: Add capa explorer rule generator plugin for IDA Pro. Now capa explorer helps you build new capa rules! #426, #438, #439 @mike-hunhoff -- python: Python 3 support in vivisect #421 @Ana06 -- main: Add backend option in Python 3 to select the backend to be used (either SMDA or vivisect) #421 @Ana06 -- python: Python 3 support in IDA #429, #437 @mike-hunhoff -- ci: test pyinstaller CI #452 @williballenthin -- scripts: enable multiple backends in `show-features.py` #429 @mike-hunhoff -- scripts: add `scripts/vivisect-py2-vs-py3.sh` to compare vivisect Python 2 vs 3 (can easily be modified to test run times and compare different versions) #421 @Ana06 - -### New Rules (12) - -- patch process command line @re-fox @williballenthin (graduated from nursery) -- compiled with dmd @re-fox -- compiled with exe4j @johnk3r -- compiled from Visual Basic @williballenthin -- capture screenshot in Go @TcM1911 -- compiled with Nim @mike-hunhoff -- linked against Go process enumeration library @TcM1911 -- linked against Go registry library @TcM1911 -- linked against Go WMI library @TcM1911 -- linked against Go static asset library @TcM1911 -- inspect load icon resource @mike-hunhoff -- linked against XZip @mr-tz - -### Bug Fixes - -- ida: check for unmapped addresses when resolving data references #436 @mike-hunhoff - -### Changes - -- setup: vivisect v1.0.0 is the default backend for Python3 (it was SMDA before) #421 @Ana06 -- setup: bump vivisect to 0.2.1 #454 @mr-tz -- linter: adding ntoskrnl, ntdll overlap lint #428 @mike-hunhoff -- ci: use py3.9 and pyinstaller 4.2 to build standalone binaries #452 @williballenthin -- scripts: remove old migration script #450 @williballenthin - -### Development - -- main: factor out common cli argument handling #450 @williballenthin - -### Raw diffs - - - [capa v1.5.1...v1.6.0](https://github.com/fireeye/capa/compare/v1.5.1...v1.6.0) - - [capa-rules v1.5.1...v1.6.0](https://github.com/fireeye/capa-rules/compare/v1.5.1...v1.6.0) - - -## v1.5.1 (2021-02-09) - -This release fixes the version number that we forgot to update for v1.5.0 (therefore, v1.5.0 was not published to pypi). It also includes 1 new rule and some rule improvements. - -### New Rules (1) - -- encrypt data using vest @re-fox - -### Raw diffs - - - [capa v1.5.0...v1.5.1](https://github.com/fireeye/capa/compare/v1.5.1...v1.6.0) - - [capa-rules v1.5.0...v1.5.1](https://github.com/fireeye/capa-rules/compare/v1.5.1...v1.6.0) - - -## v1.5.0 (2021-02-05) - -This release brings support for running capa under Python 3 via [SMDA](https://github.com/danielplohmann/smda), more thorough CI testing and linting, better extraction of strings and byte features, and 50 (!) new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: - - - @johnk3r - - @doomedraven - - @stvemillertime - - @itreallynick - - @0x534a - -@dzbeck also added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for many rules. - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - - -### New Features - - - py3 support via SMDA #355 @danielplohmann @jcrussell - - scripts: example of using capa as a library #372, #380 @doomedraven - - ci: enable dependabot #373 @mr-tz - - ci: lint rules @mr-tz - - ci: lint rule format #401 @mr-tz - - freeze: add base address #391 @mr-tz - - json: meta: add base address #412 @mr-tz - -### New Rules (50) - - - 64-bit execution via heavens gate @recvfrom - - contain anti-disasm techniques @mr-tz - - check for microsoft office emulation @re-fox - - check for windows sandbox via device @re-fox - - check for windows sandbox via dns suffix @re-fox - - check for windows sandbox via genuine state @re-fox - - check for windows sandbox via process name @re-fox - - check for windows sandbox via registry @re-fox - - capture microphone audio @re-fox - - capture public ip @re-fox - - get domain trust relationships @johnk3r - - check HTTP status code @mr-tz - - compiled with perl2exe @re-fox - - compiled with ps2exe @re-fox - - compiled with pyarmor @stvemillertime, @itreallynick - - validate payment card number using luhn algorithm @re-fox - - hash data using fnv @re-fox @mr-tz - - generate random numbers via WinAPI @mike-hunhoff @johnk3r - - enumerate files recursively @re-fox - - get file system object information @mike-hunhoff - - read virtual disk @re-fox - - register minifilter driver @mike-hunhoff - - start minifilter driver @mike-hunhoff - - enumerate gui resources @johnk3r - - simulate CTRL ALT DEL @mike-hunhoff - - hijack thread execution @0x534a - - inject dll @0x534a - - inject pe @0x534a - - create or open registry key @mike-hunhoff - - delete registry value @mike-hunhoff - - query or enumerate registry key @mike-hunhoff - - query or enumerate registry value @mike-hunhoff - - resume thread @0x534a - - suspend thread @0x534a - - allocate memory @0x534a - - allocate RW memory @0x534a - - contain pusha popa sequence @mr-tz - - create or open file @mike-hunhoff - - open process @0x534a - - open thread @0x534a - - get kernel32 base address @mr-tz - - get ntdll base address @mr-tz - - encrypt or decrypt data via BCrypt @mike-hunhoff - - generate random numbers using the Delphi LCG @williballenthin - - hash data via BCrypt @mike-hunhoff - - migrate process to active window station @williballenthin - - patch process command line @williballenthin - - resolve function by hash @williballenthin - - persist via Winlogon Helper DLL registry key @0x534a - - schedule task via command line @0x534a - -### Bug Fixes - - - doc: pyinstaller build process @mr-tz - - ida: better bytes extraction #409 @mike-hunhoff - - viv: better unicode string extraction #364 @mike-hunhoff - - viv: better unicode string extraction #378 @mr-tz - - viv: more xor instructions #379 @mr-tz - - viv: decrease logging verbosity #381 @mr-tz - - rules: fix api description syntax #403 @mike-hunhoff - - main: disable progress background thread #410 @mike-hunhoff - -### Changes - - - rules: return lib rules for scopes #398 @mr-tz - -### Raw diffs - - - [capa v1.4.1...v1.5.0](https://github.com/fireeye/capa/compare/v1.4.1...v1.5.0) - - [capa-rules v1.4.0...v1.5.0](https://github.com/fireeye/capa-rules/compare/v1.4.0...v1.5.0) - -## v1.4.1 (2020-10-23) - -This release fixes an issue building capa on our CI server, which prevented us from building standalone binaries for v1.4.1. - -### Bug Fixes - - - install VC dependencies for Python 2.7 during Windows build - -### Raw diffs - - - [capa v1.4.0...v1.4.1](https://github.com/fireeye/capa/compare/v1.4.0...v1.4.1) - - [capa-rules v1.4.0...v1.4.1](https://github.com/fireeye/capa-rules/compare/v1.4.0...v1.4.1) - -## v1.4.0 (2020-10-23) - -This capa release includes changes to the rule parsing, enhanced feature extraction, various bug fixes, and improved capa scripts. Everyone should benefit from the improved functionality and performance. The community helped to add 69 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: - - - @mwilliams31 - - @yt0ng - -@dzbeck added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for 86 rules. - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - -### New features - - - script that demonstrates bulk processing @williballenthin #307 - - main: render MBC table @mr-tz #332 - - ida backend: improve detection of APIs called via two or more chained thunks @mike-hunhoff #340 - - viv backend: improve detection of APIs called via two or more chained thunks @mr-tz #341 - - features: extract APIs called via jmp instruction @mr-tz #337 - -### New rules - - - clear the Windows event log @mike-hunhoff - - crash the Windows event logging service @mike-hunhoff - - packed with kkrunchy @re-fox - - packed with nspack @re-fox - - packed with pebundle @re-fox - - packed with pelocknt @re-fox - - packed with peshield @re-fox - - packed with petite @re-fox - - packed with rlpack @re-fox - - packed with upack @re-fox - - packed with y0da crypter @re-fox - - compiled with rust @re-fox - - compute adler32 checksum @mwilliams31 - - encrypt-data-using-hc-128 @recvfrom - - manipulate console @williballenthin - - references logon banner @re-fox - - terminate process via fastfail @re-fox - - delete volume shadow copies @mr-tz - - authenticate HMAC @mr-tz - - compiled from EPL @williballenthin - - compiled with Go @williballenthin - - create Restart Manager session @mike-hunhoff - - decode data using Base64 via WinAPI @mike-hunhoff - - empty recycle bin quietly @mwilliams31 - - enumerate network shares @mike-hunhoff - - hook routines via microsoft detours @williballenthin - - hooked by API Override @williballenthin - - impersonate user @mike-hunhoff - - the @williballenthin packer detection package, thanks to Hexacorn for the data, see https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/ - - packed with CCG - - packed with Crunch - - packed with Dragon Armor - - packed with enigma - - packed with Epack - - packed with MaskPE - - packed with MEW - - packed with Mpress - - packed with Neolite - - packed with PECompact - - packed with Pepack - - packed with Perplex - - packed with ProCrypt - - packed with RPCrypt - - packed with SeauSFX - - packed with Shrinker - - packed with Simple Pack - - packed with StarForce - - packed with SVKP - - packed with Themida - - packed with TSULoader - - packed with VProtect - - packed with WWPACK - - rebuilt by ImpRec - - packaged as a Pintool - - packaged as a CreateInstall installer - - packaged as a WinZip self-extracting archive - - reference 114DNS DNS server @williballenthin - - reference AliDNS DNS server @williballenthin - - reference Cloudflare DNS server @williballenthin - - reference Comodo Secure DNS server @williballenthin - - reference Google Public DNS server @williballenthin - - reference Hurricane Electric DNS server @williballenthin - - reference kornet DNS server @williballenthin - - reference L3 DNS server @williballenthin - - reference OpenDNS DNS server @williballenthin - - reference Quad9 DNS server @williballenthin - - reference Verisign DNS server @williballenthin - - run as service @mike-hunhoff - - schedule task via ITaskService @mike-hunhoff - - references DNS over HTTPS endpoints @yt0ng - -### Bug fixes - - - ida plugin: fix tree-view exception @mike-hunhoff #315 - - ida plugin: fix feature count @mike-hunhoff - - main: fix reported total rule count @williballenthin #325 - - features: fix handling of API names with multiple periods @mike-hunhoff #329 - - ida backend: find all byte sequences instead of only first @mike-hunhoff #335 - - features: display 0 value @mr-tz #338 - - ida backend: extract ordinal and name imports @mr-tz #343 - - show-features: improvements and support within IDA @mr-tz #342 - - main: sanity check MBC rendering @williballenthin - - main: handle sample path that contains non-ASCII characters @mr-tz #328 - -### Changes - - - rules: use yaml.CLoader for better performance @williballenthin #306 - - rules: parse descriptions for statements @mr-tz #312 - -### Raw diffs - - - [capa v1.3.0...v1.4.0](https://github.com/fireeye/capa/compare/v1.3.0...v1.4.0) - - [capa-rules v1.3.0...v1.4.0](https://github.com/fireeye/capa-rules/compare/v1.3.0...v1.4.0) - -## v1.3.0 (2020-09-14) - -This release brings newly updated mappings to the [Malware Behavior Catalog version 2.0](https://github.com/MBCProject/mbc-markdown), many enhancements to the IDA Pro plugin, [flare-capa on PyPI](https://pypi.org/project/flare-capa/), a bunch of bug fixes to improve feature extraction, and four new rules. We received contributions from ten reverse engineers, including seven new ones: - - - @dzbeck - - @recvfrom - - @toomanybananas - - @cclauss - - @adamprescott91 - - @weslambert - - @stevemk14ebr - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - -### Key changes to IDA Plugin - -The IDA Pro integration is now distributed as a real plugin, instead of a script. This enables a few things: - - - keyboard shortcuts and file menu integration - - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%` - - generally doing thing the "right way" - -How to get this new version? Its easy: download [capa_explorer.py](https://raw.githubusercontent.com/fireeye/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. - -Please refer to the plugin [readme](https://github.com/fireeye/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin. - -Please open an issue in this repository if you notice anything weird. - -### New features - - - ida plugin: now a real plugin, not a script @mike-hunhoff - - core: distributed via PyPI as [flare-capa](https://pypi.org/project/flare-capa/) @williballenthin - - features: enable automatic A/W handling for imports @williballenthin @Ana06 #246 - - ida plugin: persist rules directory setting via [ida-settings](https://github.com/williballenthin/ida-settings) @williballenthin #268 - - ida plugin: add search bar to results view @williballenthin #285 - - ida plugin: add `Analyze` and `Reset` buttons to tree view @mike-hunhoff #304 - - ida plugin: add status label to tree view @mike-hunhoff - - ida plugin: add progress indicator @mike-hunhoff, @mr-tz - -### New rules - - - compiled with py2exe @re-fox - - resolve path using msvcrt @re-fox - - decompress data using QuickLZ @edeca - - encrypt data using sosemanuk @recvfrom - -### Bug fixes - - - rule: reduce FP in DNS resolution @toomanybananas - - engine: report correct strings matched via regex @williballenthin #262 - - formatter: correctly format descriptions in two-line syntax @williballenthin @recvfrom #263 - - viv: better extract offsets from SibOper operands @williballenthin @edeca #276 - - import-to-ida: fix import error @cclauss - - viv: don't write settings to ~/.viv/viv.json @williballenthin @rakuy0 @weslambert #244 - - ida plugin: remove dependency loop that resulted in unnecessary overhead @mike-hunhoff #303 - - ida plugin: correctly highlight regex matches in IDA Disassembly view @mike-hunhoff #305 - - ida plugin: better handle rule directory prompt and failure case @stevemk14ebr @mike-hunhoff #309 - -### Changes - - - rules: update meta mapping to MBC 2.0! @dzbeck - - render: don't display rules that are also matched by other rules @williballenthin @Ana06 #224 - - ida plugin: simplify tabs, removing summary and adding detail to results view @williballenthin #286 - - ida plugin: analysis is no longer automatically started when plugin is first opened @mike-hunhoff #304 - - ida plugin: user must manually select a capa rules directory before analysis can be performed @mike-hunhoff - - ida plugin: user interface controls are disabled until analysis is performed @mike-hunhoff #304 - -### Raw diffs - - - [capa v1.2.0...v1.3.0](https://github.com/fireeye/capa/compare/v1.2.0...v1.3.0) - - [capa-rules v1.2.0...v1.3.0](https://github.com/fireeye/capa-rules/compare/v1.2.0...v1.3.0) - -## v1.2.0 (2020-08-31) - -This release brings UI enhancements, especially for the IDA Pro plugin, -investment towards py3 support, -fixes some bugs identified by the community, -and 46 (!) new rules. -We received contributions from ten reverse engineers, including five new ones: - - - @agithubuserlol - - @recvfrom - - @D4nch3n - - @edeca - - @winniepe - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). -Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) -and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - -### New features - - - ida plugin: display arch flavors @mike-hunhoff - - ida plugin: display block descriptions @mike-hunhoff - - ida backend: extract features from nested pointers @mike-hunhoff - - main: show more progress output @williballenthin - - core: pin dependency versions #258 @recvfrom - -### New rules - - bypass UAC via AppInfo ALPC @agithubuserlol - - bypass UAC via token manipulation @agithubuserlol - - check for sandbox and av modules @re-fox - - check for sandbox username @re-fox - - check if process is running under wine @re-fox - - validate credit card number using luhn algorithm @re-fox - - validate credit card number using luhn algorithm with no lookup table @re-fox - - hash data using FNV @edeca @mr-tz - - link many functions at runtime @mr-tz - - reference public RSA key @mr-tz - - packed with ASPack @williballenthin - - delete internet cache @mike-hunhoff - - enumerate internet cache @mike-hunhoff - - send ICMP echo request @mike-hunhoff - - check for debugger via API @mike-hunhoff - - check for hardware breakpoints @mike-hunhoff - - check for kernel debugger via shared user data structure @mike-hunhoff - - check for protected handle exception @mike-hunhoff - - check for software breakpoints @mike-hunhoff - - check for trap flag exception @mike-hunhoff - - check for unexpected memory writes @mike-hunhoff - - check process job object @mike-hunhoff - - reference anti-VM strings targeting Parallels @mike-hunhoff - - reference anti-VM strings targeting Qemu @mike-hunhoff - - reference anti-VM strings targeting VirtualBox @mike-hunhoff - - reference anti-VM strings targeting VirtualPC @mike-hunhoff - - reference anti-VM strings targeting VMWare @mike-hunhoff - - reference anti-VM strings targeting Xen @mike-hunhoff - - reference analysis tools strings @mike-hunhoff - - reference WMI statements @mike-hunhoff - - get number of processor cores @mike-hunhoff - - get number of processors @mike-hunhoff - - enumerate disk properties @mike-hunhoff - - get disk size @mike-hunhoff - - get process heap flags @mike-hunhoff - - get process heap force flags @mike-hunhoff - - get Explorer PID @mike-hunhoff - - delay execution @mike-hunhoff - - check for process debug object @mike-hunhoff - - check license value @mike-hunhoff - - check ProcessDebugFlags @mike-hunhoff - - check ProcessDebugPort @mike-hunhoff - - check SystemKernelDebuggerInformation @mike-hunhoff - - check thread yield allowed @mike-hunhoff - - enumerate system firmware tables @mike-hunhoff - - get system firmware table @mike-hunhoff - - hide thread from debugger @mike-hunhoff - -### Bug fixes - - - ida backend: extract unmapped immediate number features @mike-hunhoff - - ida backend: fix stack cookie check #257 @mike-hunhoff - - viv backend: better extract gs segment access @williballenthin - - core: enable counting of string features #241 @D4nch3n @williballenthin - - core: enable descriptions on feature with arch flavors @mike-hunhoff - - core: update git links for non-SSH access #259 @recvfrom - -### Changes - - - ida plugin: better default display showing first level nesting @winniepe - - remove unused `characteristic(switch)` feature @ana06 - - prepare testing infrastructure for multiple backends/py3 @williballenthin - - ci: zip build artifacts @ana06 - - ci: build all supported python versions @ana06 - - code style and formatting @mr-tz - -### Raw diffs - - - [capa v1.1.0...v1.2.0](https://github.com/fireeye/capa/compare/v1.1.0...v1.2.0) - - [capa-rules v1.1.0...v1.2.0](https://github.com/fireeye/capa-rules/compare/v1.1.0...v1.2.0) - -## v1.1.0 (2020-08-05) - -This release brings new rule format updates, such as adding `offset/x32` and negative offsets, -fixes some bugs identified by the community, and 28 (!) new rules. -We received contributions from eight reverse engineers, including four new ones: - - - @re-fox - - @psifertex - - @bitsofbinary - - @threathive - -Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). - -### New features - - - import: add Binary Ninja import script #205 #207 @psifertex - - rules: offsets can be negative #197 #208 @williballenthin - - rules: enable descriptions for statement nodes #194 #209 @Ana06 - - rules: add arch flavors to number and offset features #210 #216 @williballenthin - - render: show SHA1/SHA256 in default report #164 @threathive - - tests: add tests for IDA Pro backend #202 @williballenthin - -### New rules - - - check for unmoving mouse cursor @BitsOfBinary - - check mutex and exit @re-fox - - parse credit card information @re-fox - - read ini file @re-fox - - validate credit card number with luhn algorithm @re-fox - - change the wallpaper @re-fox - - acquire debug privileges @williballenthin - - import public key @williballenthin - - terminate process by name @williballenthin - - encrypt data using DES @re-fox - - encrypt data using DES via WinAPI @re-fox - - hash data using sha1 via x86 extensions @re-fox - - hash data using sha256 via x86 extensions @re-fox - - capture network configuration via ipconfig @re-fox - - hash data via WinCrypt @mike-hunhoff - - get file attributes @mike-hunhoff - - allocate thread local storage @mike-hunhoff - - get thread local storage value @mike-hunhoff - - set thread local storage @mike-hunhoff - - get session integrity level @mike-hunhoff - - add file to cabinet file @mike-hunhoff - - flush cabinet file @mike-hunhoff - - open cabinet file @mike-hunhoff - - gather firefox profile information @re-fox - - encrypt data using skipjack @re-fox - - encrypt data using camellia @re-fox - - hash data using tiger @re-fox - - encrypt data using blowfish @re-fox - - encrypt data using twofish @re-fox - -### Bug fixes - - - linter: fix exception when examples is `None` @Ana06 - - linter: fix suggested recommendations via templating @williballenthin - - render: fix exception when rendering counts @williballenthin - - render: fix render of negative offsets @williballenthin - - extractor: fix segmentation violation from vivisect @williballenthin - - main: fix crash when .viv cannot be saved #168 @secshoggoth @williballenthin - - main: fix shellcode .viv save path @williballenthin - -### Changes - - - doc: explain how to bypass gatekeeper on macOS @psifertex - - doc: explain supported linux distributions @Ana06 - - doc: explain submodule update with --init @psifertex - - main: improve program help output @mr-tz - - main: disable progress when run in quiet mode @mr-tz - - main: assert supported IDA versions @mr-tz - - extractor: better identify nested pointers to strings @williballenthin - - setup: specify vivisect download url @Ana06 - - setup: pin vivisect version @williballenthin - - setup: bump vivisect dependency version @williballenthin - - setup: set Python project name to `flare-capa` @williballenthin - - ci: run tests and linter via Github Actions @Ana06 - - hooks: run style checkers and hide stashed output @Ana06 - - linter: ignore period in rule filename @williballenthin - - linter: warn on nursery rule with no changes needed @williballenthin - -### Raw diffs - - - [capa v1.0.0...v1.1.0](https://github.com/fireeye/capa/compare/v1.0.0...v1.1.0) - - [capa-rules v1.0.0...v1.1.0](https://github.com/fireeye/capa-rules/compare/v1.0.0...v1.1.0) +# Change Log + +## v1.6.0 (2021-03-09) + +This release adds the capa explorer rule generator plugin for IDA Pro, vivisect support for Python 3 and 12 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. Thank you also to the vivisect development team (@rakuy0, @atlas0fd00m) for the Python 3 support (`vivisect==1.0.0`) and the fixes for Python 2 (`vivisect==0.2.1`). + +### Rule Generator IDA Plugin + +The capa explorer IDA plugin now helps you quickly build new capa rules using features extracted directly from your IDA database. Without leaving the plugin interface you can use the features extracted by capa explorer to develop and test new rules and save your work directly to your capa rules directory. To get started select the new `Rule Generator` tab, navigate to a function in the IDA `Disassembly` view, and click `Analyze`. For more information check out the capa explorer [readme](https://github.com/fireeye/capa/blob/master/capa/ida/plugin/README.md). + +![](doc/img/rulegen_expanded.png) + +### Python 2/3 vivisect workspace compatibility + +This version of capa adds Python 3 support in vivisect. Note that `.viv` files (generated by vivisect) are not compatible between Python 2 and Python 3. When updating to Python 3 you need to delete all the `.viv` files for capa to work. + +If you get the following error (or a similar one), you most likely need to delete `.viv` files: +``` +UnicodeDecodeError: 'ascii' codec can't decode byte 0x90 in position 2: ordinal not in range(128) +``` + +### Upcoming changes + +**This is the last capa release that supports Python 2.** The next release will be v2.0 and will have breaking changes, including the removal of Python 2 support. + +If you have workflows that rely on the Python 2 version and need future maintenance, please reach out. We may be able to supply limited backports of key fixes and features. + +### New features + +- explorer: Add capa explorer rule generator plugin for IDA Pro. Now capa explorer helps you build new capa rules! #426, #438, #439 @mike-hunhoff +- python: Python 3 support in vivisect #421 @Ana06 +- main: Add backend option in Python 3 to select the backend to be used (either SMDA or vivisect) #421 @Ana06 +- python: Python 3 support in IDA #429, #437 @mike-hunhoff +- ci: test pyinstaller CI #452 @williballenthin +- scripts: enable multiple backends in `show-features.py` #429 @mike-hunhoff +- scripts: add `scripts/vivisect-py2-vs-py3.sh` to compare vivisect Python 2 vs 3 (can easily be modified to test run times and compare different versions) #421 @Ana06 + +### New Rules (12) + +- patch process command line @re-fox @williballenthin (graduated from nursery) +- compiled with dmd @re-fox +- compiled with exe4j @johnk3r +- compiled from Visual Basic @williballenthin +- capture screenshot in Go @TcM1911 +- compiled with Nim @mike-hunhoff +- linked against Go process enumeration library @TcM1911 +- linked against Go registry library @TcM1911 +- linked against Go WMI library @TcM1911 +- linked against Go static asset library @TcM1911 +- inspect load icon resource @mike-hunhoff +- linked against XZip @mr-tz + +### Bug Fixes + +- ida: check for unmapped addresses when resolving data references #436 @mike-hunhoff + +### Changes + +- setup: vivisect v1.0.0 is the default backend for Python3 (it was SMDA before) #421 @Ana06 +- setup: bump vivisect to 0.2.1 #454 @mr-tz +- linter: adding ntoskrnl, ntdll overlap lint #428 @mike-hunhoff +- ci: use py3.9 and pyinstaller 4.2 to build standalone binaries #452 @williballenthin +- scripts: remove old migration script #450 @williballenthin + +### Development + +- main: factor out common cli argument handling #450 @williballenthin + +### Raw diffs + + - [capa v1.5.1...v1.6.0](https://github.com/fireeye/capa/compare/v1.5.1...v1.6.0) + - [capa-rules v1.5.1...v1.6.0](https://github.com/fireeye/capa-rules/compare/v1.5.1...v1.6.0) + + +## v1.5.1 (2021-02-09) + +This release fixes the version number that we forgot to update for v1.5.0 (therefore, v1.5.0 was not published to pypi). It also includes 1 new rule and some rule improvements. + +### New Rules (1) + +- encrypt data using vest @re-fox + +### Raw diffs + + - [capa v1.5.0...v1.5.1](https://github.com/fireeye/capa/compare/v1.5.1...v1.6.0) + - [capa-rules v1.5.0...v1.5.1](https://github.com/fireeye/capa-rules/compare/v1.5.1...v1.6.0) + + +## v1.5.0 (2021-02-05) + +This release brings support for running capa under Python 3 via [SMDA](https://github.com/danielplohmann/smda), more thorough CI testing and linting, better extraction of strings and byte features, and 50 (!) new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: + + - @johnk3r + - @doomedraven + - @stvemillertime + - @itreallynick + - @0x534a + +@dzbeck also added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for many rules. + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + + +### New Features + + - py3 support via SMDA #355 @danielplohmann @jcrussell + - scripts: example of using capa as a library #372, #380 @doomedraven + - ci: enable dependabot #373 @mr-tz + - ci: lint rules @mr-tz + - ci: lint rule format #401 @mr-tz + - freeze: add base address #391 @mr-tz + - json: meta: add base address #412 @mr-tz + +### New Rules (50) + + - 64-bit execution via heavens gate @recvfrom + - contain anti-disasm techniques @mr-tz + - check for microsoft office emulation @re-fox + - check for windows sandbox via device @re-fox + - check for windows sandbox via dns suffix @re-fox + - check for windows sandbox via genuine state @re-fox + - check for windows sandbox via process name @re-fox + - check for windows sandbox via registry @re-fox + - capture microphone audio @re-fox + - capture public ip @re-fox + - get domain trust relationships @johnk3r + - check HTTP status code @mr-tz + - compiled with perl2exe @re-fox + - compiled with ps2exe @re-fox + - compiled with pyarmor @stvemillertime, @itreallynick + - validate payment card number using luhn algorithm @re-fox + - hash data using fnv @re-fox @mr-tz + - generate random numbers via WinAPI @mike-hunhoff @johnk3r + - enumerate files recursively @re-fox + - get file system object information @mike-hunhoff + - read virtual disk @re-fox + - register minifilter driver @mike-hunhoff + - start minifilter driver @mike-hunhoff + - enumerate gui resources @johnk3r + - simulate CTRL ALT DEL @mike-hunhoff + - hijack thread execution @0x534a + - inject dll @0x534a + - inject pe @0x534a + - create or open registry key @mike-hunhoff + - delete registry value @mike-hunhoff + - query or enumerate registry key @mike-hunhoff + - query or enumerate registry value @mike-hunhoff + - resume thread @0x534a + - suspend thread @0x534a + - allocate memory @0x534a + - allocate RW memory @0x534a + - contain pusha popa sequence @mr-tz + - create or open file @mike-hunhoff + - open process @0x534a + - open thread @0x534a + - get kernel32 base address @mr-tz + - get ntdll base address @mr-tz + - encrypt or decrypt data via BCrypt @mike-hunhoff + - generate random numbers using the Delphi LCG @williballenthin + - hash data via BCrypt @mike-hunhoff + - migrate process to active window station @williballenthin + - patch process command line @williballenthin + - resolve function by hash @williballenthin + - persist via Winlogon Helper DLL registry key @0x534a + - schedule task via command line @0x534a + +### Bug Fixes + + - doc: pyinstaller build process @mr-tz + - ida: better bytes extraction #409 @mike-hunhoff + - viv: better unicode string extraction #364 @mike-hunhoff + - viv: better unicode string extraction #378 @mr-tz + - viv: more xor instructions #379 @mr-tz + - viv: decrease logging verbosity #381 @mr-tz + - rules: fix api description syntax #403 @mike-hunhoff + - main: disable progress background thread #410 @mike-hunhoff + +### Changes + + - rules: return lib rules for scopes #398 @mr-tz + +### Raw diffs + + - [capa v1.4.1...v1.5.0](https://github.com/fireeye/capa/compare/v1.4.1...v1.5.0) + - [capa-rules v1.4.0...v1.5.0](https://github.com/fireeye/capa-rules/compare/v1.4.0...v1.5.0) + +## v1.4.1 (2020-10-23) + +This release fixes an issue building capa on our CI server, which prevented us from building standalone binaries for v1.4.1. + +### Bug Fixes + + - install VC dependencies for Python 2.7 during Windows build + +### Raw diffs + + - [capa v1.4.0...v1.4.1](https://github.com/fireeye/capa/compare/v1.4.0...v1.4.1) + - [capa-rules v1.4.0...v1.4.1](https://github.com/fireeye/capa-rules/compare/v1.4.0...v1.4.1) + +## v1.4.0 (2020-10-23) + +This capa release includes changes to the rule parsing, enhanced feature extraction, various bug fixes, and improved capa scripts. Everyone should benefit from the improved functionality and performance. The community helped to add 69 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: + + - @mwilliams31 + - @yt0ng + +@dzbeck added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for 86 rules. + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + +### New features + + - script that demonstrates bulk processing @williballenthin #307 + - main: render MBC table @mr-tz #332 + - ida backend: improve detection of APIs called via two or more chained thunks @mike-hunhoff #340 + - viv backend: improve detection of APIs called via two or more chained thunks @mr-tz #341 + - features: extract APIs called via jmp instruction @mr-tz #337 + +### New rules + + - clear the Windows event log @mike-hunhoff + - crash the Windows event logging service @mike-hunhoff + - packed with kkrunchy @re-fox + - packed with nspack @re-fox + - packed with pebundle @re-fox + - packed with pelocknt @re-fox + - packed with peshield @re-fox + - packed with petite @re-fox + - packed with rlpack @re-fox + - packed with upack @re-fox + - packed with y0da crypter @re-fox + - compiled with rust @re-fox + - compute adler32 checksum @mwilliams31 + - encrypt-data-using-hc-128 @recvfrom + - manipulate console @williballenthin + - references logon banner @re-fox + - terminate process via fastfail @re-fox + - delete volume shadow copies @mr-tz + - authenticate HMAC @mr-tz + - compiled from EPL @williballenthin + - compiled with Go @williballenthin + - create Restart Manager session @mike-hunhoff + - decode data using Base64 via WinAPI @mike-hunhoff + - empty recycle bin quietly @mwilliams31 + - enumerate network shares @mike-hunhoff + - hook routines via microsoft detours @williballenthin + - hooked by API Override @williballenthin + - impersonate user @mike-hunhoff + - the @williballenthin packer detection package, thanks to Hexacorn for the data, see https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/ + - packed with CCG + - packed with Crunch + - packed with Dragon Armor + - packed with enigma + - packed with Epack + - packed with MaskPE + - packed with MEW + - packed with Mpress + - packed with Neolite + - packed with PECompact + - packed with Pepack + - packed with Perplex + - packed with ProCrypt + - packed with RPCrypt + - packed with SeauSFX + - packed with Shrinker + - packed with Simple Pack + - packed with StarForce + - packed with SVKP + - packed with Themida + - packed with TSULoader + - packed with VProtect + - packed with WWPACK + - rebuilt by ImpRec + - packaged as a Pintool + - packaged as a CreateInstall installer + - packaged as a WinZip self-extracting archive + - reference 114DNS DNS server @williballenthin + - reference AliDNS DNS server @williballenthin + - reference Cloudflare DNS server @williballenthin + - reference Comodo Secure DNS server @williballenthin + - reference Google Public DNS server @williballenthin + - reference Hurricane Electric DNS server @williballenthin + - reference kornet DNS server @williballenthin + - reference L3 DNS server @williballenthin + - reference OpenDNS DNS server @williballenthin + - reference Quad9 DNS server @williballenthin + - reference Verisign DNS server @williballenthin + - run as service @mike-hunhoff + - schedule task via ITaskService @mike-hunhoff + - references DNS over HTTPS endpoints @yt0ng + +### Bug fixes + + - ida plugin: fix tree-view exception @mike-hunhoff #315 + - ida plugin: fix feature count @mike-hunhoff + - main: fix reported total rule count @williballenthin #325 + - features: fix handling of API names with multiple periods @mike-hunhoff #329 + - ida backend: find all byte sequences instead of only first @mike-hunhoff #335 + - features: display 0 value @mr-tz #338 + - ida backend: extract ordinal and name imports @mr-tz #343 + - show-features: improvements and support within IDA @mr-tz #342 + - main: sanity check MBC rendering @williballenthin + - main: handle sample path that contains non-ASCII characters @mr-tz #328 + +### Changes + + - rules: use yaml.CLoader for better performance @williballenthin #306 + - rules: parse descriptions for statements @mr-tz #312 + +### Raw diffs + + - [capa v1.3.0...v1.4.0](https://github.com/fireeye/capa/compare/v1.3.0...v1.4.0) + - [capa-rules v1.3.0...v1.4.0](https://github.com/fireeye/capa-rules/compare/v1.3.0...v1.4.0) + +## v1.3.0 (2020-09-14) + +This release brings newly updated mappings to the [Malware Behavior Catalog version 2.0](https://github.com/MBCProject/mbc-markdown), many enhancements to the IDA Pro plugin, [flare-capa on PyPI](https://pypi.org/project/flare-capa/), a bunch of bug fixes to improve feature extraction, and four new rules. We received contributions from ten reverse engineers, including seven new ones: + + - @dzbeck + - @recvfrom + - @toomanybananas + - @cclauss + - @adamprescott91 + - @weslambert + - @stevemk14ebr + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + +### Key changes to IDA Plugin + +The IDA Pro integration is now distributed as a real plugin, instead of a script. This enables a few things: + + - keyboard shortcuts and file menu integration + - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%` + - generally doing thing the "right way" + +How to get this new version? Its easy: download [capa_explorer.py](https://raw.githubusercontent.com/fireeye/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. + +Please refer to the plugin [readme](https://github.com/fireeye/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin. + +Please open an issue in this repository if you notice anything weird. + +### New features + + - ida plugin: now a real plugin, not a script @mike-hunhoff + - core: distributed via PyPI as [flare-capa](https://pypi.org/project/flare-capa/) @williballenthin + - features: enable automatic A/W handling for imports @williballenthin @Ana06 #246 + - ida plugin: persist rules directory setting via [ida-settings](https://github.com/williballenthin/ida-settings) @williballenthin #268 + - ida plugin: add search bar to results view @williballenthin #285 + - ida plugin: add `Analyze` and `Reset` buttons to tree view @mike-hunhoff #304 + - ida plugin: add status label to tree view @mike-hunhoff + - ida plugin: add progress indicator @mike-hunhoff, @mr-tz + +### New rules + + - compiled with py2exe @re-fox + - resolve path using msvcrt @re-fox + - decompress data using QuickLZ @edeca + - encrypt data using sosemanuk @recvfrom + +### Bug fixes + + - rule: reduce FP in DNS resolution @toomanybananas + - engine: report correct strings matched via regex @williballenthin #262 + - formatter: correctly format descriptions in two-line syntax @williballenthin @recvfrom #263 + - viv: better extract offsets from SibOper operands @williballenthin @edeca #276 + - import-to-ida: fix import error @cclauss + - viv: don't write settings to ~/.viv/viv.json @williballenthin @rakuy0 @weslambert #244 + - ida plugin: remove dependency loop that resulted in unnecessary overhead @mike-hunhoff #303 + - ida plugin: correctly highlight regex matches in IDA Disassembly view @mike-hunhoff #305 + - ida plugin: better handle rule directory prompt and failure case @stevemk14ebr @mike-hunhoff #309 + +### Changes + + - rules: update meta mapping to MBC 2.0! @dzbeck + - render: don't display rules that are also matched by other rules @williballenthin @Ana06 #224 + - ida plugin: simplify tabs, removing summary and adding detail to results view @williballenthin #286 + - ida plugin: analysis is no longer automatically started when plugin is first opened @mike-hunhoff #304 + - ida plugin: user must manually select a capa rules directory before analysis can be performed @mike-hunhoff + - ida plugin: user interface controls are disabled until analysis is performed @mike-hunhoff #304 + +### Raw diffs + + - [capa v1.2.0...v1.3.0](https://github.com/fireeye/capa/compare/v1.2.0...v1.3.0) + - [capa-rules v1.2.0...v1.3.0](https://github.com/fireeye/capa-rules/compare/v1.2.0...v1.3.0) + +## v1.2.0 (2020-08-31) + +This release brings UI enhancements, especially for the IDA Pro plugin, +investment towards py3 support, +fixes some bugs identified by the community, +and 46 (!) new rules. +We received contributions from ten reverse engineers, including five new ones: + + - @agithubuserlol + - @recvfrom + - @D4nch3n + - @edeca + - @winniepe + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). +Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) +and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + +### New features + + - ida plugin: display arch flavors @mike-hunhoff + - ida plugin: display block descriptions @mike-hunhoff + - ida backend: extract features from nested pointers @mike-hunhoff + - main: show more progress output @williballenthin + - core: pin dependency versions #258 @recvfrom + +### New rules + - bypass UAC via AppInfo ALPC @agithubuserlol + - bypass UAC via token manipulation @agithubuserlol + - check for sandbox and av modules @re-fox + - check for sandbox username @re-fox + - check if process is running under wine @re-fox + - validate credit card number using luhn algorithm @re-fox + - validate credit card number using luhn algorithm with no lookup table @re-fox + - hash data using FNV @edeca @mr-tz + - link many functions at runtime @mr-tz + - reference public RSA key @mr-tz + - packed with ASPack @williballenthin + - delete internet cache @mike-hunhoff + - enumerate internet cache @mike-hunhoff + - send ICMP echo request @mike-hunhoff + - check for debugger via API @mike-hunhoff + - check for hardware breakpoints @mike-hunhoff + - check for kernel debugger via shared user data structure @mike-hunhoff + - check for protected handle exception @mike-hunhoff + - check for software breakpoints @mike-hunhoff + - check for trap flag exception @mike-hunhoff + - check for unexpected memory writes @mike-hunhoff + - check process job object @mike-hunhoff + - reference anti-VM strings targeting Parallels @mike-hunhoff + - reference anti-VM strings targeting Qemu @mike-hunhoff + - reference anti-VM strings targeting VirtualBox @mike-hunhoff + - reference anti-VM strings targeting VirtualPC @mike-hunhoff + - reference anti-VM strings targeting VMWare @mike-hunhoff + - reference anti-VM strings targeting Xen @mike-hunhoff + - reference analysis tools strings @mike-hunhoff + - reference WMI statements @mike-hunhoff + - get number of processor cores @mike-hunhoff + - get number of processors @mike-hunhoff + - enumerate disk properties @mike-hunhoff + - get disk size @mike-hunhoff + - get process heap flags @mike-hunhoff + - get process heap force flags @mike-hunhoff + - get Explorer PID @mike-hunhoff + - delay execution @mike-hunhoff + - check for process debug object @mike-hunhoff + - check license value @mike-hunhoff + - check ProcessDebugFlags @mike-hunhoff + - check ProcessDebugPort @mike-hunhoff + - check SystemKernelDebuggerInformation @mike-hunhoff + - check thread yield allowed @mike-hunhoff + - enumerate system firmware tables @mike-hunhoff + - get system firmware table @mike-hunhoff + - hide thread from debugger @mike-hunhoff + +### Bug fixes + + - ida backend: extract unmapped immediate number features @mike-hunhoff + - ida backend: fix stack cookie check #257 @mike-hunhoff + - viv backend: better extract gs segment access @williballenthin + - core: enable counting of string features #241 @D4nch3n @williballenthin + - core: enable descriptions on feature with arch flavors @mike-hunhoff + - core: update git links for non-SSH access #259 @recvfrom + +### Changes + + - ida plugin: better default display showing first level nesting @winniepe + - remove unused `characteristic(switch)` feature @ana06 + - prepare testing infrastructure for multiple backends/py3 @williballenthin + - ci: zip build artifacts @ana06 + - ci: build all supported python versions @ana06 + - code style and formatting @mr-tz + +### Raw diffs + + - [capa v1.1.0...v1.2.0](https://github.com/fireeye/capa/compare/v1.1.0...v1.2.0) + - [capa-rules v1.1.0...v1.2.0](https://github.com/fireeye/capa-rules/compare/v1.1.0...v1.2.0) + +## v1.1.0 (2020-08-05) + +This release brings new rule format updates, such as adding `offset/x32` and negative offsets, +fixes some bugs identified by the community, and 28 (!) new rules. +We received contributions from eight reverse engineers, including four new ones: + + - @re-fox + - @psifertex + - @bitsofbinary + - @threathive + +Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/fireeye/capa/). Report issues on our [issue tracker](https://github.com/fireeye/capa/issues) and contribute new rules at [capa-rules](https://github.com/fireeye/capa-rules/). + +### New features + + - import: add Binary Ninja import script #205 #207 @psifertex + - rules: offsets can be negative #197 #208 @williballenthin + - rules: enable descriptions for statement nodes #194 #209 @Ana06 + - rules: add arch flavors to number and offset features #210 #216 @williballenthin + - render: show SHA1/SHA256 in default report #164 @threathive + - tests: add tests for IDA Pro backend #202 @williballenthin + +### New rules + + - check for unmoving mouse cursor @BitsOfBinary + - check mutex and exit @re-fox + - parse credit card information @re-fox + - read ini file @re-fox + - validate credit card number with luhn algorithm @re-fox + - change the wallpaper @re-fox + - acquire debug privileges @williballenthin + - import public key @williballenthin + - terminate process by name @williballenthin + - encrypt data using DES @re-fox + - encrypt data using DES via WinAPI @re-fox + - hash data using sha1 via x86 extensions @re-fox + - hash data using sha256 via x86 extensions @re-fox + - capture network configuration via ipconfig @re-fox + - hash data via WinCrypt @mike-hunhoff + - get file attributes @mike-hunhoff + - allocate thread local storage @mike-hunhoff + - get thread local storage value @mike-hunhoff + - set thread local storage @mike-hunhoff + - get session integrity level @mike-hunhoff + - add file to cabinet file @mike-hunhoff + - flush cabinet file @mike-hunhoff + - open cabinet file @mike-hunhoff + - gather firefox profile information @re-fox + - encrypt data using skipjack @re-fox + - encrypt data using camellia @re-fox + - hash data using tiger @re-fox + - encrypt data using blowfish @re-fox + - encrypt data using twofish @re-fox + +### Bug fixes + + - linter: fix exception when examples is `None` @Ana06 + - linter: fix suggested recommendations via templating @williballenthin + - render: fix exception when rendering counts @williballenthin + - render: fix render of negative offsets @williballenthin + - extractor: fix segmentation violation from vivisect @williballenthin + - main: fix crash when .viv cannot be saved #168 @secshoggoth @williballenthin + - main: fix shellcode .viv save path @williballenthin + +### Changes + + - doc: explain how to bypass gatekeeper on macOS @psifertex + - doc: explain supported linux distributions @Ana06 + - doc: explain submodule update with --init @psifertex + - main: improve program help output @mr-tz + - main: disable progress when run in quiet mode @mr-tz + - main: assert supported IDA versions @mr-tz + - extractor: better identify nested pointers to strings @williballenthin + - setup: specify vivisect download url @Ana06 + - setup: pin vivisect version @williballenthin + - setup: bump vivisect dependency version @williballenthin + - setup: set Python project name to `flare-capa` @williballenthin + - ci: run tests and linter via Github Actions @Ana06 + - hooks: run style checkers and hide stashed output @Ana06 + - linter: ignore period in rule filename @williballenthin + - linter: warn on nursery rule with no changes needed @williballenthin + +### Raw diffs + + - [capa v1.0.0...v1.1.0](https://github.com/fireeye/capa/compare/v1.0.0...v1.1.0) + - [capa-rules v1.0.0...v1.1.0](https://github.com/fireeye/capa-rules/compare/v1.0.0...v1.1.0) diff --git a/scripts/bulk-process.py b/scripts/bulk-process.py index ac7b6c8e..584e8d66 100644 --- a/scripts/bulk-process.py +++ b/scripts/bulk-process.py @@ -1,220 +1,220 @@ -#!/usr/bin/env python -""" -bulk-process - -Invoke capa recursively against a directory of samples -and emit a JSON document mapping the file paths to their results. - -By default, this will use subprocesses for parallelism. -Use `-n/--parallelism` to change the subprocess count from - the default of current CPU count. -Use `--no-mp` to use threads instead of processes, - which is probably not useful unless you set `--parallelism=1`. - -example: - - $ python scripts/bulk-process /tmp/suspicious - { - "/tmp/suspicious/suspicious.dll_": { - "rules": { - "encode data using XOR": { - "matches": { - "268440358": { - [...] - "/tmp/suspicious/1.dll_": { ... } - "/tmp/suspicious/2.dll_": { ... } - } - - -usage: - - usage: bulk-process.py [-h] [-r RULES] [-d] [-q] [-n PARALLELISM] [--no-mp] - input - - detect capabilities in programs. - - positional arguments: - input Path to directory of files to recursively analyze - - optional arguments: - -h, --help show this help message and exit - -r RULES, --rules RULES - Path to rule file or directory, use embedded rules by - default - -d, --debug Enable debugging output on STDERR - -q, --quiet Disable all output but errors - -n PARALLELISM, --parallelism PARALLELISM - parallelism factor - --no-mp disable subprocesses - -Copyright (C) 2020 FireEye, Inc. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. -You may obtain a copy of the License at: [package root]/LICENSE.txt -Unless required by applicable law or agreed to in writing, software distributed under the License - is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. -""" -import sys -import json -import logging -import os.path -import argparse -import multiprocessing -import multiprocessing.pool - -import capa -import capa.main -import capa.rules -import capa.render - -logger = logging.getLogger("capa") - - -def get_capa_results(args): - """ - run capa against the file at the given path, using the given rules. - - args is a tuple, containing: - rules (capa.rules.RuleSet): the rules to match - format (str): the name of the sample file format - path (str): the file system path to the sample to process - - args is a tuple because i'm not quite sure how to unpack multiple arguments using `map`. - - returns an dict with two required keys: - path (str): the file system path of the sample to process - status (str): either "error" or "ok" - - when status == "error", then a human readable message is found in property "error". - when status == "ok", then the capa results are found in the property "ok". - - the capa results are a dictionary with the following keys: - meta (dict): the meta analysis results - capabilities (dict): the matched capabilities and their result objects - """ - rules, format, path = args - logger.info("computing capa results for: %s", path) - try: - extractor = capa.main.get_extractor(path, format, capa.main.BACKEND_VIV, disable_progress=True) - except capa.main.UnsupportedFormatError: - # i'm 100% sure if multiprocessing will reliably raise exceptions across process boundaries. - # so instead, return an object with explicit success/failure status. - # - # if success, then status=ok, and results found in property "ok" - # if error, then status=error, and human readable message in property "error" - return { - "path": path, - "status": "error", - "error": "input file does not appear to be a PE file: %s" % path, - } - except capa.main.UnsupportedRuntimeError: - return { - "path": path, - "status": "error", - "error": "unsupported runtime or Python interpreter", - } - except Exception as e: - return { - "path": path, - "status": "error", - "error": "unexpected error: %s" % (e), - } - - meta = capa.main.collect_metadata("", path, "", format, extractor) - capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True) - meta["analysis"].update(counts) - - return { - "path": path, - "status": "ok", - "ok": { - "meta": meta, - "capabilities": capabilities, - }, - } - - -def main(argv=None): - if argv is None: - argv = sys.argv[1:] - - parser = argparse.ArgumentParser(description="detect capabilities in programs.") - capa.main.install_common_args(parser, wanted={"rules"}) - parser.add_argument("input", type=str, help="Path to directory of files to recursively analyze") - parser.add_argument( - "-n", "--parallelism", type=int, default=multiprocessing.cpu_count(), help="parallelism factor" - ) - parser.add_argument("--no-mp", action="store_true", help="disable subprocesses") - args = parser.parse_args(args=argv) - capa.main.handle_common_args(args) - - if args.rules == "(embedded rules)": - logger.info("using default embedded rules") - logger.debug("detected running from source") - args.rules = os.path.join(os.path.dirname(__file__), "..", "rules") - logger.debug("default rule path (source method): %s", args.rules) - else: - logger.info("using rules path: %s", args.rules) - - try: - rules = capa.main.get_rules(args.rules) - rules = capa.rules.RuleSet(rules) - logger.info("successfully loaded %s rules", len(rules)) - except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e: - logger.error("%s", str(e)) - return -1 - - samples = [] - for (base, directories, files) in os.walk(args.input): - for file in files: - samples.append(os.path.join(base, file)) - - def pmap(f, args, parallelism=multiprocessing.cpu_count()): - """apply the given function f to the given args using subprocesses""" - return multiprocessing.Pool(parallelism).imap(f, args) - - def tmap(f, args, parallelism=multiprocessing.cpu_count()): - """apply the given function f to the given args using threads""" - return multiprocessing.pool.ThreadPool(parallelism).imap(f, args) - - def map(f, args, parallelism=None): - """apply the given function f to the given args in the current thread""" - for arg in args: - yield f(arg) - - if args.no_mp: - if args.parallelism == 1: - logger.debug("using current thread mapper") - mapper = map - else: - logger.debug("using threading mapper") - mapper = tmap - else: - logger.debug("using process mapper") - mapper = pmap - - results = {} - for result in mapper( - get_capa_results, [(rules, "pe", sample) for sample in samples], parallelism=args.parallelism - ): - if result["status"] == "error": - logger.warning(result["error"]) - elif result["status"] == "ok": - meta = result["ok"]["meta"] - capabilities = result["ok"]["capabilities"] - # our renderer expects to emit a json document for a single sample - # so we deserialize the json document, store it in a larger dict, and we'll subsequently re-encode. - results[result["path"]] = json.loads(capa.render.render_json(meta, rules, capabilities)) - else: - raise ValueError("unexpected status: %s" % (result["status"])) - - print(json.dumps(results)) - - logger.info("done.") - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +#!/usr/bin/env python +""" +bulk-process + +Invoke capa recursively against a directory of samples +and emit a JSON document mapping the file paths to their results. + +By default, this will use subprocesses for parallelism. +Use `-n/--parallelism` to change the subprocess count from + the default of current CPU count. +Use `--no-mp` to use threads instead of processes, + which is probably not useful unless you set `--parallelism=1`. + +example: + + $ python scripts/bulk-process /tmp/suspicious + { + "/tmp/suspicious/suspicious.dll_": { + "rules": { + "encode data using XOR": { + "matches": { + "268440358": { + [...] + "/tmp/suspicious/1.dll_": { ... } + "/tmp/suspicious/2.dll_": { ... } + } + + +usage: + + usage: bulk-process.py [-h] [-r RULES] [-d] [-q] [-n PARALLELISM] [--no-mp] + input + + detect capabilities in programs. + + positional arguments: + input Path to directory of files to recursively analyze + + optional arguments: + -h, --help show this help message and exit + -r RULES, --rules RULES + Path to rule file or directory, use embedded rules by + default + -d, --debug Enable debugging output on STDERR + -q, --quiet Disable all output but errors + -n PARALLELISM, --parallelism PARALLELISM + parallelism factor + --no-mp disable subprocesses + +Copyright (C) 2020 FireEye, Inc. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. +You may obtain a copy of the License at: [package root]/LICENSE.txt +Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. +""" +import sys +import json +import logging +import os.path +import argparse +import multiprocessing +import multiprocessing.pool + +import capa +import capa.main +import capa.rules +import capa.render + +logger = logging.getLogger("capa") + + +def get_capa_results(args): + """ + run capa against the file at the given path, using the given rules. + + args is a tuple, containing: + rules (capa.rules.RuleSet): the rules to match + format (str): the name of the sample file format + path (str): the file system path to the sample to process + + args is a tuple because i'm not quite sure how to unpack multiple arguments using `map`. + + returns an dict with two required keys: + path (str): the file system path of the sample to process + status (str): either "error" or "ok" + + when status == "error", then a human readable message is found in property "error". + when status == "ok", then the capa results are found in the property "ok". + + the capa results are a dictionary with the following keys: + meta (dict): the meta analysis results + capabilities (dict): the matched capabilities and their result objects + """ + rules, format, path = args + logger.info("computing capa results for: %s", path) + try: + extractor = capa.main.get_extractor(path, format, capa.main.BACKEND_VIV, disable_progress=True) + except capa.main.UnsupportedFormatError: + # i'm 100% sure if multiprocessing will reliably raise exceptions across process boundaries. + # so instead, return an object with explicit success/failure status. + # + # if success, then status=ok, and results found in property "ok" + # if error, then status=error, and human readable message in property "error" + return { + "path": path, + "status": "error", + "error": "input file does not appear to be a PE file: %s" % path, + } + except capa.main.UnsupportedRuntimeError: + return { + "path": path, + "status": "error", + "error": "unsupported runtime or Python interpreter", + } + except Exception as e: + return { + "path": path, + "status": "error", + "error": "unexpected error: %s" % (e), + } + + meta = capa.main.collect_metadata("", path, "", format, extractor) + capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True) + meta["analysis"].update(counts) + + return { + "path": path, + "status": "ok", + "ok": { + "meta": meta, + "capabilities": capabilities, + }, + } + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + + parser = argparse.ArgumentParser(description="detect capabilities in programs.") + capa.main.install_common_args(parser, wanted={"rules"}) + parser.add_argument("input", type=str, help="Path to directory of files to recursively analyze") + parser.add_argument( + "-n", "--parallelism", type=int, default=multiprocessing.cpu_count(), help="parallelism factor" + ) + parser.add_argument("--no-mp", action="store_true", help="disable subprocesses") + args = parser.parse_args(args=argv) + capa.main.handle_common_args(args) + + if args.rules == "(embedded rules)": + logger.info("using default embedded rules") + logger.debug("detected running from source") + args.rules = os.path.join(os.path.dirname(__file__), "..", "rules") + logger.debug("default rule path (source method): %s", args.rules) + else: + logger.info("using rules path: %s", args.rules) + + try: + rules = capa.main.get_rules(args.rules) + rules = capa.rules.RuleSet(rules) + logger.info("successfully loaded %s rules", len(rules)) + except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e: + logger.error("%s", str(e)) + return -1 + + samples = [] + for (base, directories, files) in os.walk(args.input): + for file in files: + samples.append(os.path.join(base, file)) + + def pmap(f, args, parallelism=multiprocessing.cpu_count()): + """apply the given function f to the given args using subprocesses""" + return multiprocessing.Pool(parallelism).imap(f, args) + + def tmap(f, args, parallelism=multiprocessing.cpu_count()): + """apply the given function f to the given args using threads""" + return multiprocessing.pool.ThreadPool(parallelism).imap(f, args) + + def map(f, args, parallelism=None): + """apply the given function f to the given args in the current thread""" + for arg in args: + yield f(arg) + + if args.no_mp: + if args.parallelism == 1: + logger.debug("using current thread mapper") + mapper = map + else: + logger.debug("using threading mapper") + mapper = tmap + else: + logger.debug("using process mapper") + mapper = pmap + + results = {} + for result in mapper( + get_capa_results, [(rules, "pe", sample) for sample in samples], parallelism=args.parallelism + ): + if result["status"] == "error": + logger.warning(result["error"]) + elif result["status"] == "ok": + meta = result["ok"]["meta"] + capabilities = result["ok"]["capabilities"] + # our renderer expects to emit a json document for a single sample + # so we deserialize the json document, store it in a larger dict, and we'll subsequently re-encode. + results[result["path"]] = json.loads(capa.render.render_json(meta, rules, capabilities)) + else: + raise ValueError("unexpected status: %s" % (result["status"])) + + print(json.dumps(results)) + + logger.info("done.") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_ida_features.py b/tests/test_ida_features.py index 51de2139..4539a1ad 100644 --- a/tests/test_ida_features.py +++ b/tests/test_ida_features.py @@ -1,104 +1,104 @@ -# run this script from within IDA with ./tests/data/mimikatz.exe open -import sys -import logging -import os.path -import binascii -import traceback - -import pytest - -try: - sys.path.append(os.path.dirname(__file__)) - from fixtures import * -finally: - sys.path.pop() - - -logger = logging.getLogger("test_ida_features") - - -def check_input_file(wanted): - import idautils - - # some versions (7.4) of IDA return a truncated version of the MD5. - # https://github.com/idapython/bin/issues/11 - try: - found = idautils.GetInputFileMD5()[:31].decode("ascii").lower() - except UnicodeDecodeError: - # in IDA 7.5 or so, GetInputFileMD5 started returning raw binary - # rather than the hex digest - found = binascii.hexlify(idautils.GetInputFileMD5()[:15]).decode("ascii").lower() - - if not wanted.startswith(found): - raise RuntimeError("please run the tests against sample with MD5: `%s`" % (wanted)) - - -def get_ida_extractor(_path): - check_input_file("5f66b82558ca92e54e77f216ef4c066c") - - # have to import import this inline so pytest doesn't bail outside of IDA - import capa.features.extractors.ida - - return capa.features.extractors.ida.IdaFeatureExtractor() - - -@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") -def test_ida_features(): - for (sample, scope, feature, expected) in FEATURE_PRESENCE_TESTS + FEATURE_PRESENCE_TESTS_IDA: - id = make_test_id((sample, scope, feature, expected)) - - try: - check_input_file(get_sample_md5_by_name(sample)) - except RuntimeError: - print("SKIP %s" % (id)) - continue - - scope = resolve_scope(scope) - sample = resolve_sample(sample) - - try: - do_test_feature_presence(get_ida_extractor, sample, scope, feature, expected) - except Exception as e: - print("FAIL %s" % (id)) - traceback.print_exc() - else: - print("OK %s" % (id)) - - -@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") -def test_ida_feature_counts(): - for (sample, scope, feature, expected) in FEATURE_COUNT_TESTS: - id = make_test_id((sample, scope, feature, expected)) - - try: - check_input_file(get_sample_md5_by_name(sample)) - except RuntimeError: - print("SKIP %s" % (id)) - continue - - scope = resolve_scope(scope) - sample = resolve_sample(sample) - - try: - do_test_feature_count(get_ida_extractor, sample, scope, feature, expected) - except Exception as e: - print("FAIL %s" % (id)) - traceback.print_exc() - else: - print("OK %s" % (id)) - - -if __name__ == "__main__": - print("-" * 80) - - # invoke all functions in this module that start with `test_` - for name in dir(sys.modules[__name__]): - if not name.startswith("test_"): - continue - - test = getattr(sys.modules[__name__], name) - logger.debug("invoking test: %s", name) - sys.stderr.flush() - test() - - print("DONE") +# run this script from within IDA with ./tests/data/mimikatz.exe open +import sys +import logging +import os.path +import binascii +import traceback + +import pytest + +try: + sys.path.append(os.path.dirname(__file__)) + from fixtures import * +finally: + sys.path.pop() + + +logger = logging.getLogger("test_ida_features") + + +def check_input_file(wanted): + import idautils + + # some versions (7.4) of IDA return a truncated version of the MD5. + # https://github.com/idapython/bin/issues/11 + try: + found = idautils.GetInputFileMD5()[:31].decode("ascii").lower() + except UnicodeDecodeError: + # in IDA 7.5 or so, GetInputFileMD5 started returning raw binary + # rather than the hex digest + found = binascii.hexlify(idautils.GetInputFileMD5()[:15]).decode("ascii").lower() + + if not wanted.startswith(found): + raise RuntimeError("please run the tests against sample with MD5: `%s`" % (wanted)) + + +def get_ida_extractor(_path): + check_input_file("5f66b82558ca92e54e77f216ef4c066c") + + # have to import import this inline so pytest doesn't bail outside of IDA + import capa.features.extractors.ida + + return capa.features.extractors.ida.IdaFeatureExtractor() + + +@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") +def test_ida_features(): + for (sample, scope, feature, expected) in FEATURE_PRESENCE_TESTS + FEATURE_PRESENCE_TESTS_IDA: + id = make_test_id((sample, scope, feature, expected)) + + try: + check_input_file(get_sample_md5_by_name(sample)) + except RuntimeError: + print("SKIP %s" % (id)) + continue + + scope = resolve_scope(scope) + sample = resolve_sample(sample) + + try: + do_test_feature_presence(get_ida_extractor, sample, scope, feature, expected) + except Exception as e: + print("FAIL %s" % (id)) + traceback.print_exc() + else: + print("OK %s" % (id)) + + +@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") +def test_ida_feature_counts(): + for (sample, scope, feature, expected) in FEATURE_COUNT_TESTS: + id = make_test_id((sample, scope, feature, expected)) + + try: + check_input_file(get_sample_md5_by_name(sample)) + except RuntimeError: + print("SKIP %s" % (id)) + continue + + scope = resolve_scope(scope) + sample = resolve_sample(sample) + + try: + do_test_feature_count(get_ida_extractor, sample, scope, feature, expected) + except Exception as e: + print("FAIL %s" % (id)) + traceback.print_exc() + else: + print("OK %s" % (id)) + + +if __name__ == "__main__": + print("-" * 80) + + # invoke all functions in this module that start with `test_` + for name in dir(sys.modules[__name__]): + if not name.startswith("test_"): + continue + + test = getattr(sys.modules[__name__], name) + logger.debug("invoking test: %s", name) + sys.stderr.flush() + test() + + print("DONE") From 13306b71e03ea5ec7b805f3b4b3919140a438243 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Fri, 19 Mar 2021 09:39:38 +0100 Subject: [PATCH 13/18] add file --- .gitattributes | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6b8916d3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Explicitly declare text files you want to always be normalized and converted +# to native line endings on checkout. +*.py text +*.yml text +*.md text +*.txt text From 1dbb34df9f780c5555926f703733b2c4514be693 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Sun, 21 Mar 2021 19:28:58 +0000 Subject: [PATCH 14/18] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index 21501a5a..cd6defdb 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 21501a5a4b1f8b92518559b27e6ad33a012f0893 +Subproject commit cd6defdb2c46b309142b0867f4f97af6c48a311a From 8b5dc54397eab95ee17c74e34fadc5a27041780b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Mar 2021 06:20:47 +0000 Subject: [PATCH 15/18] Bump viv-utils from 0.5.0 to 0.6.0 Bumps [viv-utils](https://github.com/williballenthin/viv-utils) from 0.5.0 to 0.6.0. - [Release notes](https://github.com/williballenthin/viv-utils/releases) - [Commits](https://github.com/williballenthin/viv-utils/compare/v0.5.0...v0.6.0) Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d564f74c..effa60b7 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ requirements = [ "ruamel.yaml", "wcwidth", "ida-settings==2.1.0", - "viv-utils==0.5.0", + "viv-utils==0.6.0", ] if sys.version_info >= (3, 0): From 8afc3f46f6715d520d4dcb409866502a9b4e39ef Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 22 Mar 2021 08:41:21 +0000 Subject: [PATCH 16/18] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 5bc15f9c..7d164212 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 5bc15f9ccd0d341f25def181e30c5523ad3657e4 +Subproject commit 7d1642122a8e751794dc909f65ebd06450e5cc42 From 4775e124db96671401aafaaf2748606940a0fab2 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 22 Mar 2021 09:02:35 +0000 Subject: [PATCH 17/18] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 7d164212..0fd707b7 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 7d1642122a8e751794dc909f65ebd06450e5cc42 +Subproject commit 0fd707b7dcd90968a62ca96b80accbf7c893ee50 From a2ff87af8afbe5b0f23a35250e239ffa1b43db23 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 22 Mar 2021 15:45:10 +0000 Subject: [PATCH 18/18] Sync capa rules submodule --- README.md | 2 +- rules | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 498a111d..0b1b26c7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-470-blue.svg)](https://github.com/fireeye/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-471-blue.svg)](https://github.com/fireeye/capa-rules) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index 0fd707b7..a6ec6686 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 0fd707b7dcd90968a62ca96b80accbf7c893ee50 +Subproject commit a6ec6686905be33a665099bb7046c6f5a4c4e1d1