From 5d16a7789171241a038761a1d5194f0e9158d67b Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Fri, 12 Mar 2021 15:32:12 +0100 Subject: [PATCH 01/14] ci: tag capa-rules on release Add GitHub Action to tag capa-rules when releasing capa. The used tag name is the same as the one in capa. --- .github/workflows/tag.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/tag.yml diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml new file mode 100644 index 00000000..77c66d0e --- /dev/null +++ b/.github/workflows/tag.yml @@ -0,0 +1,24 @@ +name: tag + +on: + release: + types: [published] + +jobs: + tag: + name: Tag capa rules + runs-on: ubuntu-20.04 + steps: + - name: Checkout capa-rules + uses: actions/checkout@v2 + with: + repository: fireeye/capa-rules + token: ${{ secrets.CAPA_TOKEN }} + - name: Tag capa-rules + run: git tag ${{ github.event.release.tag_name }} + - name: Push tag to capa-rules + uses: ad-m/github-push-action@master + with: + repository: fireeye/capa-rules + github_token: ${{ secrets.CAPA_TOKEN }} + tags: true From b880d419a35dc4bd30d998d7b51aebbc8afbb327 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 22 Mar 2021 09:50:04 -0600 Subject: [PATCH 02/14] ci: build: pin OS versions --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5469889a..35f47c51 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,10 +15,10 @@ jobs: # use old linux so that the shared library versioning is more portable artifact_name: capa asset_name: linux - - os: windows-latest + - os: windows-2019 artifact_name: capa.exe asset_name: windows - - os: macos-latest + - os: macos-10.15 artifact_name: capa asset_name: macos steps: @@ -30,7 +30,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - if: matrix.os == 'ubuntu-latest' + - if: matrix.os == 'ubuntu-16.04' run: sudo apt-get install -y libyaml-dev - name: Install PyInstaller run: pip install 'pyinstaller==4.2' @@ -47,7 +47,7 @@ jobs: zip: name: zip ${{ matrix.asset_name }} - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 needs: build strategy: matrix: From a6c472bb2acb7395216fa936537626a02b4d7a91 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 22 Mar 2021 09:50:47 -0600 Subject: [PATCH 03/14] ci: publish: pin OS version --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 53768b91..a0ecb770 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ on: jobs: deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Set up Python @@ -26,4 +26,4 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python setup.py sdist bdist_wheel - twine upload --skip-existing dist/* \ No newline at end of file + twine upload --skip-existing dist/* From 29d8f1fd27e72883ac32910731b958f0e40be7ae Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 22 Mar 2021 09:51:20 -0600 Subject: [PATCH 04/14] ci: tests: pin OS version --- .github/workflows/tests.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c3faac83..14b64894 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,7 @@ on: jobs: code_style: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout capa uses: actions/checkout@v2 @@ -24,7 +24,7 @@ jobs: run: black -l 120 --check . rule_linter: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout capa with rules submodule uses: actions/checkout@v2 @@ -42,7 +42,7 @@ jobs: tests: name: Tests in ${{ matrix.python }} - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 needs: [code_style, rule_linter] strategy: fail-fast: false @@ -67,4 +67,3 @@ jobs: run: pip install -e .[dev] - name: Run tests run: pytest tests/ - From 44f517c20d8ec7b5623899d8ed41f037982748e8 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Mon, 22 Mar 2021 18:11:29 +0100 Subject: [PATCH 05/14] rule loading: ignore files starting with .git --- capa/ida/plugin/form.py | 5 +++-- capa/main.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/capa/ida/plugin/form.py b/capa/ida/plugin/form.py index 212d8b90..583d9530 100644 --- a/capa/ida/plugin/form.py +++ b/capa/ida/plugin/form.py @@ -617,6 +617,7 @@ class CapaExplorerForm(idaapi.PluginForm): rule_path = settings.user[CAPA_SETTINGS_RULE_PATH] try: + # TODO refactor: this first part is identical to capa.main.get_rules if not os.path.exists(rule_path): raise IOError("rule path %s does not exist or cannot be accessed" % rule_path) @@ -632,8 +633,8 @@ class CapaExplorerForm(idaapi.PluginForm): continue for file in files: if not file.endswith(".yml"): - if not (file.endswith(".md") or file.endswith(".git") or file.endswith(".txt")): - # expect to see readme.md, format.md, and maybe a .git directory + if not (file.startswith(".git") or file.endswith((".git", ".md", ".txt"))): + # expect to see .git* files, readme.md, format.md, and maybe a .git directory # other things maybe are rules, but are mis-named. logger.warning("skipping non-.yml file: %s", file) continue diff --git a/capa/main.py b/capa/main.py index 19fda0d4..41910ece 100644 --- a/capa/main.py +++ b/capa/main.py @@ -379,8 +379,8 @@ def get_rules(rule_path, disable_progress=False): for file in files: if not file.endswith(".yml"): - if not (file.endswith(".md") or file.endswith(".git") or file.endswith(".txt")): - # expect to see readme.md, format.md, and maybe a .git directory + if not (file.startswith(".git") or file.endswith((".git", ".md", ".txt"))): + # expect to see .git* files, readme.md, format.md, and maybe a .git directory # other things maybe are rules, but are mis-named. logger.warning("skipping non-.yml file: %s", file) continue From 7e5cbddf5df88013cd33cf07a73d95c9b695d5f4 Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Tue, 16 Mar 2021 16:13:07 +0100 Subject: [PATCH 06/14] doc: document release process Add a release checklist. Closes https://github.com/fireeye/capa/issues/184 --- doc/release.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 doc/release.md diff --git a/doc/release.md b/doc/release.md new file mode 100644 index 00000000..729b648b --- /dev/null +++ b/doc/release.md @@ -0,0 +1,25 @@ +# Release checklist + +- [ ] Ensure all milestoned issues/PRs are addressed, or reassign to a new milestone. +- [ ] Add the `dont merge` label to all PRs that are close to be ready to merge (or merge them if they are ready) in [capa](https://github.com/fireeye/capa/pulls) and [capa-rules](https://github.com/fireeye/capa-rules/pulls). +- [ ] Ensure the [CI workflow succeeds in master](https://github.com/fireeye/capa/actions/workflows/tests.yml?query=branch%3Amaster). +- [ ] Ensure that `python scripts/lint.py rules/ --thorough` succeeds (only `missing examples` offenses are allowed in the nursery). +- [ ] Review changes + - capa https://github.com/fireeye/capa/compare/\...master + - capa-rules https://github.com/fireeye/capa-rules/compare/\\...master +- [ ] Update [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md) + - Do not forget to add a nice introduction thanking contributors + - Remember that we need a major release if we introduce breaking changes + - Sections + - New Features + - New Rules + - Bug Fixes + - Changes + - Development + - Raw diffs +- [ ] Update [capa/version.py](https://github.com/fireeye/capa/blob/master/capa/version.py) +- [ ] Create a PR with the updated [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md) and [capa/version.py](https://github.com/fireeye/capa/blob/master/capa/version.py). Copy this checklist in the PR description. +- [ ] After PR review, merge the PR and [create the release in GH](https://github.com/fireeye/capa/releases/new) using text from the [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md). +- [ ] Verify GH actions [upload artifacts](https://github.com/fireeye/capa/releases), [publish to PyPI](https://pypi.org/project/flare-capa) and [create a tag in capa rules](https://github.com/fireeye/capa-rules/tags) upon completion. +- [ ] [Spread the word](https://twitter.com) + From 7f3e8f1fb1c4d168926e7e0676aabc7ab92ee59a Mon Sep 17 00:00:00 2001 From: Michael Hunhoff Date: Mon, 22 Mar 2021 17:12:13 -0600 Subject: [PATCH 07/14] adding support to match subscope rules and auto insert child statements when creating a new basic block subscope --- capa/ida/plugin/form.py | 6 ++++++ capa/ida/plugin/view.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/capa/ida/plugin/form.py b/capa/ida/plugin/form.py index 212d8b90..5025ccdf 100644 --- a/capa/ida/plugin/form.py +++ b/capa/ida/plugin/form.py @@ -1019,6 +1019,12 @@ class CapaExplorerForm(idaapi.PluginForm): # create deep copy of current rules, add our new rule rules = copy.copy(self.rules_cache) + + # ensure subscope rules are included + for sub in rule.extract_subscope_rules(): + rules.append(sub) + + # include our new rule in the list rules.append(rule) try: diff --git a/capa/ida/plugin/view.py b/capa/ida/plugin/view.py index 87eb5d8a..37077a98 100644 --- a/capa/ida/plugin/view.py +++ b/capa/ida/plugin/view.py @@ -415,6 +415,11 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget): # create a new parent under root node, by default; new node added last position in tree new_parent = self.new_expression_node(self.root, (action.data()[0], "")) + if "basic block" in action.data()[0]: + # add default child expression when nesting under basic block + new_parent.setExpanded(True) + new_parent = self.new_expression_node(new_parent, ("- or:", "")) + for o in self.get_features(selected=True): # take child from its parent by index, add to new parent new_parent.addChild(o.parent().takeChild(o.parent().indexOfChild(o))) @@ -425,6 +430,15 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget): def slot_edit_expression(self, action): """ """ expression, o = action.data() + if "basic block" in expression and "basic block" not in o.text( + CapaExplorerRulgenEditor.get_column_feature_index() + ): + # current expression is "basic block", and not changing to "basic block" expression + children = o.takeChildren() + new_parent = self.new_expression_node(o, ("- or:", "")) + for child in children: + new_parent.addChild(child) + new_parent.setExpanded(True) o.setText(CapaExplorerRulgenEditor.get_column_feature_index(), expression) def slot_clear_all(self, action): From 51d125642f8f9355b0ec546002b4d71e362c7b20 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 23 Mar 2021 07:14:21 +0000 Subject: [PATCH 08/14] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index a6ec6686..989c72b8 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit a6ec6686905be33a665099bb7046c6f5a4c4e1d1 +Subproject commit 989c72b851b4c892af0ad0bc24cfec948110475e From 42a87d4eaa0a2c08add372c5ebf88f77a0f4d685 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 23 Mar 2021 07:14:58 +0000 Subject: [PATCH 09/14] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index cd6defdb..84c47672 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit cd6defdb2c46b309142b0867f4f97af6c48a311a +Subproject commit 84c476728f8a6dd74ac9c6b98a149b65316f2290 From 195bae903f8ae528eb75ec39113f1320446fd366 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 23 Mar 2021 12:25:20 +0000 Subject: [PATCH 10/14] Sync capa rules submodule --- README.md | 2 +- rules | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0b1b26c7..498a111d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-471-blue.svg)](https://github.com/fireeye/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-470-blue.svg)](https://github.com/fireeye/capa-rules) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index 989c72b8..ac8f833a 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 989c72b851b4c892af0ad0bc24cfec948110475e +Subproject commit ac8f833a031c2be4f84d63cf91179b87801e1447 From c48b46e9326e7d5fdb135b00658e4fdd2abd9d80 Mon Sep 17 00:00:00 2001 From: Michael Hunhoff Date: Wed, 24 Mar 2021 15:33:20 -0600 Subject: [PATCH 11/14] explorer: adding checks to validate matched data when searching --- capa/ida/plugin/view.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/capa/ida/plugin/view.py b/capa/ida/plugin/view.py index 37077a98..9dc161a4 100644 --- a/capa/ida/plugin/view.py +++ b/capa/ida/plugin/view.py @@ -803,9 +803,11 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget): if text: for o in iterate_tree(self): data = o.data(0, 0x100) - if data and text.lower() not in data.get_value_str().lower(): - o.setHidden(True) - continue + if data: + to_match = data.get_value_str() + if not to_match or text.lower() not in to_match.lower(): + o.setHidden(True) + continue o.setHidden(False) o.setExpanded(True) else: From c0fe96cec64ed3c90a8e9050a06fb206a68a8153 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 25 Mar 2021 07:17:41 +0000 Subject: [PATCH 12/14] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index 84c47672..e63a71b3 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 84c476728f8a6dd74ac9c6b98a149b65316f2290 +Subproject commit e63a71b394fca79f209ab718796316cce2b1a82c From 280754956405212a7b3ad12400217d4386a52f4a Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 25 Mar 2021 07:21:21 +0000 Subject: [PATCH 13/14] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index ac8f833a..689c56ba 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit ac8f833a031c2be4f84d63cf91179b87801e1447 +Subproject commit 689c56ba745c158dfe5c39f2fcfe4a60f3f6e761 From ea4c7d6403e47b1dc2dc5c006b34682b01d77362 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 25 Mar 2021 18:37:22 +0000 Subject: [PATCH 14/14] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 689c56ba..eb8221d9 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 689c56ba745c158dfe5c39f2fcfe4a60f3f6e761 +Subproject commit eb8221d9ad44a3d10851a281614ed8dcefb8495c