From 7e0846e66aa3c3db11efc900dcc221f02808d490 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 12 Jan 2021 17:55:13 +0000 Subject: [PATCH 01/18] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 3b4377aa..900aab1b 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 3b4377aabb0734966b720088db89f002681558d7 +Subproject commit 900aab1b2b75b7fb0e476d37ce227896bb4d5d05 From 69670102813de57efd9ddff320f2030f09b17c6e Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 12 Jan 2021 18:26:12 +0000 Subject: [PATCH 02/18] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index 83b81067..dca979b4 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 83b81067982a8cd1c568fac61b619245d6bdd78d +Subproject commit dca979b4979597ef9b01f58ddc719ca50c1f866c From b25120280413eb4e0c32257e6edd1358bc36c81e Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 12 Jan 2021 18:27:11 +0000 Subject: [PATCH 03/18] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index dca979b4..c8bdffd0 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit dca979b4979597ef9b01f58ddc719ca50c1f866c +Subproject commit c8bdffd0cc5f3aa02c0d1cd26b72c8f0a9e28861 From 0fcc9f3df68072372113c570f8862f2c40c46931 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 12 Jan 2021 18:27:32 +0000 Subject: [PATCH 04/18] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index c8bdffd0..3f438ab5 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit c8bdffd0cc5f3aa02c0d1cd26b72c8f0a9e28861 +Subproject commit 3f438ab56e65d1d8a6088fb99d9cf9463007bf89 From 2b385ead7f0e83cad6acf4834bfb713aa8e1a7a8 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 12 Jan 2021 18:30:11 +0000 Subject: [PATCH 05/18] Sync capa rules submodule --- README.md | 2 +- rules | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2acdd329..664eebbd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![capa](.github/logo.png) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) -[![Number of rules](https://img.shields.io/badge/rules-443-blue.svg)](https://github.com/fireeye/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-446-blue.svg)](https://github.com/fireeye/capa-rules) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) capa detects capabilities in executable files. diff --git a/rules b/rules index 900aab1b..96f27f25 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 900aab1b2b75b7fb0e476d37ce227896bb4d5d05 +Subproject commit 96f27f2559951f4c6b6f4e552fe868e69b735b11 From 48c045d381654f242c47b794db4a41f4530562b0 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 12 Jan 2021 18:30:44 +0000 Subject: [PATCH 06/18] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 96f27f25..89e556f4 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 96f27f2559951f4c6b6f4e552fe868e69b735b11 +Subproject commit 89e556f4825756869dfb9de13238faf7ad0a5a64 From 4cde2e1a7870fd6e49fa41ef2cf984132603defb Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Sat, 16 Jan 2021 15:39:09 +0000 Subject: [PATCH 07/18] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 89e556f4..c5ef712a 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 89e556f4825756869dfb9de13238faf7ad0a5a64 +Subproject commit c5ef712a16494cf83fa3f45059321bfe6d20e108 From c9bf7f424d03fb980ee01ff671a39542efb78c43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jan 2021 06:44:33 +0000 Subject: [PATCH 08/18] Bump smda from 1.5.10 to 1.5.11 Bumps [smda](https://github.com/danielplohmann/smda) from 1.5.10 to 1.5.11. - [Release notes](https://github.com/danielplohmann/smda/releases) - [Commits](https://github.com/danielplohmann/smda/commits) Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1b5e7e71..8caca973 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ if sys.version_info >= (3, 0): # py3 requirements.append("halo") requirements.append("networkx") - requirements.append("smda==1.5.10") + requirements.append("smda==1.5.11") else: # py2 requirements.append("enum34==1.1.6") # v1.1.6 is needed by halo 0.0.30 / spinners 0.0.24 From 4bd93a680e2d4b7bf536e6d6e28700ae475b1fb1 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 18 Jan 2021 08:02:29 +0000 Subject: [PATCH 09/18] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index 3f438ab5..fac3eb57 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 3f438ab56e65d1d8a6088fb99d9cf9463007bf89 +Subproject commit fac3eb5708269f2439ccb7e03a7f4f65770c47b4 From 905fff041ba0536991e5700b05f37515714f04e7 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 21 Jan 2021 21:32:42 +0000 Subject: [PATCH 10/18] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index c5ef712a..f23f2798 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit c5ef712a16494cf83fa3f45059321bfe6d20e108 +Subproject commit f23f27983772fe33748134d5c40de551f0d675c8 From 2c5508febdc605324d61ad19fb3ca5aed0c53887 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Fri, 22 Jan 2021 10:00:25 +0100 Subject: [PATCH 11/18] bump smda, enable Python 3.9 --- .github/workflows/tests.yml | 3 +-- setup.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4799b1ae..36842b85 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,8 +52,7 @@ jobs: - python: 3.6 - python: 3.7 - python: 3.8 - #- python: '3.9.0-rc.1' # Python latest - # disabled due to LIEF, see #362 + - python: 3.9.1 steps: - name: Checkout capa with submodules uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 8caca973..a6546599 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ if sys.version_info >= (3, 0): # py3 requirements.append("halo") requirements.append("networkx") - requirements.append("smda==1.5.11") + requirements.append("smda==1.5.13") else: # py2 requirements.append("enum34==1.1.6") # v1.1.6 is needed by halo 0.0.30 / spinners 0.0.24 From c750447d62a492210d262ffc45eef5025e3279a1 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Wed, 27 Jan 2021 17:59:56 +0100 Subject: [PATCH 12/18] potential fix for #398 --- capa/rules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/capa/rules.py b/capa/rules.py index 196458d7..01ab1bfa 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -866,7 +866,8 @@ class RuleSet(object): given a collection of rules, collect the rules that are needed at the given scope. these rules are ordered topologically. - don't include "lib" rules, unless they are dependencies of other rules. + don't include auto-generated "subscope"/"lib" rules. + we want to include general "lib" rules here - even if they are not dependencies of other rules, see #398 """ scope_rules = set([]) @@ -875,7 +876,7 @@ class RuleSet(object): # at lower scope, e.g. function scope. # so, we find all dependencies of all rules, and later will filter them down. for rule in rules: - if rule.meta.get("lib", False): + if rule.meta.get("capa/subscope-rule", False): continue scope_rules.update(get_rules_and_dependencies(rules, rule.name)) From 44c9d6a22b83e1abef1914097016e3414be2fcf5 Mon Sep 17 00:00:00 2001 From: Michael Hunhoff Date: Wed, 27 Jan 2021 18:29:53 -0700 Subject: [PATCH 13/18] fixing #403 --- capa/features/insn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/features/insn.py b/capa/features/insn.py index 9b13b861..ca612e56 100644 --- a/capa/features/insn.py +++ b/capa/features/insn.py @@ -16,7 +16,7 @@ class API(Feature): modname, _, impname = name.rpartition(".") name = modname.lower() + "." + impname - super(API, self).__init__(name, description) + super(API, self).__init__(name, description=description) class Number(Feature): From 7b64425c24ad9b388f7bcfaeb7098e93f09f50ac Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Thu, 28 Jan 2021 08:18:23 +0100 Subject: [PATCH 14/18] update doc and test case --- capa/rules.py | 2 +- tests/test_rules.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/capa/rules.py b/capa/rules.py index 01ab1bfa..09929636 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -866,7 +866,7 @@ class RuleSet(object): given a collection of rules, collect the rules that are needed at the given scope. these rules are ordered topologically. - don't include auto-generated "subscope"/"lib" rules. + don't include auto-generated "subscope" rules. we want to include general "lib" rules here - even if they are not dependencies of other rules, see #398 """ scope_rules = set([]) diff --git a/tests/test_rules.py b/tests/test_rules.py index a7059025..c08d7212 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -282,7 +282,8 @@ def test_lib_rules(): ), ] ) - assert len(rules.function_rules) == 1 + # lib rules are added to the rule set + assert len(rules.function_rules) == 2 def test_subscope_rules(): From 4bc06aa8cd903ae4e4dc33c5a2481cb03031a85b Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Thu, 28 Jan 2021 08:23:15 +0100 Subject: [PATCH 15/18] closes #405 --- .github/workflows/tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36842b85..c3faac83 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,7 +49,6 @@ jobs: matrix: include: - python: 2.7 - - python: 3.6 - python: 3.7 - python: 3.8 - python: 3.9.1 From 0d439c0f55f3074a943a9db9780c29acbe73c52d Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Thu, 28 Jan 2021 09:22:15 +0100 Subject: [PATCH 16/18] disable extractor progress --- scripts/lint.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/lint.py b/scripts/lint.py index 0b803998..e5b621a2 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -194,7 +194,7 @@ class DoesntMatchExample(Lint): continue try: - extractor = capa.main.get_extractor(path, "auto") + extractor = capa.main.get_extractor(path, "auto", disable_progress=True) capabilities, meta = capa.main.find_capabilities(ctx["rules"], extractor, disable_progress=True) except Exception as e: logger.error("failed to extract capabilities: %s %s %s", rule.name, path, e) @@ -226,6 +226,7 @@ class LibRuleNotInLibDirectory(Lint): recommendation = "Move the rule to the `lib` subdirectory of the rules path" def check_rule(self, ctx, rule): + logger.debug(rule.meta) if is_nursery_rule(rule): return False From f18a8f5b31860622662b0705181930be3560c888 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Thu, 28 Jan 2021 10:08:13 +0100 Subject: [PATCH 17/18] adjust expected lib path and log time --- scripts/lint.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/lint.py b/scripts/lint.py index e5b621a2..97f5f562 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -15,6 +15,7 @@ See the License for the specific language governing permissions and limitations """ import os import sys +import time import string import hashlib import logging @@ -226,14 +227,13 @@ class LibRuleNotInLibDirectory(Lint): recommendation = "Move the rule to the `lib` subdirectory of the rules path" def check_rule(self, ctx, rule): - logger.debug(rule.meta) if is_nursery_rule(rule): return False if "lib" not in rule.meta: return False - return "/lib/" not in get_normpath(rule.meta["capa/path"]) + return "lib/" not in get_normpath(rule.meta["capa/path"]) class LibRuleHasNamespace(Lint): @@ -519,8 +519,10 @@ def main(argv=None): capa.main.set_vivisect_log_level(logging.CRITICAL) logging.getLogger("capa").setLevel(logging.CRITICAL) + time0 = time.time() + try: - rules = capa.main.get_rules(args.rules) + rules = capa.main.get_rules(args.rules, disable_progress=True) rules = capa.rules.RuleSet(rules) logger.info("successfully loaded %s rules", len(rules)) if args.tag: @@ -546,6 +548,10 @@ def main(argv=None): } did_violate = lint(ctx, rules) + + diff = time.time() - time0 + logger.debug("lint ran for ~ %02d:%02d", (diff // 60), diff) + if not did_violate: logger.info("no suggestions, nice!") return 0 From 9406e3dbfb51a6e55af7ee6adbcd88d41b24368e Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 28 Jan 2021 09:52:43 +0000 Subject: [PATCH 18/18] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index f23f2798..37351674 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit f23f27983772fe33748134d5c40de551f0d675c8 +Subproject commit 37351674f65a50e845ad637418c408932676139a