From 58e94a35cbaa384307410ef846b5965868b051e2 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 9 Jan 2023 10:51:08 +0100 Subject: [PATCH 01/12] features: string: better __str__ embedded whitespace --- CHANGELOG.md | 1 + capa/features/common.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d76f33d..031002d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ - dotnet: address unhandled exceptions with improved type checking #1230 @mike-hunhoff - fix import-to-ida script formatting #1208 @williballenthin - render: fix verbose rendering of scopes #1263 @williballenthin +- show-features: better render strings with embedded whitespace #1267 @williballenthin ### capa explorer IDA Pro plugin - fix: display instruction items #1154 @mr-tz diff --git a/capa/features/common.py b/capa/features/common.py index 5d30f10b..782094d8 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -179,6 +179,9 @@ class String(Feature): def __init__(self, value: str, description=None): super().__init__(value, description=description) + def get_value_str(self) -> str: + return escape_string(self.value) + class Class(Feature): def __init__(self, value: str, description=None): @@ -232,9 +235,12 @@ class Substring(String): else: return Result(False, _MatchedSubstring(self, {}), []) + def get_value_str(self) -> str: + return escape_string(self.value) + def __str__(self): assert isinstance(self.value, str) - return "substring(%s)" % self.value + return "substring(%s)" % escape_string(self.value) class _MatchedSubstring(Substring): From e34fdfae1ab8efccc89a14b4a3d3fc78a7caa3c1 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 9 Jan 2023 13:01:41 +0100 Subject: [PATCH 02/12] mypy --- capa/features/common.py | 2 ++ capa/features/extractors/common.py | 1 + 2 files changed, 3 insertions(+) diff --git a/capa/features/common.py b/capa/features/common.py index 782094d8..cf2c02f3 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -180,6 +180,7 @@ class String(Feature): super().__init__(value, description=description) def get_value_str(self) -> str: + assert isinstance(self.value, str) return escape_string(self.value) @@ -236,6 +237,7 @@ class Substring(String): return Result(False, _MatchedSubstring(self, {}), []) def get_value_str(self) -> str: + assert isinstance(self.value, str) return escape_string(self.value) def __str__(self): diff --git a/capa/features/extractors/common.py b/capa/features/extractors/common.py index 585c0040..d72fcefd 100644 --- a/capa/features/extractors/common.py +++ b/capa/features/extractors/common.py @@ -9,6 +9,7 @@ import pefile import capa.features import capa.features.extractors.elf import capa.features.extractors.pefile +import capa.features.extractors.strings from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, FORMAT_FREEZE, Arch, Format, String, Feature from capa.features.freeze import is_freeze from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress From 2a61e357de7dcba5c616bd1edd6d5f443568acb2 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 9 Jan 2023 13:08:27 +0000 Subject: [PATCH 03/12] Sync capa rules submodule --- CHANGELOG.md | 2 +- README.md | 2 +- rules | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d76f33d..1d4c0c7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ ### Breaking Changes - remove SMDA backend #1062 @williballenthin -### New Rules (48) +### New Rules (49) - collection/use-dotnet-library-sharpclipboard @johnk3r - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r diff --git a/README.md b/README.md index 699f885f..32a1628c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-747-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-746-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index 4c93fad4..dc81cb49 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 4c93fad4aaed8249767a9b65dee51eaa12d0ff37 +Subproject commit dc81cb4904fb4a7cbb7c6526939231e1ff7a2d14 From 85dd065f91cc48c9684809a062623cdedde44e11 Mon Sep 17 00:00:00 2001 From: Moritz Date: Mon, 9 Jan 2023 14:14:08 +0100 Subject: [PATCH 04/12] only show first lib match to reduce vverbose output noise (#1266) * only show first lib match to reduce vverbose output noise * improve rendering and wording --- CHANGELOG.md | 1 + capa/render/utils.py | 4 ++++ capa/render/vverbose.py | 17 ++++++++++++++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d4c0c7d..2470af58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - dotnet: emit namespace/class features for ldvirtftn/ldftn instructions #1241 @mike-hunhoff - dotnet: emit namespace/class features for type references #1242 @mike-hunhoff - dotnet: extract dotnet and pe format #1187 @mr-tz +- don't render all library rule matches in vverbose output #1174 @mr-tz ### Breaking Changes - remove SMDA backend #1062 @williballenthin diff --git a/capa/render/utils.py b/capa/render/utils.py index 2cf480c9..96abadcd 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -24,6 +24,10 @@ def bold2(s: str) -> str: return termcolor.colored(s, "green") +def warn(s: str) -> str: + return termcolor.colored(s, "yellow") + + def format_parts_id(data: Union[rd.AttackSpec, rd.MBCSpec]): """ format canonical representation of ATT&CK/MBC parts and ID diff --git a/capa/render/vverbose.py b/capa/render/vverbose.py index 5950275a..74be65da 100644 --- a/capa/render/vverbose.py +++ b/capa/render/vverbose.py @@ -285,17 +285,24 @@ def render_rules(ostream, doc: rd.ResultDocument): if rule.meta.is_subscope_rule: continue + lib_info = "" count = len(rule.matches) if count == 1: - capability = rutils.bold(rule.meta.name) + if rule.meta.lib: + lib_info = " (library rule)" + capability = "%s%s" % (rutils.bold(rule.meta.name), lib_info) else: - capability = "%s (%d matches)" % (rutils.bold(rule.meta.name), count) + if rule.meta.lib: + lib_info = ", only showing first match of library rule" + capability = "%s (%d matches%s)" % (rutils.bold(rule.meta.name), count, lib_info) ostream.writeln(capability) had_match = True rows = [] - rows.append(("namespace", rule.meta.namespace)) + if not rule.meta.lib: + # library rules should not have a namespace + rows.append(("namespace", rule.meta.namespace)) if rule.meta.maec.analysis_conclusion or rule.meta.maec.analysis_conclusion_ov: rows.append( @@ -355,6 +362,10 @@ def render_rules(ostream, doc: rd.ResultDocument): ostream.write("\n") render_match(ostream, match, indent=1) + if rule.meta.lib: + # only show first match + break + ostream.write("\n") if not had_match: From 810e2d70d3a7c1d19caad07a9146e33783cebf47 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 9 Jan 2023 13:38:25 +0000 Subject: [PATCH 05/12] Sync capa rules submodule --- CHANGELOG.md | 5 ++++- README.md | 2 +- rules | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2470af58..0f5e5614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ ### Breaking Changes - remove SMDA backend #1062 @williballenthin -### New Rules (49) +### New Rules (52) - collection/use-dotnet-library-sharpclipboard @johnk3r - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r @@ -66,6 +66,9 @@ - compiler/nuitka/compiled-with-nuitka @williballenthin - nursery/authenticate-data-with-md5-mac william.ballenthin@mandiant.com - nursery/resolve-function-by-djb2-hash still@teamt5.org +- host-interaction/mutex/create-semaphore-on-linux @ramen0x3f +- host-interaction/mutex/lock-semaphore-on-linux @ramen0x3f +- host-interaction/mutex/unlock-semaphore-on-linux @ramen0x3f - ### Bug Fixes diff --git a/README.md b/README.md index 32a1628c..09569bf0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-746-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-749-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index dc81cb49..cf41d4fe 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit dc81cb4904fb4a7cbb7c6526939231e1ff7a2d14 +Subproject commit cf41d4fe4ad6e807b26a3669fa3eca364009502a From 5f31bdbb3e76ab23557114abdbfbd3034a48dc4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:03:44 +0000 Subject: [PATCH 06/12] build(deps): bump networkx from 2.5.1 to 3.0 Bumps [networkx](https://github.com/networkx/networkx) from 2.5.1 to 3.0. - [Release notes](https://github.com/networkx/networkx/releases) - [Commits](https://github.com/networkx/networkx/compare/networkx-2.5.1...networkx-3.0) --- updated-dependencies: - dependency-name: networkx dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a52f8f5b..46a4d35e 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ requirements = [ "ida-settings==2.1.0", "viv-utils[flirt]==0.7.7", "halo==0.0.31", - "networkx==2.5.1", + "networkx==3.0", "ruamel.yaml==0.17.21", "vivisect==1.0.8", "pefile==2022.5.30", From ef86160d884047465e2c5bda1848a895821bf21b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:03:48 +0000 Subject: [PATCH 07/12] build(deps): bump termcolor from 2.1.1 to 2.2.0 Bumps [termcolor](https://github.com/termcolor/termcolor) from 2.1.1 to 2.2.0. - [Release notes](https://github.com/termcolor/termcolor/releases) - [Changelog](https://github.com/termcolor/termcolor/blob/main/CHANGES.md) - [Commits](https://github.com/termcolor/termcolor/compare/2.1.1...2.2.0) --- updated-dependencies: - dependency-name: termcolor dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a52f8f5b..92225ed3 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ requirements = [ "pyyaml==6.0", "tabulate==0.9.0", "colorama==0.4.5", - "termcolor==2.1.1", + "termcolor==2.2.0", "wcwidth==0.2.5", "ida-settings==2.1.0", "viv-utils[flirt]==0.7.7", From 37a4dbf8220ec146060c20ea8510c429dfc68716 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 9 Jan 2023 15:53:03 +0000 Subject: [PATCH 08/12] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index cf41d4fe..9a514c76 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit cf41d4fe4ad6e807b26a3669fa3eca364009502a +Subproject commit 9a514c7620377ded2ede17b3c6b11276e5d8e1eb From b6fd95c7b8cb292dfbd8fd289a8527d8b68d70c5 Mon Sep 17 00:00:00 2001 From: Moritz Date: Tue, 10 Jan 2023 13:14:23 +0100 Subject: [PATCH 09/12] use positive error return code numbers (#1274) --- CHANGELOG.md | 3 ++- capa/main.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d59c32c9..c98cdd44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,8 @@ - don't render all library rule matches in vverbose output #1174 @mr-tz ### Breaking Changes -- remove SMDA backend #1062 @williballenthin +- remove SMDA backend #1062 @williballenthin +- error return codes are now positive numbers #1269 @mr-tz ### New Rules (52) diff --git a/capa/main.py b/capa/main.py index b3a654a4..6262b7d0 100644 --- a/capa/main.py +++ b/capa/main.py @@ -73,16 +73,16 @@ SIGNATURES_PATH_DEFAULT_STRING = "(embedded signatures)" BACKEND_VIV = "vivisect" BACKEND_DOTNET = "dotnet" -E_MISSING_RULES = -10 -E_MISSING_FILE = -11 -E_INVALID_RULE = -12 -E_CORRUPT_FILE = -13 -E_FILE_LIMITATION = -14 -E_INVALID_SIG = -15 -E_INVALID_FILE_TYPE = -16 -E_INVALID_FILE_ARCH = -17 -E_INVALID_FILE_OS = -18 -E_UNSUPPORTED_IDA_VERSION = -19 +E_MISSING_RULES = 10 +E_MISSING_FILE = 11 +E_INVALID_RULE = 12 +E_CORRUPT_FILE = 13 +E_FILE_LIMITATION = 14 +E_INVALID_SIG = 15 +E_INVALID_FILE_TYPE = 16 +E_INVALID_FILE_ARCH = 17 +E_INVALID_FILE_OS = 18 +E_UNSUPPORTED_IDA_VERSION = 19 logger = logging.getLogger("capa") From 05879dc02a6d4f4a3971f5231e8fe746bc1715e8 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Wed, 11 Jan 2023 10:49:04 +0100 Subject: [PATCH 10/12] Revert "build(deps): bump networkx from 2.5.1 to 3.0" --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 010c227a..92225ed3 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ requirements = [ "ida-settings==2.1.0", "viv-utils[flirt]==0.7.7", "halo==0.0.31", - "networkx==3.0", + "networkx==2.5.1", "ruamel.yaml==0.17.21", "vivisect==1.0.8", "pefile==2022.5.30", From 5fb9de775f627dd9f1c94e8fd67a2338c87b4627 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Wed, 11 Jan 2023 10:50:55 +0100 Subject: [PATCH 11/12] setup: document networkx dep version pin --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 92225ed3..d94dbf38 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ requirements = [ "ida-settings==2.1.0", "viv-utils[flirt]==0.7.7", "halo==0.0.31", - "networkx==2.5.1", + "networkx==2.5.1", # newer versions no longer support py3.7. "ruamel.yaml==0.17.21", "vivisect==1.0.8", "pefile==2022.5.30", From a273ad31d4e5fc6d556d99e10a1b9915a373e2a8 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 11 Jan 2023 17:17:04 +0100 Subject: [PATCH 12/12] make read consistent with file object behavior (#1281) --- CHANGELOG.md | 1 + capa/ida/helpers.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c98cdd44..86be5431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,7 @@ - generator: refactor caching and matching #1251 @mike-hunhoff - fix: improve exception handling to prevent IDA from locking up when errors occur #1262 @mike-hunhoff - verify rule metadata using Pydantic #1167 @mr-tz +- extractor: make read consistent with file object behavior #1254 @mr-tz ### Development diff --git a/capa/ida/helpers.py b/capa/ida/helpers.py index d1ef3093..2d12e931 100644 --- a/capa/ida/helpers.py +++ b/capa/ida/helpers.py @@ -181,11 +181,13 @@ class IDAIO: def read(self, size): ea = ida_loader.get_fileregion_ea(self.offset) if ea == idc.BADADDR: - # best guess, such as if file is mapped at address 0x0. - ea = self.offset + logger.debug("cannot read 0x%x bytes at 0x%x (ea: BADADDR)", size, self.offset) + return b"" logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, ea) - return ida_bytes.get_bytes(ea, size) + + # get_bytes returns None on error, for consistency with read always return bytes + return ida_bytes.get_bytes(ea, size) or b"" def close(self): return