Merge branch 'master' into fix/issue-1282

CHANGELOG.md

@@ -12,11 +12,13 @@
 - dotnet: emit namespace/class features for ldvirtftn/ldftn instructions #1241 @mike-hunhoff
 - dotnet: emit namespace/class features for type references #1242 @mike-hunhoff
 - dotnet: extract dotnet and pe format #1187 @mr-tz
+- don't render all library rule matches in vverbose output #1174 @mr-tz
 
 ### Breaking Changes
-- remove SMDA backend #1062 @williballenthin 
+- remove SMDA backend #1062 @williballenthin
+- error return codes are now positive numbers #1269 @mr-tz
 
-### New Rules (48)
+### New Rules (52)
 
 - collection/use-dotnet-library-sharpclipboard @johnk3r
 - data-manipulation/encryption/aes/use-dotnet-library-encryptdecryptutils @johnk3r
@@ -65,6 +67,9 @@
 - compiler/nuitka/compiled-with-nuitka @williballenthin
 - nursery/authenticate-data-with-md5-mac william.ballenthin@mandiant.com
 - nursery/resolve-function-by-djb2-hash still@teamt5.org
+- host-interaction/mutex/create-semaphore-on-linux @ramen0x3f
+- host-interaction/mutex/lock-semaphore-on-linux @ramen0x3f
+- host-interaction/mutex/unlock-semaphore-on-linux @ramen0x3f
 -
 
 ### Bug Fixes
@@ -78,6 +83,7 @@
 - fix import-to-ida script formatting #1208 @williballenthin
 - render: fix verbose rendering of scopes #1263 @williballenthin
 - rules: better detect invalid rules #1282 @williballenthin
+- show-features: better render strings with embedded whitespace #1267 @williballenthin
 
 ### capa explorer IDA Pro plugin
 - fix: display instruction items #1154 @mr-tz
@@ -89,6 +95,7 @@
 - generator: refactor caching and matching #1251 @mike-hunhoff
 - fix: improve exception handling to prevent IDA from locking up when errors occur #1262 @mike-hunhoff
 - verify rule metadata using Pydantic #1167 @mr-tz
+- extractor: make read consistent with file object behavior #1254 @mr-tz
 
 ### Development
 

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-747-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-749-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)

capa/features/common.py

@@ -179,6 +179,10 @@ class String(Feature):
     def __init__(self, value: str, description=None):
         super().__init__(value, description=description)
 
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, str)
+        return escape_string(self.value)
+
 
 class Class(Feature):
     def __init__(self, value: str, description=None):
@@ -232,9 +236,13 @@ class Substring(String):
         else:
             return Result(False, _MatchedSubstring(self, {}), [])
 
+    def get_value_str(self) -> str:
+        assert isinstance(self.value, str)
+        return escape_string(self.value)
+
     def __str__(self):
         assert isinstance(self.value, str)
-        return "substring(%s)" % self.value
+        return "substring(%s)" % escape_string(self.value)
 
 
 class _MatchedSubstring(Substring):

capa/features/extractors/common.py

@@ -9,6 +9,7 @@ import pefile
 import capa.features
 import capa.features.extractors.elf
 import capa.features.extractors.pefile
+import capa.features.extractors.strings
 from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, FORMAT_FREEZE, Arch, Format, String, Feature
 from capa.features.freeze import is_freeze
 from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress

capa/ida/helpers.py

@@ -181,11 +181,13 @@ class IDAIO:
     def read(self, size):
         ea = ida_loader.get_fileregion_ea(self.offset)
         if ea == idc.BADADDR:
-            # best guess, such as if file is mapped at address 0x0.
-            ea = self.offset
+            logger.debug("cannot read 0x%x bytes at 0x%x (ea: BADADDR)", size, self.offset)
+            return b""
 
         logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, ea)
-        return ida_bytes.get_bytes(ea, size)
+
+        # get_bytes returns None on error, for consistency with read always return bytes
+        return ida_bytes.get_bytes(ea, size) or b""
 
     def close(self):
         return

capa/main.py

@@ -73,16 +73,16 @@ SIGNATURES_PATH_DEFAULT_STRING = "(embedded signatures)"
 BACKEND_VIV = "vivisect"
 BACKEND_DOTNET = "dotnet"
 
-E_MISSING_RULES = -10
-E_MISSING_FILE = -11
-E_INVALID_RULE = -12
-E_CORRUPT_FILE = -13
-E_FILE_LIMITATION = -14
-E_INVALID_SIG = -15
-E_INVALID_FILE_TYPE = -16
-E_INVALID_FILE_ARCH = -17
-E_INVALID_FILE_OS = -18
-E_UNSUPPORTED_IDA_VERSION = -19
+E_MISSING_RULES = 10
+E_MISSING_FILE = 11
+E_INVALID_RULE = 12
+E_CORRUPT_FILE = 13
+E_FILE_LIMITATION = 14
+E_INVALID_SIG = 15
+E_INVALID_FILE_TYPE = 16
+E_INVALID_FILE_ARCH = 17
+E_INVALID_FILE_OS = 18
+E_UNSUPPORTED_IDA_VERSION = 19
 
 logger = logging.getLogger("capa")
 

capa/render/utils.py

@@ -24,6 +24,10 @@ def bold2(s: str) -> str:
     return termcolor.colored(s, "green")
 
 
+def warn(s: str) -> str:
+    return termcolor.colored(s, "yellow")
+
+
 def format_parts_id(data: Union[rd.AttackSpec, rd.MBCSpec]):
     """
     format canonical representation of ATT&CK/MBC parts and ID

capa/render/vverbose.py

@@ -285,17 +285,24 @@ def render_rules(ostream, doc: rd.ResultDocument):
         if rule.meta.is_subscope_rule:
             continue
 
+        lib_info = ""
         count = len(rule.matches)
         if count == 1:
-            capability = rutils.bold(rule.meta.name)
+            if rule.meta.lib:
+                lib_info = " (library rule)"
+            capability = "%s%s" % (rutils.bold(rule.meta.name), lib_info)
         else:
-            capability = "%s (%d matches)" % (rutils.bold(rule.meta.name), count)
+            if rule.meta.lib:
+                lib_info = ", only showing first match of library rule"
+            capability = "%s (%d matches%s)" % (rutils.bold(rule.meta.name), count, lib_info)
 
         ostream.writeln(capability)
         had_match = True
 
         rows = []
-        rows.append(("namespace", rule.meta.namespace))
+        if not rule.meta.lib:
+            # library rules should not have a namespace
+            rows.append(("namespace", rule.meta.namespace))
 
         if rule.meta.maec.analysis_conclusion or rule.meta.maec.analysis_conclusion_ov:
             rows.append(
@@ -355,6 +362,10 @@ def render_rules(ostream, doc: rd.ResultDocument):
 
                 ostream.write("\n")
                 render_match(ostream, match, indent=1)
+                if rule.meta.lib:
+                    # only show first match
+                    break
+
         ostream.write("\n")
 
     if not had_match:

setup.py

@@ -15,12 +15,12 @@ requirements = [
     "pyyaml==6.0",
     "tabulate==0.9.0",
     "colorama==0.4.5",
-    "termcolor==2.1.1",
+    "termcolor==2.2.0",
     "wcwidth==0.2.5",
     "ida-settings==2.1.0",
     "viv-utils[flirt]==0.7.7",
     "halo==0.0.31",
-    "networkx==2.5.1",
+    "networkx==2.5.1",  # newer versions no longer support py3.7.
     "ruamel.yaml==0.17.21",
     "vivisect==1.0.8",
     "pefile==2022.5.30",