diff --git a/README.md b/README.md index 75dd0f1a..ea373375 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![capa](.github/logo.png) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) -[![Number of rules](https://img.shields.io/badge/rules-342-blue.svg)](https://github.com/fireeye/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-343-blue.svg)](https://github.com/fireeye/capa-rules) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) capa detects capabilities in executable files. diff --git a/capa/features/__init__.py b/capa/features/__init__.py index 5d59e4da..0a9a59c7 100644 --- a/capa/features/__init__.py +++ b/capa/features/__init__.py @@ -139,7 +139,6 @@ class Regex(String): raise ValueError( "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value ) - self.match = None def evaluate(self, ctx): for feature, locations in ctx.items(): @@ -151,10 +150,38 @@ class Regex(String): # using this mode cleans is more convenient for rule authors, # so that they don't have to prefix/suffix their terms like: /.*foo.*/. if self.re.search(feature.value): - self.match = feature.value - return capa.engine.Result(True, self, [], locations=locations) + # unlike other features, we cannot return put a reference to `self` directly in a `Result`. + # this is because `self` may match on many strings, so we can't stuff the matched value into it. + # instead, return a new instance that has a reference to both the regex and the matched value. + # see #262. + return capa.engine.Result(True, _MatchedRegex(self, feature.value), [], locations=locations) - return capa.engine.Result(False, self, []) + return capa.engine.Result(False, _MatchedRegex(self, None), []) + + def __str__(self): + return "regex(string =~ %s)" % self.value + + +class _MatchedRegex(Regex): + """ + this represents a specific instance of a regular expression feature match. + treat it the same as a `Regex` except it has the `match` field that contains the complete string that matched. + + note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API. + """ + + def __init__(self, regex, match): + """ + args: + regex (Regex): the regex feature that matches + match (string|None): the matching string or None if it doesn't match + """ + super(_MatchedRegex, self).__init__(regex.value, description=regex.description) + # we want this to collide with the name of `Regex` above, + # so that it works nicely with the renderers. + self.name = "regex" + # this may be None if the regex doesn't match + self.match = match def __str__(self): return 'regex(string =~ %s, matched = "%s")' % (self.value, self.match) diff --git a/capa/features/extractors/viv/insn.py b/capa/features/extractors/viv/insn.py index 0e091ffa..7375bc37 100644 --- a/capa/features/extractors/viv/insn.py +++ b/capa/features/extractors/viv/insn.py @@ -318,25 +318,38 @@ def extract_insn_offset_features(f, bb, insn): # # .text:0040112F cmp [esi+4], ebx for oper in insn.opers: + # this is for both x32 and x64 - if not isinstance(oper, envi.archs.i386.disasm.i386RegMemOper): - continue + # like [esi + 4] + # reg ^ + # disp + if isinstance(oper, envi.archs.i386.disasm.i386RegMemOper): + if oper.reg == envi.archs.i386.disasm.REG_ESP: + continue - if oper.reg == envi.archs.i386.disasm.REG_ESP: - continue + if oper.reg == envi.archs.i386.disasm.REG_EBP: + continue - if oper.reg == envi.archs.i386.disasm.REG_EBP: - continue + # TODO: do x64 support for real. + if oper.reg == envi.archs.amd64.disasm.REG_RBP: + continue - # TODO: do x64 support for real. - if oper.reg == envi.archs.amd64.disasm.REG_RBP: - continue + # viv already decodes offsets as signed + v = oper.disp - # viv already decodes offsets as signed - v = oper.disp + yield Offset(v), insn.va + yield Offset(v, arch=get_arch(f.vw)), insn.va - yield Offset(v), insn.va - yield Offset(v, arch=get_arch(f.vw)), insn.va + # like: [esi + ecx + 16384] + # reg ^ ^ + # index ^ + # disp + elif isinstance(oper, envi.archs.i386.disasm.i386SibOper): + # viv already decodes offsets as signed + v = oper.disp + + yield Offset(v), insn.va + yield Offset(v, arch=get_arch(f.vw)), insn.va def is_security_cookie(f, bb, insn): diff --git a/capa/rules.py b/capa/rules.py index 6b9eba58..81df0c80 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -624,7 +624,25 @@ class Rule(object): continue meta[key] = value - return ostream.getvalue().decode("utf-8").rstrip("\n") + "\n" + doc = ostream.getvalue().decode("utf-8").rstrip("\n") + "\n" + # when we have something like: + # + # and: + # - string: foo + # description: bar + # + # we want the `description` horizontally aligned with the start of the `string` (like above). + # however, ruamel will give us (which I don't think is even valid yaml): + # + # and: + # - string: foo + # description: bar + # + # tweaking `ruamel.indent()` doesn't quite give us the control we want. + # so, add the two extra spaces that we've determined we need through experimentation. + # see #263 + doc = doc.replace(" description:", " description:") + return doc def get_rules_with_scope(rules, scope): diff --git a/rules b/rules index 1549f6f8..d5467445 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 1549f6f885710580efe5c17cbaeeb2f8877c39d5 +Subproject commit d54674456840cfa558624efb17a0100576deb269 diff --git a/tests/data b/tests/data index e6f6ca89..c3a35d4b 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit e6f6ca898d323c38040af9b012533cca04c46d88 +Subproject commit c3a35d4b6430ed61ffef59d672a2a8b6061e23fe diff --git a/tests/fixtures.py b/tests/fixtures.py index 2351730f..d0754b2d 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -122,6 +122,8 @@ def get_data_path_by_name(name): return os.path.join(CD, "data", "kernel32-64.dll_") elif name == "pma12-04": return os.path.join(CD, "data", "Practical Malware Analysis Lab 12-04.exe_") + elif name == "pma16-01": + return os.path.join(CD, "data", "Practical Malware Analysis Lab 16-01.exe_") elif name == "pma21-01": return os.path.join(CD, "data", "Practical Malware Analysis Lab 21-01.exe_") elif name == "al-khaser x86": @@ -140,6 +142,8 @@ def get_data_path_by_name(name): return os.path.join(CD, "data", "bfb9b5391a13d0afd787e87ab90f14f5.dll_") elif name.startswith("c9188"): return os.path.join(CD, "data", "c91887d861d9bd4a5872249b641bc9f9.exe_") + elif name.startswith("64d9f"): + return os.path.join(CD, "data", "64d9f7d96b99467f36e22fada623c3bb.dll_") else: raise ValueError("unexpected sample fixture") @@ -154,6 +158,8 @@ def get_sample_md5_by_name(name): return "a8565440629ac87f6fef7d588fe3ff0f" elif name == "pma12-04": return "56bed8249e7c2982a90e54e1e55391a2" + elif name == "pma16-01": + return "7faafc7e4a5c736ebfee6abbbc812d80" elif name == "pma21-01": return "c8403fb05244e23a7931c766409b5e22" elif name == "al-khaser x86": @@ -172,6 +178,8 @@ def get_sample_md5_by_name(name): return "bfb9b5391a13d0afd787e87ab90f14f5" elif name.startswith("c9188"): return "c91887d861d9bd4a5872249b641bc9f9" + elif name.startswith("64d9f"): + return "64d9f7d96b99467f36e22fada623c3bb" else: raise ValueError("unexpected sample fixture") @@ -317,6 +325,8 @@ FEATURE_PRESENCE_TESTS = [ ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0), True), ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x4), True), ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0xC), True), + # insn/offset, issue #276 + ("64d9f", "function=0x10001510,bb=0x100015B0", capa.features.insn.Offset(0x4000), True), # insn/offset: stack references ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x8), False), ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x10), False), @@ -369,6 +379,9 @@ FEATURE_PRESENCE_TESTS = [ ("mimikatz", "function=0x40105D", capa.features.String("SCardTransmit"), True), ("mimikatz", "function=0x40105D", capa.features.String("ACR > "), True), ("mimikatz", "function=0x40105D", capa.features.String("nope"), False), + # insn/regex, issue #262 + ("pma16-01", "function=0x4021B0", capa.features.Regex("HTTP/1.0"), True), + ("pma16-01", "function=0x4021B0", capa.features.Regex("www.practicalmalwareanalysis.com"), False), # insn/string, pointer to string ("mimikatz", "function=0x44EDEF", capa.features.String("INPUTEVENT"), True), # insn/bytes @@ -480,6 +493,11 @@ def pma12_04_extractor(): return get_extractor(get_data_path_by_name("pma12-04")) +@pytest.fixture +def pma16_01_extractor(): + return get_extractor(get_data_path_by_name("pma16-01")) + + @pytest.fixture def bfb9b_extractor(): return get_extractor(get_data_path_by_name("bfb9b...")) diff --git a/tests/test_fmt.py b/tests/test_fmt.py index 1ca4725a..92bd4ffa 100644 --- a/tests/test_fmt.py +++ b/tests/test_fmt.py @@ -92,6 +92,8 @@ def test_rule_reformat_order(): def test_rule_reformat_meta_update(): + # test updating the rule content after parsing + rule = textwrap.dedent( """ rule: @@ -112,3 +114,24 @@ def test_rule_reformat_meta_update(): rule = capa.rules.Rule.from_yaml(rule) rule.name = "test rule" assert rule.to_yaml() == EXPECTED + + +def test_rule_reformat_string_description(): + # the `description` should be aligned with the preceding feature name. + # see #263 + src = textwrap.dedent( + """ + rule: + meta: + name: test rule + author: user@domain.com + scope: function + features: + - and: + - string: foo + description: bar + """ + ).lstrip() + + rule = capa.rules.Rule.from_yaml(src) + assert rule.to_yaml() == src diff --git a/tests/test_main.py b/tests/test_main.py index 4c92c7f0..6b8eaf57 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -309,3 +309,14 @@ def test_count_bb(z9324d_extractor): ) capabilities, meta = capa.main.find_capabilities(rules, z9324d_extractor) assert "count bb" in capabilities + + +@pytest.mark.xfail(sys.version_info >= (3, 0), reason="vivsect only works on py2") +def test_fix262(pma16_01_extractor, capsys): + # tests rules can be loaded successfully and all output modes + path = pma16_01_extractor.path + assert capa.main.main([path, "-vv", "-t", "send HTTP request", "-q"]) == 0 + + std = capsys.readouterr() + assert "HTTP/1.0" in std.out + assert "www.practicalmalwareanalysis.com" not in std.out