From 8a386b690949fe0f454b283a96935f8ff6299350 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Tue, 18 May 2021 13:23:19 +0200 Subject: [PATCH 1/2] improve progress bar output --- CHANGELOG.md | 4 +++- capa/main.py | 24 +++++++++++++++++++----- capa/render/verbose.py | 1 + tests/test_main.py | 2 ++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3566bb00..a391db12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,11 @@ It includes many new rules, including all new techniques introduced in MITRE ATT ### New Features -- main: auto detect shellcode based on file extension #516 @mr-tz - main: use FLIRT signatures to identify and ignore library code #446 @williballenthin - explorer: IDA 7.6 support #497 @williballenthin - scripts: capa2yara.py convert capa rules to YARA rules #561 @ruppde +- main: auto detect shellcode based on file extension #516 @mr-tz +- main: more detailed progress bar output when loading rules and matching functions #562 @mr-tz ### New Rules (69) @@ -97,6 +98,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - rules: update ATT&CK and MBC mappings https://github.com/fireeye/capa-rules/pull/317 @williballenthin - tests: update test cases and caching #545 @mr-tz - show-features: don't show features from library functions #569 @williballenthin +- meta: added `library_functions` field, `feature_counts.functions` does not include library functions any more #562 @mr-tz ### Development diff --git a/capa/main.py b/capa/main.py index 61a0eaa3..3c1bacd3 100644 --- a/capa/main.py +++ b/capa/main.py @@ -128,7 +128,8 @@ def find_capabilities(ruleset, extractor, disable_progress=None): "feature_counts": { "file": 0, "functions": {}, - } + }, + "library_functions": {}, } pbar = tqdm.tqdm @@ -138,13 +139,20 @@ def find_capabilities(ruleset, extractor, disable_progress=None): pbar = lambda s, *args, **kwargs: s functions = list(extractor.get_functions()) + n_funcs = len(functions) - for f in pbar(functions, desc="matching", unit=" functions"): + pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions") + for f in pb: function_address = int(f) if extractor.is_library_function(function_address): function_name = extractor.get_function_name(function_address) logger.debug("skipping library function 0x%x (%s)", function_address, function_name) + meta["library_functions"][function_address] = function_name + n_libs = len(meta["library_functions"]) + percentage = 100 * (n_libs / n_funcs) + if isinstance(pb, tqdm.tqdm): + pb.set_postfix_str("skipped %d library functions (%d%%)" % (n_libs, percentage)) continue function_matches, bb_matches, feature_count = find_function_capabilities(ruleset, extractor, f) @@ -494,7 +502,7 @@ def get_rules(rule_path, disable_progress=False): # to disable progress completely pbar = lambda s, *args, **kwargs: s - for rule_path in pbar(list(rule_paths), desc="loading ", unit=" rules"): + for rule_path in pbar(list(rule_paths), desc="loading ", unit=" rules"): try: rule = capa.rules.Rule.from_yaml_file(rule_path) except capa.rules.InvalidRule: @@ -778,7 +786,7 @@ def main(argv=None): logger.debug("using rules path: %s", rules_path) try: - rules = get_rules(rules_path, disable_progress=args.quiet) + rules = get_rules(rules_path, disable_progress=args.quiet or args.tag) rules = capa.rules.RuleSet(rules) logger.debug( "successfully loaded %s rules", @@ -788,9 +796,15 @@ def main(argv=None): len([i for i in filter(lambda r: "capa/subscope-rule" not in r.meta, rules.rules.values())]), ) if args.tag: + n_rules_all = len(rules) rules = rules.filter_rules_by_meta(args.tag) - logger.debug("selected %s rules", len(rules)) + n_rules = len(rules) + diff = n_rules_all - n_rules + if not args.quiet: + for _ in tqdm.trange(n_rules, desc="loading ", unit=" rules", postfix="skipped %d rules" % diff): + pass for i, r in enumerate(rules.rules, 1): + logger.debug("selected %d rules", len(rules)) # TODO don't display subscope rules? logger.debug(" %d. %s", i, r) except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e: diff --git a/capa/render/verbose.py b/capa/render/verbose.py index 9f43d439..d07845b2 100644 --- a/capa/render/verbose.py +++ b/capa/render/verbose.py @@ -57,6 +57,7 @@ def render_meta(ostream, doc): ("base address", hex(doc["meta"]["analysis"]["base_address"])), ("rules", doc["meta"]["analysis"]["rules"]), ("function count", len(doc["meta"]["analysis"]["feature_counts"]["functions"])), + ("library function count", len(doc["meta"]["analysis"]["library_functions"])), ( "total feature count", doc["meta"]["analysis"]["feature_counts"]["file"] diff --git a/tests/test_main.py b/tests/test_main.py index cde291d1..03f755c4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -25,6 +25,7 @@ def test_main(z9324d_extractor): assert capa.main.main([path, "-vv"]) == 0 assert capa.main.main([path, "-v"]) == 0 assert capa.main.main([path, "-j"]) == 0 + assert capa.main.main([path, "-q"]) == 0 assert capa.main.main([path]) == 0 @@ -79,6 +80,7 @@ def test_main_shellcode(z499c2_extractor): assert capa.main.main([path, "-vv", "-f", "sc32"]) == 0 assert capa.main.main([path, "-v", "-f", "sc32"]) == 0 assert capa.main.main([path, "-j", "-f", "sc32"]) == 0 + assert capa.main.main([path, "-q", "-f", "sc32"]) == 0 assert capa.main.main([path, "-f", "sc32"]) == 0 # auto detect shellcode based on file extension assert capa.main.main([path]) == 0 From a8e353fe3134f5751bd3dc4c07545cf6ef34c0ae Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Thu, 20 May 2021 14:00:01 +0200 Subject: [PATCH 2/2] revert rule loading pbar --- CHANGELOG.md | 2 +- capa/main.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a391db12..27964fc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - explorer: IDA 7.6 support #497 @williballenthin - scripts: capa2yara.py convert capa rules to YARA rules #561 @ruppde - main: auto detect shellcode based on file extension #516 @mr-tz -- main: more detailed progress bar output when loading rules and matching functions #562 @mr-tz +- main: more detailed progress bar output when matching functions #562 @mr-tz ### New Rules (69) diff --git a/capa/main.py b/capa/main.py index 3c1bacd3..0da06ea6 100644 --- a/capa/main.py +++ b/capa/main.py @@ -786,7 +786,7 @@ def main(argv=None): logger.debug("using rules path: %s", rules_path) try: - rules = get_rules(rules_path, disable_progress=args.quiet or args.tag) + rules = get_rules(rules_path, disable_progress=args.quiet) rules = capa.rules.RuleSet(rules) logger.debug( "successfully loaded %s rules", @@ -796,13 +796,7 @@ def main(argv=None): len([i for i in filter(lambda r: "capa/subscope-rule" not in r.meta, rules.rules.values())]), ) if args.tag: - n_rules_all = len(rules) rules = rules.filter_rules_by_meta(args.tag) - n_rules = len(rules) - diff = n_rules_all - n_rules - if not args.quiet: - for _ in tqdm.trange(n_rules, desc="loading ", unit=" rules", postfix="skipped %d rules" % diff): - pass for i, r in enumerate(rules.rules, 1): logger.debug("selected %d rules", len(rules)) # TODO don't display subscope rules?