Merge pull request #562 from fireeye/lib-meta-info

improve progress bar output

CHANGELOG.md

@@ -7,10 +7,11 @@ It includes many new rules, including all new techniques introduced in MITRE ATT
 
 ### New Features
 
-- main: auto detect shellcode based on file extension #516 @mr-tz
 - main: use FLIRT signatures to identify and ignore library code #446 @williballenthin
 - explorer: IDA 7.6 support #497 @williballenthin
 - scripts: capa2yara.py convert capa rules to YARA rules #561 @ruppde
+- main: auto detect shellcode based on file extension #516 @mr-tz
+- main: more detailed progress bar output when matching functions #562 @mr-tz
 
 ### New Rules (70)
 
@@ -98,6 +99,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT
 - rules: update ATT&CK and MBC mappings https://github.com/fireeye/capa-rules/pull/317 @williballenthin
 - tests: update test cases and caching #545 @mr-tz
 - show-features: don't show features from library functions #569 @williballenthin
+- meta: added `library_functions` field, `feature_counts.functions` does not include library functions any more #562 @mr-tz
 
 ### Development
 

capa/main.py

@@ -128,7 +128,8 @@ def find_capabilities(ruleset, extractor, disable_progress=None):
         "feature_counts": {
             "file": 0,
             "functions": {},
-        }
+        },
+        "library_functions": {},
     }
 
     pbar = tqdm.tqdm
@@ -138,13 +139,20 @@ def find_capabilities(ruleset, extractor, disable_progress=None):
         pbar = lambda s, *args, **kwargs: s
 
     functions = list(extractor.get_functions())
+    n_funcs = len(functions)
 
-    for f in pbar(functions, desc="matching", unit=" functions"):
+    pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions")
+    for f in pb:
         function_address = int(f)
 
         if extractor.is_library_function(function_address):
             function_name = extractor.get_function_name(function_address)
             logger.debug("skipping library function 0x%x (%s)", function_address, function_name)
+            meta["library_functions"][function_address] = function_name
+            n_libs = len(meta["library_functions"])
+            percentage = 100 * (n_libs / n_funcs)
+            if isinstance(pb, tqdm.tqdm):
+                pb.set_postfix_str("skipped %d library functions (%d%%)" % (n_libs, percentage))
             continue
 
         function_matches, bb_matches, feature_count = find_function_capabilities(ruleset, extractor, f)
@@ -494,7 +502,7 @@ def get_rules(rule_path, disable_progress=False):
         # to disable progress completely
         pbar = lambda s, *args, **kwargs: s
 
-    for rule_path in pbar(list(rule_paths), desc="loading ", unit="     rules"):
+    for rule_path in pbar(list(rule_paths), desc="loading ", unit=" rules"):
         try:
             rule = capa.rules.Rule.from_yaml_file(rule_path)
         except capa.rules.InvalidRule:
@@ -789,8 +797,8 @@ def main(argv=None):
         )
         if args.tag:
             rules = rules.filter_rules_by_meta(args.tag)
-            logger.debug("selected %s rules", len(rules))
             for i, r in enumerate(rules.rules, 1):
+                logger.debug("selected %d rules", len(rules))
                 # TODO don't display subscope rules?
                 logger.debug(" %d. %s", i, r)
     except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e:

capa/render/verbose.py

@@ -57,6 +57,7 @@ def render_meta(ostream, doc):
         ("base address", hex(doc["meta"]["analysis"]["base_address"])),
         ("rules", doc["meta"]["analysis"]["rules"]),
         ("function count", len(doc["meta"]["analysis"]["feature_counts"]["functions"])),
+        ("library function count", len(doc["meta"]["analysis"]["library_functions"])),
         (
             "total feature count",
             doc["meta"]["analysis"]["feature_counts"]["file"]

tests/test_main.py

@@ -25,6 +25,7 @@ def test_main(z9324d_extractor):
     assert capa.main.main([path, "-vv"]) == 0
     assert capa.main.main([path, "-v"]) == 0
     assert capa.main.main([path, "-j"]) == 0
+    assert capa.main.main([path, "-q"]) == 0
     assert capa.main.main([path]) == 0
 
 
@@ -79,6 +80,7 @@ def test_main_shellcode(z499c2_extractor):
     assert capa.main.main([path, "-vv", "-f", "sc32"]) == 0
     assert capa.main.main([path, "-v", "-f", "sc32"]) == 0
     assert capa.main.main([path, "-j", "-f", "sc32"]) == 0
+    assert capa.main.main([path, "-q", "-f", "sc32"]) == 0
     assert capa.main.main([path, "-f", "sc32"]) == 0
     # auto detect shellcode based on file extension
     assert capa.main.main([path]) == 0