From ff88393248b3a235309a1e18e7583b619ee9d220 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 18 May 2021 15:19:34 -0600 Subject: [PATCH 1/6] linter: summarize status at end closes #571 --- scripts/lint.py | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/scripts/lint.py b/scripts/lint.py index 893b8cfd..2f9db98b 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -528,7 +528,8 @@ def lint_rule(ctx, rule): print("") - lints_failed = any(map(lambda v: v.level == Lint.FAIL, violations)) + lints_failed = len(tuple(filter(lambda v: v.level == Lint.FAIL and not is_nursery_rule(rule), violations))) + lints_warned = len(tuple(filter(lambda v: v.level == Lint.WARN or (v.level == Lint.FAIL and is_nursery_rule(rule)), violations))) if not lints_failed and is_nursery_rule(rule): print("") @@ -536,7 +537,7 @@ def lint_rule(ctx, rule): print("%s %s: %s: %s" % (" ", Lint.WARN, "no lint failures", "Graduate the rule")) print("") - return lints_failed and not is_nursery_rule(rule) + return (lints_failed, lints_warned, is_nursery_rule(rule)) def lint(ctx, rules): @@ -546,15 +547,21 @@ def lint(ctx, rules): for each sample, record sample id of sha256, md5, and filename. see `collect_samples(path)`. rules (List[Rule]): the rules to lint. + + Returns: Dict[string, Tuple(int, int, bool)] + - # lints failed + - # lints warned + - is nursery rule """ - did_suggest_fix = False - for rule in rules.rules.values(): + ret = {} + + for name, rule in rules.rules.items(): if rule.meta.get("capa/subscope-rule", False): continue - did_suggest_fix = lint_rule(ctx, rule) or did_suggest_fix + ret[name] = lint_rule(ctx, rule) - return did_suggest_fix + return ret def collect_samples(path): @@ -651,16 +658,33 @@ def main(argv=None): "is_thorough": args.thorough, } - did_violate = lint(ctx, rules) + results_by_name = lint(ctx, rules) + failed_rules = [] + warned_rules = [] + for name, (fail_count, warn_count, is_nursery_rule) in results_by_name.items(): + if fail_count > 0: + failed_rules.append(name) + + if warn_count > 0 and not is_nursery_rule: + warned_rules.append(name) min, sec = divmod(time.time() - time0, 60) logger.debug("lints ran for ~ %02d:%02dm", min, sec) - if not did_violate: + if warned_rules: + print("rules with WARN:") + for warned_rule in sorted(warned_rules): + print(" - " + warned_rule) + print() + + if failed_rules: + print("rules with FAIL:") + for failed_rule in sorted(failed_rules): + print(" - " + failed_rule) + return 1 + else: logger.info("no lints failed, nice!") return 0 - else: - return 1 if __name__ == "__main__": From 2f2849dee0a12ae64ccc28b35d7cb842ee342610 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 18 May 2021 15:20:54 -0600 Subject: [PATCH 2/6] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cafeda7f..5ce50fcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - explorer: document IDA 7.6sp1 as alternative to the patch #536 @Ana06 - rules: update ATT&CK and MBC mappings https://github.com/fireeye/capa-rules/pull/317 @williballenthin - tests: update test cases and caching #545 @mr-tz +- linter: summarize results at the end #571 @williballenthin ### Development From 3eef034a943c57e83934a70729121c4ebcbcb338 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 19 May 2021 16:06:07 -0600 Subject: [PATCH 3/6] lint: better handling of nursery rule summary --- scripts/lint.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/scripts/lint.py b/scripts/lint.py index 2f9db98b..7dee395b 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -528,16 +528,23 @@ def lint_rule(ctx, rule): print("") - lints_failed = len(tuple(filter(lambda v: v.level == Lint.FAIL and not is_nursery_rule(rule), violations))) - lints_warned = len(tuple(filter(lambda v: v.level == Lint.WARN or (v.level == Lint.FAIL and is_nursery_rule(rule)), violations))) + if is_nursery_rule(rule): + has_examples = not any(map(lambda v: v.level == Lint.FAIL and v.name == "missing examples", violations)) + lints_failed = len( + tuple(filter(lambda v: v.level == Lint.FAIL and not v.name == "missing examples", violations)) + ) + lints_warned = len(tuple(filter(lambda v: v.level == Lint.WARN, violations))) - if not lints_failed and is_nursery_rule(rule): - print("") - print("%s%s" % (" (nursery) ", rule.name)) - print("%s %s: %s: %s" % (" ", Lint.WARN, "no lint failures", "Graduate the rule")) - print("") + if (not lints_failed) and has_examples: + print("") + print("%s%s" % (" (nursery) ", rule.name)) + print("%s %s: %s: %s" % (" ", Lint.WARN, "no lint failures", "Graduate the rule")) + print("") + else: + lints_failed = len(tuple(filter(lambda v: v.level == Lint.FAIL, violations))) + lints_warned = len(tuple(filter(lambda v: v.level == Lint.WARN, violations))) - return (lints_failed, lints_warned, is_nursery_rule(rule)) + return (lints_failed, lints_warned) def lint(ctx, rules): @@ -548,10 +555,9 @@ def lint(ctx, rules): see `collect_samples(path)`. rules (List[Rule]): the rules to lint. - Returns: Dict[string, Tuple(int, int, bool)] + Returns: Dict[string, Tuple(int, int)] - # lints failed - # lints warned - - is nursery rule """ ret = {} @@ -661,11 +667,11 @@ def main(argv=None): results_by_name = lint(ctx, rules) failed_rules = [] warned_rules = [] - for name, (fail_count, warn_count, is_nursery_rule) in results_by_name.items(): + for name, (fail_count, warn_count) in results_by_name.items(): if fail_count > 0: failed_rules.append(name) - if warn_count > 0 and not is_nursery_rule: + if warn_count > 0: warned_rules.append(name) min, sec = divmod(time.time() - time0, 60) From 0a1adb99e09486981070dc65ca8aebb823ec1dc3 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 19 May 2021 16:13:45 -0600 Subject: [PATCH 4/6] lint: cleanup handling of nursery rules further --- scripts/lint.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/lint.py b/scripts/lint.py index 7dee395b..cf025c39 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -531,11 +531,11 @@ def lint_rule(ctx, rule): if is_nursery_rule(rule): has_examples = not any(map(lambda v: v.level == Lint.FAIL and v.name == "missing examples", violations)) lints_failed = len( - tuple(filter(lambda v: v.level == Lint.FAIL and not v.name == "missing examples", violations)) + tuple(filter(lambda v: v.level == Lint.FAIL and not (v.name == "missing examples" or v.name == "referenced example doesn't exist"), violations)) ) - lints_warned = len(tuple(filter(lambda v: v.level == Lint.WARN, violations))) + lints_warned = len(tuple(filter(lambda v: v.level == Lint.WARN or (v.level == Lint.FAIL and v.name == "referenced example doesn't exist"), violations))) - if (not lints_failed) and has_examples: + if (not lints_failed) and (not lints_warned) and has_examples: print("") print("%s%s" % (" (nursery) ", rule.name)) print("%s %s: %s: %s" % (" ", Lint.WARN, "no lint failures", "Graduate the rule")) From 9b1400c23a66b8aa5d821742cca26ce459cab924 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 19 May 2021 16:14:37 -0600 Subject: [PATCH 5/6] pep8 --- scripts/lint.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/lint.py b/scripts/lint.py index 0d757429..5813c9af 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -557,9 +557,23 @@ def lint_rule(ctx, rule): if is_nursery_rule(rule): has_examples = not any(map(lambda v: v.level == Lint.FAIL and v.name == "missing examples", violations)) lints_failed = len( - tuple(filter(lambda v: v.level == Lint.FAIL and not (v.name == "missing examples" or v.name == "referenced example doesn't exist"), violations)) + tuple( + filter( + lambda v: v.level == Lint.FAIL + and not (v.name == "missing examples" or v.name == "referenced example doesn't exist"), + violations, + ) + ) + ) + lints_warned = len( + tuple( + filter( + lambda v: v.level == Lint.WARN + or (v.level == Lint.FAIL and v.name == "referenced example doesn't exist"), + violations, + ) + ) ) - lints_warned = len(tuple(filter(lambda v: v.level == Lint.WARN or (v.level == Lint.FAIL and v.name == "referenced example doesn't exist"), violations))) if (not lints_failed) and (not lints_warned) and has_examples: print("") From 0a203b54cd3f7751c46a10b25ce095366b039ce9 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Fri, 21 May 2021 11:13:48 -0600 Subject: [PATCH 6/6] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3566bb00..af01bf52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - rules: update ATT&CK and MBC mappings https://github.com/fireeye/capa-rules/pull/317 @williballenthin - tests: update test cases and caching #545 @mr-tz - show-features: don't show features from library functions #569 @williballenthin +- linter: summarize results at the end #571 @williballenthin ### Development