gitea-mirror/capa
1
0
Fork 0
mirror of https://github.com/mandiant/capa.git synced 2025-12-21 14:50:33 -08:00
Code Issues Packages Projects Releases Wiki Activity

introduce flake8-comprehensions

Browse Source
This commit is contained in:
Willi Ballenthin
2023-07-06 20:04:27 +02:00
parent ff47270681
commit 13a8e252f0
22 changed files with 64 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
capa/features/common.py
View File

@@ -270,7 +270,7 @@ class _MatchedSubstring(Substring):
self.matches = matches
def __str__(self):
matches = ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys()))
matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
assert isinstance(self.value, str)
return f'substring("{self.value}", matches = {matches})'
@@ -362,7 +362,7 @@ class _MatchedRegex(Regex):
self.matches = matches
def __str__(self):
matches = ", ".join(map(lambda s: '"' + s + '"', (self.matches or {}).keys()))
matches = ", ".join(f'"{s}"' for s in (self.matches or {}).keys())
assert isinstance(self.value, str)
return f"regex(string =~ {self.value}, matches = {matches})"

4
capa/features/extractors/elf.py
View File

@@ -824,7 +824,7 @@ def guess_os_from_abi_versions_needed(elf: ELF) -> Optional[OS]:
# this will let us guess about linux/hurd in some cases.
versions_needed = elf.versions_needed
if any(map(lambda abi: abi.startswith("GLIBC"), itertools.chain(*versions_needed.values()))):
if any(abi.startswith("GLIBC") for abi in itertools.chain(*versions_needed.values())):
# there are any GLIBC versions needed
if elf.e_machine != "i386":
@@ -881,7 +881,7 @@ def guess_os_from_symtab(elf: ELF) -> Optional[OS]:
sym_name = symtab.get_name(symbol)
for os, hints in keywords.items():
if any(map(lambda x: x in sym_name, hints)):
if any(hint in sym_name for hint in hints):
return os
return None

4
capa/features/extractors/ida/insn.py
View File

@@ -398,7 +398,7 @@ def extract_insn_peb_access_characteristic_features(
if insn.itype not in (idaapi.NN_push, idaapi.NN_mov):
return
if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
if all(op.type != idaapi.o_mem for op in insn.ops):
# try to optimize for only memory references
return
@@ -419,7 +419,7 @@ def extract_insn_segment_access_features(
"""
insn: idaapi.insn_t = ih.inner
if all(map(lambda op: op.type != idaapi.o_mem, insn.ops)):
if all(op.type != idaapi.o_mem for op in insn.ops):
# try to optimize for only memory references
return

2
capa/features/extractors/viv/indirect_calls.py
View File

@@ -88,7 +88,7 @@ def find_definition(vw: VivWorkspace, va: int, reg: int) -> Tuple[int, Union[int
NotFoundError: when the definition cannot be found.
"""
q: Deque[int] = collections.deque()
seen: Set[int] = set([])
seen: Set[int] = set()
q.extend(get_previous_instructions(vw, va))
while q:

6
capa/ida/helpers.py
View File

@@ -159,15 +159,15 @@ def collect_metadata(rules):
rules=rules,
base_address=capa.features.freeze.Address.from_capa(idaapi.get_imagebase()),
layout=rdoc.Layout(
functions=tuple()
functions=(),
# this is updated after capabilities have been collected.
# will look like:
#
# "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
),
# ignore these for now - not used by IDA plugin.
feature_counts=rdoc.FeatureCounts(file=0, functions=tuple()),
library_functions=tuple(),
feature_counts=rdoc.FeatureCounts(file=0, functions=()),
library_functions=(),
),
)

2
capa/ida/plugin/model.py
View File

@@ -426,7 +426,7 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
# optional statement with no successful children is empty
if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
if not any(map(lambda m: m.success, match.children)):
if not any(m.success for m in match.children):
return
if isinstance(match.node, rd.StatementNode):

4
capa/ida/plugin/view.py
View File

@@ -64,7 +64,7 @@ def parse_yaml_line(feature):
feature, _, comment = feature.partition("#")
feature, _, description = feature.partition("=")
return map(lambda o: o.strip(), (feature, description, comment))
return (o.strip() for o in (feature, description, comment))
def parse_node_for_feature(feature, description, comment, depth):
@@ -499,7 +499,7 @@ class CapaExplorerRulegenEditor(QtWidgets.QTreeWidget):
rule_text += "\n features:\n"
for o in iterate_tree(self):
feature, description, comment = map(lambda o: o.strip(), tuple(o.text(i) for i in range(3)))
feature, description, comment = (o.strip() for o in tuple(o.text(i) for i in range(3)))
rule_text += parse_node_for_feature(feature, description, comment, calc_item_depth(o))
# FIXME we avoid circular update by disabling signals when updating

21
capa/main.py
View File

@@ -249,8 +249,8 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
all_bb_matches = collections.defaultdict(list) # type: MatchResults
all_insn_matches = collections.defaultdict(list) # type: MatchResults
feature_counts = rdoc.FeatureCounts(file=0, functions=tuple())
library_functions: Tuple[rdoc.LibraryFunction, ...] = tuple()
feature_counts = rdoc.FeatureCounts(file=0, functions=())
library_functions: Tuple[rdoc.LibraryFunction, ...] = ()
with redirecting_print_to_tqdm(disable_progress):
with tqdm.contrib.logging.logging_redirect_tqdm():
@@ -299,16 +299,15 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
for rule_name, results in itertools.chain(
all_function_matches.items(), all_bb_matches.items(), all_insn_matches.items()
):
locations = set(map(lambda p: p[0], results))
locations = {p[0] for p in results}
rule = ruleset[rule_name]
capa.engine.index_rule_matches(function_and_lower_features, rule, locations)
all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, function_and_lower_features)
feature_counts.file = feature_count
matches = {
rule_name: results
for rule_name, results in itertools.chain(
matches = dict(
itertools.chain(
# each rule exists in exactly one scope,
# so there won't be any overlap among these following MatchResults,
# and we can merge the dictionaries naively.
@@ -317,7 +316,7 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
all_function_matches.items(),
all_file_matches.items(),
)
}
)
meta = {
"feature_counts": feature_counts,
@@ -589,7 +588,7 @@ def get_extractor(
def get_file_extractors(sample: str, format_: str) -> List[FeatureExtractor]:
file_extractors: List[FeatureExtractor] = list()
file_extractors: List[FeatureExtractor] = []
if format_ == FORMAT_PE:
file_extractors.append(capa.features.extractors.pefile.PefileFeatureExtractor(sample))
@@ -785,14 +784,14 @@ def collect_metadata(
rules=tuple(rules_path),
base_address=frz.Address.from_capa(extractor.get_base_address()),
layout=rdoc.Layout(
functions=tuple(),
functions=(),
# this is updated after capabilities have been collected.
# will look like:
#
# "functions": { 0x401000: { "matched_basic_blocks": [ 0x401000, 0x401005, ... ] }, ... }
),
feature_counts=rdoc.FeatureCounts(file=0, functions=tuple()),
library_functions=tuple(),
feature_counts=rdoc.FeatureCounts(file=0, functions=()),
library_functions=(),
),
)

2
capa/render/default.py
View File

@@ -48,7 +48,7 @@ def find_subrule_matches(doc: rd.ResultDocument):
collect the rule names that have been matched as a subrule match.
this way we can avoid displaying entries for things that are too specific.
"""
matches = set([])
matches = set()
def rec(match: rd.Match):
if not match.success:

2
capa/render/proto/__init__.py
View File

@@ -39,7 +39,7 @@ from capa.features.freeze import AddressType
def dict_tuple_to_list_values(d: Dict) -> Dict:
o = dict()
o = {}
for k, v in d.items():
if isinstance(v, tuple):
o[k] = list(v)

4
capa/render/result_document.py
View File

@@ -302,7 +302,7 @@ class Match(FrozenModel):
# pull matches from the referenced rule into our tree here.
rule_name = name
rule = rules[rule_name]
rule_matches = {address: result for (address, result) in capabilities[rule_name]}
rule_matches = dict(capabilities[rule_name])
if rule.is_subscope_rule():
# for a subscope rule, fixup the node to be a scope node, rather than a match feature node.
@@ -347,7 +347,7 @@ class Match(FrozenModel):
# we could introduce an intermediate node here.
# this would be a breaking change and require updates to the renderers.
# in the meantime, the above might be sufficient.
rule_matches = {address: result for (address, result) in capabilities[rule.name]}
rule_matches = dict(capabilities[rule.name])
for location in result.locations:
# doc[locations] contains all matches for the given namespace.
# for example, the feature might be `match: anti-analysis/packer`

2
capa/render/utils.py
View File

@@ -37,7 +37,7 @@ def format_parts_id(data: Union[rd.AttackSpec, rd.MBCSpec]):
def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]:
"""enumerate the rules in (namespace, name) order that are 'capability' rules (not lib/subscope/disposition/etc)."""
for _, _, rule in sorted(map(lambda rule: (rule.meta.namespace or "", rule.meta.name, rule), doc.rules.values())):
for _, _, rule in sorted((rule.meta.namespace or "", rule.meta.name, rule) for rule in doc.rules.values()):
if rule.meta.lib:
continue
if rule.meta.is_subscope_rule:

5
capa/render/verbose.py
View File

@@ -96,8 +96,7 @@ def render_meta(ostream, doc: rd.ResultDocument):
("library function count", len(doc.meta.analysis.library_functions)),
(
"total feature count",
doc.meta.analysis.feature_counts.file
+ sum(map(lambda f: f.count, doc.meta.analysis.feature_counts.functions)),
doc.meta.analysis.feature_counts.file + sum(f.count for f in doc.meta.analysis.feature_counts.functions),
),
]
@@ -141,7 +140,7 @@ def render_rules(ostream, doc: rd.ResultDocument):
rows.append((key, v))
if rule.meta.scope != capa.rules.FILE_SCOPE:
locations = list(map(lambda m: m[0], doc.rules[rule.meta.name].matches))
locations = [m[0] for m in doc.rules[rule.meta.name].matches]
rows.append(("matches", "\n".join(map(format_address, locations))))
ostream.writeln(tabulate.tabulate(rows, tablefmt="plain"))

8
capa/render/vverbose.py
View File

@@ -29,7 +29,7 @@ def render_locations(ostream, locations: Iterable[frz.Address]):
# its possible to have an empty locations array here,
# such as when we're in MODE_FAILURE and showing the logic
# under a `not` statement (which will have no matched locations).
locations = list(sorted(locations))
locations = sorted(locations)
if len(locations) == 0:
return
@@ -222,7 +222,7 @@ def render_match(ostream, match: rd.Match, indent=0, mode=MODE_SUCCESS):
# optional statement with no successful children is empty
if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
if not any(map(lambda m: m.success, match.children)):
if not any(m.success for m in match.children):
return
# not statement, so invert the child mode to show failed evaluations
@@ -236,7 +236,7 @@ def render_match(ostream, match: rd.Match, indent=0, mode=MODE_SUCCESS):
# optional statement with successful children is not relevant
if isinstance(match.node, rd.StatementNode) and match.node.statement.type == rd.CompoundStatementType.OPTIONAL:
if any(map(lambda m: m.success, match.children)):
if any(m.success for m in match.children):
return
# not statement, so invert the child mode to show successful evaluations
@@ -277,7 +277,7 @@ def render_rules(ostream, doc: rd.ResultDocument):
had_match = False
for _, _, rule in sorted(map(lambda rule: (rule.meta.namespace or "", rule.meta.name, rule), doc.rules.values())):
for _, _, rule in sorted((rule.meta.namespace or "", rule.meta.name, rule) for rule in doc.rules.values()):
# default scope hides things like lib rules, malware-category rules, etc.
# but in vverbose mode, we really want to show everything.
#

16
capa/rules/__init__.py
View File

@@ -634,7 +634,7 @@ class Rule:
Returns:
List[str]: names of rules upon which this rule depends.
"""
deps: Set[str] = set([])
deps: Set[str] = set()
def rec(statement):
if isinstance(statement, capa.features.common.MatchedRule):
@@ -648,7 +648,7 @@ class Rule:
# but, namespaces tend to use `-` while rule names use ` `. so, unlikely, but possible.
if statement.value in namespaces:
# matches a namespace, so take precedence and don't even check rule names.
deps.update(map(lambda r: r.name, namespaces[statement.value]))
deps.update(r.name for r in namespaces[statement.value])
else:
# not a namespace, assume its a rule name.
assert isinstance(statement.value, str)
@@ -950,7 +950,7 @@ def get_rules_with_scope(rules, scope) -> List[Rule]:
from the given collection of rules, select those with the given scope.
`scope` is one of the capa.rules.*_SCOPE constants.
"""
return list(rule for rule in rules if rule.scope == scope)
return [rule for rule in rules if rule.scope == scope]
def get_rules_and_dependencies(rules: List[Rule], rule_name: str) -> Iterator[Rule]:
@@ -961,7 +961,7 @@ def get_rules_and_dependencies(rules: List[Rule], rule_name: str) -> Iterator[Ru
rules = list(rules)
namespaces = index_rules_by_namespace(rules)
rules_by_name = {rule.name: rule for rule in rules}
wanted = set([rule_name])
wanted = {rule_name}
def rec(rule):
wanted.add(rule.name)
@@ -976,7 +976,7 @@ def get_rules_and_dependencies(rules: List[Rule], rule_name: str) -> Iterator[Ru
def ensure_rules_are_unique(rules: List[Rule]) -> None:
seen = set([])
seen = set()
for rule in rules:
if rule.name in seen:
raise InvalidRule("duplicate rule name: " + rule.name)
@@ -1041,7 +1041,7 @@ def topologically_order_rules(rules: List[Rule]) -> List[Rule]:
rules = list(rules)
namespaces = index_rules_by_namespace(rules)
rules_by_name = {rule.name: rule for rule in rules}
seen = set([])
seen = set()
ret = []
def rec(rule):
@@ -1284,7 +1284,7 @@ class RuleSet:
don't include auto-generated "subscope" rules.
we want to include general "lib" rules here - even if they are not dependencies of other rules, see #398
"""
scope_rules: Set[Rule] = set([])
scope_rules: Set[Rule] = set()
# we need to process all rules, not just rules with the given scope.
# this is because rules with a higher scope, e.g. file scope, may have subscope rules
@@ -1329,7 +1329,7 @@ class RuleSet:
TODO support -t=metafield <k>
"""
rules = list(self.rules.values())
rules_filtered = set([])
rules_filtered = set()
for rule in rules:
for k, v in rule.meta.items():
if isinstance(v, str) and tag in v:

2
capa/rules/cache.py
View File

@@ -28,7 +28,7 @@ def compute_cache_identifier(rule_content: List[bytes]) -> CacheIdentifier:
hash.update(version.encode("utf-8"))
hash.update(b"\x00")
rule_hashes = list(sorted([hashlib.sha256(buf).hexdigest() for buf in rule_content]))
rule_hashes = sorted([hashlib.sha256(buf).hexdigest() for buf in rule_content])
for rule_hash in rule_hashes:
hash.update(rule_hash.encode("ascii"))
hash.update(b"\x00")

12
scripts/capa_as_library.py
View File

@@ -29,7 +29,7 @@ def find_subrule_matches(doc: rd.ResultDocument) -> Set[str]:
collect the rule names that have been matched as a subrule match.
this way we can avoid displaying entries for things that are too specific.
"""
matches = set([])
matches = set()
def rec(node: rd.Match):
if not node.success:
@@ -65,7 +65,7 @@ def render_capabilities(doc: rd.ResultDocument, result):
"""
subrule_matches = find_subrule_matches(doc)
result["CAPABILITY"] = dict()
result["CAPABILITY"] = {}
for rule in rutils.capability_rules(doc):
if rule.meta.name in subrule_matches:
# rules that are also matched by other rules should not get rendered by default.
@@ -79,7 +79,7 @@ def render_capabilities(doc: rd.ResultDocument, result):
else:
capability = f"{rule.meta.name} ({count} matches)"
result["CAPABILITY"].setdefault(rule.meta.namespace, list())
result["CAPABILITY"].setdefault(rule.meta.namespace, [])
result["CAPABILITY"][rule.meta.namespace].append(capability)
@@ -96,7 +96,7 @@ def render_attack(doc, result):
'EXECUTION': ['Shared Modules [T1129]']}
}
"""
result["ATTCK"] = dict()
result["ATTCK"] = {}
tactics = collections.defaultdict(set)
for rule in rutils.capability_rules(doc):
if not rule.meta.attack:
@@ -129,7 +129,7 @@ def render_mbc(doc, result):
'[C0021.004]']}
}
"""
result["MBC"] = dict()
result["MBC"] = {}
objectives = collections.defaultdict(set)
for rule in rutils.capability_rules(doc):
if not rule.meta.mbc:
@@ -149,7 +149,7 @@ def render_mbc(doc, result):
def render_dictionary(doc: rd.ResultDocument) -> Dict[str, Any]:
result: Dict[str, Any] = dict()
result: Dict[str, Any] = {}
render_meta(doc, result)
render_attack(doc, result)
render_mbc(doc, result)

2
scripts/detect_duplicate_features.py
View File

@@ -73,7 +73,7 @@ def find_overlapping_rules(new_rule_path, rules_path):
continue
count += 1
# Checks if any features match between existing and new rule.
if any([feature in rule_features for feature in new_rule_features]):
if any(feature in rule_features for feature in new_rule_features):
overlapping_rules.append(rule_name)
result = {"overlapping_rules": overlapping_rules, "count": count}

2
scripts/lint.py
View File

@@ -825,7 +825,7 @@ def lint_rule(ctx: Context, rule: Rule):
print("")
if is_nursery_rule(rule):
has_examples = not any(map(lambda v: v.level == Lint.FAIL and v.name == "missing examples", violations))
has_examples = not any(v.level == Lint.FAIL and v.name == "missing examples" for v in violations)
lints_failed = len(
tuple(
filter(

12
tests/test_freeze.py
View File

@@ -61,7 +61,7 @@ EXTRACTOR = capa.features.extractors.null.NullFeatureExtractor(
def addresses(s) -> List[Address]:
return list(sorted(map(lambda i: i.address, s)))
return sorted(i.address for i in s)
def test_null_feature_extractor():
@@ -104,17 +104,17 @@ def compare_extractors(a, b):
assert addresses(a.get_functions()) == addresses(b.get_functions())
for f in a.get_functions():
assert addresses(a.get_basic_blocks(f)) == addresses(b.get_basic_blocks(f))
assert list(sorted(set(a.extract_function_features(f)))) == list(sorted(set(b.extract_function_features(f))))
assert sorted(set(a.extract_function_features(f))) == sorted(set(b.extract_function_features(f)))
for bb in a.get_basic_blocks(f):
assert addresses(a.get_instructions(f, bb)) == addresses(b.get_instructions(f, bb))
assert list(sorted(set(a.extract_basic_block_features(f, bb)))) == list(
sorted(set(b.extract_basic_block_features(f, bb)))
assert sorted(set(a.extract_basic_block_features(f, bb))) == sorted(
set(b.extract_basic_block_features(f, bb))
)
for insn in a.get_instructions(f, bb):
assert list(sorted(set(a.extract_insn_features(f, bb, insn)))) == list(
sorted(set(b.extract_insn_features(f, bb, insn)))
assert sorted(set(a.extract_insn_features(f, bb, insn))) == sorted(
set(b.extract_insn_features(f, bb, insn))
)

10
tests/test_main.py
View File

@@ -359,7 +359,7 @@ def test_instruction_scope(z9324d_extractor):
)
capabilities, meta = capa.main.find_capabilities(rules, z9324d_extractor)
assert "push 1000" in capabilities
assert 0x4071A4 in set(map(lambda result: result[0], capabilities["push 1000"]))
assert 0x4071A4 in {result[0] for result in capabilities["push 1000"]}
def test_instruction_subscope(z9324d_extractor):
@@ -389,7 +389,7 @@ def test_instruction_subscope(z9324d_extractor):
)
capabilities, meta = capa.main.find_capabilities(rules, z9324d_extractor)
assert "push 1000 on i386" in capabilities
assert 0x406F60 in set(map(lambda result: result[0], capabilities["push 1000 on i386"]))
assert 0x406F60 in {result[0] for result in capabilities["push 1000 on i386"]}
def test_fix262(pma16_01_extractor, capsys):
@@ -433,9 +433,9 @@ def test_json_meta(capsys):
std = capsys.readouterr()
std_json = json.loads(std.out)
assert {"type": "absolute", "value": 0x10001010} in list(
map(lambda f: f["address"], std_json["meta"]["analysis"]["layout"]["functions"])
)
assert {"type": "absolute", "value": 0x10001010} in [
f["address"] for f in std_json["meta"]["analysis"]["layout"]["functions"]
]
for addr, info in std_json["meta"]["analysis"]["layout"]["functions"]:
if addr == ["absolute", 0x10001010]:

4
tests/test_rules.py
View File

@@ -874,12 +874,12 @@ def test_rules_namespace_dependencies():
),
]
r3 = set(map(lambda r: r.name, capa.rules.get_rules_and_dependencies(rules, "rule 3")))
r3 = {r.name for r in capa.rules.get_rules_and_dependencies(rules, "rule 3")}
assert "rule 1" in r3
assert "rule 2" not in r3
assert "rule 4" not in r3
r4 = set(map(lambda r: r.name, capa.rules.get_rules_and_dependencies(rules, "rule 4")))
r4 = {r.name for r in capa.rules.get_rules_and_dependencies(rules, "rule 4")}
assert "rule 1" in r4
assert "rule 2" in r4
assert "rule 3" not in r4