From 4929d5936ee82c1409393fcf02c167095ab47f02 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 26 Jun 2024 16:03:45 +0200 Subject: [PATCH 01/14] Update macos 12 (#2174) * update CI to use macos-12 instead of macos-11 --- .github/workflows/build.yml | 2 +- .github/workflows/tests.yml | 2 +- CHANGELOG.md | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85b898b4..691850ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,7 @@ jobs: artifact_name: capa.exe asset_name: windows python_version: 3.8 - - os: macos-11 + - os: macos-12 # use older macOS for assumed better portability artifact_name: capa asset_name: macos diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eb8ec1ce..84c4f2fe 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -76,7 +76,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, windows-2019, macos-11] + os: [ubuntu-20.04, windows-2019, macos-12] # across all operating systems python-version: ["3.8", "3.11"] include: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c64822e..531aaa75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ ### capa explorer IDA Pro plugin ### Development +- CI: use macos-12 since macos-11 is deprecated and will be removed on June 28th, 2024 #2173 @mr-tz ### Raw diffs - [capa v7.1.0...master](https://github.com/mandiant/capa/compare/v7.1.0...master) From e68bcddfe0def63a96f73082c4d847907e1cfb27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:05:14 +0000 Subject: [PATCH 02/14] build(deps): bump psutil from 5.9.2 to 6.0.0 Bumps [psutil](https://github.com/giampaolo/psutil) from 5.9.2 to 6.0.0. - [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) - [Commits](https://github.com/giampaolo/psutil/compare/release-5.9.2...release-6.0.0) --- updated-dependencies: - dependency-name: psutil dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3afcb54d..cca16a46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,7 +162,7 @@ build = [ ] scripts = [ "jschema_to_python==1.2.3", - "psutil==5.9.2", + "psutil==6.0.0", "stix2==3.0.1", "sarif_om==1.0.4", "requests==2.31.0", From d79ea074f20e968d041db2f77fe627e1173469dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:23:05 +0000 Subject: [PATCH 03/14] build(deps): bump flake8 from 7.0.0 to 7.1.0 Bumps [flake8](https://github.com/pycqa/flake8) from 7.0.0 to 7.1.0. - [Commits](https://github.com/pycqa/flake8/compare/7.0.0...7.1.0) --- updated-dependencies: - dependency-name: flake8 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3afcb54d..48a161ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,7 +123,7 @@ dev = [ "pytest-sugar==1.0.0", "pytest-instafail==0.5.0", "pytest-cov==5.0.0", - "flake8==7.0.0", + "flake8==7.1.0", "flake8-bugbear==24.4.26", "flake8-encodings==0.5.1", "flake8-comprehensions==3.14.0", From 1ec680856d80a2cc2c7b7eaa8d3cf23449c7e40b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:24:17 +0000 Subject: [PATCH 04/14] build(deps): bump requests from 2.31.0 to 2.32.3 Bumps [requests](https://github.com/psf/requests) from 2.31.0 to 2.32.3. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.31.0...v2.32.3) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3afcb54d..62c7badc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,7 +165,7 @@ scripts = [ "psutil==5.9.2", "stix2==3.0.1", "sarif_om==1.0.4", - "requests==2.31.0", + "requests==2.32.3", ] [tool.deptry] From fd811d13875f7daccd6b651910d6ba181f20540e Mon Sep 17 00:00:00 2001 From: Yacine Elhamer Date: Mon, 1 Jul 2024 09:55:24 +0100 Subject: [PATCH 05/14] scripts/show-features.py: use extractor.get_process_name() interface for getting process name --- scripts/show-features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/show-features.py b/scripts/show-features.py index 9813a26d..30ad2a4b 100644 --- a/scripts/show-features.py +++ b/scripts/show-features.py @@ -227,13 +227,13 @@ def print_static_features(functions, extractor: StaticFeatureExtractor): def print_dynamic_features(processes, extractor: DynamicFeatureExtractor): for p in processes: - print(f"proc: {p.inner.process_name} (ppid={p.address.ppid}, pid={p.address.pid})") + print(f"proc: {extractor.get_process_name(p)} (ppid={p.address.ppid}, pid={p.address.pid})") for feature, addr in extractor.extract_process_features(p): if is_global_feature(feature): continue - print(f" proc: {p.inner.process_name}: {feature}") + print(f" proc: {extractor.get_process_name(p)}: {feature}") for t in extractor.get_threads(p): print(f" thread: {t.address.tid}") From 6de22a0264f1bbf4066240ecebdca25ced06cc53 Mon Sep 17 00:00:00 2001 From: Yacine Elhamer Date: Mon, 1 Jul 2024 10:34:19 +0100 Subject: [PATCH 06/14] show-features.py: fix process filtering bug --- scripts/show-features.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/show-features.py b/scripts/show-features.py index 30ad2a4b..e1bd87ba 100644 --- a/scripts/show-features.py +++ b/scripts/show-features.py @@ -171,8 +171,8 @@ def print_dynamic_analysis(extractor: DynamicFeatureExtractor, args): process_handles = tuple(extractor.get_processes()) if args.process: - process_handles = tuple(filter(lambda ph: ph.inner["name"] == args.process, process_handles)) - if args.process not in [ph.inner["name"] for ph in args.process]: + process_handles = tuple(filter(lambda ph: extractor.get_process_name(ph) == args.process, process_handles)) + if args.process not in [extractor.get_process_name(ph) for ph in process_handles]: print(f"{args.process} not a process") return -1 @@ -227,13 +227,13 @@ def print_static_features(functions, extractor: StaticFeatureExtractor): def print_dynamic_features(processes, extractor: DynamicFeatureExtractor): for p in processes: - print(f"proc: {extractor.get_process_name(p)} (ppid={p.address.ppid}, pid={p.address.pid})") + print(f"proc: {p.inner.process_name} (ppid={p.address.ppid}, pid={p.address.pid})") for feature, addr in extractor.extract_process_features(p): if is_global_feature(feature): continue - print(f" proc: {extractor.get_process_name(p)}: {feature}") + print(f" proc: {p.inner.process_name}: {feature}") for t in extractor.get_threads(p): print(f" thread: {t.address.tid}") From 0b70abca9374265e655d6bc1db7b0d73e157f3c1 Mon Sep 17 00:00:00 2001 From: Yacine Elhamer Date: Mon, 1 Jul 2024 12:03:12 +0100 Subject: [PATCH 07/14] show-features.py: add other usage of get_process_name() --- scripts/show-features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/show-features.py b/scripts/show-features.py index e1bd87ba..188aa974 100644 --- a/scripts/show-features.py +++ b/scripts/show-features.py @@ -227,13 +227,13 @@ def print_static_features(functions, extractor: StaticFeatureExtractor): def print_dynamic_features(processes, extractor: DynamicFeatureExtractor): for p in processes: - print(f"proc: {p.inner.process_name} (ppid={p.address.ppid}, pid={p.address.pid})") + print(f"proc: {extractor.get_process_name(p)} (ppid={p.address.ppid}, pid={p.address.pid})") for feature, addr in extractor.extract_process_features(p): if is_global_feature(feature): continue - print(f" proc: {p.inner.process_name}: {feature}") + print(f" proc: {extractor.get_process_name(p)}: {feature}") for t in extractor.get_threads(p): print(f" thread: {t.address.tid}") From 202b5ddae74e96efa3d441875c44cc0fdc170096 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:48:15 +0000 Subject: [PATCH 08/14] build(deps): bump ruff from 0.4.8 to 0.5.0 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.4.8 to 0.5.0. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/v0.4.8...0.5.0) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1367e69e..6baf172f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,7 +134,7 @@ dev = [ "flake8-simplify==0.21.0", "flake8-use-pathlib==0.3.0", "flake8-copyright==0.2.4", - "ruff==0.4.8", + "ruff==0.5.0", "black==24.4.2", "isort==5.13.2", "mypy==1.10.0", From cd5199f8735c82e148f6b9155cc92dad7fd0345c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:48:28 +0000 Subject: [PATCH 09/14] build(deps): bump flake8-comprehensions from 3.14.0 to 3.15.0 Bumps [flake8-comprehensions](https://github.com/adamchainz/flake8-comprehensions) from 3.14.0 to 3.15.0. - [Changelog](https://github.com/adamchainz/flake8-comprehensions/blob/main/CHANGELOG.rst) - [Commits](https://github.com/adamchainz/flake8-comprehensions/compare/3.14.0...3.15.0) --- updated-dependencies: - dependency-name: flake8-comprehensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1367e69e..5a355e2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,7 @@ dev = [ "flake8==7.1.0", "flake8-bugbear==24.4.26", "flake8-encodings==0.5.1", - "flake8-comprehensions==3.14.0", + "flake8-comprehensions==3.15.0", "flake8-logging-format==0.9.0", "flake8-no-implicit-concat==0.3.5", "flake8-print==5.0.0", From 3b165c3d8e4cfffe3b22aadc803f200f488d15d8 Mon Sep 17 00:00:00 2001 From: Yacine Elhamer Date: Mon, 1 Jul 2024 21:41:46 +0100 Subject: [PATCH 10/14] test:scripts.py: add tests for show-features.py process filtering --- tests/test_scripts.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 9bad3013..b3c45844 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -23,10 +23,21 @@ def get_script_path(s: str): return str(CD / ".." / "scripts" / s) -def get_file_path(): +def get_binary_file_path(): return str(CD / "data" / "9324d1a8ae37a36ae560c37448c9705a.exe_") +def get_report_file_path(): + return str( + CD + / "data" + / "dynamic" + / "cape" + / "v2.4" + / "fb7ade52dc5a1d6128b9c217114a46d0089147610f99f5122face29e429a1e74.json.gz" + ) + + def get_rules_path(): return str(CD / ".." / "rules") @@ -48,12 +59,13 @@ def get_rule_path(): pytest.param("lint.py", ["-t", "create directory", get_rules_path()]), # `create directory` rule has native and .NET example PEs pytest.param("lint.py", ["--thorough", "-t", "create directory", get_rules_path()]), - pytest.param("match-function-id.py", [get_file_path()]), - pytest.param("show-capabilities-by-function.py", [get_file_path()]), - pytest.param("show-features.py", [get_file_path()]), - pytest.param("show-features.py", ["-F", "0x407970", get_file_path()]), - pytest.param("show-unused-features.py", [get_file_path()]), - pytest.param("capa_as_library.py", [get_file_path()]), + pytest.param("match-function-id.py", [get_binary_file_path()]), + pytest.param("show-capabilities-by-function.py", [get_binary_file_path()]), + pytest.param("show-features.py", [get_binary_file_path()]), + pytest.param("show-features.py", ["-F", "0x407970", get_binary_file_path()]), + pytest.param("show-features.py", ["-P", "MicrosoftEdgeUpdate.exe", get_report_file_path]), + pytest.param("show-unused-features.py", [get_binary_file_path()]), + pytest.param("capa_as_library.py", [get_binary_file_path()]), ], ) def test_scripts(script, args): From fccb5338411a33ae9415b5243d1c25ccac4082e6 Mon Sep 17 00:00:00 2001 From: Yacine Elhamer Date: Mon, 1 Jul 2024 21:59:28 +0100 Subject: [PATCH 11/14] test/scripts.py: bugfix --- tests/test_scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_scripts.py b/tests/test_scripts.py index b3c45844..35bf5347 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -63,7 +63,7 @@ def get_rule_path(): pytest.param("show-capabilities-by-function.py", [get_binary_file_path()]), pytest.param("show-features.py", [get_binary_file_path()]), pytest.param("show-features.py", ["-F", "0x407970", get_binary_file_path()]), - pytest.param("show-features.py", ["-P", "MicrosoftEdgeUpdate.exe", get_report_file_path]), + pytest.param("show-features.py", ["-P", "MicrosoftEdgeUpdate.exe", get_report_file_path()]), pytest.param("show-unused-features.py", [get_binary_file_path()]), pytest.param("capa_as_library.py", [get_binary_file_path()]), ], From 5495a8555cfd205abb39b479b18622c6183e662a Mon Sep 17 00:00:00 2001 From: Ilyas Osman Date: Tue, 2 Jul 2024 10:53:41 +0300 Subject: [PATCH 12/14] Fix incomplete f-strings (#2188) * Fix incomplete f-strings * Fix incomplete f-strings * Apply black formatting to fix linting errors * Apply black formatting to fix linting errors --- capa/features/extractors/cape/global_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/capa/features/extractors/cape/global_.py b/capa/features/extractors/cape/global_.py index 62eeff20..b73e5ab4 100644 --- a/capa/features/extractors/cape/global_.py +++ b/capa/features/extractors/cape/global_.py @@ -48,7 +48,7 @@ def extract_format(report: CapeReport) -> Iterator[Tuple[Feature, Address]]: else: logger.warning("unknown file format, file command output: %s", report.target.file.type) raise ValueError( - "unrecognized file format from the CAPE report; output of file command: {report.target.file.type}" + f"unrecognized file format from the CAPE report; output of file command: {report.target.file.type}" ) @@ -73,7 +73,7 @@ def extract_os(report: CapeReport) -> Iterator[Tuple[Feature, Address]]: else: # if the operating system information is missing from the cape report, it's likely a bug logger.warning("unrecognized OS: %s", file_output) - raise ValueError("unrecognized OS from the CAPE report; output of file command: {file_output}") + raise ValueError(f"unrecognized OS from the CAPE report; output of file command: {file_output}") else: # the sample is shellcode logger.debug("unsupported file format, file command output: %s", file_output) From 120f34e8efb794cd5199ab6bdc87999dd26b05a6 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Tue, 2 Jul 2024 07:56:15 +0000 Subject: [PATCH 13/14] Sync capa-testfiles submodule --- tests/data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data b/tests/data index 3a769017..5c3b7a8d 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 3a7690178bcb05671bf4d33f3d117715272fe538 +Subproject commit 5c3b7a8da47cff8e45538da8a6d74d405b4f4537 From 76913af20b133721b5d5722b6120942ac8b19fc4 Mon Sep 17 00:00:00 2001 From: Maxime Berthault <2143487+Maxou56800@users.noreply.github.com> Date: Fri, 12 Jul 2024 12:25:19 +0200 Subject: [PATCH 14/14] Binary Ninja update and fix (#2205) * Fix binja warning (use of a deprecated API method) * Update binja plugin > Fix json openning and parsing > Fix base address * Fix code_style * lint black update --- capa/features/extractors/binja/extractor.py | 2 +- scripts/import-to-bn.py | 26 +++++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/capa/features/extractors/binja/extractor.py b/capa/features/extractors/binja/extractor.py index e8d42908..e542494a 100644 --- a/capa/features/extractors/binja/extractor.py +++ b/capa/features/extractors/binja/extractor.py @@ -28,7 +28,7 @@ from capa.features.extractors.base_extractor import ( class BinjaFeatureExtractor(StaticFeatureExtractor): def __init__(self, bv: binja.BinaryView): - super().__init__(hashes=SampleHashes.from_bytes(bv.file.raw.read(0, len(bv.file.raw)))) + super().__init__(hashes=SampleHashes.from_bytes(bv.file.raw.read(0, bv.file.raw.length))) self.bv = bv self.global_features: List[Tuple[Feature, Address]] = [] self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv)) diff --git a/scripts/import-to-bn.py b/scripts/import-to-bn.py index 7fefcd2f..b09780a6 100644 --- a/scripts/import-to-bn.py +++ b/scripts/import-to-bn.py @@ -69,7 +69,8 @@ def load_analysis(bv): return 0 binaryninja.log_info(f"Using capa file {path}") - doc = json.loads(path.read_bytes().decode("utf-8")) + with Path(path).open("r", encoding="utf-8") as file: + doc = json.load(file) if "meta" not in doc or "rules" not in doc: binaryninja.log_error("doesn't appear to be a capa report") @@ -83,20 +84,35 @@ def load_analysis(bv): binaryninja.log_error("sample mismatch") return -2 + # Retreive base address + capa_base_address = 0 + if "analysis" in doc["meta"] and "base_address" in doc["meta"]["analysis"]: + if doc["meta"]["analysis"]["base_address"]["type"] == "absolute": + capa_base_address = int(doc["meta"]["analysis"]["base_address"]["value"]) + rows = [] for rule in doc["rules"].values(): if rule["meta"].get("lib"): continue if rule["meta"].get("capa/subscope"): continue - if rule["meta"]["scope"] != "function": + if rule["meta"]["scopes"].get("static") != "function": continue name = rule["meta"]["name"] ns = rule["meta"].get("namespace", "") - for va in rule["matches"].keys(): - va = int(va) - rows.append((ns, name, va)) + for matches in rule["matches"]: + for match in matches: + if "type" not in match.keys(): + continue + if "value" not in match.keys(): + continue + va = match["value"] + # Substract va and CAPA base_address + va = int(va) - capa_base_address + # Add binja base address + va = va + bv.start + rows.append((ns, name, va)) # order by (namespace, name) so that like things show up together rows = sorted(rows)