From 45c22a24a62ea349a250d923c9bb89f982bd375b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Apr 2022 14:12:38 +0000 Subject: [PATCH 01/30] build(deps-dev): bump types-requests from 2.27.16 to 2.27.19 Bumps [types-requests](https://github.com/python/typeshed) from 2.27.16 to 2.27.19. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9dea5043..147899e9 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ setuptools.setup( "types-tabulate==0.8.6", "types-termcolor==1.1.3", "types-psutil==5.8.20", - "types_requests==2.27.16", + "types_requests==2.27.19", ], }, zip_safe=False, From fd1785fe653c10378b8c2a139a4d9f8dcee20344 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Apr 2022 14:12:43 +0000 Subject: [PATCH 02/30] build(deps-dev): bump types-pyyaml from 6.0.5 to 6.0.6 Bumps [types-pyyaml](https://github.com/python/typeshed) from 6.0.5 to 6.0.6. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyyaml dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9dea5043..48768969 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ setuptools.setup( # type stubs for mypy "types-backports==0.1.3", "types-colorama==0.4.10", - "types-PyYAML==6.0.5", + "types-PyYAML==6.0.6", "types-tabulate==0.8.6", "types-termcolor==1.1.3", "types-psutil==5.8.20", From 65cf8509f91959b13d266163cca228cce938f83b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Apr 2022 15:48:15 +0000 Subject: [PATCH 03/30] build(deps-dev): bump types-colorama from 0.4.10 to 0.4.12 Bumps [types-colorama](https://github.com/python/typeshed) from 0.4.10 to 0.4.12. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-colorama dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 123c6d14..c7833c07 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ setuptools.setup( "requests==2.27.1", # type stubs for mypy "types-backports==0.1.3", - "types-colorama==0.4.10", + "types-colorama==0.4.12", "types-PyYAML==6.0.6", "types-tabulate==0.8.6", "types-termcolor==1.1.3", From 2226bf0faa0e2acb1cac3b429dd8663c143a4ed9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Apr 2022 15:48:17 +0000 Subject: [PATCH 04/30] build(deps-dev): bump types-psutil from 5.8.20 to 5.8.22 Bumps [types-psutil](https://github.com/python/typeshed) from 5.8.20 to 5.8.22. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-psutil dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 123c6d14..e1cafb1c 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ setuptools.setup( "types-PyYAML==6.0.6", "types-tabulate==0.8.6", "types-termcolor==1.1.3", - "types-psutil==5.8.20", + "types-psutil==5.8.22", "types_requests==2.27.19", ], }, From 308a47a7846a425d2804a22bb3f71861e4112379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Apr 2022 15:48:18 +0000 Subject: [PATCH 05/30] build(deps-dev): bump types-tabulate from 0.8.6 to 0.8.7 Bumps [types-tabulate](https://github.com/python/typeshed) from 0.8.6 to 0.8.7. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-tabulate dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 123c6d14..0d3d6357 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ setuptools.setup( "types-backports==0.1.3", "types-colorama==0.4.10", "types-PyYAML==6.0.6", - "types-tabulate==0.8.6", + "types-tabulate==0.8.7", "types-termcolor==1.1.3", "types-psutil==5.8.20", "types_requests==2.27.19", From 054bcc9cb8d63a8d8e5576adb7a36cab39c48128 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:13:26 +0000 Subject: [PATCH 06/30] build(deps-dev): bump pytest from 7.1.1 to 7.1.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.1.1 to 7.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.1.1...7.1.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c4be8605..f62fef88 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ setuptools.setup( install_requires=requirements, extras_require={ "dev": [ - "pytest==7.1.1", + "pytest==7.1.2", "pytest-sugar==0.9.4", "pytest-instafail==0.4.2", "pytest-cov==3.0.0", From 45738773ca8ac78640502d604190282d737e0c1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:13:28 +0000 Subject: [PATCH 07/30] build(deps-dev): bump types-pyyaml from 6.0.6 to 6.0.7 Bumps [types-pyyaml](https://github.com/python/typeshed) from 6.0.6 to 6.0.7. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyyaml dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c4be8605..d889a81d 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ setuptools.setup( # type stubs for mypy "types-backports==0.1.3", "types-colorama==0.4.12", - "types-PyYAML==6.0.6", + "types-PyYAML==6.0.7", "types-tabulate==0.8.7", "types-termcolor==1.1.3", "types-psutil==5.8.22", From e950932e434e5e2d2b6389ab0f36e3ef691e56b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:13:31 +0000 Subject: [PATCH 08/30] build(deps-dev): bump types-requests from 2.27.19 to 2.27.20 Bumps [types-requests](https://github.com/python/typeshed) from 2.27.19 to 2.27.20. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c4be8605..de4ec6e4 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ setuptools.setup( "types-tabulate==0.8.7", "types-termcolor==1.1.3", "types-psutil==5.8.22", - "types_requests==2.27.19", + "types_requests==2.27.20", ], }, zip_safe=False, From 0e18cea11ac334eac68acc6e0555da22448dcf8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 14:13:34 +0000 Subject: [PATCH 09/30] build(deps): bump viv-utils[flirt] from 0.6.11 to 0.7.1 Bumps [viv-utils[flirt]](https://github.com/williballenthin/viv-utils) from 0.6.11 to 0.7.1. - [Release notes](https://github.com/williballenthin/viv-utils/releases) - [Commits](https://github.com/williballenthin/viv-utils/compare/v0.6.11...v0.7.1) --- updated-dependencies: - dependency-name: viv-utils[flirt] dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c4be8605..a67c3709 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ requirements = [ "termcolor==1.1.0", "wcwidth==0.2.5", "ida-settings==2.1.0", - "viv-utils[flirt]==0.6.11", + "viv-utils[flirt]==0.7.1", "halo==0.0.31", "networkx==2.5.1", "ruamel.yaml==0.17.21", From 10852a5d96c7fcf86096450289461139fef42c1f Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Wed, 27 Apr 2022 11:36:08 +0000 Subject: [PATCH 10/30] Sync capa rules submodule --- CHANGELOG.md | 3 ++- rules | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bf139e6..b4f2bf86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,13 +17,14 @@ - remove /x32 and /x64 flavors of number and operand features #932 @williballenthin - the tool now accepts multiple paths to rules, and JSON doc updated accordingly @williballenthin -### New Rules (5) +### New Rules (6) - data-manipulation/encryption/aes/manually-build-aes-constants huynh.t.nhan@gmail.com - nursery/get-process-image-filename michael.hunhoff@mandiant.com - compiler/v/compiled-with-v jakub.jozwiak@mandiant.com - compiler/zig/compiled-with-zig jakub.jozwiak@mandiant.com - anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com +- internal/limitation/file/internal-dotnet-file-limitation william.ballenthin@mandiant.com - ### Bug Fixes diff --git a/rules b/rules index f8a03a30..3b72e490 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit f8a03a3014c9e7fa97cfd5b681cfe089d6083de0 +Subproject commit 3b72e490b3d8dcec25fc2fe1cff32e82bc002dbe From 9f12f069ee5fe1ba622f1a040366a3caed6292f9 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 27 Apr 2022 15:09:58 +0200 Subject: [PATCH 11/30] ci: fix build (#980) * ci: fix build * fix: newest PyInstaller version * fix: logo path * fix: logo path 2 * fix: logo path 3 * fix: icon another way * fix: remove icon for now * ci: only build after tests succeed * ci: add workflow_run check --- .github/pyinstaller/pyinstaller.spec | 2 +- .github/workflows/build.yml | 18 +++++++++++------- setup.py | 3 +++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/pyinstaller/pyinstaller.spec b/.github/pyinstaller/pyinstaller.spec index 6d0854a9..f1c4fb1b 100644 --- a/.github/pyinstaller/pyinstaller.spec +++ b/.github/pyinstaller/pyinstaller.spec @@ -95,7 +95,7 @@ exe = EXE(pyz, a.datas, exclude_binaries=False, name='capa', - icon='logo.ico', + # TODO not working anymore for unknown reason icon='logo.ico', debug=False, strip=None, upx=True, diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ef58012..0d85fdce 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,16 +1,22 @@ name: build on: - push: - branches: [master] + workflow_run: + workflows: [CI] + types: + - completed release: types: [edited, published] jobs: build: + # only build on release or if tests pass + if: ${{ github.event_name == 'release' }} || ${{ github.event.workflow_run.conclusion == 'success' }} name: PyInstaller for ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: + # set to false for debugging + fail-fast: true matrix: include: - os: ubuntu-18.04 @@ -37,12 +43,10 @@ jobs: run: sudo apt-get install -y libyaml-dev - name: Upgrade pip, setuptools run: pip install --upgrade pip setuptools - - name: Install PyInstaller - run: pip install 'pyinstaller==4.10' - - name: Install capa - run: pip install -e . + - name: Install capa with build requirements + run: pip install -e .[build] - name: Build standalone executable - run: pyinstaller .github/pyinstaller/pyinstaller.spec + run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec - name: Does it run (PE)? run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_" - name: Does it run (Shellcode)? diff --git a/setup.py b/setup.py index db9f5381..29e3acf1 100644 --- a/setup.py +++ b/setup.py @@ -88,6 +88,9 @@ setuptools.setup( "types-psutil==5.8.22", "types_requests==2.27.20", ], + "build": [ + "pyinstaller==5.0", + ], }, zip_safe=False, keywords="capa malware analysis capability detection FLARE", From 49b1296d6eb494d4a98ed1dc4df93cad86c53b9c Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Wed, 27 Apr 2022 17:18:08 +0000 Subject: [PATCH 12/30] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 3b72e490..6728fb0d 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 3b72e490b3d8dcec25fc2fe1cff32e82bc002dbe +Subproject commit 6728fb0d5ad4d452959f7d4da49032fa48b3e49f From daf483309eb74c1b9bf04c10efdd0869de6ae000 Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 5 May 2022 20:19:35 +0200 Subject: [PATCH 13/30] fix: temporarily accept x32/x64 flavors but ignore (#1014) --- capa/rules.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/capa/rules.py b/capa/rules.py index f42b56a4..abc06b19 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -259,6 +259,13 @@ def parse_feature(key: str): return capa.features.insn.Number elif key == "offset": return capa.features.insn.Offset + # TODO remove x32/x64 flavor keys once fixed master/rules + elif key.startswith("number/"): + logger.warning("x32/x64 flavor currently not supported and deprecated") + return capa.features.insn.Number + elif key.startswith("offset/"): + logger.warning("x32/x64 flavor currently not supported and deprecated") + return capa.features.insn.Offset elif key == "mnemonic": return capa.features.insn.Mnemonic elif key == "basic blocks": From 0066b3f33ab17eb96c654c29a45eb2b3b6215775 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 13:28:48 -0600 Subject: [PATCH 14/30] build(deps): bump dnfile from 0.10.0 to 0.11.0 (#1004) --- capa/features/extractors/dnfile/helpers.py | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/capa/features/extractors/dnfile/helpers.py b/capa/features/extractors/dnfile/helpers.py index c7304462..4457afe9 100644 --- a/capa/features/extractors/dnfile/helpers.py +++ b/capa/features/extractors/dnfile/helpers.py @@ -105,7 +105,7 @@ def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]: TypeName (index into String heap) TypeNamespace (index into String heap) """ - if not hasattr(pe.net.mdtables, "MemberRef"): + if not hasattr(pe.net.mdtables, "MemberRef") or pe.net.mdtables.MemberRef is None: return for (rid, row) in enumerate(pe.net.mdtables.MemberRef): @@ -130,7 +130,7 @@ def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]: ImportName (index into the String heap) ImportScope (index into the ModuleRef table) """ - if not hasattr(pe.net.mdtables, "ImplMap"): + if not hasattr(pe.net.mdtables, "ImplMap") or pe.net.mdtables.ImplMap is None: return for row in pe.net.mdtables.ImplMap: @@ -154,7 +154,7 @@ def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]: def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[CilMethodBody]: """get managed methods from MethodDef table""" - if not hasattr(pe.net.mdtables, "MethodDef"): + if not hasattr(pe.net.mdtables, "MethodDef") or pe.net.mdtables.MethodDef is None: return for row in pe.net.mdtables.MethodDef: diff --git a/setup.py b/setup.py index 29e3acf1..b9cabb91 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ requirements = [ "smda==1.7.1", "pefile==2021.9.3", "pyelftools==0.28", - "dnfile==0.10.0", + "dnfile==0.11.0", "dncil==1.0.0", ] From 24c4215820a005dcd03353c3d5494989f947356e Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Thu, 5 May 2022 13:39:29 -0600 Subject: [PATCH 15/30] dotnet: add file string parsing (#1012) --- CHANGELOG.md | 1 + capa/features/extractors/dnfile/file.py | 8 ++++++-- capa/features/extractors/dotnetfile.py | 19 +++++++++++++++++-- tests/fixtures.py | 3 +++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4f2bf86..15d56672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - add new feature "operand[{0, 1, 2}].offset" for matching instruction operand offsets #767 @williballenthin - extract additional offset/number features in certain circumstances #320 @williballenthin - add detection and basic feature extraction for dotnet #987 @mr-tz, @mike-hunhoff, @williballenthin + - add file string extraction for dotnet files #1012 @mike-hunhoff ### Breaking Changes diff --git a/capa/features/extractors/dnfile/file.py b/capa/features/extractors/dnfile/file.py index 99e2643c..f0af0085 100644 --- a/capa/features/extractors/dnfile/file.py +++ b/capa/features/extractors/dnfile/file.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Tuple, Iterator if TYPE_CHECKING: import dnfile - from capa.features.common import Feature, Format + from capa.features.common import Feature, Format, String from capa.features.file import Import import capa.features.extractors @@ -26,6 +26,10 @@ def extract_file_format(pe: dnfile.dnPE) -> Iterator[Tuple[Format, int]]: yield from capa.features.extractors.dotnetfile.extract_file_format(pe=pe) +def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, int]]: + yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe) + + def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, int]]: for file_handler in FILE_HANDLERS: for (feature, token) in file_handler(pe): @@ -34,7 +38,7 @@ def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, int]]: FILE_HANDLERS = ( extract_file_import_names, - # TODO extract_file_strings, + extract_file_strings, # TODO extract_file_function_names, extract_file_format, ) diff --git a/capa/features/extractors/dotnetfile.py b/capa/features/extractors/dotnetfile.py index a9a2c600..07a88147 100644 --- a/capa/features/extractors/dotnetfile.py +++ b/capa/features/extractors/dotnetfile.py @@ -7,7 +7,18 @@ import pefile import capa.features.extractors.helpers from capa.features.file import Import -from capa.features.common import OS, OS_ANY, ARCH_ANY, ARCH_I386, ARCH_AMD64, FORMAT_DOTNET, Arch, Format, Feature +from capa.features.common import ( + OS, + OS_ANY, + ARCH_ANY, + ARCH_I386, + ARCH_AMD64, + FORMAT_DOTNET, + Arch, + Format, + String, + Feature, +) from capa.features.extractors.base_extractor import FeatureExtractor from capa.features.extractors.dnfile.helpers import get_dotnet_managed_imports, get_dotnet_unmanaged_imports @@ -45,6 +56,10 @@ def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Arch, int]]: yield Arch(ARCH_ANY), 0x0 +def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[String, int]]: + yield from capa.features.extractors.common.extract_file_strings(pe.__data__) + + def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, int]]: for file_handler in FILE_HANDLERS: for feature, va in file_handler(pe=pe): # type: ignore @@ -53,7 +68,7 @@ def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, int]]: FILE_HANDLERS = ( extract_file_import_names, - # TODO extract_file_strings, + extract_file_strings, # TODO extract_file_function_names, extract_file_format, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index 7f5abc73..af56cb40 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -671,10 +671,13 @@ FEATURE_PRESENCE_TESTS_DOTNET = sorted( ("mixed-mode-64", "file", Arch(ARCH_I386), False), ("b9f5b", "file", OS(OS_ANY), True), ("b9f5b", "file", Format(FORMAT_DOTNET), True), + ("hello-world", "file", capa.features.common.String("Hello World!"), True), ("hello-world", "function=0x250", capa.features.common.String("Hello World!"), True), ("hello-world", "function=0x250, bb=0x250, insn=0x252", capa.features.common.String("Hello World!"), True), ("hello-world", "function=0x250", capa.features.insn.API("System.Console::WriteLine"), True), ("hello-world", "file", capa.features.file.Import("System.Console::WriteLine"), True), + ("_1c444", "file", capa.features.common.String(r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall"), True), + ("_1c444", "file", capa.features.common.String("get_IsAlive"), True), ("_1c444", "file", capa.features.file.Import("gdi32.CreateCompatibleBitmap"), True), ("_1c444", "file", capa.features.file.Import("CreateCompatibleBitmap"), True), ("_1c444", "file", capa.features.file.Import("gdi32::CreateCompatibleBitmap"), False), From 20d80c1a2e1ddfb3fe1848257b81584659e78e48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 14:19:21 -0600 Subject: [PATCH 16/30] build(deps-dev): bump types-colorama from 0.4.12 to 0.4.13 (#1010) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b9cabb91..006bbeba 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ setuptools.setup( "requests==2.27.1", # type stubs for mypy "types-backports==0.1.3", - "types-colorama==0.4.12", + "types-colorama==0.4.13", "types-PyYAML==6.0.7", "types-tabulate==0.8.7", "types-termcolor==1.1.3", From 6d218aaf0dd739ae4de9447d980a2068c55eebaf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 14:19:35 -0600 Subject: [PATCH 17/30] build(deps-dev): bump types-requests from 2.27.20 to 2.27.25 (#1007) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 006bbeba..17519585 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ setuptools.setup( "types-tabulate==0.8.7", "types-termcolor==1.1.3", "types-psutil==5.8.22", - "types_requests==2.27.20", + "types_requests==2.27.25", ], "build": [ "pyinstaller==5.0", From dcf43b6feedf44e7800a2f0ffa29ee449e12b8fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 20:19:53 +0000 Subject: [PATCH 18/30] build(deps): bump vivisect from 1.0.7 to 1.0.8 Bumps [vivisect](https://github.com/vivisect/vivisect) from 1.0.7 to 1.0.8. - [Release notes](https://github.com/vivisect/vivisect/releases) - [Changelog](https://github.com/vivisect/vivisect/blob/master/CHANGELOG.rst) - [Commits](https://github.com/vivisect/vivisect/compare/v1.0.7...v1.0.8) --- updated-dependencies: - dependency-name: vivisect dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index b9cabb91..bf1205aa 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ requirements = [ "halo==0.0.31", "networkx==2.5.1", "ruamel.yaml==0.17.21", - "vivisect==1.0.7", + "vivisect==1.0.8", "smda==1.7.1", "pefile==2021.9.3", "pyelftools==0.28", @@ -81,12 +81,12 @@ setuptools.setup( "requests==2.27.1", # type stubs for mypy "types-backports==0.1.3", - "types-colorama==0.4.12", + "types-colorama==0.4.13", "types-PyYAML==6.0.7", "types-tabulate==0.8.7", "types-termcolor==1.1.3", "types-psutil==5.8.22", - "types_requests==2.27.20", + "types_requests==2.27.25", ], "build": [ "pyinstaller==5.0", From eefc0a9632fb81638384a2a9d5eab011a9e06e2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 14:19:56 -0600 Subject: [PATCH 19/30] build(deps-dev): bump pyinstaller from 5.0 to 5.0.1 (#1008) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 17519585..65e5a091 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ setuptools.setup( "types_requests==2.27.25", ], "build": [ - "pyinstaller==5.0", + "pyinstaller==5.0.1", ], }, zip_safe=False, From bcd00004b8f5bfd4f44134cbb1895f14e7822c56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 14:20:16 -0600 Subject: [PATCH 20/30] build(deps-dev): bump types-tabulate from 0.8.7 to 0.8.8 (#1009) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 65e5a091..67c2da6c 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ setuptools.setup( "types-backports==0.1.3", "types-colorama==0.4.13", "types-PyYAML==6.0.7", - "types-tabulate==0.8.7", + "types-tabulate==0.8.8", "types-termcolor==1.1.3", "types-psutil==5.8.22", "types_requests==2.27.25", From aca4f27ee88be2f6fa6b1aac96e4b65969b0bf3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 20:20:18 +0000 Subject: [PATCH 21/30] build(deps-dev): bump mypy from 0.942 to 0.950 Bumps [mypy](https://github.com/python/mypy) from 0.942 to 0.950. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.942...v0.950) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 65e5a091..dd43d968 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ setuptools.setup( "pycodestyle==2.8.0", "black==22.3.0", "isort==5.10.1", - "mypy==0.942", + "mypy==0.950", "psutil==5.9.0", "stix2==3.0.1", "requests==2.27.1", From d0a1313f33bcd942e38dc49f08c529f626e7806a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 20:21:15 +0000 Subject: [PATCH 22/30] build(deps-dev): bump types-termcolor from 1.1.3 to 1.1.4 Bumps [types-termcolor](https://github.com/python/typeshed) from 1.1.3 to 1.1.4. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-termcolor dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67c2da6c..75f82b26 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ setuptools.setup( "types-colorama==0.4.13", "types-PyYAML==6.0.7", "types-tabulate==0.8.8", - "types-termcolor==1.1.3", + "types-termcolor==1.1.4", "types-psutil==5.8.22", "types_requests==2.27.25", ], From 5573794a1f4fe688d31d583254b2e9695a5efe91 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Fri, 6 May 2022 15:49:04 +0200 Subject: [PATCH 23/30] dep: bump viv-utils --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf1205aa..7f699f06 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ requirements = [ "termcolor==1.1.0", "wcwidth==0.2.5", "ida-settings==2.1.0", - "viv-utils[flirt]==0.7.1", + "viv-utils[flirt]==0.7.4", "halo==0.0.31", "networkx==2.5.1", "ruamel.yaml==0.17.21", From 80e4161b40ddb1e63ffe5d265b97bdd5810eee30 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Fri, 6 May 2022 16:29:54 +0200 Subject: [PATCH 24/30] ci: build on PR --- .github/workflows/build.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0d85fdce..022fc0ee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,17 +1,13 @@ name: build on: - workflow_run: - workflows: [CI] - types: - - completed + pull_request: + branches: [ master ] release: types: [edited, published] jobs: build: - # only build on release or if tests pass - if: ${{ github.event_name == 'release' }} || ${{ github.event.workflow_run.conclusion == 'success' }} name: PyInstaller for ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: From a9c9b3cea8e6b3d436fc3294d50d6f41e3aaf70e Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Fri, 6 May 2022 08:34:50 -0600 Subject: [PATCH 25/30] dotnet: extract file function names (#1015) --- CHANGELOG.md | 1 + capa/features/extractors/dnfile/file.py | 10 ++-- capa/features/extractors/dnfile/helpers.py | 57 ++++++++++++++++++---- capa/features/extractors/dnfile/insn.py | 26 ++++++++-- capa/features/extractors/dotnetfile.py | 35 ++++++++----- tests/fixtures.py | 3 ++ 6 files changed, 102 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15d56672..aba76951 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - extract additional offset/number features in certain circumstances #320 @williballenthin - add detection and basic feature extraction for dotnet #987 @mr-tz, @mike-hunhoff, @williballenthin - add file string extraction for dotnet files #1012 @mike-hunhoff + - add file function-name extraction for dotnet files #1015 @mike-hunhoff ### Breaking Changes diff --git a/capa/features/extractors/dnfile/file.py b/capa/features/extractors/dnfile/file.py index f0af0085..4168249a 100644 --- a/capa/features/extractors/dnfile/file.py +++ b/capa/features/extractors/dnfile/file.py @@ -13,19 +13,23 @@ from typing import TYPE_CHECKING, Tuple, Iterator if TYPE_CHECKING: import dnfile from capa.features.common import Feature, Format, String - from capa.features.file import Import + from capa.features.file import Import, FunctionName import capa.features.extractors def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[Tuple[Import, int]]: - yield from capa.features.extractors.dotnetfile.extract_file_import_names(pe) + yield from capa.features.extractors.dotnetfile.extract_file_import_names(pe=pe) def extract_file_format(pe: dnfile.dnPE) -> Iterator[Tuple[Format, int]]: yield from capa.features.extractors.dotnetfile.extract_file_format(pe=pe) +def extract_file_function_names(pe: dnfile.dnPE) -> Iterator[Tuple[FunctionName, int]]: + yield from capa.features.extractors.dotnetfile.extract_file_function_names(pe=pe) + + def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, int]]: yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe) @@ -38,7 +42,7 @@ def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, int]]: FILE_HANDLERS = ( extract_file_import_names, + extract_file_function_names, extract_file_strings, - # TODO extract_file_function_names, extract_file_format, ) diff --git a/capa/features/extractors/dnfile/helpers.py b/capa/features/extractors/dnfile/helpers.py index 4457afe9..ddca5e61 100644 --- a/capa/features/extractors/dnfile/helpers.py +++ b/capa/features/extractors/dnfile/helpers.py @@ -105,18 +105,24 @@ def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]: TypeName (index into String heap) TypeNamespace (index into String heap) """ - if not hasattr(pe.net.mdtables, "MemberRef") or pe.net.mdtables.MemberRef is None: + if not is_dotnet_table_valid(pe, "MemberRef"): return for (rid, row) in enumerate(pe.net.mdtables.MemberRef): if not isinstance(row.Class.row, (dnfile.mdtable.TypeRefRow,)): continue - token: int = calculate_dotnet_token_value(dnfile.enums.MetadataTables.MemberRef.value, rid + 1) - # like System.IO.File::OpenRead - imp: str = f"{row.Class.row.TypeNamespace}.{row.Class.row.TypeName}::{row.Name}" + # like File::OpenRead + name = f"{row.Class.row.TypeName}::{row.Name}" - yield token, imp + # ECMA II.22.38: TypeNamespace can be null or non-null + if row.Class.row.TypeNamespace: + # like System.IO.File::OpenRead + name = f"{row.Class.row.TypeNamespace}.{name}" + + token: int = calculate_dotnet_token_value(pe.net.mdtables.MemberRef.number, rid + 1) + + yield token, name def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]: @@ -130,7 +136,7 @@ def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]: ImportName (index into the String heap) ImportScope (index into the ModuleRef table) """ - if not hasattr(pe.net.mdtables, "ImplMap") or pe.net.mdtables.ImplMap is None: + if not is_dotnet_table_valid(pe, "ImplMap"): return for row in pe.net.mdtables.ImplMap: @@ -147,14 +153,14 @@ def get_dotnet_unmanaged_imports(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]: dll = dll.split(".")[0] # like kernel32.CreateFileA - imp: str = f"{dll}.{symbol}" + name: str = f"{dll}.{symbol}" - yield token, imp + yield token, name def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[CilMethodBody]: """get managed methods from MethodDef table""" - if not hasattr(pe.net.mdtables, "MethodDef") or pe.net.mdtables.MethodDef is None: + if not is_dotnet_table_valid(pe, "MethodDef"): return for row in pe.net.mdtables.MethodDef: @@ -167,3 +173,36 @@ def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[CilMethodBody] continue yield body + + +def is_dotnet_table_valid(pe: dnfile.dnPE, table_name: str) -> bool: + return bool(getattr(pe.net.mdtables, table_name, None)) + + +def get_dotnet_managed_method_names(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]: + """get managed method names from TypeDef table + + see https://www.ntcore.com/files/dotnetformat.htm + + 02 - TypeDef Table + Each row represents a class in the current assembly. + TypeName (index into String heap) + TypeNamespace (index into String heap) + MethodList (index into MethodDef table; it marks the first of a continguous run of Methods owned by this Type) + """ + if not is_dotnet_table_valid(pe, "TypeDef"): + return + + for row in pe.net.mdtables.TypeDef: + for index in row.MethodList: + # like File::OpenRead + name = f"{row.TypeName}::{index.row.Name}" + + # ECMA II.22.37: TypeNamespace can be null or non-null + if row.TypeNamespace: + # like System.IO.File::OpenRead + name = f"{row.TypeNamespace}.{name}" + + token = calculate_dotnet_token_value(index.table.number, index.row_index) + + yield token, name diff --git a/capa/features/extractors/dnfile/insn.py b/capa/features/extractors/dnfile/insn.py index 262b9779..15a75ae0 100644 --- a/capa/features/extractors/dnfile/insn.py +++ b/capa/features/extractors/dnfile/insn.py @@ -9,7 +9,6 @@ from __future__ import annotations from typing import TYPE_CHECKING, Dict, Tuple, Iterator, Optional -from itertools import chain if TYPE_CHECKING: from dncil.cil.instruction import Instruction @@ -26,24 +25,41 @@ from capa.features.extractors.dnfile.helpers import ( read_dotnet_user_string, get_dotnet_managed_imports, get_dotnet_unmanaged_imports, + get_dotnet_managed_method_names, ) def get_imports(ctx: Dict) -> Dict: if "imports_cache" not in ctx: - ctx["imports_cache"] = { - token: imp - for (token, imp) in chain(get_dotnet_managed_imports(ctx["pe"]), get_dotnet_unmanaged_imports(ctx["pe"])) - } + ctx["imports_cache"] = {} + + for (token, name) in get_dotnet_managed_imports(ctx["pe"]): + ctx["imports_cache"][token] = name + for (token, name) in get_dotnet_unmanaged_imports(ctx["pe"]): + ctx["imports_cache"][token] = name + return ctx["imports_cache"] +def get_methods(ctx: Dict) -> Dict: + if "methods_cache" not in ctx: + ctx["methods_cache"] = {} + + for (token, name) in get_dotnet_managed_method_names(ctx["pe"]): + ctx["methods_cache"][token] = name + + return ctx["methods_cache"] + + def extract_insn_api_features(f: CilMethodBody, bb: CilMethodBody, insn: Instruction) -> Iterator[Tuple[API, int]]: """parse instruction API features""" if insn.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp, OpCodes.Calli): return name: str = get_imports(f.ctx).get(insn.operand.value, "") + if not name: + name = get_methods(f.ctx).get(insn.operand.value, "") + if not name: return diff --git a/capa/features/extractors/dotnetfile.py b/capa/features/extractors/dotnetfile.py index 07a88147..6c6adb0d 100644 --- a/capa/features/extractors/dotnetfile.py +++ b/capa/features/extractors/dotnetfile.py @@ -1,12 +1,11 @@ import logging from typing import Tuple, Iterator -from itertools import chain import dnfile import pefile import capa.features.extractors.helpers -from capa.features.file import Import +from capa.features.file import Import, FunctionName from capa.features.common import ( OS, OS_ANY, @@ -20,7 +19,12 @@ from capa.features.common import ( Feature, ) from capa.features.extractors.base_extractor import FeatureExtractor -from capa.features.extractors.dnfile.helpers import get_dotnet_managed_imports, get_dotnet_unmanaged_imports +from capa.features.extractors.dnfile.helpers import ( + get_dotnet_managed_imports, + calculate_dotnet_token_value, + get_dotnet_unmanaged_imports, + get_dotnet_managed_method_names, +) logger = logging.getLogger(__name__) @@ -30,15 +34,20 @@ def extract_file_format(**kwargs) -> Iterator[Tuple[Format, int]]: def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Import, int]]: - for (token, imp) in chain(get_dotnet_managed_imports(pe), get_dotnet_unmanaged_imports(pe)): - if "::" in imp: - # like System.IO.File::OpenRead - yield Import(imp), token - else: - # like kernel32.CreateFileA - dll, _, symbol = imp.rpartition(".") - for symbol_variant in capa.features.extractors.helpers.generate_symbols(dll, symbol): - yield Import(symbol_variant), token + for (token, name) in get_dotnet_managed_imports(pe): + # like System.IO.File::OpenRead + yield Import(name), token + + for (token, name) in get_dotnet_unmanaged_imports(pe): + # like kernel32.CreateFileA + dll, _, symbol = name.rpartition(".") + for name_variant in capa.features.extractors.helpers.generate_symbols(dll, symbol): + yield Import(name_variant), token + + +def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[FunctionName, int]]: + for (token, name) in get_dotnet_managed_method_names(pe): + yield FunctionName(name), token def extract_file_os(**kwargs) -> Iterator[Tuple[OS, int]]: @@ -68,8 +77,8 @@ def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, int]]: FILE_HANDLERS = ( extract_file_import_names, + extract_file_function_names, extract_file_strings, - # TODO extract_file_function_names, extract_file_format, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index af56cb40..f4a91580 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -671,6 +671,9 @@ FEATURE_PRESENCE_TESTS_DOTNET = sorted( ("mixed-mode-64", "file", Arch(ARCH_I386), False), ("b9f5b", "file", OS(OS_ANY), True), ("b9f5b", "file", Format(FORMAT_DOTNET), True), + ("hello-world", "file", capa.features.file.FunctionName("HelloWorld::Main"), True), + ("hello-world", "file", capa.features.file.FunctionName("HelloWorld::.ctor"), True), + ("hello-world", "file", capa.features.file.FunctionName("HelloWorld::.cctor"), False), ("hello-world", "file", capa.features.common.String("Hello World!"), True), ("hello-world", "function=0x250", capa.features.common.String("Hello World!"), True), ("hello-world", "function=0x250, bb=0x250, insn=0x252", capa.features.common.String("Hello World!"), True), From 6fb9dd961a7917e2bb5b334d59fadaf7ac56a502 Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Fri, 6 May 2022 13:05:48 -0600 Subject: [PATCH 26/30] dotnet: emit unmanaged call characteristic (#1023) --- CHANGELOG.md | 1 + capa/features/extractors/dnfile/insn.py | 64 ++++++++++++++++++------- capa/rules.py | 1 + tests/fixtures.py | 7 +++ 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aba76951..6b38aa34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - add detection and basic feature extraction for dotnet #987 @mr-tz, @mike-hunhoff, @williballenthin - add file string extraction for dotnet files #1012 @mike-hunhoff - add file function-name extraction for dotnet files #1015 @mike-hunhoff + - add unmanaged call characteristic for dotnet files #1023 @mike-hunhoff ### Breaking Changes diff --git a/capa/features/extractors/dnfile/insn.py b/capa/features/extractors/dnfile/insn.py index 15a75ae0..5974cde1 100644 --- a/capa/features/extractors/dnfile/insn.py +++ b/capa/features/extractors/dnfile/insn.py @@ -8,20 +8,22 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Tuple, Iterator, Optional +from typing import TYPE_CHECKING, Any, Dict, Tuple, Iterator, Optional if TYPE_CHECKING: from dncil.cil.instruction import Instruction from dncil.cil.body import CilMethodBody from capa.features.common import Feature -from dncil.clr.token import StringToken +import dnfile +from dncil.clr.token import StringToken, InvalidToken from dncil.cil.opcode import OpCodes import capa.features.extractors.helpers from capa.features.insn import API, Number -from capa.features.common import String +from capa.features.common import String, Characteristic from capa.features.extractors.dnfile.helpers import ( + resolve_dotnet_token, read_dotnet_user_string, get_dotnet_managed_imports, get_dotnet_unmanaged_imports, @@ -29,37 +31,48 @@ from capa.features.extractors.dnfile.helpers import ( ) -def get_imports(ctx: Dict) -> Dict: - if "imports_cache" not in ctx: - ctx["imports_cache"] = {} - +def get_managed_imports(ctx: Dict) -> Dict: + if "managed_imports_cache" not in ctx: + ctx["managed_imports_cache"] = {} for (token, name) in get_dotnet_managed_imports(ctx["pe"]): - ctx["imports_cache"][token] = name - for (token, name) in get_dotnet_unmanaged_imports(ctx["pe"]): - ctx["imports_cache"][token] = name + ctx["managed_imports_cache"][token] = name + return ctx["managed_imports_cache"] - return ctx["imports_cache"] + +def get_unmanaged_imports(ctx: Dict) -> Dict: + if "unmanaged_imports_cache" not in ctx: + ctx["unmanaged_imports_cache"] = {} + for (token, name) in get_dotnet_unmanaged_imports(ctx["pe"]): + ctx["unmanaged_imports_cache"][token] = name + return ctx["unmanaged_imports_cache"] def get_methods(ctx: Dict) -> Dict: if "methods_cache" not in ctx: ctx["methods_cache"] = {} - for (token, name) in get_dotnet_managed_method_names(ctx["pe"]): ctx["methods_cache"][token] = name - return ctx["methods_cache"] +def get_callee_name(ctx: Dict, token: int) -> str: + """map dotnet token to method name""" + name: str = get_managed_imports(ctx).get(token, "") + if not name: + # we must check unmanaged imports before managed methods because we map forwarded managed methods + # to their unmanaged imports; we prefer a forwarded managed method be mapped to its unmanaged import for analysis + name = get_unmanaged_imports(ctx).get(token, "") + if not name: + name = get_methods(ctx).get(token, "") + return name + + def extract_insn_api_features(f: CilMethodBody, bb: CilMethodBody, insn: Instruction) -> Iterator[Tuple[API, int]]: """parse instruction API features""" if insn.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp, OpCodes.Calli): return - name: str = get_imports(f.ctx).get(insn.operand.value, "") - if not name: - name = get_methods(f.ctx).get(insn.operand.value, "") - + name: str = get_callee_name(f.ctx, insn.operand.value) if not name: return @@ -98,6 +111,22 @@ def extract_insn_string_features( yield String(user_string), insn.offset +def extract_unmanaged_call_characteristic_features( + f: CilMethodBody, bb: CilMethodBody, insn: Instruction +) -> Iterator[Tuple[Characteristic, int]]: + if insn.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp, OpCodes.Calli): + return + + token: Any = resolve_dotnet_token(f.ctx["pe"], insn.operand) + if isinstance(token, InvalidToken): + return + if not isinstance(token, dnfile.mdtable.MethodDefRow): + return + + if any((token.Flags.mdPinvokeImpl, token.ImplFlags.miUnmanaged, token.ImplFlags.miNative)): + yield Characteristic("unmanaged call"), insn.offset + + def extract_features(f: CilMethodBody, bb: CilMethodBody, insn: Instruction) -> Iterator[Tuple[Feature, int]]: """extract instruction features""" for inst_handler in INSTRUCTION_HANDLERS: @@ -109,4 +138,5 @@ INSTRUCTION_HANDLERS = ( extract_insn_api_features, extract_insn_number_features, extract_insn_string_features, + extract_unmanaged_call_characteristic_features, ) diff --git a/capa/rules.py b/capa/rules.py index abc06b19..59d7238f 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -134,6 +134,7 @@ SUPPORTED_FEATURES: Dict[str, Set] = { capa.features.common.Characteristic("indirect call"), capa.features.common.Characteristic("call $+5"), capa.features.common.Characteristic("cross section flow"), + capa.features.common.Characteristic("unmanaged call"), }, } diff --git a/tests/fixtures.py b/tests/fixtures.py index f4a91580..0b4feea5 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -689,6 +689,13 @@ FEATURE_PRESENCE_TESTS_DOTNET = sorted( ("_1c444", "function=0x1F68", capa.features.insn.Number(0xCC0020), True), ("_1c444", "function=0x1F68", capa.features.insn.Number(0x0), True), ("_1c444", "function=0x1F68", capa.features.insn.Number(0x1), False), + ( + "_1c444", + "function=0x1F59, bb=0x1F59, insn=0x1F5B", + capa.features.common.Characteristic("unmanaged call"), + True, + ), + ("_1c444", "function=0x2544", capa.features.common.Characteristic("unmanaged call"), False), ( "_1c444", "function=0x1F68, bb=0x1F68, insn=0x1FF9", From 0d849142bab8639727efe4842397a9efa881005c Mon Sep 17 00:00:00 2001 From: Mike Hunhoff Date: Fri, 6 May 2022 14:32:06 -0600 Subject: [PATCH 27/30] dotnet: emit mixed mode characteristic (#1024) --- CHANGELOG.md | 1 + capa/features/extractors/dnfile/file.py | 7 ++++++- capa/features/extractors/dnfile/helpers.py | 4 ++++ capa/features/extractors/dotnetfile.py | 10 +++++++++- capa/rules.py | 1 + tests/fixtures.py | 2 ++ 6 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b38aa34..b52eb948 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - add file string extraction for dotnet files #1012 @mike-hunhoff - add file function-name extraction for dotnet files #1015 @mike-hunhoff - add unmanaged call characteristic for dotnet files #1023 @mike-hunhoff + - add mixed mode characteristic feature extraction for dotnet files #1024 @mike-hunhoff ### Breaking Changes diff --git a/capa/features/extractors/dnfile/file.py b/capa/features/extractors/dnfile/file.py index 4168249a..248d8108 100644 --- a/capa/features/extractors/dnfile/file.py +++ b/capa/features/extractors/dnfile/file.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Tuple, Iterator if TYPE_CHECKING: import dnfile - from capa.features.common import Feature, Format, String + from capa.features.common import Feature, Format, String, Characteristic from capa.features.file import Import, FunctionName import capa.features.extractors @@ -34,6 +34,10 @@ def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, int]]: yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe) +def extract_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[Tuple[Characteristic, int]]: + yield from capa.features.extractors.dotnetfile.extract_mixed_mode_characteristic_features(pe=pe) + + def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, int]]: for file_handler in FILE_HANDLERS: for (feature, token) in file_handler(pe): @@ -45,4 +49,5 @@ FILE_HANDLERS = ( extract_file_function_names, extract_file_strings, extract_file_format, + extract_mixed_mode_characteristic_features, ) diff --git a/capa/features/extractors/dnfile/helpers.py b/capa/features/extractors/dnfile/helpers.py index ddca5e61..7c8adcbd 100644 --- a/capa/features/extractors/dnfile/helpers.py +++ b/capa/features/extractors/dnfile/helpers.py @@ -206,3 +206,7 @@ def get_dotnet_managed_method_names(pe: dnfile.dnPE) -> Iterator[Tuple[int, str] token = calculate_dotnet_token_value(index.table.number, index.row_index) yield token, name + + +def is_dotnet_mixed_mode(pe: dnfile.dnPE) -> bool: + return not bool(pe.net.Flags.CLR_ILONLY) diff --git a/capa/features/extractors/dotnetfile.py b/capa/features/extractors/dotnetfile.py index 6c6adb0d..965c4ea4 100644 --- a/capa/features/extractors/dotnetfile.py +++ b/capa/features/extractors/dotnetfile.py @@ -17,9 +17,11 @@ from capa.features.common import ( Format, String, Feature, + Characteristic, ) from capa.features.extractors.base_extractor import FeatureExtractor from capa.features.extractors.dnfile.helpers import ( + is_dotnet_mixed_mode, get_dotnet_managed_imports, calculate_dotnet_token_value, get_dotnet_unmanaged_imports, @@ -69,6 +71,11 @@ def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[String, in yield from capa.features.extractors.common.extract_file_strings(pe.__data__) +def extract_mixed_mode_characteristic_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Characteristic, int]]: + if is_dotnet_mixed_mode(pe): + yield Characteristic("mixed mode"), 0x0 + + def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, int]]: for file_handler in FILE_HANDLERS: for feature, va in file_handler(pe=pe): # type: ignore @@ -80,6 +87,7 @@ FILE_HANDLERS = ( extract_file_function_names, extract_file_strings, extract_file_format, + extract_mixed_mode_characteristic_features, ) @@ -120,7 +128,7 @@ class DotnetFileFeatureExtractor(FeatureExtractor): return bool(self.pe.net) def is_mixed_mode(self) -> bool: - return not bool(self.pe.net.Flags.CLR_ILONLY) + return is_dotnet_mixed_mode(self.pe) def get_runtime_version(self) -> Tuple[int, int]: return self.pe.net.struct.MajorRuntimeVersion, self.pe.net.struct.MinorRuntimeVersion diff --git a/capa/rules.py b/capa/rules.py index 59d7238f..1421c70a 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -101,6 +101,7 @@ SUPPORTED_FEATURES: Dict[str, Set] = { capa.features.common.Characteristic("embedded pe"), capa.features.common.String, capa.features.common.Format, + capa.features.common.Characteristic("mixed mode"), }, FUNCTION_SCOPE: { capa.features.common.MatchedRule, diff --git a/tests/fixtures.py b/tests/fixtures.py index 0b4feea5..9fbc57a0 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -669,6 +669,8 @@ FEATURE_PRESENCE_TESTS_DOTNET = sorted( ("b9f5b", "file", Arch(ARCH_AMD64), False), ("mixed-mode-64", "file", Arch(ARCH_AMD64), True), ("mixed-mode-64", "file", Arch(ARCH_I386), False), + ("mixed-mode-64", "file", capa.features.common.Characteristic("mixed mode"), True), + ("hello-world", "file", capa.features.common.Characteristic("mixed mode"), False), ("b9f5b", "file", OS(OS_ANY), True), ("b9f5b", "file", Format(FORMAT_DOTNET), True), ("hello-world", "file", capa.features.file.FunctionName("HelloWorld::Main"), True), From 95b3c6a594555b661729d30f730dd6908b3739f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 14:13:40 +0000 Subject: [PATCH 28/30] build(deps-dev): bump types-tabulate from 0.8.8 to 0.8.9 Bumps [types-tabulate](https://github.com/python/typeshed) from 0.8.8 to 0.8.9. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-tabulate dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1d912f89..25c75f83 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ setuptools.setup( "types-backports==0.1.3", "types-colorama==0.4.13", "types-PyYAML==6.0.7", - "types-tabulate==0.8.8", + "types-tabulate==0.8.9", "types-termcolor==1.1.4", "types-psutil==5.8.22", "types_requests==2.27.25", From 7971b940015f9407045ae13ed574ce3dd513f0e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 14:13:43 +0000 Subject: [PATCH 29/30] build(deps-dev): bump types-colorama from 0.4.13 to 0.4.14 Bumps [types-colorama](https://github.com/python/typeshed) from 0.4.13 to 0.4.14. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-colorama dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1d912f89..c855838f 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ setuptools.setup( "requests==2.27.1", # type stubs for mypy "types-backports==0.1.3", - "types-colorama==0.4.13", + "types-colorama==0.4.14", "types-PyYAML==6.0.7", "types-tabulate==0.8.8", "types-termcolor==1.1.4", From 141da27715efbe92408bd2f90af516a073a71034 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Wed, 11 May 2022 16:34:46 +0000 Subject: [PATCH 30/30] Sync capa rules submodule --- rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules b/rules index 6728fb0d..52ff654c 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 6728fb0d5ad4d452959f7d4da49032fa48b3e49f +Subproject commit 52ff654ca0a73235df7d2e9bfbd52961f957cbc8