From 7f03db9fe44fd69e6660cf5fef5bf9e21dc46b98 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 15 Jun 2021 12:24:01 -0600 Subject: [PATCH 01/23] main: dont save .viv by default, unless CAPA_SAVE_WORKSPACE set closes #507 --- capa/features/freeze.py | 2 +- capa/main.py | 20 +++++++++++++------- scripts/bulk-process.py | 3 ++- scripts/capa_as_library.py | 2 +- scripts/lint.py | 2 +- scripts/show-capabilities-by-function.py | 3 ++- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/capa/features/freeze.py b/capa/features/freeze.py index c6d4d7f5..4fcf1f52 100644 --- a/capa/features/freeze.py +++ b/capa/features/freeze.py @@ -254,7 +254,7 @@ def main(argv=None): args = parser.parse_args(args=argv) capa.main.handle_common_args(args) - extractor = capa.main.get_extractor(args.sample, args.format, args.backend, sigpaths=args.signatures) + extractor = capa.main.get_extractor(args.sample, args.format, args.backend, args.signatures, False) with open(args.output, "wb") as f: f.write(dump(extractor)) diff --git a/capa/main.py b/capa/main.py index f12aeb29..1c42a50a 100644 --- a/capa/main.py +++ b/capa/main.py @@ -401,7 +401,7 @@ class UnsupportedRuntimeError(RuntimeError): pass -def get_extractor(path, format, backend, sigpaths, disable_progress=False): +def get_extractor(path, format, backend, sigpaths, should_save_workspace, disable_progress=False): """ raises: UnsupportedFormatError: @@ -430,11 +430,15 @@ def get_extractor(path, format, backend, sigpaths, disable_progress=False): format = "sc64" vw = get_workspace(path, format, sigpaths) - try: - vw.saveWorkspace() - except IOError: - # see #168 for discussion around how to handle non-writable directories - logger.info("source directory is not writable, won't save intermediate workspace") + if should_save_workspace: + logger.debug("saving workspace") + try: + vw.saveWorkspace() + except IOError: + # see #168 for discussion around how to handle non-writable directories + logger.info("source directory is not writable, won't save intermediate workspace") + else: + logger.debug("CAPA_SAVE_WORKSPACE unset, not saving workspace") return capa.features.extractors.viv.extractor.VivisectFeatureExtractor(vw, path) @@ -813,8 +817,10 @@ def main(argv=None): extractor = capa.features.freeze.load(f.read()) else: format = args.format + should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) + try: - extractor = get_extractor(args.sample, format, args.backend, args.signatures, disable_progress=args.quiet) + extractor = get_extractor(args.sample, format, args.backend, args.signatures, should_save_workspace, disable_progress=args.quiet) except UnsupportedFormatError: logger.error("-" * 80) logger.error(" Input file does not appear to be a PE file.") diff --git a/scripts/bulk-process.py b/scripts/bulk-process.py index 5939744f..864e8f05 100644 --- a/scripts/bulk-process.py +++ b/scripts/bulk-process.py @@ -95,9 +95,10 @@ def get_capa_results(args): capabilities (dict): the matched capabilities and their result objects """ rules, format, path = args + should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) logger.info("computing capa results for: %s", path) try: - extractor = capa.main.get_extractor(path, format, capa.main.BACKEND_VIV, args.signatures, disable_progress=True) + extractor = capa.main.get_extractor(path, format, capa.main.BACKEND_VIV, args.signatures, should_save_workspace, disable_progress=True) except capa.main.UnsupportedFormatError: # i'm 100% sure if multiprocessing will reliably raise exceptions across process boundaries. # so instead, return an object with explicit success/failure status. diff --git a/scripts/capa_as_library.py b/scripts/capa_as_library.py index 8cc4dee8..3a9eb295 100644 --- a/scripts/capa_as_library.py +++ b/scripts/capa_as_library.py @@ -193,7 +193,7 @@ def render_dictionary(doc): def capa_details(file_path, output_format="dictionary"): # extract features and find capabilities - extractor = capa.main.get_extractor(file_path, "auto", capa.main.BACKEND_VIV, sigpaths=[], disable_progress=True) + extractor = capa.main.get_extractor(file_path, "auto", capa.main.BACKEND_VIV, [], False, disable_progress=True) capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True) # collect metadata (used only to make rendering more complete) diff --git a/scripts/lint.py b/scripts/lint.py index 19eb93eb..0d765849 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -220,7 +220,7 @@ class DoesntMatchExample(Lint): try: extractor = capa.main.get_extractor( - path, "auto", capa.main.BACKEND_VIV, sigpaths=DEFAULT_SIGNATURES, disable_progress=True + path, "auto", capa.main.BACKEND_VIV, DEFAULT_SIGNATURES, False, disable_progress=True ) capabilities, meta = capa.main.find_capabilities(ctx["rules"], extractor, disable_progress=True) except Exception as e: diff --git a/scripts/show-capabilities-by-function.py b/scripts/show-capabilities-by-function.py index ad0af057..153c567a 100644 --- a/scripts/show-capabilities-by-function.py +++ b/scripts/show-capabilities-by-function.py @@ -152,9 +152,10 @@ def main(argv=None): extractor = capa.features.freeze.load(f.read()) else: format = args.format + should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) try: - extractor = capa.main.get_extractor(args.sample, args.format, args.backend, args.signatures) + extractor = capa.main.get_extractor(args.sample, args.format, args.backend, args.signatures, should_save_workspace) except capa.main.UnsupportedFormatError: logger.error("-" * 80) logger.error(" Input file does not appear to be a PE file.") From 09ad0ec1844fd6545452d18236433c03637df53f Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 15 Jun 2021 12:24:29 -0600 Subject: [PATCH 02/23] tests: save .viv by default, hopefully improve test performance --- tests/fixtures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fixtures.py b/tests/fixtures.py index 0a9d62d0..fda8a9f1 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -84,6 +84,7 @@ def get_viv_extractor(path): vw = capa.main.get_workspace(path, "sc64", sigpaths=sigpaths) else: vw = capa.main.get_workspace(path, "auto", sigpaths=sigpaths) + vw.saveWorkspace() extractor = capa.features.extractors.viv.extractor.VivisectFeatureExtractor(vw, path) fixup_viv(path, extractor) return extractor From d510840bb7e8d728c4e73c40838e6ede64242383 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 15 Jun 2021 12:26:37 -0600 Subject: [PATCH 03/23] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b39f7e74..6a23897c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - main: implement file limitations via rules not code #390 @williballenthin - json: breaking change: correctly render negative offsets #619 @williballenthin - library: breaking change: remove logic from `__init__.py` throughout #622 @williballenthin +- main: don't create .viv files unless CAPA_SAVE_WORKSPACE is set #507 @williballenthin ### Development From 40c7401f0a78ee457b5d0046630f60d989e7d4fb Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 15 Jun 2021 12:28:45 -0600 Subject: [PATCH 04/23] pep8 --- capa/main.py | 4 +++- scripts/bulk-process.py | 4 +++- scripts/show-capabilities-by-function.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/capa/main.py b/capa/main.py index d4e47fc6..53addd5b 100644 --- a/capa/main.py +++ b/capa/main.py @@ -846,7 +846,9 @@ def main(argv=None): should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) try: - extractor = get_extractor(args.sample, format, args.backend, args.signatures, should_save_workspace, disable_progress=args.quiet) + extractor = get_extractor( + args.sample, format, args.backend, args.signatures, should_save_workspace, disable_progress=args.quiet + ) except UnsupportedFormatError: logger.error("-" * 80) logger.error(" Input file does not appear to be a PE file.") diff --git a/scripts/bulk-process.py b/scripts/bulk-process.py index 864e8f05..f120fe66 100644 --- a/scripts/bulk-process.py +++ b/scripts/bulk-process.py @@ -98,7 +98,9 @@ def get_capa_results(args): should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) logger.info("computing capa results for: %s", path) try: - extractor = capa.main.get_extractor(path, format, capa.main.BACKEND_VIV, args.signatures, should_save_workspace, disable_progress=True) + extractor = capa.main.get_extractor( + path, format, capa.main.BACKEND_VIV, args.signatures, should_save_workspace, disable_progress=True + ) except capa.main.UnsupportedFormatError: # i'm 100% sure if multiprocessing will reliably raise exceptions across process boundaries. # so instead, return an object with explicit success/failure status. diff --git a/scripts/show-capabilities-by-function.py b/scripts/show-capabilities-by-function.py index 153c567a..5ec8d6f3 100644 --- a/scripts/show-capabilities-by-function.py +++ b/scripts/show-capabilities-by-function.py @@ -155,7 +155,9 @@ def main(argv=None): should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) try: - extractor = capa.main.get_extractor(args.sample, args.format, args.backend, args.signatures, should_save_workspace) + extractor = capa.main.get_extractor( + args.sample, args.format, args.backend, args.signatures, should_save_workspace + ) except capa.main.UnsupportedFormatError: logger.error("-" * 80) logger.error(" Input file does not appear to be a PE file.") From 7bce20212202fc2ff28cca56071c6d555256bb0e Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 15 Jun 2021 12:31:56 -0600 Subject: [PATCH 05/23] doc: explain CAPA_SAVE_WORKSPACE --- doc/usage.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/usage.md b/doc/usage.md index 75a71b4f..74b163f4 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -11,3 +11,9 @@ For example, `capa -t william.ballenthin@mandiant.com` runs rules that reference ### IDA Pro plugin: capa explorer Please check out the [capa explorer documentation](/capa/ida/plugin/README.md). + +### save time by reusing .viv files +Set the environment variable `CAPA_SAVE_WORKSPACE` to instruct the underlying analysis engine to +cache its intermediate results to the file system. For example, vivisect will create `.viv` files. +Subsequently, capa may run faster when reprocessing the same input file. +This is particularly useful during rule development as you repeatedly test a rule against a known sample. \ No newline at end of file From 466bc4995be3cba5235359a249cf828c0a3b16a4 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Tue, 15 Jun 2021 15:06:34 -0600 Subject: [PATCH 06/23] Update CHANGELOG.md Co-authored-by: Moritz --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d1361b..e0d87314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,7 +130,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - main: implement file limitations via rules not code #390 @williballenthin - json: breaking change: correctly render negative offsets #619 @williballenthin - library: breaking change: remove logic from `__init__.py` throughout #622 @williballenthin -- main: don't create .viv files unless CAPA_SAVE_WORKSPACE is set #507 @williballenthin +- main: don't create .viv files unless CAPA_SAVE_WORKSPACE environment variable is set #507 @williballenthin - library: add type annotations for use with mypy #447 @williballenthin ### Development From f5a0e1cd089cc701df6d74d33bb9ccca0af580b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 14:02:06 +0000 Subject: [PATCH 07/23] build(deps-dev): bump types-colorama from 0.4.0 to 0.4.2 Bumps [types-colorama](https://github.com/python/typeshed) from 0.4.0 to 0.4.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-colorama dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7ab9d9e9..163477e5 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setuptools.setup( "mypy==0.901", # type stubs for mypy "types-backports==0.1.2", - "types-colorama==0.4.0", + "types-colorama==0.4.2", "types-PyYAML==0.1.6", "types-tabulate==0.1.0", "types-termcolor==0.1.0", From d1407f0a1e1c6cdb57f81af6cdb336f0362b590f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 14:02:08 +0000 Subject: [PATCH 08/23] build(deps-dev): bump types-termcolor from 0.1.0 to 0.1.1 Bumps [types-termcolor](https://github.com/python/typeshed) from 0.1.0 to 0.1.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-termcolor dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7ab9d9e9..ab7da752 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ setuptools.setup( "types-colorama==0.4.0", "types-PyYAML==0.1.6", "types-tabulate==0.1.0", - "types-termcolor==0.1.0", + "types-termcolor==0.1.1", ], }, zip_safe=False, From 7fe53073fe5bd95ddc10fa94160fd473fa8986a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 14:02:11 +0000 Subject: [PATCH 09/23] build(deps): bump viv-utils[flirt] from 0.6.4 to 0.6.5 Bumps [viv-utils[flirt]](https://github.com/williballenthin/viv-utils) from 0.6.4 to 0.6.5. - [Release notes](https://github.com/williballenthin/viv-utils/releases) - [Commits](https://github.com/williballenthin/viv-utils/compare/v0.6.4...v0.6.5) --- updated-dependencies: - dependency-name: viv-utils[flirt] dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7ab9d9e9..76ca0fa0 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ requirements = [ "termcolor==1.1.0", "wcwidth==0.2.5", "ida-settings==2.1.0", - "viv-utils[flirt]==0.6.4", + "viv-utils[flirt]==0.6.5", "halo==0.0.31", "networkx==2.5.1", "ruamel.yaml==0.17.9", From 2d7ac73caa0298bd3a541befd8ebec8265c8c990 Mon Sep 17 00:00:00 2001 From: Moritz Date: Mon, 21 Jun 2021 16:24:43 +0200 Subject: [PATCH 10/23] update dependabot actor name --- .github/workflows/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 07870383..c028ce82 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -10,7 +10,7 @@ on: jobs: check_changelog: # no need to check for dependency updates via dependabot - if: github.actor != 'dependabot' + if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' runs-on: ubuntu-20.04 env: NO_CHANGELOG: '[x] No CHANGELOG update needed' From 03b15ce28977ffcc617d3c67c9dff20de7ee6196 Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Mon, 21 Jun 2021 14:30:00 +0000 Subject: [PATCH 11/23] Sync capa rules submodule --- CHANGELOG.md | 3 ++- README.md | 2 +- rules | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0d87314..e8a0ae84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - main: more detailed progress bar output when matching functions #562 @mr-tz - main: detect file limitations without doing code analysis for better performance #583 @williballenthin -### New Rules (88) +### New Rules (89) - anti-analysis/packer/amber/packed-with-amber @gormaniac - collection/file-managers/gather-3d-ftp-information @re-fox @@ -104,6 +104,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - compiler/autohotkey/compiled-with-autohotkey awillia2@cisco.com - internal/limitation/file/internal-autohotkey-file-limitation @mr-tz - host-interaction/process/dump/create-process-memory-minidump michael.hunhoff@fireeye.com +- nursery/get-storage-device-properties michael.hunhoff@fireeye.com - diff --git a/README.md b/README.md index d0152578..de2f23c5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-574-blue.svg)](https://github.com/fireeye/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-575-blue.svg)](https://github.com/fireeye/capa-rules) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index b7a5a15e..037169f2 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit b7a5a15e72d8d9abe0298df5631e42d5c16d20ba +Subproject commit 037169f2bddfbf52e9b5c17db096ba141651d67c From e4b89f1d7b2d4cc5673ff26effc73e4d4366886c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 16:43:23 +0000 Subject: [PATCH 12/23] build(deps-dev): bump types-tabulate from 0.1.0 to 0.1.1 Bumps [types-tabulate](https://github.com/python/typeshed) from 0.1.0 to 0.1.1. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-tabulate dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 34ee5e64..66aec49d 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ setuptools.setup( "types-backports==0.1.2", "types-colorama==0.4.2", "types-PyYAML==0.1.6", - "types-tabulate==0.1.0", + "types-tabulate==0.1.1", "types-termcolor==0.1.1", ], }, From a00c3b6d320c6a7fe72954ec05db99ee2978041d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 16:44:34 +0000 Subject: [PATCH 13/23] build(deps-dev): bump types-pyyaml from 0.1.6 to 5.4.3 Bumps [types-pyyaml](https://github.com/python/typeshed) from 0.1.6 to 5.4.3. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-pyyaml dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 66aec49d..c307ac80 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ setuptools.setup( # type stubs for mypy "types-backports==0.1.2", "types-colorama==0.4.2", - "types-PyYAML==0.1.6", + "types-PyYAML==5.4.3", "types-tabulate==0.1.1", "types-termcolor==0.1.1", ], From 86b3438a2d985ccb2e40746df056018a41864ca2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 16:44:34 +0000 Subject: [PATCH 14/23] build(deps-dev): bump isort from 5.8.0 to 5.9.1 Bumps [isort](https://github.com/pycqa/isort) from 5.8.0 to 5.9.1. - [Release notes](https://github.com/pycqa/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/main/CHANGELOG.md) - [Commits](https://github.com/pycqa/isort/compare/5.8.0...5.9.1) --- updated-dependencies: - dependency-name: isort dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 66aec49d..15c15a57 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setuptools.setup( "pytest-cov==2.12.1", "pycodestyle==2.7.0", "black==21.6b0", - "isort==5.8.0", + "isort==5.9.1", "mypy==0.901", # type stubs for mypy "types-backports==0.1.2", From 6977477a3909664f65a34acd7719397b41a16e04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 16:44:34 +0000 Subject: [PATCH 15/23] build(deps-dev): bump types-backports from 0.1.2 to 0.1.3 Bumps [types-backports](https://github.com/python/typeshed) from 0.1.2 to 0.1.3. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-backports dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 66aec49d..87083df4 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ setuptools.setup( "isort==5.8.0", "mypy==0.901", # type stubs for mypy - "types-backports==0.1.2", + "types-backports==0.1.3", "types-colorama==0.4.2", "types-PyYAML==0.1.6", "types-tabulate==0.1.1", From 565bb96c9e873b9dbe37df7b36a739cdf323e3c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 16:47:28 +0000 Subject: [PATCH 16/23] build(deps-dev): bump mypy from 0.901 to 0.902 Bumps [mypy](https://github.com/python/mypy) from 0.901 to 0.902. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.901...v0.902) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 35d9fe5f..d23fa03a 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ setuptools.setup( "pycodestyle==2.7.0", "black==21.6b0", "isort==5.9.1", - "mypy==0.901", + "mypy==0.902", # type stubs for mypy "types-backports==0.1.3", "types-colorama==0.4.2", From 2872db8b239646f2d019eeed18f6b545aec2dca7 Mon Sep 17 00:00:00 2001 From: Michael Hunhoff Date: Tue, 22 Jun 2021 16:06:14 -0600 Subject: [PATCH 17/23] resolve circular import failure --- CHANGELOG.md | 1 + capa/ida/plugin/__init__.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8a0ae84..1f5bcde4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,6 +113,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - build: use Python 3.8 for PyInstaller to support consistently running across multiple operating systems including Windows 7 #505 @mr-tz - main: correctly match BB-scope matches at file scope #605 @williballenthin - explorer: add support for function-name feature #618 @mike-hunhoff +- explorer: circular import workaround #654 @mike-hunhoff ### Changes diff --git a/capa/ida/plugin/__init__.py b/capa/ida/plugin/__init__.py index f262f486..2738a97e 100644 --- a/capa/ida/plugin/__init__.py +++ b/capa/ida/plugin/__init__.py @@ -11,7 +11,6 @@ import logging import idaapi import ida_kernwin -from capa.ida.helpers import is_supported_file_type, is_supported_ida_version from capa.ida.plugin.form import CapaExplorerForm from capa.ida.plugin.icon import ICON @@ -41,10 +40,12 @@ class CapaExplorerPlugin(idaapi.plugin_t): """called when IDA is loading the plugin""" logging.basicConfig(level=logging.INFO) + import capa.ida.helpers + # do not load plugin if IDA version/file type not supported - if not is_supported_ida_version(): + if not capa.ida.helpers.is_supported_ida_version(): return idaapi.PLUGIN_SKIP - if not is_supported_file_type(): + if not capa.ida.helpers.is_supported_file_type(): return idaapi.PLUGIN_SKIP return idaapi.PLUGIN_OK From 831dc577f40408f5954fbbd94be40da570f82172 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Tue, 8 Jun 2021 14:17:14 +0200 Subject: [PATCH 18/23] add breaking change section and reorg --- CHANGELOG.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f5bcde4..8fcde2f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,16 @@ It includes many new rules, including all new techniques introduced in MITRE ATT ### New Features +- rules: update ATT&CK and MBC mappings https://github.com/fireeye/capa-rules/pull/317 @williballenthin - main: use FLIRT signatures to identify and ignore library code #446 @williballenthin - explorer: IDA 7.6 support #497 @williballenthin +- tests: update test cases and caching #545 @mr-tz - scripts: capa2yara.py convert capa rules to YARA rules #561 @ruppde - rule: add file-scope feature (`function-name`) for recognized library functions #567 @williballenthin - main: auto detect shellcode based on file extension #516 @mr-tz - main: more detailed progress bar output when matching functions #562 @mr-tz - main: detect file limitations without doing code analysis for better performance #583 @williballenthin +- show-features: don't show features from library functions #569 @williballenthin ### New Rules (89) @@ -115,26 +118,22 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - explorer: add support for function-name feature #618 @mike-hunhoff - explorer: circular import workaround #654 @mike-hunhoff -### Changes +### Breaking Changes - py3: drop Python 2 support #480 @Ana06 -- deps: bump ruamel yaml parser to 0.17.4 #519 @williballenthin -- explorer: explain how to install IDA 7.6 patch to enable the plugin #528 @williballenthin -- explorer: document IDA 7.6sp1 as alternative to the patch #536 @Ana06 -- rules: update ATT&CK and MBC mappings https://github.com/fireeye/capa-rules/pull/317 @williballenthin -- tests: update test cases and caching #545 @mr-tz -- show-features: don't show features from library functions #569 @williballenthin -- linter: summarize results at the end #571 @williballenthin - meta: added `library_functions` field, `feature_counts.functions` does not include library functions any more #562 @mr-tz -- linter: check for `or` with always true child statement, e.g. `optional`, colors #348 @mr-tz -- json: breaking change in results document; now contains parsed ATT&CK and MBC fields instead of canonical representation #526 @mr-tz -- json: breaking change: record all matching strings for regex #159 @williballenthin +- json: results document now contains parsed ATT&CK and MBC fields instead of canonical representation #526 @mr-tz +- json: record all matching strings for regex #159 @williballenthin - main: implement file limitations via rules not code #390 @williballenthin - json: breaking change: correctly render negative offsets #619 @williballenthin - library: breaking change: remove logic from `__init__.py` throughout #622 @williballenthin - main: don't create .viv files unless CAPA_SAVE_WORKSPACE environment variable is set #507 @williballenthin - library: add type annotations for use with mypy #447 @williballenthin +### Documentation +- explorer: explain how to install IDA 7.6 patch to enable the plugin #528 @williballenthin +- explorer: document IDA 7.6sp1 as alternative to the patch #536 @Ana06 + ### Development - ci: add capa release link to capa-rules tag #517 @Ana06 @@ -142,6 +141,8 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - ci, changelog: support multiple author in sync GH https://github.com/fireeye/capa-rules/pull/378 @Ana06 - ci, lint: check statements for single child statements #563 @mr-tz - ci: reject PRs without CHANGELOG update to ensure CHANGELOG is kept up-to-date. #584 @Ana06 +- linter: summarize results at the end #571 @williballenthin +- linter: check for `or` with always true child statement, e.g. `optional`, colors #348 @mr-tz ### Raw diffs From 45c4b4019a39755325118c4d23512b45b9c96222 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Wed, 9 Jun 2021 09:43:26 +0200 Subject: [PATCH 19/23] move breaking changes to top --- CHANGELOG.md | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fcde2f0..53226dc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,6 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - rules: update ATT&CK and MBC mappings https://github.com/fireeye/capa-rules/pull/317 @williballenthin - main: use FLIRT signatures to identify and ignore library code #446 @williballenthin -- explorer: IDA 7.6 support #497 @williballenthin - tests: update test cases and caching #545 @mr-tz - scripts: capa2yara.py convert capa rules to YARA rules #561 @ruppde - rule: add file-scope feature (`function-name`) for recognized library functions #567 @williballenthin @@ -17,6 +16,18 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - main: more detailed progress bar output when matching functions #562 @mr-tz - main: detect file limitations without doing code analysis for better performance #583 @williballenthin - show-features: don't show features from library functions #569 @williballenthin +- linter: summarize results at the end #571 @williballenthin +- linter: check for `or` with always true child statement, e.g. `optional`, colors #348 @mr-tz + +### Breaking Changes + +- py3: drop Python 2 support #480 @Ana06 +- meta: added `library_functions` field, `feature_counts.functions` does not include library functions any more #562 @mr-tz +- json: results document now contains parsed ATT&CK and MBC fields instead of canonical representation #526 @mr-tz +- json: record all matching strings for regex #159 @williballenthin +- main: implement file limitations via rules not code #390 @williballenthin +- json: correctly render negative offsets #619 @williballenthin +- library: remove logic from `__init__.py` throughout #622 @williballenthin ### New Rules (89) @@ -110,29 +121,17 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - nursery/get-storage-device-properties michael.hunhoff@fireeye.com - - ### Bug Fixes - build: use Python 3.8 for PyInstaller to support consistently running across multiple operating systems including Windows 7 #505 @mr-tz - main: correctly match BB-scope matches at file scope #605 @williballenthin -- explorer: add support for function-name feature #618 @mike-hunhoff -- explorer: circular import workaround #654 @mike-hunhoff -### Breaking Changes - -- py3: drop Python 2 support #480 @Ana06 -- meta: added `library_functions` field, `feature_counts.functions` does not include library functions any more #562 @mr-tz -- json: results document now contains parsed ATT&CK and MBC fields instead of canonical representation #526 @mr-tz -- json: record all matching strings for regex #159 @williballenthin -- main: implement file limitations via rules not code #390 @williballenthin -- json: breaking change: correctly render negative offsets #619 @williballenthin -- library: breaking change: remove logic from `__init__.py` throughout #622 @williballenthin -- main: don't create .viv files unless CAPA_SAVE_WORKSPACE environment variable is set #507 @williballenthin -- library: add type annotations for use with mypy #447 @williballenthin - -### Documentation +### capa explorer IDA Pro plugin +- explorer: IDA 7.6 support #497 @williballenthin - explorer: explain how to install IDA 7.6 patch to enable the plugin #528 @williballenthin - explorer: document IDA 7.6sp1 as alternative to the patch #536 @Ana06 +- explorer: add support for function-name feature #618 @mike-hunhoff +- explorer: circular import workaround #654 @mike-hunhoff ### Development @@ -141,8 +140,6 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - ci, changelog: support multiple author in sync GH https://github.com/fireeye/capa-rules/pull/378 @Ana06 - ci, lint: check statements for single child statements #563 @mr-tz - ci: reject PRs without CHANGELOG update to ensure CHANGELOG is kept up-to-date. #584 @Ana06 -- linter: summarize results at the end #571 @williballenthin -- linter: check for `or` with always true child statement, e.g. `optional`, colors #348 @mr-tz ### Raw diffs From 44b47eb39cd9eb46f9a77555670669a9155d9397 Mon Sep 17 00:00:00 2001 From: Moritz Raabe Date: Fri, 11 Jun 2021 09:17:06 +0200 Subject: [PATCH 20/23] update release checklist --- doc/release.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/doc/release.md b/doc/release.md index efd60aa9..fb154ff7 100644 --- a/doc/release.md +++ b/doc/release.md @@ -10,13 +10,7 @@ - [ ] Update [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md) - Do not forget to add a nice introduction thanking contributors - Remember that we need a major release if we introduce breaking changes - - Sections - - New Features - - New Rules - - Bug Fixes - - Changes - - Development - - Raw diffs + - Sections: see template below - Update `Raw diffs` links - Create placeholder for `master (unreleased)` section ``` @@ -24,13 +18,15 @@ ### New Features + ### Breaking Changes + ### New Rules (0) - ### Bug Fixes - ### Changes + ### capa explorer IDA Pro plugin ### Development From 2158be0a2ef7b52132ac74acbb209f67f86c7692 Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Tue, 4 May 2021 15:53:40 +0200 Subject: [PATCH 21/23] explorer: add analyze option I would like to load capa explorer with an script and that it runs the analysis without needing extra clicks. Introduce an analyze option for this. Loading capa explorer from the UI or with Alt+F5 behaves as before. The following command as well: ``` ida_loader.load_and_run_plugin("capa_explorer", 0) ``` But the following command automatically runs the analysis without extra clicks: ``` ida_loader.load_and_run_plugin("capa_explorer", 1) ``` Example of where I am using this: https://github.com/Ana06/idapython/blob/master/idapythonrc.py#L22 --- CHANGELOG.md | 1 + capa/ida/plugin/README.md | 1 + capa/ida/plugin/__init__.py | 10 ++++++++-- capa/ida/plugin/form.py | 5 ++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53226dc8..37162912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - show-features: don't show features from library functions #569 @williballenthin - linter: summarize results at the end #571 @williballenthin - linter: check for `or` with always true child statement, e.g. `optional`, colors #348 @mr-tz +- explorer: add argument to control whether to automatically analyze when running capa explorer #548 @Ana06 ### Breaking Changes diff --git a/capa/ida/plugin/README.md b/capa/ida/plugin/README.md index b40d6ff5..846e74a0 100644 --- a/capa/ida/plugin/README.md +++ b/capa/ida/plugin/README.md @@ -79,6 +79,7 @@ You can install capa explorer using the following steps: 1. Open IDA and analyze a supported file type (select the `Manual Load` and `Load Resources` options in IDA for best results) 2. Open capa explorer in IDA by navigating to `Edit > Plugins > FLARE capa explorer` or using the keyboard shortcut `Alt+F5` + You can also use `ida_loader.load_and_run_plugin("capa_explorer", arg)`. `arg` is a bitflag for which setting the LSB enables automatic analysis. See `capa.ida.plugin.form.Options` for more details. 3. Select the `Program Analysis` tab 4. Click the `Analyze` button diff --git a/capa/ida/plugin/__init__.py b/capa/ida/plugin/__init__.py index 2738a97e..97c11e18 100644 --- a/capa/ida/plugin/__init__.py +++ b/capa/ida/plugin/__init__.py @@ -54,8 +54,14 @@ class CapaExplorerPlugin(idaapi.plugin_t): pass def run(self, arg): - """called when IDA is running the plugin as a script""" - self.form = CapaExplorerForm(self.PLUGIN_NAME) + """ + called when IDA is running the plugin as a script + + args: + arg (int): bitflag. Setting LSB enables automatic analysis upon + loading. The other bits are currently undefined. See `form.Options`. + """ + self.form = CapaExplorerForm(self.PLUGIN_NAME, arg) return True diff --git a/capa/ida/plugin/form.py b/capa/ida/plugin/form.py index 215ec2d7..29d8305c 100644 --- a/capa/ida/plugin/form.py +++ b/capa/ida/plugin/form.py @@ -230,7 +230,7 @@ class CapaSettingsInputDialog(QtWidgets.QDialog): class CapaExplorerForm(idaapi.PluginForm): """form element for plugin interface""" - def __init__(self, name): + def __init__(self, name, option=0): """initialize form elements""" super(CapaExplorerForm, self).__init__() @@ -278,6 +278,9 @@ class CapaExplorerForm(idaapi.PluginForm): self.Show() + if option == 1: + self.analyze_program() + def OnCreate(self, form): """called when plugin form is created From 0685d36220717c3efc65e8610cae397cdc54e3e9 Mon Sep 17 00:00:00 2001 From: Ana Maria Martinez Gomez Date: Tue, 25 May 2021 10:58:35 +0200 Subject: [PATCH 22/23] explorer: use bitmask + enum for analyze option --- capa/ida/plugin/form.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/capa/ida/plugin/form.py b/capa/ida/plugin/form.py index 29d8305c..99cdafd9 100644 --- a/capa/ida/plugin/form.py +++ b/capa/ida/plugin/form.py @@ -44,6 +44,13 @@ CAPA_SETTINGS_RULE_PATH = "rule_path" CAPA_SETTINGS_RULEGEN_AUTHOR = "rulegen_author" CAPA_SETTINGS_RULEGEN_SCOPE = "rulegen_scope" +from enum import IntFlag + + +class Options(IntFlag): + DEFAULT = 0 + ANALYZE = 1 # Runs the analysis when starting the explorer + def write_file(path, data): """ """ @@ -230,7 +237,7 @@ class CapaSettingsInputDialog(QtWidgets.QDialog): class CapaExplorerForm(idaapi.PluginForm): """form element for plugin interface""" - def __init__(self, name, option=0): + def __init__(self, name, option=Options.DEFAULT): """initialize form elements""" super(CapaExplorerForm, self).__init__() @@ -278,7 +285,7 @@ class CapaExplorerForm(idaapi.PluginForm): self.Show() - if option == 1: + if (option & Options.ANALYZE) == Options.ANALYZE: self.analyze_program() def OnCreate(self, form): From 58aa3e33bf647929bd7ab03b66901133223e506a Mon Sep 17 00:00:00 2001 From: Capa Bot Date: Thu, 24 Jun 2021 00:33:45 +0000 Subject: [PATCH 23/23] Sync capa rules submodule --- CHANGELOG.md | 6 +++++- README.md | 2 +- rules | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53226dc8..310f1a84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - json: correctly render negative offsets #619 @williballenthin - library: remove logic from `__init__.py` throughout #622 @williballenthin -### New Rules (89) +### New Rules (93) - anti-analysis/packer/amber/packed-with-amber @gormaniac - collection/file-managers/gather-3d-ftp-information @re-fox @@ -119,6 +119,10 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - internal/limitation/file/internal-autohotkey-file-limitation @mr-tz - host-interaction/process/dump/create-process-memory-minidump michael.hunhoff@fireeye.com - nursery/get-storage-device-properties michael.hunhoff@fireeye.com +- nursery/execute-shell-command-via-windows-remote-management michael.hunhoff@fireeye.com +- nursery/get-token-privileges michael.hunhoff@fireeye.com +- nursery/prompt-user-for-credentials michael.hunhoff@fireeye.com +- nursery/spoof-parent-pid michael.hunhoff@fireeye.com - ### Bug Fixes diff --git a/README.md b/README.md index de2f23c5..64049694 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-575-blue.svg)](https://github.com/fireeye/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-579-blue.svg)](https://github.com/fireeye/capa-rules) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/rules b/rules index 037169f2..30086076 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 037169f2bddfbf52e9b5c17db096ba141651d67c +Subproject commit 30086076974245c23806ce7089f684d64b173459