diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 07870383..c028ce82 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -10,7 +10,7 @@ on: jobs: check_changelog: # no need to check for dependency updates via dependabot - if: github.actor != 'dependabot' + if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' runs-on: ubuntu-20.04 env: NO_CHANGELOG: '[x] No CHANGELOG update needed' diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ce682c6..f66116e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,15 +7,30 @@ It includes many new rules, including all new techniques introduced in MITRE ATT ### New Features +- rules: update ATT&CK and MBC mappings https://github.com/fireeye/capa-rules/pull/317 @williballenthin - main: use FLIRT signatures to identify and ignore library code #446 @williballenthin -- explorer: IDA 7.6 support #497 @williballenthin +- tests: update test cases and caching #545 @mr-tz - scripts: capa2yara.py convert capa rules to YARA rules #561 @ruppde - rule: add file-scope feature (`function-name`) for recognized library functions #567 @williballenthin - main: auto detect shellcode based on file extension #516 @mr-tz - main: more detailed progress bar output when matching functions #562 @mr-tz - main: detect file limitations without doing code analysis for better performance #583 @williballenthin +- show-features: don't show features from library functions #569 @williballenthin +- linter: summarize results at the end #571 @williballenthin +- linter: check for `or` with always true child statement, e.g. `optional`, colors #348 @mr-tz +- explorer: add argument to control whether to automatically analyze when running capa explorer #548 @Ana06 -### New Rules (88) +### Breaking Changes + +- py3: drop Python 2 support #480 @Ana06 +- meta: added `library_functions` field, `feature_counts.functions` does not include library functions any more #562 @mr-tz +- json: results document now contains parsed ATT&CK and MBC fields instead of canonical representation #526 @mr-tz +- json: record all matching strings for regex #159 @williballenthin +- main: implement file limitations via rules not code #390 @williballenthin +- json: correctly render negative offsets #619 @williballenthin +- library: remove logic from `__init__.py` throughout #622 @williballenthin + +### New Rules (93) - anti-analysis/packer/amber/packed-with-amber @gormaniac - collection/file-managers/gather-3d-ftp-information @re-fox @@ -104,33 +119,24 @@ It includes many new rules, including all new techniques introduced in MITRE ATT - compiler/autohotkey/compiled-with-autohotkey awillia2@cisco.com - internal/limitation/file/internal-autohotkey-file-limitation @mr-tz - host-interaction/process/dump/create-process-memory-minidump michael.hunhoff@fireeye.com +- nursery/get-storage-device-properties michael.hunhoff@fireeye.com +- nursery/execute-shell-command-via-windows-remote-management michael.hunhoff@fireeye.com +- nursery/get-token-privileges michael.hunhoff@fireeye.com +- nursery/prompt-user-for-credentials michael.hunhoff@fireeye.com +- nursery/spoof-parent-pid michael.hunhoff@fireeye.com - - ### Bug Fixes - build: use Python 3.8 for PyInstaller to support consistently running across multiple operating systems including Windows 7 #505 @mr-tz - main: correctly match BB-scope matches at file scope #605 @williballenthin -- explorer: add support for function-name feature #618 @mike-hunhoff -### Changes - -- py3: drop Python 2 support #480 @Ana06 -- deps: bump ruamel yaml parser to 0.17.4 #519 @williballenthin +### capa explorer IDA Pro plugin +- explorer: IDA 7.6 support #497 @williballenthin - explorer: explain how to install IDA 7.6 patch to enable the plugin #528 @williballenthin - explorer: document IDA 7.6sp1 as alternative to the patch #536 @Ana06 -- rules: update ATT&CK and MBC mappings https://github.com/fireeye/capa-rules/pull/317 @williballenthin -- tests: update test cases and caching #545 @mr-tz -- show-features: don't show features from library functions #569 @williballenthin -- linter: summarize results at the end #571 @williballenthin -- meta: added `library_functions` field, `feature_counts.functions` does not include library functions any more #562 @mr-tz -- linter: check for `or` with always true child statement, e.g. `optional`, colors #348 @mr-tz -- json: breaking change in results document; now contains parsed ATT&CK and MBC fields instead of canonical representation #526 @mr-tz -- json: breaking change: record all matching strings for regex #159 @williballenthin -- main: implement file limitations via rules not code #390 @williballenthin -- json: breaking change: correctly render negative offsets #619 @williballenthin -- library: breaking change: remove logic from `__init__.py` throughout #622 @williballenthin -- library: add type annotations for use with mypy #447 @williballenthin +- explorer: add support for function-name feature #618 @mike-hunhoff +- explorer: circular import workaround #654 @mike-hunhoff ### Development diff --git a/README.md b/README.md index d0152578..64049694 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-574-blue.svg)](https://github.com/fireeye/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-579-blue.svg)](https://github.com/fireeye/capa-rules) [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/capa/features/freeze.py b/capa/features/freeze.py index 04514559..47fbcf3a 100644 --- a/capa/features/freeze.py +++ b/capa/features/freeze.py @@ -257,7 +257,8 @@ def main(argv=None): sigpaths = capa.main.get_signatures(args.signatures) - extractor = capa.main.get_extractor(args.sample, args.format, args.backend, sigpaths=sigpaths) + extractor = capa.main.get_extractor(args.sample, args.format, args.backend, sigpaths, False) + with open(args.output, "wb") as f: f.write(dump(extractor)) diff --git a/capa/ida/plugin/README.md b/capa/ida/plugin/README.md index b40d6ff5..846e74a0 100644 --- a/capa/ida/plugin/README.md +++ b/capa/ida/plugin/README.md @@ -79,6 +79,7 @@ You can install capa explorer using the following steps: 1. Open IDA and analyze a supported file type (select the `Manual Load` and `Load Resources` options in IDA for best results) 2. Open capa explorer in IDA by navigating to `Edit > Plugins > FLARE capa explorer` or using the keyboard shortcut `Alt+F5` + You can also use `ida_loader.load_and_run_plugin("capa_explorer", arg)`. `arg` is a bitflag for which setting the LSB enables automatic analysis. See `capa.ida.plugin.form.Options` for more details. 3. Select the `Program Analysis` tab 4. Click the `Analyze` button diff --git a/capa/ida/plugin/__init__.py b/capa/ida/plugin/__init__.py index f262f486..97c11e18 100644 --- a/capa/ida/plugin/__init__.py +++ b/capa/ida/plugin/__init__.py @@ -11,7 +11,6 @@ import logging import idaapi import ida_kernwin -from capa.ida.helpers import is_supported_file_type, is_supported_ida_version from capa.ida.plugin.form import CapaExplorerForm from capa.ida.plugin.icon import ICON @@ -41,10 +40,12 @@ class CapaExplorerPlugin(idaapi.plugin_t): """called when IDA is loading the plugin""" logging.basicConfig(level=logging.INFO) + import capa.ida.helpers + # do not load plugin if IDA version/file type not supported - if not is_supported_ida_version(): + if not capa.ida.helpers.is_supported_ida_version(): return idaapi.PLUGIN_SKIP - if not is_supported_file_type(): + if not capa.ida.helpers.is_supported_file_type(): return idaapi.PLUGIN_SKIP return idaapi.PLUGIN_OK @@ -53,8 +54,14 @@ class CapaExplorerPlugin(idaapi.plugin_t): pass def run(self, arg): - """called when IDA is running the plugin as a script""" - self.form = CapaExplorerForm(self.PLUGIN_NAME) + """ + called when IDA is running the plugin as a script + + args: + arg (int): bitflag. Setting LSB enables automatic analysis upon + loading. The other bits are currently undefined. See `form.Options`. + """ + self.form = CapaExplorerForm(self.PLUGIN_NAME, arg) return True diff --git a/capa/ida/plugin/form.py b/capa/ida/plugin/form.py index 215ec2d7..99cdafd9 100644 --- a/capa/ida/plugin/form.py +++ b/capa/ida/plugin/form.py @@ -44,6 +44,13 @@ CAPA_SETTINGS_RULE_PATH = "rule_path" CAPA_SETTINGS_RULEGEN_AUTHOR = "rulegen_author" CAPA_SETTINGS_RULEGEN_SCOPE = "rulegen_scope" +from enum import IntFlag + + +class Options(IntFlag): + DEFAULT = 0 + ANALYZE = 1 # Runs the analysis when starting the explorer + def write_file(path, data): """ """ @@ -230,7 +237,7 @@ class CapaSettingsInputDialog(QtWidgets.QDialog): class CapaExplorerForm(idaapi.PluginForm): """form element for plugin interface""" - def __init__(self, name): + def __init__(self, name, option=Options.DEFAULT): """initialize form elements""" super(CapaExplorerForm, self).__init__() @@ -278,6 +285,9 @@ class CapaExplorerForm(idaapi.PluginForm): self.Show() + if (option & Options.ANALYZE) == Options.ANALYZE: + self.analyze_program() + def OnCreate(self, form): """called when plugin form is created diff --git a/capa/main.py b/capa/main.py index 716cba17..1f960a01 100644 --- a/capa/main.py +++ b/capa/main.py @@ -433,7 +433,7 @@ class UnsupportedRuntimeError(RuntimeError): def get_extractor( - path: str, format: str, backend: str, sigpaths: List[str], disable_progress=False + path: str, format: str, backend: str, sigpaths: List[str], should_save_workspace, disable_progress=False ) -> FeatureExtractor: """ raises: @@ -463,11 +463,15 @@ def get_extractor( format = "sc64" vw = get_workspace(path, format, sigpaths) - try: - vw.saveWorkspace() - except IOError: - # see #168 for discussion around how to handle non-writable directories - logger.info("source directory is not writable, won't save intermediate workspace") + if should_save_workspace: + logger.debug("saving workspace") + try: + vw.saveWorkspace() + except IOError: + # see #168 for discussion around how to handle non-writable directories + logger.info("source directory is not writable, won't save intermediate workspace") + else: + logger.debug("CAPA_SAVE_WORKSPACE unset, not saving workspace") return capa.features.extractors.viv.extractor.VivisectFeatureExtractor(vw, path) @@ -883,8 +887,12 @@ def main(argv=None): extractor = capa.features.freeze.load(f.read()) else: format = args.format + should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) + try: - extractor = get_extractor(args.sample, format, args.backend, sig_paths, disable_progress=args.quiet) + extractor = get_extractor( + args.sample, format, args.backend, sig_paths, should_save_workspace, disable_progress=args.quiet + ) except UnsupportedFormatError: logger.error("-" * 80) logger.error(" Input file does not appear to be a PE file.") diff --git a/doc/release.md b/doc/release.md index efd60aa9..fb154ff7 100644 --- a/doc/release.md +++ b/doc/release.md @@ -10,13 +10,7 @@ - [ ] Update [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md) - Do not forget to add a nice introduction thanking contributors - Remember that we need a major release if we introduce breaking changes - - Sections - - New Features - - New Rules - - Bug Fixes - - Changes - - Development - - Raw diffs + - Sections: see template below - Update `Raw diffs` links - Create placeholder for `master (unreleased)` section ``` @@ -24,13 +18,15 @@ ### New Features + ### Breaking Changes + ### New Rules (0) - ### Bug Fixes - ### Changes + ### capa explorer IDA Pro plugin ### Development diff --git a/doc/usage.md b/doc/usage.md index 75a71b4f..74b163f4 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -11,3 +11,9 @@ For example, `capa -t william.ballenthin@mandiant.com` runs rules that reference ### IDA Pro plugin: capa explorer Please check out the [capa explorer documentation](/capa/ida/plugin/README.md). + +### save time by reusing .viv files +Set the environment variable `CAPA_SAVE_WORKSPACE` to instruct the underlying analysis engine to +cache its intermediate results to the file system. For example, vivisect will create `.viv` files. +Subsequently, capa may run faster when reprocessing the same input file. +This is particularly useful during rule development as you repeatedly test a rule against a known sample. \ No newline at end of file diff --git a/rules b/rules index b7a5a15e..30086076 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit b7a5a15e72d8d9abe0298df5631e42d5c16d20ba +Subproject commit 30086076974245c23806ce7089f684d64b173459 diff --git a/scripts/bulk-process.py b/scripts/bulk-process.py index 88def2c9..da80a477 100644 --- a/scripts/bulk-process.py +++ b/scripts/bulk-process.py @@ -96,9 +96,12 @@ def get_capa_results(args): capabilities (dict): the matched capabilities and their result objects """ rules, sigpaths, format, path = args + should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) logger.info("computing capa results for: %s", path) try: - extractor = capa.main.get_extractor(path, format, capa.main.BACKEND_VIV, sigpaths, disable_progress=True) + extractor = capa.main.get_extractor( + path, format, capa.main.BACKEND_VIV, sigpaths, should_save_workspace, disable_progress=True + ) except capa.main.UnsupportedFormatError: # i'm 100% sure if multiprocessing will reliably raise exceptions across process boundaries. # so instead, return an object with explicit success/failure status. diff --git a/scripts/capa_as_library.py b/scripts/capa_as_library.py index e770b348..36244a3f 100644 --- a/scripts/capa_as_library.py +++ b/scripts/capa_as_library.py @@ -192,7 +192,7 @@ def render_dictionary(doc): def capa_details(file_path, output_format="dictionary"): # extract features and find capabilities - extractor = capa.main.get_extractor(file_path, "auto", capa.main.BACKEND_VIV, sigpaths=[], disable_progress=True) + extractor = capa.main.get_extractor(file_path, "auto", capa.main.BACKEND_VIV, [], False, disable_progress=True) capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True) # collect metadata (used only to make rendering more complete) diff --git a/scripts/lint.py b/scripts/lint.py index f5a2c023..d62d764e 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -220,7 +220,7 @@ class DoesntMatchExample(Lint): try: extractor = capa.main.get_extractor( - path, "auto", capa.main.BACKEND_VIV, sigpaths=DEFAULT_SIGNATURES, disable_progress=True + path, "auto", capa.main.BACKEND_VIV, DEFAULT_SIGNATURES, False, disable_progress=True ) capabilities, meta = capa.main.find_capabilities(ctx["rules"], extractor, disable_progress=True) except Exception as e: diff --git a/scripts/show-capabilities-by-function.py b/scripts/show-capabilities-by-function.py index affba3e3..b95481df 100644 --- a/scripts/show-capabilities-by-function.py +++ b/scripts/show-capabilities-by-function.py @@ -144,9 +144,12 @@ def main(argv=None): extractor = capa.features.freeze.load(f.read()) else: format = args.format + should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) try: - extractor = capa.main.get_extractor(args.sample, args.format, args.backend, sig_paths) + extractor = capa.main.get_extractor( + args.sample, args.format, args.backend, sig_paths, should_save_workspace + ) except capa.main.UnsupportedFormatError: logger.error("-" * 80) logger.error(" Input file does not appear to be a PE file.") diff --git a/setup.py b/setup.py index 7ab9d9e9..d23fa03a 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ requirements = [ "termcolor==1.1.0", "wcwidth==0.2.5", "ida-settings==2.1.0", - "viv-utils[flirt]==0.6.4", + "viv-utils[flirt]==0.6.5", "halo==0.0.31", "networkx==2.5.1", "ruamel.yaml==0.17.9", @@ -72,14 +72,14 @@ setuptools.setup( "pytest-cov==2.12.1", "pycodestyle==2.7.0", "black==21.6b0", - "isort==5.8.0", - "mypy==0.901", + "isort==5.9.1", + "mypy==0.902", # type stubs for mypy - "types-backports==0.1.2", - "types-colorama==0.4.0", - "types-PyYAML==0.1.6", - "types-tabulate==0.1.0", - "types-termcolor==0.1.0", + "types-backports==0.1.3", + "types-colorama==0.4.2", + "types-PyYAML==5.4.3", + "types-tabulate==0.1.1", + "types-termcolor==0.1.1", ], }, zip_safe=False, diff --git a/tests/fixtures.py b/tests/fixtures.py index 0a9d62d0..fda8a9f1 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -84,6 +84,7 @@ def get_viv_extractor(path): vw = capa.main.get_workspace(path, "sc64", sigpaths=sigpaths) else: vw = capa.main.get_workspace(path, "auto", sigpaths=sigpaths) + vw.saveWorkspace() extractor = capa.features.extractors.viv.extractor.VivisectFeatureExtractor(vw, path) fixup_viv(path, extractor) return extractor