diff --git a/.github/mypy/mypy.ini b/.github/mypy/mypy.ini index 3bd4b2e2..163731eb 100644 --- a/.github/mypy/mypy.ini +++ b/.github/mypy/mypy.ini @@ -1,8 +1,5 @@ [mypy] -[mypy-tqdm.*] -ignore_missing_imports = True - [mypy-ruamel.*] ignore_missing_imports = True diff --git a/.github/pyinstaller/pyinstaller.spec b/.github/pyinstaller/pyinstaller.spec index e392eb5a..5cd51435 100644 --- a/.github/pyinstaller/pyinstaller.spec +++ b/.github/pyinstaller/pyinstaller.spec @@ -2,7 +2,6 @@ # Copyright (C) 2020 Mandiant, Inc. All Rights Reserved. import sys -import wcwidth import capa.rules.cache from pathlib import Path @@ -29,13 +28,6 @@ a = Analysis( ("../../rules", "rules"), ("../../sigs", "sigs"), ("../../cache", "cache"), - # capa.render.default uses tabulate that depends on wcwidth. - # it seems wcwidth uses a json file `version.json` - # and this doesn't get picked up by pyinstaller automatically. - # so we manually embed the wcwidth resources here. - # - # ref: https://stackoverflow.com/a/62278462/87207 - (Path(wcwidth.__file__).parent, "wcwidth"), ], # when invoking pyinstaller from the project root, # this gets run from the project root. @@ -48,11 +40,6 @@ a = Analysis( "tkinter", "_tkinter", "Tkinter", - # tqdm provides renderers for ipython, - # however, this drags in a lot of dependencies. - # since we don't spawn a notebook, we can safely remove these. - "IPython", - "ipywidgets", # these are pulled in by networkx # but we don't need to compute the strongly connected components. "numpy", diff --git a/CHANGELOG.md b/CHANGELOG.md index 2edd08c5..77bfaa3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,29 +27,36 @@ - use Python 3.12 to build extra standalone build on Linux #2383 @williballenthin - bump minimum Python version to 3.8.1 to satisfy uv #2387 @williballenthin - vmray: collect more process information from flog.xml #2394 @mr-tz @mike-hunhoff +- replace tabulate, tqdm, and termcolor with rich #2374 @s-ff ### capa explorer IDA Pro plugin ### Development ### Raw diffs + - [capa v7.3.0...master](https://github.com/mandiant/capa/compare/v7.3.0...master) - [capa-rules v7.3.0...master](https://github.com/mandiant/capa-rules/compare/v7.3.0...master) ## v7.3.0 + This release comes with the following three major enhancements: ### 1. Support for VMRay sandbox analysis archives + Unlock powerful malware analysis with capa's new [VMRay sandbox](https://www.vmray.com/) integration! Simply provide a VMRay analysis archive, and capa will automatically extract and match capabilities to streamline your workflow. This is the second support for the analysis of dynamic analysis results after [CAPE](https://www.mandiant.com/resources/blog/dynamic-capa-executable-behavior-cape-sandbox). ### 2. Support for BinExport files generated by Ghidra + [BinExport](https://github.com/google/binexport) files store disassembled data into a Protocol Buffer format. capa now supports the analysis of BinExport files generated by Ghidra. Using Ghidra and the BinExport file format users can now analyze ARM (AARCH64) ELF files targeting Android. ### 3. Introducing the capa rules website + You can now browse capa's default rule set at https://mandiant.github.io/capa/rules. In modern terminals the CLI capa tool hyperlinks to resources on the web, including entries on the capa rules website. Furthermore, https://mandiant.github.io/capa provides a landing page for the capa tool project. ### Additional updates + - [capa Explorer Web](https://mandiant.github.io/capa/explorer/) received several enhancements and bug fixes. - Support for the IDA Pro 9.0 IDAPython API while keeping compatibility to older IDA Pro versions - Six rules have been added and two rules have been updated @@ -57,6 +64,7 @@ Furthermore, https://mandiant.github.io/capa provides a landing page for the cap Thanks to @r-sm2024 for their contribution in https://github.com/mandiant/capa/pull/2155 and their further work. And of course a big thanks to the community for reporting issues, participating in discussions, and supporting the capa tool and capa rules. ### New Features + - regenerate ruleset cache automatically on source change (only in dev mode) #2133 @s-ff - add landing page https://mandiant.github.io/capa/ @williballenthin #2310 - add rules website https://mandiant.github.io/capa/rules @DeeyaSingh #2310 @@ -82,14 +90,17 @@ Thanks to @r-sm2024 for their contribution in https://github.com/mandiant/capa/p - fix code path reference in linter @williballenthin #2350 ### capa explorer IDA Pro plugin + - update IDAPython to IDA Pro 9.0 @mr-tz - fix byte search IDA Pro 7.5 compatibility @mr-tz #2371 ### Raw diffs + - [capa v7.2.0...v7.3.0](https://github.com/mandiant/capa/compare/v7.2.0...v7.3.0) - [capa-rules v7.2.0...v7.3.0](https://github.com/mandiant/capa-rules/compare/v7.2.0...v7.3.0) ## v7.2.0 + capa v7.2.0 introduces a first version of capa explorer web: a web-based user interface to inspect capa results using your browser. Users can inspect capa result JSON documents in an online web instance or a standalone HTML page for offline usage. capa explorer supports interactive exploring of capa results to make it easier to understand them. Users can filter, sort, and see the details of all identified capabilities. capa explorer web was worked on by @s-ff as part of a [GSoC project](https://summerofcode.withgoogle.com/programs/2024/projects/cR3hjbsq), and it is available at https://mandiant.github.io/capa/explorer/#/. This release also adds a feature extractor for output from the DRAKVUF sandbox. Now, analysts can pass the resulting `drakmon.log` file to capa and extract capabilities from the artifacts captured by the sandbox. This feature extractor will also be added to the DRAKVUF sandbox as a post-processing script, and it was worked on by @yelhamer as part of a [GSoC project](https://summerofcode.withgoogle.com/programs/2024/projects/fCnBGuEC). @@ -97,9 +108,10 @@ This release also adds a feature extractor for output from the DRAKVUF sandbox. Additionally, we fixed several bugs handling ELF files, and added the ability to filter capa analysis by functions or processes. We also added support to the IDA Pro extractor to leverage analyst recovered API names. Special thanks to our repeat and new contributors: -* @lakshayletsgo for their first contribution in https://github.com/mandiant/capa/pull/2248 -* @msm-cert for their first contribution in https://github.com/mandiant/capa/pull/2143 -* @VascoSch92 for their first contribution in https://github.com/mandiant/capa/pull/2143 + +- @lakshayletsgo for their first contribution in https://github.com/mandiant/capa/pull/2248 +- @msm-cert for their first contribution in https://github.com/mandiant/capa/pull/2143 +- @VascoSch92 for their first contribution in https://github.com/mandiant/capa/pull/2143 ### New Features @@ -126,38 +138,43 @@ Special thanks to our repeat and new contributors: ### capa explorer IDA Pro plugin ### Development + - CI: use macos-12 since macos-11 is deprecated and will be removed on June 28th, 2024 #2173 @mr-tz - CI: update Binary Ninja version to 4.1 and use Python 3.9 to test it #2211 @xusheng6 - CI: update tests.yml workflow to exclude web and documentation files #2263 @s-ff - CI: update build.yml workflow to exclude web and documentation files #2270 @s-ff ### Raw diffs + - [capa v7.1.0...7.2.0](https://github.com/mandiant/capa/compare/v7.1.0...7.2.0) - [capa-rules v7.1.0...7.2.0](https://github.com/mandiant/capa-rules/compare/v7.1.0...7.2.0) ## v7.1.0 + The v7.1.0 release brings large performance improvements to capa's rule matching engine. Additionally, we've fixed various bugs and added new features for people using and developing capa. Special thanks to our repeat and new contributors: -* @sjha2048 made their first contribution in https://github.com/mandiant/capa/pull/2000 -* @Rohit1123 made their first contribution in https://github.com/mandiant/capa/pull/1990 -* @psahithireddy made their first contribution in https://github.com/mandiant/capa/pull/2020 -* @Atlas-64 made their first contribution in https://github.com/mandiant/capa/pull/2018 -* @s-ff made their first contribution in https://github.com/mandiant/capa/pull/2011 -* @samadpls made their first contribution in https://github.com/mandiant/capa/pull/2024 -* @acelynnzhang made their first contribution in https://github.com/mandiant/capa/pull/2044 -* @RainRat made their first contribution in https://github.com/mandiant/capa/pull/2058 -* @ReversingWithMe made their first contribution in https://github.com/mandiant/capa/pull/2093 -* @malwarefrank made their first contribution in https://github.com/mandiant/capa/pull/2037 + +- @sjha2048 made their first contribution in https://github.com/mandiant/capa/pull/2000 +- @Rohit1123 made their first contribution in https://github.com/mandiant/capa/pull/1990 +- @psahithireddy made their first contribution in https://github.com/mandiant/capa/pull/2020 +- @Atlas-64 made their first contribution in https://github.com/mandiant/capa/pull/2018 +- @s-ff made their first contribution in https://github.com/mandiant/capa/pull/2011 +- @samadpls made their first contribution in https://github.com/mandiant/capa/pull/2024 +- @acelynnzhang made their first contribution in https://github.com/mandiant/capa/pull/2044 +- @RainRat made their first contribution in https://github.com/mandiant/capa/pull/2058 +- @ReversingWithMe made their first contribution in https://github.com/mandiant/capa/pull/2093 +- @malwarefrank made their first contribution in https://github.com/mandiant/capa/pull/2037 ### New Features + - Emit "dotnet" as format to ResultDocument when processing .NET files #2024 @samadpls - ELF: detect OS from statically-linked Go binaries #1978 @williballenthin - add function in capa/helpers to load plain and compressed JSON reports #1883 @Rohit1123 - document Antivirus warnings and VirusTotal false positive detections #2028 @RionEV @mr-tz - Add json to sarif conversion script @reversingwithme -- render maec/* fields #843 @s-ff +- render maec/\* fields #843 @s-ff - replace Halo spinner with Rich #2086 @s-ff - optimize rule matching #2080 @williballenthin - add aarch64 as a valid architecture #2144 mehunhoff@google.com @williballenthin @@ -200,6 +217,7 @@ Special thanks to our repeat and new contributors: - cape: support more report formats #2035 @mr-tz ### capa explorer IDA Pro plugin + - replace deprecated IDA API find_binary with bin_search #1606 @s-ff ### Development @@ -208,11 +226,12 @@ Special thanks to our repeat and new contributors: - ci: use rules number badge stored in our bot gist and generated using `schneegans/dynamic-badges-action` #2001 capa-rules#882 @Ana06 - ci: update github workflows to use latest version of actions that were using a deprecated version of node #1967 #2003 capa-rules#883 @sjha2048 @Ana06 - ci: update binja version to stable 4.0 #2016 @xusheng6 -- ci: update github workflows to reflect the latest ghidrathon installation and bumped up jep, ghidra versions #2020 @psahithireddy +- ci: update github workflows to reflect the latest ghidrathon installation and bumped up jep, ghidra versions #2020 @psahithireddy - ci: include rule caching in PyInstaller build process #2097 @s-ff - add deptry support #1497 @s-ff ### Raw diffs + - [capa v7.0.1...v7.1.0](https://github.com/mandiant/capa/compare/v7.0.1...v7.1.0) - [capa-rules v7.0.1...v7.1.0](https://github.com/mandiant/capa-rules/compare/v7.0.1...v7.1.0) @@ -225,10 +244,12 @@ This release fixes a circular import error when using capa as a library. - fix potentially circular import errors #1969 @williballenthin ### Raw diffs + - [capa v7.0.0...v7.0.1](https://github.com/mandiant/capa/compare/v7.0.0...v7.0.1) - [capa-rules v7.0.0...v7.0.1](https://github.com/mandiant/capa-rules/compare/v7.0.0...v7.0.1) ## v7.0.0 + This is the v7.0.0 release of capa which was mainly worked on during the Google Summer of Code (GSoC) 2023. A huge shoutout to our GSoC contributors @colton-gabertan and @yelhamer for their amazing work. @@ -304,6 +325,7 @@ Also, a big thanks to the other contributors: @aaronatp, @Aayush-Goel-04, @bkoju - host-interaction/process/inject/process-ghostly-hollowing sara.rincon@mandiant.com ### Bug Fixes + - ghidra: fix `ints_to_bytes` performance #1761 @mike-hunhoff - binja: improve function call site detection @xusheng6 - binja: use `binaryninja.load` to open files @xusheng6 @@ -314,12 +336,15 @@ Also, a big thanks to the other contributors: @aaronatp, @Aayush-Goel-04, @bkoju - remove unnecessary scripts/vivisect-py2-vs-py3.sh file #1949 @JCoonradt ### capa explorer IDA Pro plugin + - various integration updates and minor bug fixes ### Development + - update ATT&CK/MBC data for linting #1932 @mr-tz #### Developer Notes + With this new release, many classes and concepts have been split up into static (mostly identical to the prior implementations) and dynamic ones. For example, the legacy FeatureExtractor class has been renamed to StaticFeatureExtractor and the DynamicFeatureExtractor has been added. @@ -340,6 +365,7 @@ format and backend is more consistent. We've documented that the input file is n (cape/freeze/etc.) inputs are not actually the sample. ### Raw diffs + - [capa v6.1.0...v7.0.0](https://github.com/mandiant/capa/compare/v6.1.0...v7.0.0) - [capa-rules v6.1.0...v7.0.0](https://github.com/mandiant/capa-rules/compare/v6.1.0...v7.0.0) @@ -353,6 +379,7 @@ You could use this script to find opportunities for new rules. Speaking of new rules, we have eight additions, coming from Ronnie, Jakub, Moritz, Ervin, and still@teamt5.org! ### New Features + - ELF: implement import and export name extractor #1607 #1608 @Aayush-Goel-04 - bump pydantic from 1.10.9 to 2.1.1 #1582 @Aayush-Goel-04 - develop script to highlight features not used during matching #331 @Aayush-Goel-04 @@ -371,16 +398,18 @@ Speaking of new rules, we have eight additions, coming from Ronnie, Jakub, Morit - rules: fix forwarded export characteristic #1656 @RonnieSalomonsen - Binary Ninja: Fix stack string detection #1473 @xusheng6 -- linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin +- linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin - OS: detect Android ELF files #1705 @williballenthin - ELF: fix parsing of symtab #1704 @williballenthin - result document: don't use deprecated pydantic functions #1718 @williballenthin - pytest: don't mark IDA tests as pytest tests #1719 @williballenthin ### capa explorer IDA Pro plugin + - fix unhandled exception when resolving rule path #1693 @mike-hunhoff ### Raw diffs + - [capa v6.0.0...v6.1.0](https://github.com/mandiant/capa/compare/v6.0.0...v6.1.0) - [capa-rules v6.0.0...v6.1.0](https://github.com/mandiant/capa-rules/compare/v6.0.0...v6.1.0) @@ -391,12 +420,14 @@ capa v6.0 brings many bug fixes and quality improvements, including 64 rule upda For those that use capa as a library, we've introduced some limited breaking changes that better represent data types (versus less-structured data like dictionaries and strings). With the recent deprecation, we've also dropped support for Python 3.7. ### New Features + - add script to detect feature overlap between new and existing capa rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04) - extract forwarded exports from PE files #1624 @williballenthin - extract function and API names from ELF symtab entries @yelhamer https://github.com/mandiant/capa-rules/issues/736 - use fancy box drawing characters for default output #1586 @williballenthin ### Breaking Changes + - use a class to represent Metadata (not dict) #1411 @Aayush-Goel-04 @manasghandat - use pathlib.Path to represent file paths #1534 @Aayush-Goel-04 - Python 3.8 is now the minimum supported Python version #1578 @williballenthin @@ -431,10 +462,11 @@ For those that use capa as a library, we've introduced some limited breaking cha - linking/runtime-linking/resolve-function-by-brute-ratel-badger-hash jakub.jozwiak@mandiant.com ### Bug Fixes + - extractor: add a Binary Ninja test that asserts its version #1487 @xusheng6 - extractor: update Binary Ninja stack string detection after the new constant outlining feature #1473 @xusheng6 - extractor: update vivisect Arch extraction #1334 @mr-tz -- extractor: avoid Binary Ninja exception when analyzing certain files #1441 @xusheng6 +- extractor: avoid Binary Ninja exception when analyzing certain files #1441 @xusheng6 - symtab: fix struct.unpack() format for 64-bit ELF files @yelhamer - symtab: safeguard against ZeroDivisionError for files containing a symtab with a null entry size @yelhamer - improve ELF strtab and needed parsing @mr-tz @@ -451,6 +483,7 @@ For those that use capa as a library, we've introduced some limited breaking cha ### capa explorer IDA Pro plugin ### Development + - update ATT&CK/MBC data for linting #1568 @mr-tz - log time taken to analyze each function #1290 @williballenthin - tests: make fixture available via conftest.py #1592 @williballenthin @@ -458,12 +491,13 @@ For those that use capa as a library, we've introduced some limited breaking cha - migrate to pyproject.toml #1301 @williballenthin - use [pre-commit](https://pre-commit.com/) to invoke linters #1579 @williballenthin - ### Raw diffs + - [capa v5.1.0...v6.0.0](https://github.com/mandiant/capa/compare/v5.1.0...v6.0.0) - [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0) ## v5.1.0 + capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and an output color format that's easier to read. Over 25 capa rules have been added and improved. @@ -471,7 +505,8 @@ Over 25 capa rules have been added and improved. Thanks for all the support, especially to @xusheng6, @captainGeech42, @ggold7046, @manasghandat, @ooprathamm, @linpeiyu164, @yelhamer, @HongThatCong, @naikordian, @stevemk14ebr, @emtuls, @raymondlleong, @bkojusner, @joren485, and everyone else who submitted bugs and provided feedback! ### New Features -- add protobuf format for result documents #1219 @williballenthin @mr-tz + +- add protobuf format for result documents #1219 @williballenthin @mr-tz - extractor: add Binary Ninja feature extractor @xusheng6 - new cli flag `--os` to override auto-detected operating system for a sample @captainGeech42 - change colour/highlight to "cyan" instead of "blue" for better readability #1384 @ggold7046 @@ -508,24 +543,27 @@ Thanks for all the support, especially to @xusheng6, @captainGeech42, @ggold7046 - nursery/contain-a-thread-local-storage-tls-section-in-dotnet michael.hunhoff@mandiant.com ### Bug Fixes + - extractor: interface of cache modified to prevent extracting file and global features multiple times @stevemk14ebr -- extractor: removed '.dynsym' as the library name for ELF imports #1318 @stevemk14ebr +- extractor: removed '.dynsym' as the library name for ELF imports #1318 @stevemk14ebr - extractor: fix vivisect loop detection corner case #1310 @mr-tz - match: extend OS characteristic to match OS_ANY to all supported OSes #1324 @mike-hunhoff - extractor: fix IDA and vivisect string and bytes features overlap and tests #1327 #1336 @xusheng6 ### capa explorer IDA Pro plugin + - rule generator plugin now loads faster when jumping between functions @stevemk14ebr - fix exception when plugin loaded in IDA hosted under idat #1341 @mike-hunhoff - improve embedded PE detection performance and reduce FP potential #1344 @mike-hunhoff ### Raw diffs + - [capa v5.0.0...v5.1.0](https://github.com/mandiant/capa/compare/v5.0.0...v5.1.0) - [capa-rules v5.0.0...v5.1.0](https://github.com/mandiant/capa-rules/compare/v5.0.0...v5.1.0) - ## v5.0.0 (2023-02-08) -This capa version comes with major improvements and additions to better handle .NET binaries. To showcase this we've updated and added over 30 .NET rules. + +This capa version comes with major improvements and additions to better handle .NET binaries. To showcase this we've updated and added over 30 .NET rules. Additionally, capa now caches its rule set for better performance. The capa explorer also caches its analysis results, so that multiple IDA Pro or plugin invocations don't need to repeat the same analysis. @@ -536,6 +574,7 @@ Other improvements to highlight include better ELF OS detection, various renderi Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardweiss80, @joren485, @ryantxu1, @mwilliams31, @anushkavirgaonkar, @MalwareMechanic, @Still34, @dzbeck, @johnk3r, and everyone else who submitted bugs and provided feedback! ### New Features + - verify rule metadata format on load #1160 @mr-tz - dotnet: emit property features #1168 @anushkavirgaonkar - dotnet: emit API features for objects created via the newobj instruction #1186 @mike-hunhoff @@ -550,6 +589,7 @@ Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardw - update ATT&CK/MBC data for linting #1297 @mr-tz ### Breaking Changes + - remove SMDA backend #1062 @williballenthin - error return codes are now positive numbers #1269 @mr-tz @@ -629,6 +669,7 @@ Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardw - nursery/unmanaged-call-via-dynamic-pinvoke-in-dotnet michael.hunhoff@mandiant.com ### Bug Fixes + - render: convert feature attributes to aliased dictionary for vverbose #1152 @mike-hunhoff - decouple Token dependency / extractor and features #1139 @mr-tz - update pydantic model to guarantee type coercion #1176 @mike-hunhoff @@ -647,6 +688,7 @@ Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardw - extractor: don't extract byte features for strings #1293 @mr-tz ### capa explorer IDA Pro plugin + - fix: display instruction items #1154 @mr-tz - fix: accept only plaintext pasted content #1194 @williballenthin - fix: UnboundLocalError #1217 @williballenthin @@ -661,27 +703,30 @@ Thanks for all the support, especially to @jsoref, @bkojusner, @edeca, @richardw - cache capa results across IDA sessions #1279 @mr-tz ### Raw diffs + - [capa v4.0.1...v5.0.0](https://github.com/mandiant/capa/compare/v4.0.1...v5.0.0) - [capa-rules v4.0.1...v5.0.0](https://github.com/mandiant/capa-rules/compare/v4.0.1...v5.0.0) - ## v4.0.1 (2022-08-15) + Some rules contained invalid metadata fields that caused an error when rendering rule hits. We've updated all rules and enhanced the rule linter to catch such issues. ### New Rules (1) - anti-analysis/obfuscation/obfuscated-with-vs-obfuscation jakub.jozwiak@mandiant.com - ### Bug Fixes + - linter: use pydantic to validate rule metadata #1141 @mike-hunhoff - build binaries using PyInstaller no longer overwrites functions in version.py #1136 @mr-tz ### Raw diffs + - [capa v4.0.0...v4.0.1](https://github.com/mandiant/capa/compare/v4.0.0...v4.0.1) - [capa-rules v4.0.0...v4.0.1](https://github.com/mandiant/capa-rules/compare/v4.0.0...v4.0.1) ## v4.0.0 (2022-08-10) + Version 4 adds support for analyzing .NET executables. capa will autodetect .NET modules, or you can explicitly invoke the new feature extractor via `--format dotnet`. We've also extended the rule syntax for .NET features including `namespace` and `class`. Additionally, new `instruction` scope and `operand` features enable users to create more explicit rules. These features are not backwards compatible. We removed the previously used `/x32` and `/x64` flavors of number and operand features. @@ -690,33 +735,33 @@ We updated 49 existing rules and added 22 new rules leveraging these new feature More breaking changes include updates to the JSON results document, freeze file format schema (now format version v2), and the internal handling of addresses. -Thanks for all the support, especially to @htnhan, @jtothej, @sara-rn, @anushkavirgaonkar, and @_re_fox! +Thanks for all the support, especially to @htnhan, @jtothej, @sara-rn, @anushkavirgaonkar, and @\_re_fox! -*Deprecation warning: v4.0 will be the last capa version to support the SMDA backend.* +_Deprecation warning: v4.0 will be the last capa version to support the SMDA backend._ ### New Features - - add new scope "instruction" for matching mnemonics and operands #767 @williballenthin - - add new feature "operand[{0, 1, 2}].number" for matching instruction operand immediate values #767 @williballenthin - - add new feature "operand[{0, 1, 2}].offset" for matching instruction operand offsets #767 @williballenthin - - extract additional offset/number features in certain circumstances #320 @williballenthin - - add detection and basic feature extraction for dotnet #987 @mr-tz, @mike-hunhoff, @williballenthin - - add file string extraction for dotnet files #1012 @mike-hunhoff - - add file function-name extraction for dotnet files #1015 @mike-hunhoff - - add unmanaged call characteristic for dotnet files #1023 @mike-hunhoff - - add mixed mode characteristic feature extraction for dotnet files #1024 @mike-hunhoff - - emit class and namespace features for dotnet files #1030 @mike-hunhoff - - render: support Addresses that aren't simple integers, like .NET token+offset #981 @williballenthin - - document rule tags and branches #1006 @williballenthin, @mr-tz +- add new scope "instruction" for matching mnemonics and operands #767 @williballenthin +- add new feature "operand[{0, 1, 2}].number" for matching instruction operand immediate values #767 @williballenthin +- add new feature "operand[{0, 1, 2}].offset" for matching instruction operand offsets #767 @williballenthin +- extract additional offset/number features in certain circumstances #320 @williballenthin +- add detection and basic feature extraction for dotnet #987 @mr-tz, @mike-hunhoff, @williballenthin +- add file string extraction for dotnet files #1012 @mike-hunhoff +- add file function-name extraction for dotnet files #1015 @mike-hunhoff +- add unmanaged call characteristic for dotnet files #1023 @mike-hunhoff +- add mixed mode characteristic feature extraction for dotnet files #1024 @mike-hunhoff +- emit class and namespace features for dotnet files #1030 @mike-hunhoff +- render: support Addresses that aren't simple integers, like .NET token+offset #981 @williballenthin +- document rule tags and branches #1006 @williballenthin, @mr-tz ### Breaking Changes - - instruction scope and operand feature are new and are not backwards compatible with older versions of capa - - Python 3.7 is now the minimum supported Python version #866 @williballenthin - - remove /x32 and /x64 flavors of number and operand features #932 @williballenthin - - the tool now accepts multiple paths to rules, and JSON doc updated accordingly @williballenthin - - extractors must use handles to identify functions/basic blocks/instructions #981 @williballenthin - - the freeze file format schema was updated, including format version bump to v2 #986 @williballenthin +- instruction scope and operand feature are new and are not backwards compatible with older versions of capa +- Python 3.7 is now the minimum supported Python version #866 @williballenthin +- remove /x32 and /x64 flavors of number and operand features #932 @williballenthin +- the tool now accepts multiple paths to rules, and JSON doc updated accordingly @williballenthin +- extractors must use handles to identify functions/basic blocks/instructions #981 @williballenthin +- the freeze file format schema was updated, including format version bump to v2 #986 @williballenthin Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issues/937), we plan to remove the SMDA backend for v5. If you rely on this backend, please reach out so we can discuss extending the support for SMDA or transitioning your workflow to use vivisect. @@ -733,11 +778,11 @@ Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issu - anti-analysis/packer/huan/packed-with-huan jakub.jozwiak@mandiant.com - nursery/execute-dotnet-assembly anushka.virgaonkar@mandiant.com - nursery/invoke-dotnet-assembly-method anushka.virgaonkar@mandiant.com -- collection/screenshot/capture-screenshot-via-keybd-event @_re_fox -- collection/browser/gather-chrome-based-browser-login-information @_re_fox +- collection/screenshot/capture-screenshot-via-keybd-event @\_re_fox +- collection/browser/gather-chrome-based-browser-login-information @\_re_fox - nursery/power-down-monitor michael.hunhoff@mandiant.com -- nursery/hash-data-using-aphash @_re_fox -- nursery/hash-data-using-jshash @_re_fox +- nursery/hash-data-using-aphash @\_re_fox +- nursery/hash-data-using-jshash @\_re_fox - host-interaction/file-system/files/list/enumerate-files-on-windows moritz.raabe@mandiant.com anushka.virgaonkar@mandiant.com - nursery/check-clipboard-data anushka.virgaonkar@mandiant.com - nursery/clear-clipboard-data anushka.virgaonkar@mandiant.com @@ -749,16 +794,18 @@ Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issu - nursery/send-keystrokes anushka.virgaonkar@mandiant.com - nursery/send-request-in-dotnet anushka.virgaonakr@mandiant.com - nursery/terminate-process-by-name-in-dotnet anushka.virgaonkar@mandiant.com -- nursery/hash-data-using-rshash @_re_fox +- nursery/hash-data-using-rshash @\_re_fox - persistence/authentication-process/act-as-credential-manager-dll jakub.jozwiak@mandiant.com - persistence/authentication-process/act-as-password-filter-dll jakub.jozwiak@mandiant.com ### Bug Fixes -- improve handling _ prefix compile/link artifact #924 @mike-hunhoff + +- improve handling \_ prefix compile/link artifact #924 @mike-hunhoff - better detect OS in ELF samples #988 @williballenthin - display number feature zero in vverbose #1097 @mike-hunhoff ### capa explorer IDA Pro plugin + - improve file format extraction #918 @mike-hunhoff - remove decorators added by IDA to ELF imports #919 @mike-hunhoff - bug fixes for Address abstraction #1091 @mike-hunhoff @@ -766,10 +813,12 @@ Deprecation notice: as described in [#937](https://github.com/mandiant/capa/issu ### Development ### Raw diffs + - [capa v3.2.0...v4.0.0](https://github.com/mandiant/capa/compare/v3.2.0...master) - [capa-rules v3.2.0...v4.0.0](https://github.com/mandiant/capa-rules/compare/v3.2.0...master) ## v3.2.1 (2022-06-06) + This out-of-band release bumps the SMDA dependency version to enable installation on Python 3.10. ### Bug Fixes @@ -777,10 +826,12 @@ This out-of-band release bumps the SMDA dependency version to enable installatio - update SMDA dependency @mike-hunhoff #922 ### Raw diffs + - [capa v3.2.0...v3.2.1](https://github.com/mandiant/capa/compare/v3.2.0...v3.2.1) - [capa-rules v3.2.0...v3.2.1](https://github.com/mandiant/capa-rules/compare/v3.2.0...v3.2.1) ## v3.2.0 (2022-03-03) + This release adds a new characteristic `characteristic: call $+5` enabling users to create more explicit rules. The linter now also validates ATT&CK and MBC categories. Additionally, many dependencies, including the vivisect backend, have been updated. One rule has been added and many more have been improved. @@ -802,20 +853,21 @@ Thanks for all the support, especially to @kn0wl3dge and first time contributor - elf: fix OS detection for Linux kernel modules #867 @williballenthin ### Raw diffs + - [capa v3.1.0...v3.2.0](https://github.com/mandiant/capa/compare/v3.1.0...v3.2.0) - [capa-rules v3.1.0...v3.2.0](https://github.com/mandiant/capa-rules/compare/v3.1.0...v3.2.0) ## v3.1.0 (2022-01-10) + This release improves the performance of capa while also adding 23 new rules and many code quality enhancements. We profiled capa's CPU usage and optimized the way that it matches rules, such as by short circuiting when appropriate. According to our testing, the matching phase is approximately 66% faster than v3.0.3! We also added support for Python 3.10, aarch64 builds, and additional MAEC metadata in the rule headers. - + This release adds 23 new rules, including nine by Jakub Jozwiak of Mandiant. @ryantxu1 and @dzbeck updated the ATT&CK and MBC mappings for many rules. Thank you! - + And as always, welcome first time contributors! - - @kn0wl3dge - - @jtothej - - @cl30 - +- @kn0wl3dge +- @jtothej +- @cl30 ### New Features @@ -854,10 +906,10 @@ And as always, welcome first time contributors! ### Rule Changes - - update ATT&CK mappings by @ryantxu1 - - update ATT&CK and MBC mappings by @dzbeck - - aplib detection by @cdong1012 - - golang runtime detection by @stevemk14eber +- update ATT&CK mappings by @ryantxu1 +- update ATT&CK and MBC mappings by @dzbeck +- aplib detection by @cdong1012 +- golang runtime detection by @stevemk14eber ### Bug Fixes @@ -875,22 +927,23 @@ And as always, welcome first time contributors! - show features script: add backend flag #430 @kn0wl3dge ### Raw diffs + - [capa v3.0.3...v3.1.0](https://github.com/mandiant/capa/compare/v3.0.3...v3.1.0) - [capa-rules v3.0.3...v3.1.0](https://github.com/mandiant/capa-rules/compare/v3.0.3...v3.1.0) - ## v3.0.3 (2021-10-27) This is primarily a rule maintenance release: - - eight new rules, including all relevant techniques from [ATT&CK v10](https://medium.com/mitre-attack/introducing-attack-v10-7743870b37e3), and - - two rules removed, due to the prevalence of false positives + +- eight new rules, including all relevant techniques from [ATT&CK v10](https://medium.com/mitre-attack/introducing-attack-v10-7743870b37e3), and +- two rules removed, due to the prevalence of false positives We've also tweaked the status codes returned by capa.exe to be more specific and added a bit more metadata to the JSON output format. - + As always, welcome first time contributors! - - still@teamt5.org - - zander.work@mandiant.com - + +- still@teamt5.org +- zander.work@mandiant.com ### New Features @@ -909,6 +962,7 @@ As always, welcome first time contributors! - targeting/language/identify-system-language-via-api william.ballenthin@mandiant.com ## Removed rules (2) + - load-code/pe/parse-pe-exports: too many false positives in unrelated structure accesses - anti-analysis/anti-vm/vm-detection/execute-anti-vm-instructions: too many false positives in junk code @@ -917,11 +971,12 @@ As always, welcome first time contributors! - update references from FireEye to Mandiant ### Raw diffs + - [capa v3.0.2...v3.0.3](https://github.com/fireeye/capa/compare/v3.0.2...v3.0.3) - [capa-rules v3.0.2...v3.0.3](https://github.com/fireeye/capa-rules/compare/v3.0.2...v3.0.3) - + ## v3.0.2 (2021-09-28) - + This release fixes an issue with the standalone executables built with PyInstaller when running capa against ELF files. ### Bug Fixes @@ -929,6 +984,7 @@ This release fixes an issue with the standalone executables built with PyInstall - fix bug in PyInstaller config preventing ELF analysis #795 @mr-tz ### Raw diffs + - [capa v3.0.1...v3.0.2](https://github.com/fireeye/capa/compare/v3.0.1...v3.0.2) - [capa-rules v3.0.1...v3.0.2](https://github.com/fireeye/capa-rules/compare/v3.0.1...v3.0.2) @@ -943,6 +999,7 @@ Thanks to the community for highlighting issues and analysis misses. Your feedba - fix many underlying bugs in vivisect analysis and update to version v1.0.5 #786 @williballenthin ### Raw diffs + - [capa v3.0.0...v3.0.1](https://github.com/fireeye/capa/compare/v3.0.0...v3.0.1) - [capa-rules v3.0.0...v3.0.1](https://github.com/fireeye/capa-rules/compare/v3.0.0...v3.0.1) @@ -951,6 +1008,7 @@ Thanks to the community for highlighting issues and analysis misses. Your feedba We are excited to announce version 3.0! :tada: capa 3.0: + - adds support for ELF files targeting Linux thanks to [Intezer](https://www.intezer.com/) - adds new features to specify OS, CPU architecture, and file format - fixes a few bugs that may have led to false negatives (missed capabilities) in older versions @@ -959,9 +1017,10 @@ capa 3.0: A huge thanks to everyone who submitted issues, provided feedback, and contributed code and rules. Special acknowledgement to @Adir-Shemesh and @TcM1911 of [Intezer](https://www.intezer.com/) for contributing the code to enable ELF support. Also, welcome first time contributors: - - @jaredscottwilson - - @cdong1012 - - @jlepore-fe + +- @jaredscottwilson +- @cdong1012 +- @jlepore-fe ### New Features @@ -1066,14 +1125,15 @@ Also, welcome first time contributors: ### Development ### Raw diffs + - [capa v2.0.0...v3.0.0](https://github.com/mandiant/capa/compare/v2.0.0...v3.0.0) - [capa-rules v2.0.0...v3.0.0](https://github.com/mandiant/capa-rules/compare/v2.0.0...v3.0.0) - ## v2.0.0 (2021-07-19) We are excited to announce version 2.0! :tada: capa 2.0: + - enables anyone to contribute rules more easily - is the first Python 3 ONLY version - provides more concise and relevant result via identification of library functions using FLIRT @@ -1083,7 +1143,6 @@ capa 2.0: A huge thanks to everyone who submitted issues, provided feedback, and contributed code and rules. Many colleagues across dozens of organizations have volunteered their experience to improve this tool! :heart: - ### New Features - rules: update ATT&CK and MBC mappings https://github.com/mandiant/capa-rules/pull/317 @williballenthin @@ -1210,6 +1269,7 @@ A huge thanks to everyone who submitted issues, provided feedback, and contribut - main: do not process non-PE files even when --format explicitly provided #664 @mr-tz ### capa explorer IDA Pro plugin + - explorer: IDA 7.6 support #497 @williballenthin - explorer: explain how to install IDA 7.6 patch to enable the plugin #528 @williballenthin - explorer: document IDA 7.6sp1 as alternative to the patch #536 @Ana06 @@ -1230,10 +1290,10 @@ A huge thanks to everyone who submitted issues, provided feedback, and contribut ### Raw diffs + - [capa v1.6.1...v2.0.0](https://github.com/mandiant/capa/compare/v1.6.1...v2.0.0) - [capa-rules v1.6.1...v2.0.0](https://github.com/mandiant/capa-rules/compare/v1.6.1...v2.0.0) - ## v1.6.3 (2021-04-29) This release adds IDA 7.6 support to capa. @@ -1244,8 +1304,7 @@ This release adds IDA 7.6 support to capa. ### Raw diffs - - [capa v1.6.2...v1.6.3](https://github.com/mandiant/capa/compare/v1.6.2...v1.6.3) - +- [capa v1.6.2...v1.6.3](https://github.com/mandiant/capa/compare/v1.6.2...v1.6.3) ## v1.6.2 (2021-04-13) @@ -1257,8 +1316,7 @@ This release backports a fix to capa 1.6: The Windows binary was built with Pyth ### Raw diffs - - [capa v1.6.1...v1.6.2](https://github.com/mandiant/capa/compare/v1.6.1...v1.6.2) - +- [capa v1.6.1...v1.6.2](https://github.com/mandiant/capa/compare/v1.6.1...v1.6.2) ## v1.6.1 (2021-04-07) @@ -1325,9 +1383,8 @@ This release includes several bug fixes, such as a vivisect issue that prevented ### Raw diffs - - [capa v1.6.0...v1.6.1](https://github.com/mandiant/capa/compare/v1.6.0...v1.6.1) - - [capa-rules v1.6.0...v1.6.1](https://github.com/mandiant/capa-rules/compare/v1.6.0...v1.6.1) - +- [capa v1.6.0...v1.6.1](https://github.com/mandiant/capa/compare/v1.6.0...v1.6.1) +- [capa-rules v1.6.0...v1.6.1](https://github.com/mandiant/capa-rules/compare/v1.6.0...v1.6.1) ## v1.6.0 (2021-03-09) @@ -1344,6 +1401,7 @@ The capa explorer IDA plugin now helps you quickly build new capa rules using fe This version of capa adds Python 3 support in vivisect. Note that `.viv` files (generated by vivisect) are not compatible between Python 2 and Python 3. When updating to Python 3 you need to delete all the `.viv` files for capa to work. If you get the following error (or a similar one), you most likely need to delete `.viv` files: + ``` UnicodeDecodeError: 'ascii' codec can't decode byte 0x90 in position 2: ordinal not in range(128) ``` @@ -1356,13 +1414,13 @@ If you have workflows that rely on the Python 2 version and need future maintena ### New features -- explorer: Add capa explorer rule generator plugin for IDA Pro. Now capa explorer helps you build new capa rules! #426, #438, #439 @mike-hunhoff +- explorer: Add capa explorer rule generator plugin for IDA Pro. Now capa explorer helps you build new capa rules! #426, #438, #439 @mike-hunhoff - python: Python 3 support in vivisect #421 @Ana06 - main: Add backend option in Python 3 to select the backend to be used (either SMDA or vivisect) #421 @Ana06 - python: Python 3 support in IDA #429, #437 @mike-hunhoff - ci: test pyinstaller CI #452 @williballenthin - scripts: enable multiple backends in `show-features.py` #429 @mike-hunhoff -- scripts: add `scripts/vivisect-py2-vs-py3.sh` to compare vivisect Python 2 vs 3 (can easily be modified to test run times and compare different versions) #421 @Ana06 +- scripts: add `scripts/vivisect-py2-vs-py3.sh` to compare vivisect Python 2 vs 3 (can easily be modified to test run times and compare different versions) #421 @Ana06 ### New Rules (12) @@ -1397,9 +1455,8 @@ If you have workflows that rely on the Python 2 version and need future maintena ### Raw diffs - - [capa v1.5.1...v1.6.0](https://github.com/mandiant/capa/compare/v1.5.1...v1.6.0) - - [capa-rules v1.5.1...v1.6.0](https://github.com/mandiant/capa-rules/compare/v1.5.1...v1.6.0) - +- [capa v1.5.1...v1.6.0](https://github.com/mandiant/capa/compare/v1.5.1...v1.6.0) +- [capa-rules v1.5.1...v1.6.0](https://github.com/mandiant/capa-rules/compare/v1.5.1...v1.6.0) ## v1.5.1 (2021-02-09) @@ -1411,107 +1468,105 @@ This release fixes the version number that we forgot to update for v1.5.0 (there ### Raw diffs - - [capa v1.5.0...v1.5.1](https://github.com/mandiant/capa/compare/v1.5.1...v1.6.0) - - [capa-rules v1.5.0...v1.5.1](https://github.com/mandiant/capa-rules/compare/v1.5.1...v1.6.0) - +- [capa v1.5.0...v1.5.1](https://github.com/mandiant/capa/compare/v1.5.1...v1.6.0) +- [capa-rules v1.5.0...v1.5.1](https://github.com/mandiant/capa-rules/compare/v1.5.1...v1.6.0) ## v1.5.0 (2021-02-05) This release brings support for running capa under Python 3 via [SMDA](https://github.com/danielplohmann/smda), more thorough CI testing and linting, better extraction of strings and byte features, and 50 (!) new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: - - @johnk3r - - @doomedraven - - @stvemillertime - - @itreallynick - - @0x534a - +- @johnk3r +- @doomedraven +- @stvemillertime +- @itreallynick +- @0x534a + @dzbeck also added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for many rules. Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/mandiant/capa/). Report issues on our [issue tracker](https://github.com/mandiant/capa/issues) and contribute new rules at [capa-rules](https://github.com/mandiant/capa-rules/). - ### New Features - - py3 support via SMDA #355 @danielplohmann @jcrussell - - scripts: example of using capa as a library #372, #380 @doomedraven - - ci: enable dependabot #373 @mr-tz - - ci: lint rules @mr-tz - - ci: lint rule format #401 @mr-tz - - freeze: add base address #391 @mr-tz - - json: meta: add base address #412 @mr-tz +- py3 support via SMDA #355 @danielplohmann @jcrussell +- scripts: example of using capa as a library #372, #380 @doomedraven +- ci: enable dependabot #373 @mr-tz +- ci: lint rules @mr-tz +- ci: lint rule format #401 @mr-tz +- freeze: add base address #391 @mr-tz +- json: meta: add base address #412 @mr-tz ### New Rules (50) - - 64-bit execution via heavens gate @recvfrom - - contain anti-disasm techniques @mr-tz - - check for microsoft office emulation @re-fox - - check for windows sandbox via device @re-fox - - check for windows sandbox via dns suffix @re-fox - - check for windows sandbox via genuine state @re-fox - - check for windows sandbox via process name @re-fox - - check for windows sandbox via registry @re-fox - - capture microphone audio @re-fox - - capture public ip @re-fox - - get domain trust relationships @johnk3r - - check HTTP status code @mr-tz - - compiled with perl2exe @re-fox - - compiled with ps2exe @re-fox - - compiled with pyarmor @stvemillertime, @itreallynick - - validate payment card number using luhn algorithm @re-fox - - hash data using fnv @re-fox @mr-tz - - generate random numbers via WinAPI @mike-hunhoff @johnk3r - - enumerate files recursively @re-fox - - get file system object information @mike-hunhoff - - read virtual disk @re-fox - - register minifilter driver @mike-hunhoff - - start minifilter driver @mike-hunhoff - - enumerate gui resources @johnk3r - - simulate CTRL ALT DEL @mike-hunhoff - - hijack thread execution @0x534a - - inject dll @0x534a - - inject pe @0x534a - - create or open registry key @mike-hunhoff - - delete registry value @mike-hunhoff - - query or enumerate registry key @mike-hunhoff - - query or enumerate registry value @mike-hunhoff - - resume thread @0x534a - - suspend thread @0x534a - - allocate memory @0x534a - - allocate RW memory @0x534a - - contain pusha popa sequence @mr-tz - - create or open file @mike-hunhoff - - open process @0x534a - - open thread @0x534a - - get kernel32 base address @mr-tz - - get ntdll base address @mr-tz - - encrypt or decrypt data via BCrypt @mike-hunhoff - - generate random numbers using the Delphi LCG @williballenthin - - hash data via BCrypt @mike-hunhoff - - migrate process to active window station @williballenthin - - patch process command line @williballenthin - - resolve function by hash @williballenthin - - persist via Winlogon Helper DLL registry key @0x534a - - schedule task via command line @0x534a +- 64-bit execution via heavens gate @recvfrom +- contain anti-disasm techniques @mr-tz +- check for microsoft office emulation @re-fox +- check for windows sandbox via device @re-fox +- check for windows sandbox via dns suffix @re-fox +- check for windows sandbox via genuine state @re-fox +- check for windows sandbox via process name @re-fox +- check for windows sandbox via registry @re-fox +- capture microphone audio @re-fox +- capture public ip @re-fox +- get domain trust relationships @johnk3r +- check HTTP status code @mr-tz +- compiled with perl2exe @re-fox +- compiled with ps2exe @re-fox +- compiled with pyarmor @stvemillertime, @itreallynick +- validate payment card number using luhn algorithm @re-fox +- hash data using fnv @re-fox @mr-tz +- generate random numbers via WinAPI @mike-hunhoff @johnk3r +- enumerate files recursively @re-fox +- get file system object information @mike-hunhoff +- read virtual disk @re-fox +- register minifilter driver @mike-hunhoff +- start minifilter driver @mike-hunhoff +- enumerate gui resources @johnk3r +- simulate CTRL ALT DEL @mike-hunhoff +- hijack thread execution @0x534a +- inject dll @0x534a +- inject pe @0x534a +- create or open registry key @mike-hunhoff +- delete registry value @mike-hunhoff +- query or enumerate registry key @mike-hunhoff +- query or enumerate registry value @mike-hunhoff +- resume thread @0x534a +- suspend thread @0x534a +- allocate memory @0x534a +- allocate RW memory @0x534a +- contain pusha popa sequence @mr-tz +- create or open file @mike-hunhoff +- open process @0x534a +- open thread @0x534a +- get kernel32 base address @mr-tz +- get ntdll base address @mr-tz +- encrypt or decrypt data via BCrypt @mike-hunhoff +- generate random numbers using the Delphi LCG @williballenthin +- hash data via BCrypt @mike-hunhoff +- migrate process to active window station @williballenthin +- patch process command line @williballenthin +- resolve function by hash @williballenthin +- persist via Winlogon Helper DLL registry key @0x534a +- schedule task via command line @0x534a ### Bug Fixes - - doc: pyinstaller build process @mr-tz - - ida: better bytes extraction #409 @mike-hunhoff - - viv: better unicode string extraction #364 @mike-hunhoff - - viv: better unicode string extraction #378 @mr-tz - - viv: more xor instructions #379 @mr-tz - - viv: decrease logging verbosity #381 @mr-tz - - rules: fix api description syntax #403 @mike-hunhoff - - main: disable progress background thread #410 @mike-hunhoff - +- doc: pyinstaller build process @mr-tz +- ida: better bytes extraction #409 @mike-hunhoff +- viv: better unicode string extraction #364 @mike-hunhoff +- viv: better unicode string extraction #378 @mr-tz +- viv: more xor instructions #379 @mr-tz +- viv: decrease logging verbosity #381 @mr-tz +- rules: fix api description syntax #403 @mike-hunhoff +- main: disable progress background thread #410 @mike-hunhoff + ### Changes - - rules: return lib rules for scopes #398 @mr-tz - +- rules: return lib rules for scopes #398 @mr-tz + ### Raw diffs - - [capa v1.4.1...v1.5.0](https://github.com/mandiant/capa/compare/v1.4.1...v1.5.0) - - [capa-rules v1.4.0...v1.5.0](https://github.com/mandiant/capa-rules/compare/v1.4.0...v1.5.0) +- [capa v1.4.1...v1.5.0](https://github.com/mandiant/capa/compare/v1.4.1...v1.5.0) +- [capa-rules v1.4.0...v1.5.0](https://github.com/mandiant/capa-rules/compare/v1.4.0...v1.5.0) ## v1.4.1 (2020-10-23) @@ -1519,19 +1574,19 @@ This release fixes an issue building capa on our CI server, which prevented us f ### Bug Fixes - - install VC dependencies for Python 2.7 during Windows build - +- install VC dependencies for Python 2.7 during Windows build + ### Raw diffs - - [capa v1.4.0...v1.4.1](https://github.com/mandiant/capa/compare/v1.4.0...v1.4.1) - - [capa-rules v1.4.0...v1.4.1](https://github.com/mandiant/capa-rules/compare/v1.4.0...v1.4.1) +- [capa v1.4.0...v1.4.1](https://github.com/mandiant/capa/compare/v1.4.0...v1.4.1) +- [capa-rules v1.4.0...v1.4.1](https://github.com/mandiant/capa-rules/compare/v1.4.0...v1.4.1) ## v1.4.0 (2020-10-23) This capa release includes changes to the rule parsing, enhanced feature extraction, various bug fixes, and improved capa scripts. Everyone should benefit from the improved functionality and performance. The community helped to add 69 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. A special shout out to the following new project contributors: - - @mwilliams31 - - @yt0ng +- @mwilliams31 +- @yt0ng @dzbeck added [Malware Behavior Catalog](https://github.com/MBCProject/mbc-markdown) (MBC) and ATT&CK mappings for 86 rules. @@ -1539,277 +1594,278 @@ Download a standalone binary below and checkout the readme [here on GitHub](http ### New features - - script that demonstrates bulk processing @williballenthin #307 - - main: render MBC table @mr-tz #332 - - ida backend: improve detection of APIs called via two or more chained thunks @mike-hunhoff #340 - - viv backend: improve detection of APIs called via two or more chained thunks @mr-tz #341 - - features: extract APIs called via jmp instruction @mr-tz #337 +- script that demonstrates bulk processing @williballenthin #307 +- main: render MBC table @mr-tz #332 +- ida backend: improve detection of APIs called via two or more chained thunks @mike-hunhoff #340 +- viv backend: improve detection of APIs called via two or more chained thunks @mr-tz #341 +- features: extract APIs called via jmp instruction @mr-tz #337 ### New rules - - clear the Windows event log @mike-hunhoff - - crash the Windows event logging service @mike-hunhoff - - packed with kkrunchy @re-fox - - packed with nspack @re-fox - - packed with pebundle @re-fox - - packed with pelocknt @re-fox - - packed with peshield @re-fox - - packed with petite @re-fox - - packed with rlpack @re-fox - - packed with upack @re-fox - - packed with y0da crypter @re-fox - - compiled with rust @re-fox - - compute adler32 checksum @mwilliams31 - - encrypt-data-using-hc-128 @recvfrom - - manipulate console @williballenthin - - references logon banner @re-fox - - terminate process via fastfail @re-fox - - delete volume shadow copies @mr-tz - - authenticate HMAC @mr-tz - - compiled from EPL @williballenthin - - compiled with Go @williballenthin - - create Restart Manager session @mike-hunhoff - - decode data using Base64 via WinAPI @mike-hunhoff - - empty recycle bin quietly @mwilliams31 - - enumerate network shares @mike-hunhoff - - hook routines via microsoft detours @williballenthin - - hooked by API Override @williballenthin - - impersonate user @mike-hunhoff - - the @williballenthin packer detection package, thanks to Hexacorn for the data, see https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/ - - packed with CCG - - packed with Crunch - - packed with Dragon Armor - - packed with enigma - - packed with Epack - - packed with MaskPE - - packed with MEW - - packed with Mpress - - packed with Neolite - - packed with PECompact - - packed with Pepack - - packed with Perplex - - packed with ProCrypt - - packed with RPCrypt - - packed with SeauSFX - - packed with Shrinker - - packed with Simple Pack - - packed with StarForce - - packed with SVKP - - packed with Themida - - packed with TSULoader - - packed with VProtect - - packed with WWPACK - - rebuilt by ImpRec - - packaged as a Pintool - - packaged as a CreateInstall installer - - packaged as a WinZip self-extracting archive - - reference 114DNS DNS server @williballenthin - - reference AliDNS DNS server @williballenthin - - reference Cloudflare DNS server @williballenthin - - reference Comodo Secure DNS server @williballenthin - - reference Google Public DNS server @williballenthin - - reference Hurricane Electric DNS server @williballenthin - - reference kornet DNS server @williballenthin - - reference L3 DNS server @williballenthin - - reference OpenDNS DNS server @williballenthin - - reference Quad9 DNS server @williballenthin - - reference Verisign DNS server @williballenthin - - run as service @mike-hunhoff - - schedule task via ITaskService @mike-hunhoff - - references DNS over HTTPS endpoints @yt0ng +- clear the Windows event log @mike-hunhoff +- crash the Windows event logging service @mike-hunhoff +- packed with kkrunchy @re-fox +- packed with nspack @re-fox +- packed with pebundle @re-fox +- packed with pelocknt @re-fox +- packed with peshield @re-fox +- packed with petite @re-fox +- packed with rlpack @re-fox +- packed with upack @re-fox +- packed with y0da crypter @re-fox +- compiled with rust @re-fox +- compute adler32 checksum @mwilliams31 +- encrypt-data-using-hc-128 @recvfrom +- manipulate console @williballenthin +- references logon banner @re-fox +- terminate process via fastfail @re-fox +- delete volume shadow copies @mr-tz +- authenticate HMAC @mr-tz +- compiled from EPL @williballenthin +- compiled with Go @williballenthin +- create Restart Manager session @mike-hunhoff +- decode data using Base64 via WinAPI @mike-hunhoff +- empty recycle bin quietly @mwilliams31 +- enumerate network shares @mike-hunhoff +- hook routines via microsoft detours @williballenthin +- hooked by API Override @williballenthin +- impersonate user @mike-hunhoff +- the @williballenthin packer detection package, thanks to Hexacorn for the data, see https://www.hexacorn.com/blog/2016/12/15/pe-section-names-re-visited/ + - packed with CCG + - packed with Crunch + - packed with Dragon Armor + - packed with enigma + - packed with Epack + - packed with MaskPE + - packed with MEW + - packed with Mpress + - packed with Neolite + - packed with PECompact + - packed with Pepack + - packed with Perplex + - packed with ProCrypt + - packed with RPCrypt + - packed with SeauSFX + - packed with Shrinker + - packed with Simple Pack + - packed with StarForce + - packed with SVKP + - packed with Themida + - packed with TSULoader + - packed with VProtect + - packed with WWPACK + - rebuilt by ImpRec + - packaged as a Pintool + - packaged as a CreateInstall installer + - packaged as a WinZip self-extracting archive +- reference 114DNS DNS server @williballenthin +- reference AliDNS DNS server @williballenthin +- reference Cloudflare DNS server @williballenthin +- reference Comodo Secure DNS server @williballenthin +- reference Google Public DNS server @williballenthin +- reference Hurricane Electric DNS server @williballenthin +- reference kornet DNS server @williballenthin +- reference L3 DNS server @williballenthin +- reference OpenDNS DNS server @williballenthin +- reference Quad9 DNS server @williballenthin +- reference Verisign DNS server @williballenthin +- run as service @mike-hunhoff +- schedule task via ITaskService @mike-hunhoff +- references DNS over HTTPS endpoints @yt0ng ### Bug fixes - - ida plugin: fix tree-view exception @mike-hunhoff #315 - - ida plugin: fix feature count @mike-hunhoff - - main: fix reported total rule count @williballenthin #325 - - features: fix handling of API names with multiple periods @mike-hunhoff #329 - - ida backend: find all byte sequences instead of only first @mike-hunhoff #335 - - features: display 0 value @mr-tz #338 - - ida backend: extract ordinal and name imports @mr-tz #343 - - show-features: improvements and support within IDA @mr-tz #342 - - main: sanity check MBC rendering @williballenthin - - main: handle sample path that contains non-ASCII characters @mr-tz #328 +- ida plugin: fix tree-view exception @mike-hunhoff #315 +- ida plugin: fix feature count @mike-hunhoff +- main: fix reported total rule count @williballenthin #325 +- features: fix handling of API names with multiple periods @mike-hunhoff #329 +- ida backend: find all byte sequences instead of only first @mike-hunhoff #335 +- features: display 0 value @mr-tz #338 +- ida backend: extract ordinal and name imports @mr-tz #343 +- show-features: improvements and support within IDA @mr-tz #342 +- main: sanity check MBC rendering @williballenthin +- main: handle sample path that contains non-ASCII characters @mr-tz #328 ### Changes - - rules: use yaml.CLoader for better performance @williballenthin #306 - - rules: parse descriptions for statements @mr-tz #312 +- rules: use yaml.CLoader for better performance @williballenthin #306 +- rules: parse descriptions for statements @mr-tz #312 ### Raw diffs - - [capa v1.3.0...v1.4.0](https://github.com/mandiant/capa/compare/v1.3.0...v1.4.0) - - [capa-rules v1.3.0...v1.4.0](https://github.com/mandiant/capa-rules/compare/v1.3.0...v1.4.0) +- [capa v1.3.0...v1.4.0](https://github.com/mandiant/capa/compare/v1.3.0...v1.4.0) +- [capa-rules v1.3.0...v1.4.0](https://github.com/mandiant/capa-rules/compare/v1.3.0...v1.4.0) ## v1.3.0 (2020-09-14) This release brings newly updated mappings to the [Malware Behavior Catalog version 2.0](https://github.com/MBCProject/mbc-markdown), many enhancements to the IDA Pro plugin, [flare-capa on PyPI](https://pypi.org/project/flare-capa/), a bunch of bug fixes to improve feature extraction, and four new rules. We received contributions from ten reverse engineers, including seven new ones: - - @dzbeck - - @recvfrom - - @toomanybananas - - @cclauss - - @adamprescott91 - - @weslambert - - @stevemk14ebr - +- @dzbeck +- @recvfrom +- @toomanybananas +- @cclauss +- @adamprescott91 +- @weslambert +- @stevemk14ebr + Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/mandiant/capa/). Report issues on our [issue tracker](https://github.com/mandiant/capa/issues) and contribute new rules at [capa-rules](https://github.com/mandiant/capa-rules/). ### Key changes to IDA Plugin The IDA Pro integration is now distributed as a real plugin, instead of a script. This enables a few things: - - keyboard shortcuts and file menu integration - - updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%` - - generally doing thing the "right way" +- keyboard shortcuts and file menu integration +- updates distributed PyPI/`pip install --upgrade` without touching your `%IDADIR%` +- generally doing thing the "right way" -How to get this new version? It's easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. +How to get this new version? It's easy: download [capa_explorer.py](https://raw.githubusercontent.com/mandiant/capa/master/capa/ida/plugin/capa_explorer.py) to your IDA plugins directory and update your capa installation (incidentally, this is a good opportunity to migrate to `pip install flare-capa` instead of git checkouts). Now you should see the plugin listed in the `Edit > Plugins > FLARE capa explorer` menu in IDA. Please refer to the plugin [readme](https://github.com/mandiant/capa/blob/master/capa/ida/plugin/README.md) for additional information on installing and using the IDA Pro plugin. Please open an issue in this repository if you notice anything weird. - + ### New features - - ida plugin: now a real plugin, not a script @mike-hunhoff - - core: distributed via PyPI as [flare-capa](https://pypi.org/project/flare-capa/) @williballenthin - - features: enable automatic A/W handling for imports @williballenthin @Ana06 #246 - - ida plugin: persist rules directory setting via [ida-settings](https://github.com/williballenthin/ida-settings) @williballenthin #268 - - ida plugin: add search bar to results view @williballenthin #285 - - ida plugin: add `Analyze` and `Reset` buttons to tree view @mike-hunhoff #304 - - ida plugin: add status label to tree view @mike-hunhoff - - ida plugin: add progress indicator @mike-hunhoff, @mr-tz +- ida plugin: now a real plugin, not a script @mike-hunhoff +- core: distributed via PyPI as [flare-capa](https://pypi.org/project/flare-capa/) @williballenthin +- features: enable automatic A/W handling for imports @williballenthin @Ana06 #246 +- ida plugin: persist rules directory setting via [ida-settings](https://github.com/williballenthin/ida-settings) @williballenthin #268 +- ida plugin: add search bar to results view @williballenthin #285 +- ida plugin: add `Analyze` and `Reset` buttons to tree view @mike-hunhoff #304 +- ida plugin: add status label to tree view @mike-hunhoff +- ida plugin: add progress indicator @mike-hunhoff, @mr-tz ### New rules - - compiled with py2exe @re-fox - - resolve path using msvcrt @re-fox - - decompress data using QuickLZ @edeca - - encrypt data using sosemanuk @recvfrom +- compiled with py2exe @re-fox +- resolve path using msvcrt @re-fox +- decompress data using QuickLZ @edeca +- encrypt data using sosemanuk @recvfrom ### Bug fixes - - rule: reduce FP in DNS resolution @toomanybananas - - engine: report correct strings matched via regex @williballenthin #262 - - formatter: correctly format descriptions in two-line syntax @williballenthin @recvfrom #263 - - viv: better extract offsets from SibOper operands @williballenthin @edeca #276 - - import-to-ida: fix import error @cclauss - - viv: don't write settings to ~/.viv/viv.json @williballenthin @rakuy0 @weslambert #244 - - ida plugin: remove dependency loop that resulted in unnecessary overhead @mike-hunhoff #303 - - ida plugin: correctly highlight regex matches in IDA Disassembly view @mike-hunhoff #305 - - ida plugin: better handle rule directory prompt and failure case @stevemk14ebr @mike-hunhoff #309 +- rule: reduce FP in DNS resolution @toomanybananas +- engine: report correct strings matched via regex @williballenthin #262 +- formatter: correctly format descriptions in two-line syntax @williballenthin @recvfrom #263 +- viv: better extract offsets from SibOper operands @williballenthin @edeca #276 +- import-to-ida: fix import error @cclauss +- viv: don't write settings to ~/.viv/viv.json @williballenthin @rakuy0 @weslambert #244 +- ida plugin: remove dependency loop that resulted in unnecessary overhead @mike-hunhoff #303 +- ida plugin: correctly highlight regex matches in IDA Disassembly view @mike-hunhoff #305 +- ida plugin: better handle rule directory prompt and failure case @stevemk14ebr @mike-hunhoff #309 ### Changes - - rules: update meta mapping to MBC 2.0! @dzbeck - - render: don't display rules that are also matched by other rules @williballenthin @Ana06 #224 - - ida plugin: simplify tabs, removing summary and adding detail to results view @williballenthin #286 - - ida plugin: analysis is no longer automatically started when plugin is first opened @mike-hunhoff #304 - - ida plugin: user must manually select a capa rules directory before analysis can be performed @mike-hunhoff - - ida plugin: user interface controls are disabled until analysis is performed @mike-hunhoff #304 +- rules: update meta mapping to MBC 2.0! @dzbeck +- render: don't display rules that are also matched by other rules @williballenthin @Ana06 #224 +- ida plugin: simplify tabs, removing summary and adding detail to results view @williballenthin #286 +- ida plugin: analysis is no longer automatically started when plugin is first opened @mike-hunhoff #304 +- ida plugin: user must manually select a capa rules directory before analysis can be performed @mike-hunhoff +- ida plugin: user interface controls are disabled until analysis is performed @mike-hunhoff #304 ### Raw diffs - - [capa v1.2.0...v1.3.0](https://github.com/mandiant/capa/compare/v1.2.0...v1.3.0) - - [capa-rules v1.2.0...v1.3.0](https://github.com/mandiant/capa-rules/compare/v1.2.0...v1.3.0) +- [capa v1.2.0...v1.3.0](https://github.com/mandiant/capa/compare/v1.2.0...v1.3.0) +- [capa-rules v1.2.0...v1.3.0](https://github.com/mandiant/capa-rules/compare/v1.2.0...v1.3.0) ## v1.2.0 (2020-08-31) -This release brings UI enhancements, especially for the IDA Pro plugin, +This release brings UI enhancements, especially for the IDA Pro plugin, investment towards py3 support, -fixes some bugs identified by the community, +fixes some bugs identified by the community, and 46 (!) new rules. We received contributions from ten reverse engineers, including five new ones: - - @agithubuserlol - - @recvfrom - - @D4nch3n - - @edeca - - @winniepe - +- @agithubuserlol +- @recvfrom +- @D4nch3n +- @edeca +- @winniepe + Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/mandiant/capa/). Report issues on our [issue tracker](https://github.com/mandiant/capa/issues) and contribute new rules at [capa-rules](https://github.com/mandiant/capa-rules/). - + ### New features - - ida plugin: display arch flavors @mike-hunhoff - - ida plugin: display block descriptions @mike-hunhoff - - ida backend: extract features from nested pointers @mike-hunhoff - - main: show more progress output @williballenthin - - core: pin dependency versions #258 @recvfrom +- ida plugin: display arch flavors @mike-hunhoff +- ida plugin: display block descriptions @mike-hunhoff +- ida backend: extract features from nested pointers @mike-hunhoff +- main: show more progress output @williballenthin +- core: pin dependency versions #258 @recvfrom ### New rules - - bypass UAC via AppInfo ALPC @agithubuserlol - - bypass UAC via token manipulation @agithubuserlol - - check for sandbox and av modules @re-fox - - check for sandbox username @re-fox - - check if process is running under wine @re-fox - - validate credit card number using luhn algorithm @re-fox - - validate credit card number using luhn algorithm with no lookup table @re-fox - - hash data using FNV @edeca @mr-tz - - link many functions at runtime @mr-tz - - reference public RSA key @mr-tz - - packed with ASPack @williballenthin - - delete internet cache @mike-hunhoff - - enumerate internet cache @mike-hunhoff - - send ICMP echo request @mike-hunhoff - - check for debugger via API @mike-hunhoff - - check for hardware breakpoints @mike-hunhoff - - check for kernel debugger via shared user data structure @mike-hunhoff - - check for protected handle exception @mike-hunhoff - - check for software breakpoints @mike-hunhoff - - check for trap flag exception @mike-hunhoff - - check for unexpected memory writes @mike-hunhoff - - check process job object @mike-hunhoff - - reference anti-VM strings targeting Parallels @mike-hunhoff - - reference anti-VM strings targeting Qemu @mike-hunhoff - - reference anti-VM strings targeting VirtualBox @mike-hunhoff - - reference anti-VM strings targeting VirtualPC @mike-hunhoff - - reference anti-VM strings targeting VMWare @mike-hunhoff - - reference anti-VM strings targeting Xen @mike-hunhoff - - reference analysis tools strings @mike-hunhoff - - reference WMI statements @mike-hunhoff - - get number of processor cores @mike-hunhoff - - get number of processors @mike-hunhoff - - enumerate disk properties @mike-hunhoff - - get disk size @mike-hunhoff - - get process heap flags @mike-hunhoff - - get process heap force flags @mike-hunhoff - - get Explorer PID @mike-hunhoff - - delay execution @mike-hunhoff - - check for process debug object @mike-hunhoff - - check license value @mike-hunhoff - - check ProcessDebugFlags @mike-hunhoff - - check ProcessDebugPort @mike-hunhoff - - check SystemKernelDebuggerInformation @mike-hunhoff - - check thread yield allowed @mike-hunhoff - - enumerate system firmware tables @mike-hunhoff - - get system firmware table @mike-hunhoff - - hide thread from debugger @mike-hunhoff + +- bypass UAC via AppInfo ALPC @agithubuserlol +- bypass UAC via token manipulation @agithubuserlol +- check for sandbox and av modules @re-fox +- check for sandbox username @re-fox +- check if process is running under wine @re-fox +- validate credit card number using luhn algorithm @re-fox +- validate credit card number using luhn algorithm with no lookup table @re-fox +- hash data using FNV @edeca @mr-tz +- link many functions at runtime @mr-tz +- reference public RSA key @mr-tz +- packed with ASPack @williballenthin +- delete internet cache @mike-hunhoff +- enumerate internet cache @mike-hunhoff +- send ICMP echo request @mike-hunhoff +- check for debugger via API @mike-hunhoff +- check for hardware breakpoints @mike-hunhoff +- check for kernel debugger via shared user data structure @mike-hunhoff +- check for protected handle exception @mike-hunhoff +- check for software breakpoints @mike-hunhoff +- check for trap flag exception @mike-hunhoff +- check for unexpected memory writes @mike-hunhoff +- check process job object @mike-hunhoff +- reference anti-VM strings targeting Parallels @mike-hunhoff +- reference anti-VM strings targeting Qemu @mike-hunhoff +- reference anti-VM strings targeting VirtualBox @mike-hunhoff +- reference anti-VM strings targeting VirtualPC @mike-hunhoff +- reference anti-VM strings targeting VMWare @mike-hunhoff +- reference anti-VM strings targeting Xen @mike-hunhoff +- reference analysis tools strings @mike-hunhoff +- reference WMI statements @mike-hunhoff +- get number of processor cores @mike-hunhoff +- get number of processors @mike-hunhoff +- enumerate disk properties @mike-hunhoff +- get disk size @mike-hunhoff +- get process heap flags @mike-hunhoff +- get process heap force flags @mike-hunhoff +- get Explorer PID @mike-hunhoff +- delay execution @mike-hunhoff +- check for process debug object @mike-hunhoff +- check license value @mike-hunhoff +- check ProcessDebugFlags @mike-hunhoff +- check ProcessDebugPort @mike-hunhoff +- check SystemKernelDebuggerInformation @mike-hunhoff +- check thread yield allowed @mike-hunhoff +- enumerate system firmware tables @mike-hunhoff +- get system firmware table @mike-hunhoff +- hide thread from debugger @mike-hunhoff ### Bug fixes - - ida backend: extract unmapped immediate number features @mike-hunhoff - - ida backend: fix stack cookie check #257 @mike-hunhoff - - viv backend: better extract gs segment access @williballenthin - - core: enable counting of string features #241 @D4nch3n @williballenthin - - core: enable descriptions on feature with arch flavors @mike-hunhoff - - core: update git links for non-SSH access #259 @recvfrom +- ida backend: extract unmapped immediate number features @mike-hunhoff +- ida backend: fix stack cookie check #257 @mike-hunhoff +- viv backend: better extract gs segment access @williballenthin +- core: enable counting of string features #241 @D4nch3n @williballenthin +- core: enable descriptions on feature with arch flavors @mike-hunhoff +- core: update git links for non-SSH access #259 @recvfrom ### Changes - - ida plugin: better default display showing first level nesting @winniepe - - remove unused `characteristic(switch)` feature @ana06 - - prepare testing infrastructure for multiple backends/py3 @williballenthin - - ci: zip build artifacts @ana06 - - ci: build all supported python versions @ana06 - - code style and formatting @mr-tz +- ida plugin: better default display showing first level nesting @winniepe +- remove unused `characteristic(switch)` feature @ana06 +- prepare testing infrastructure for multiple backends/py3 @williballenthin +- ci: zip build artifacts @ana06 +- ci: build all supported python versions @ana06 +- code style and formatting @mr-tz ### Raw diffs - - [capa v1.1.0...v1.2.0](https://github.com/mandiant/capa/compare/v1.1.0...v1.2.0) - - [capa-rules v1.1.0...v1.2.0](https://github.com/mandiant/capa-rules/compare/v1.1.0...v1.2.0) +- [capa v1.1.0...v1.2.0](https://github.com/mandiant/capa/compare/v1.1.0...v1.2.0) +- [capa-rules v1.1.0...v1.2.0](https://github.com/mandiant/capa-rules/compare/v1.1.0...v1.2.0) ## v1.1.0 (2020-08-05) @@ -1817,83 +1873,83 @@ This release brings new rule format updates, such as adding `offset/x32` and neg fixes some bugs identified by the community, and 28 (!) new rules. We received contributions from eight reverse engineers, including four new ones: - - @re-fox - - @psifertex - - @bitsofbinary - - @threathive - +- @re-fox +- @psifertex +- @bitsofbinary +- @threathive + Download a standalone binary below and checkout the readme [here on GitHub](https://github.com/mandiant/capa/). Report issues on our [issue tracker](https://github.com/mandiant/capa/issues) and contribute new rules at [capa-rules](https://github.com/mandiant/capa-rules/). - + ### New features - - import: add Binary Ninja import script #205 #207 @psifertex - - rules: offsets can be negative #197 #208 @williballenthin - - rules: enable descriptions for statement nodes #194 #209 @Ana06 - - rules: add arch flavors to number and offset features #210 #216 @williballenthin - - render: show SHA1/SHA256 in default report #164 @threathive - - tests: add tests for IDA Pro backend #202 @williballenthin - +- import: add Binary Ninja import script #205 #207 @psifertex +- rules: offsets can be negative #197 #208 @williballenthin +- rules: enable descriptions for statement nodes #194 #209 @Ana06 +- rules: add arch flavors to number and offset features #210 #216 @williballenthin +- render: show SHA1/SHA256 in default report #164 @threathive +- tests: add tests for IDA Pro backend #202 @williballenthin + ### New rules - - check for unmoving mouse cursor @BitsOfBinary - - check mutex and exit @re-fox - - parse credit card information @re-fox - - read ini file @re-fox - - validate credit card number with luhn algorithm @re-fox - - change the wallpaper @re-fox - - acquire debug privileges @williballenthin - - import public key @williballenthin - - terminate process by name @williballenthin - - encrypt data using DES @re-fox - - encrypt data using DES via WinAPI @re-fox - - hash data using sha1 via x86 extensions @re-fox - - hash data using sha256 via x86 extensions @re-fox - - capture network configuration via ipconfig @re-fox - - hash data via WinCrypt @mike-hunhoff - - get file attributes @mike-hunhoff - - allocate thread local storage @mike-hunhoff - - get thread local storage value @mike-hunhoff - - set thread local storage @mike-hunhoff - - get session integrity level @mike-hunhoff - - add file to cabinet file @mike-hunhoff - - flush cabinet file @mike-hunhoff - - open cabinet file @mike-hunhoff - - gather firefox profile information @re-fox - - encrypt data using skipjack @re-fox - - encrypt data using camellia @re-fox - - hash data using tiger @re-fox - - encrypt data using blowfish @re-fox - - encrypt data using twofish @re-fox +- check for unmoving mouse cursor @BitsOfBinary +- check mutex and exit @re-fox +- parse credit card information @re-fox +- read ini file @re-fox +- validate credit card number with luhn algorithm @re-fox +- change the wallpaper @re-fox +- acquire debug privileges @williballenthin +- import public key @williballenthin +- terminate process by name @williballenthin +- encrypt data using DES @re-fox +- encrypt data using DES via WinAPI @re-fox +- hash data using sha1 via x86 extensions @re-fox +- hash data using sha256 via x86 extensions @re-fox +- capture network configuration via ipconfig @re-fox +- hash data via WinCrypt @mike-hunhoff +- get file attributes @mike-hunhoff +- allocate thread local storage @mike-hunhoff +- get thread local storage value @mike-hunhoff +- set thread local storage @mike-hunhoff +- get session integrity level @mike-hunhoff +- add file to cabinet file @mike-hunhoff +- flush cabinet file @mike-hunhoff +- open cabinet file @mike-hunhoff +- gather firefox profile information @re-fox +- encrypt data using skipjack @re-fox +- encrypt data using camellia @re-fox +- hash data using tiger @re-fox +- encrypt data using blowfish @re-fox +- encrypt data using twofish @re-fox ### Bug fixes - - linter: fix exception when examples is `None` @Ana06 - - linter: fix suggested recommendations via templating @williballenthin - - render: fix exception when rendering counts @williballenthin - - render: fix render of negative offsets @williballenthin - - extractor: fix segmentation violation from vivisect @williballenthin - - main: fix crash when .viv cannot be saved #168 @secshoggoth @williballenthin - - main: fix shellcode .viv save path @williballenthin +- linter: fix exception when examples is `None` @Ana06 +- linter: fix suggested recommendations via templating @williballenthin +- render: fix exception when rendering counts @williballenthin +- render: fix render of negative offsets @williballenthin +- extractor: fix segmentation violation from vivisect @williballenthin +- main: fix crash when .viv cannot be saved #168 @secshoggoth @williballenthin +- main: fix shellcode .viv save path @williballenthin ### Changes - - doc: explain how to bypass gatekeeper on macOS @psifertex - - doc: explain supported linux distributions @Ana06 - - doc: explain submodule update with --init @psifertex - - main: improve program help output @mr-tz - - main: disable progress when run in quiet mode @mr-tz - - main: assert supported IDA versions @mr-tz - - extractor: better identify nested pointers to strings @williballenthin - - setup: specify vivisect download url @Ana06 - - setup: pin vivisect version @williballenthin - - setup: bump vivisect dependency version @williballenthin - - setup: set Python project name to `flare-capa` @williballenthin - - ci: run tests and linter via GitHub Actions @Ana06 - - hooks: run style checkers and hide stashed output @Ana06 - - linter: ignore period in rule filename @williballenthin - - linter: warn on nursery rule with no changes needed @williballenthin +- doc: explain how to bypass gatekeeper on macOS @psifertex +- doc: explain supported linux distributions @Ana06 +- doc: explain submodule update with --init @psifertex +- main: improve program help output @mr-tz +- main: disable progress when run in quiet mode @mr-tz +- main: assert supported IDA versions @mr-tz +- extractor: better identify nested pointers to strings @williballenthin +- setup: specify vivisect download url @Ana06 +- setup: pin vivisect version @williballenthin +- setup: bump vivisect dependency version @williballenthin +- setup: set Python project name to `flare-capa` @williballenthin +- ci: run tests and linter via GitHub Actions @Ana06 +- hooks: run style checkers and hide stashed output @Ana06 +- linter: ignore period in rule filename @williballenthin +- linter: warn on nursery rule with no changes needed @williballenthin ### Raw diffs - - [capa v1.0.0...v1.1.0](https://github.com/mandiant/capa/compare/v1.0.0...v1.1.0) - - [capa-rules v1.0.0...v1.1.0](https://github.com/mandiant/capa-rules/compare/v1.0.0...v1.1.0) +- [capa v1.0.0...v1.1.0](https://github.com/mandiant/capa/compare/v1.0.0...v1.1.0) +- [capa-rules v1.0.0...v1.1.0](https://github.com/mandiant/capa-rules/compare/v1.0.0...v1.1.0) diff --git a/capa/capabilities/dynamic.py b/capa/capabilities/dynamic.py index 98858c58..2a433be4 100644 --- a/capa/capabilities/dynamic.py +++ b/capa/capabilities/dynamic.py @@ -6,20 +6,16 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import sys import logging import itertools import collections -from typing import Any, Tuple - -import tqdm +from typing import Any, List, Tuple import capa.perf import capa.features.freeze as frz import capa.render.result_document as rdoc from capa.rules import Scope, RuleSet from capa.engine import FeatureSet, MatchResults -from capa.helpers import redirecting_print_to_tqdm from capa.capabilities.common import find_file_capabilities from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle, DynamicFeatureExtractor @@ -139,38 +135,30 @@ def find_dynamic_capabilities( feature_counts = rdoc.DynamicFeatureCounts(file=0, processes=()) assert isinstance(extractor, DynamicFeatureExtractor) - with redirecting_print_to_tqdm(disable_progress): - with tqdm.contrib.logging.logging_redirect_tqdm(): - pbar = tqdm.tqdm - if disable_progress: - # do not use tqdm to avoid unnecessary side effects when caller intends - # to disable progress completely - def pbar(s, *args, **kwargs): - return s + processes: List[ProcessHandle] = list(extractor.get_processes()) + n_processes: int = len(processes) - elif not sys.stderr.isatty(): - # don't display progress bar when stderr is redirected to a file - def pbar(s, *args, **kwargs): - return s + with capa.helpers.CapaProgressBar( + console=capa.helpers.log_console, transient=True, disable=disable_progress + ) as pbar: + task = pbar.add_task("matching", total=n_processes, unit="processes") + for p in processes: + process_matches, thread_matches, call_matches, feature_count = find_process_capabilities( + ruleset, extractor, p + ) + feature_counts.processes += ( + rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count), + ) + logger.debug("analyzed %s and extracted %d features", p.address, feature_count) - processes = list(extractor.get_processes()) + for rule_name, res in process_matches.items(): + all_process_matches[rule_name].extend(res) + for rule_name, res in thread_matches.items(): + all_thread_matches[rule_name].extend(res) + for rule_name, res in call_matches.items(): + all_call_matches[rule_name].extend(res) - pb = pbar(processes, desc="matching", unit=" processes", leave=False) - for p in pb: - process_matches, thread_matches, call_matches, feature_count = find_process_capabilities( - ruleset, extractor, p - ) - feature_counts.processes += ( - rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count), - ) - logger.debug("analyzed %s and extracted %d features", p.address, feature_count) - - for rule_name, res in process_matches.items(): - all_process_matches[rule_name].extend(res) - for rule_name, res in thread_matches.items(): - all_thread_matches[rule_name].extend(res) - for rule_name, res in call_matches.items(): - all_call_matches[rule_name].extend(res) + pbar.advance(task) # collection of features that captures the rule matches within process and thread scopes. # mapping from feature (matched rule) to set of addresses at which it matched. diff --git a/capa/capabilities/static.py b/capa/capabilities/static.py index 4f3b3b6a..aeb710ae 100644 --- a/capa/capabilities/static.py +++ b/capa/capabilities/static.py @@ -6,21 +6,18 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import sys import time import logging import itertools import collections -from typing import Any, Tuple - -import tqdm.contrib.logging +from typing import Any, List, Tuple import capa.perf +import capa.helpers import capa.features.freeze as frz import capa.render.result_document as rdoc from capa.rules import Scope, RuleSet from capa.engine import FeatureSet, MatchResults -from capa.helpers import redirecting_print_to_tqdm from capa.capabilities.common import find_file_capabilities from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, StaticFeatureExtractor @@ -143,75 +140,58 @@ def find_static_capabilities( library_functions: Tuple[rdoc.LibraryFunction, ...] = () assert isinstance(extractor, StaticFeatureExtractor) - with redirecting_print_to_tqdm(disable_progress): - with tqdm.contrib.logging.logging_redirect_tqdm(): - pbar = tqdm.tqdm - if capa.helpers.is_runtime_ghidra(): - # Ghidrathon interpreter cannot properly handle - # the TMonitor thread that is created via a monitor_interval - # > 0 - pbar.monitor_interval = 0 - if disable_progress: - # do not use tqdm to avoid unnecessary side effects when caller intends - # to disable progress completely - def pbar(s, *args, **kwargs): - return s + functions: List[FunctionHandle] = list(extractor.get_functions()) + n_funcs: int = len(functions) + n_libs: int = 0 + percentage: float = 0 - elif not sys.stderr.isatty(): - # don't display progress bar when stderr is redirected to a file - def pbar(s, *args, **kwargs): - return s - - functions = list(extractor.get_functions()) - n_funcs = len(functions) - - pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False) - for f in pb: - t0 = time.time() - if extractor.is_library_function(f.address): - function_name = extractor.get_function_name(f.address) - logger.debug("skipping library function 0x%x (%s)", f.address, function_name) - library_functions += ( - rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name), - ) - n_libs = len(library_functions) - percentage = round(100 * (n_libs / n_funcs)) - if isinstance(pb, tqdm.tqdm): - pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)") - continue - - function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities( - ruleset, extractor, f + with capa.helpers.CapaProgressBar( + console=capa.helpers.log_console, transient=True, disable=disable_progress + ) as pbar: + task = pbar.add_task( + "matching", total=n_funcs, unit="functions", postfix=f"skipped {n_libs} library functions, {percentage}%" + ) + for f in functions: + t0 = time.time() + if extractor.is_library_function(f.address): + function_name = extractor.get_function_name(f.address) + logger.debug("skipping library function 0x%x (%s)", f.address, function_name) + library_functions += ( + rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name), ) - feature_counts.functions += ( - rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count), - ) - t1 = time.time() + n_libs = len(library_functions) + percentage = round(100 * (n_libs / n_funcs)) + pbar.update(task, postfix=f"skipped {n_libs} library functions, {percentage}%") + pbar.advance(task) + continue - match_count = 0 - for name, matches_ in itertools.chain( - function_matches.items(), bb_matches.items(), insn_matches.items() - ): - # in practice, most matches are derived rules, - # like "check OS version/5bf4c7f39fd4492cbed0f6dc7d596d49" - # but when we log to the human, they really care about "real" rules. - if not ruleset.rules[name].is_subscope_rule(): - match_count += len(matches_) + function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(ruleset, extractor, f) + feature_counts.functions += ( + rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count), + ) + t1 = time.time() - logger.debug( - "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs", - f.address, - feature_count, - match_count, - t1 - t0, - ) + match_count = 0 + for name, matches_ in itertools.chain(function_matches.items(), bb_matches.items(), insn_matches.items()): + if not ruleset.rules[name].is_subscope_rule(): + match_count += len(matches_) - for rule_name, res in function_matches.items(): - all_function_matches[rule_name].extend(res) - for rule_name, res in bb_matches.items(): - all_bb_matches[rule_name].extend(res) - for rule_name, res in insn_matches.items(): - all_insn_matches[rule_name].extend(res) + logger.debug( + "analyzed function 0x%x and extracted %d features, %d matches in %0.02fs", + f.address, + feature_count, + match_count, + t1 - t0, + ) + + for rule_name, res in function_matches.items(): + all_function_matches[rule_name].extend(res) + for rule_name, res in bb_matches.items(): + all_bb_matches[rule_name].extend(res) + for rule_name, res in insn_matches.items(): + all_insn_matches[rule_name].extend(res) + + pbar.advance(task) # collection of features that captures the rule matches within function, BB, and instruction scopes. # mapping from feature (matched rule) to set of addresses at which it matched. diff --git a/capa/helpers.py b/capa/helpers.py index f185db9e..4505647c 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -10,7 +10,6 @@ import os import sys import gzip import ctypes -import inspect import logging import tempfile import contextlib @@ -20,8 +19,21 @@ from pathlib import Path from zipfile import ZipFile from datetime import datetime -import tqdm import msgspec.json +from rich.console import Console +from rich.progress import ( + Task, + Text, + Progress, + BarColumn, + TextColumn, + SpinnerColumn, + ProgressColumn, + TimeElapsedColumn, + MofNCompleteColumn, + TaskProgressColumn, + TimeRemainingColumn, +) from capa.exceptions import UnsupportedFormatError from capa.features.common import ( @@ -51,6 +63,10 @@ EXTENSIONS_FREEZE = "frz" logger = logging.getLogger("capa") +# shared console used to redirect logging to stderr +log_console: Console = Console(stderr=True) + + def hex(n: int) -> str: """render the given number using upper case hex, like: 0x123ABC""" if n < 0: @@ -247,39 +263,6 @@ def get_format(sample: Path) -> str: return FORMAT_UNKNOWN -@contextlib.contextmanager -def redirecting_print_to_tqdm(disable_progress): - """ - tqdm (progress bar) expects to have fairly tight control over console output. - so calls to `print()` will break the progress bar and make things look bad. - so, this context manager temporarily replaces the `print` implementation - with one that is compatible with tqdm. - via: https://stackoverflow.com/a/42424890/87207 - """ - old_print = print # noqa: T202 [reserved word print used] - - def new_print(*args, **kwargs): - # If tqdm.tqdm.write raises error, use builtin print - if disable_progress: - old_print(*args, **kwargs) - else: - try: - tqdm.tqdm.write(*args, **kwargs) - except Exception: - old_print(*args, **kwargs) - - try: - # Globally replace print with new_print. - # Verified this works manually on Python 3.11: - # >>> import inspect - # >>> inspect.builtins - # - inspect.builtins.print = new_print # type: ignore - yield - finally: - inspect.builtins.print = old_print # type: ignore - - def log_unsupported_format_error(): logger.error("-" * 80) logger.error(" Input file does not appear to be a supported file.") @@ -433,3 +416,47 @@ def is_cache_newer_than_rule_code(cache_dir: Path) -> bool: return False return True + + +class RateColumn(ProgressColumn): + """Renders speed column in progress bar.""" + + def render(self, task: "Task") -> Text: + speed = f"{task.speed:>.1f}" if task.speed else "00.0" + unit = task.fields.get("unit", "it") + return Text.from_markup(f"[progress.data.speed]{speed} {unit}/s") + + +class PostfixColumn(ProgressColumn): + """Renders a postfix column in progress bar.""" + + def render(self, task: "Task") -> Text: + return Text(task.fields.get("postfix", "")) + + +class MofNCompleteColumnWithUnit(MofNCompleteColumn): + """Renders completed/total count column with a unit.""" + + def render(self, task: "Task") -> Text: + ret = super().render(task) + unit = task.fields.get("unit") + return ret.append(f" {unit}") if unit else ret + + +class CapaProgressBar(Progress): + @classmethod + def get_default_columns(cls): + return ( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + TaskProgressColumn(), + BarColumn(), + MofNCompleteColumnWithUnit(), + "•", + TimeElapsedColumn(), + "<", + TimeRemainingColumn(), + "•", + RateColumn(), + PostfixColumn(), + ) diff --git a/capa/main.py b/capa/main.py index d7b45e03..518c9ce2 100644 --- a/capa/main.py +++ b/capa/main.py @@ -22,6 +22,7 @@ from pathlib import Path import colorama from pefile import PEFormatError +from rich.logging import RichHandler from elftools.common.exceptions import ELFError import capa.perf @@ -405,15 +406,23 @@ def handle_common_args(args): ShouldExitError: if the program is invoked incorrectly and should exit. """ if args.quiet: - logging.basicConfig(level=logging.WARNING) logging.getLogger().setLevel(logging.WARNING) elif args.debug: - logging.basicConfig(level=logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG) else: - logging.basicConfig(level=logging.INFO) logging.getLogger().setLevel(logging.INFO) + # use [/] after the logger name to reset any styling, + # and prevent the color from carrying over to the message + logformat = "[dim]%(name)s[/]: %(message)s" + + # set markup=True to allow the use of Rich's markup syntax in log messages + rich_handler = RichHandler(markup=True, show_time=False, show_path=True, console=capa.helpers.log_console) + rich_handler.setFormatter(logging.Formatter(logformat)) + + # use RichHandler for root logger + logging.getLogger().addHandler(rich_handler) + # disable vivisect-related logging, it's verbose and not relevant for capa users set_vivisect_log_level(logging.CRITICAL) diff --git a/capa/render/utils.py b/capa/render/utils.py index 2846e05f..73ed1d29 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -9,28 +9,29 @@ import io from typing import Dict, List, Tuple, Union, Iterator, Optional -import termcolor +import rich.console +from rich.progress import Text import capa.render.result_document as rd -def bold(s: str) -> str: +def bold(s: str) -> Text: """draw attention to the given string""" - return termcolor.colored(s, "cyan") + return Text.from_markup(f"[cyan]{s}") -def bold2(s: str) -> str: +def bold2(s: str) -> Text: """draw attention to the given string, within a `bold` section""" - return termcolor.colored(s, "green") + return Text.from_markup(f"[green]{s}") -def mute(s: str) -> str: +def mute(s: str) -> Text: """draw attention away from the given string""" - return termcolor.colored(s, "dark_grey") + return Text.from_markup(f"[dim]{s}") -def warn(s: str) -> str: - return termcolor.colored(s, "yellow") +def warn(s: str) -> Text: + return Text.from_markup(f"[yellow]{s}") def format_parts_id(data: Union[rd.AttackSpec, rd.MBCSpec]): @@ -85,3 +86,17 @@ class StringIO(io.StringIO): def writeln(self, s): self.write(s) self.write("\n") + + +class Console(rich.console.Console): + def writeln(self, *args, **kwargs) -> None: + """ + prints the text with a new line at the end. + """ + return self.print(*args, **kwargs) + + def write(self, *args, **kwargs) -> None: + """ + prints the text without a new line at the end. + """ + return self.print(*args, **kwargs, end="") diff --git a/capa/render/verbose.py b/capa/render/verbose.py index 076ad2b1..54ac53e9 100644 --- a/capa/render/verbose.py +++ b/capa/render/verbose.py @@ -25,7 +25,8 @@ See the License for the specific language governing permissions and limitations from typing import cast -import tabulate +from rich.text import Text +from rich.table import Table import capa.rules import capa.helpers @@ -34,6 +35,7 @@ import capa.features.freeze as frz import capa.render.result_document as rd from capa.rules import RuleSet from capa.engine import MatchResults +from capa.render.utils import Console def format_address(address: frz.Address) -> str: @@ -140,7 +142,7 @@ def render_call(layout: rd.DynamicLayout, addr: frz.Address) -> str: ) -def render_static_meta(ostream, meta: rd.StaticMetadata): +def render_static_meta(console: Console, meta: rd.StaticMetadata): """ like: @@ -161,12 +163,16 @@ def render_static_meta(ostream, meta: rd.StaticMetadata): total feature count 1918 """ + grid = Table.grid(padding=(0, 2)) + grid.add_column(style="dim") + grid.add_column() + rows = [ ("md5", meta.sample.md5), ("sha1", meta.sample.sha1), ("sha256", meta.sample.sha256), ("path", meta.sample.path), - ("timestamp", meta.timestamp), + ("timestamp", str(meta.timestamp)), ("capa version", meta.version), ("os", meta.analysis.os), ("format", meta.analysis.format), @@ -175,18 +181,21 @@ def render_static_meta(ostream, meta: rd.StaticMetadata): ("extractor", meta.analysis.extractor), ("base address", format_address(meta.analysis.base_address)), ("rules", "\n".join(meta.analysis.rules)), - ("function count", len(meta.analysis.feature_counts.functions)), - ("library function count", len(meta.analysis.library_functions)), + ("function count", str(len(meta.analysis.feature_counts.functions))), + ("library function count", str(len(meta.analysis.library_functions))), ( "total feature count", - meta.analysis.feature_counts.file + sum(f.count for f in meta.analysis.feature_counts.functions), + str(meta.analysis.feature_counts.file + sum(f.count for f in meta.analysis.feature_counts.functions)), ), ] - ostream.writeln(tabulate.tabulate(rows, tablefmt="plain")) + for row in rows: + grid.add_row(*row) + + console.print(grid) -def render_dynamic_meta(ostream, meta: rd.DynamicMetadata): +def render_dynamic_meta(console: Console, meta: rd.DynamicMetadata): """ like: @@ -205,12 +214,16 @@ def render_dynamic_meta(ostream, meta: rd.DynamicMetadata): total feature count 1918 """ + table = Table.grid(padding=(0, 2)) + table.add_column(style="dim") + table.add_column() + rows = [ ("md5", meta.sample.md5), ("sha1", meta.sample.sha1), ("sha256", meta.sample.sha256), ("path", meta.sample.path), - ("timestamp", meta.timestamp), + ("timestamp", str(meta.timestamp)), ("capa version", meta.version), ("os", meta.analysis.os), ("format", meta.analysis.format), @@ -218,26 +231,29 @@ def render_dynamic_meta(ostream, meta: rd.DynamicMetadata): ("analysis", meta.flavor.value), ("extractor", meta.analysis.extractor), ("rules", "\n".join(meta.analysis.rules)), - ("process count", len(meta.analysis.feature_counts.processes)), + ("process count", str(len(meta.analysis.feature_counts.processes))), ( "total feature count", - meta.analysis.feature_counts.file + sum(p.count for p in meta.analysis.feature_counts.processes), + str(meta.analysis.feature_counts.file + sum(p.count for p in meta.analysis.feature_counts.processes)), ), ] - ostream.writeln(tabulate.tabulate(rows, tablefmt="plain")) + for row in rows: + table.add_row(*row) + + console.print(table) -def render_meta(osstream, doc: rd.ResultDocument): +def render_meta(console: Console, doc: rd.ResultDocument): if doc.meta.flavor == rd.Flavor.STATIC: - render_static_meta(osstream, cast(rd.StaticMetadata, doc.meta)) + render_static_meta(console, cast(rd.StaticMetadata, doc.meta)) elif doc.meta.flavor == rd.Flavor.DYNAMIC: - render_dynamic_meta(osstream, cast(rd.DynamicMetadata, doc.meta)) + render_dynamic_meta(console, cast(rd.DynamicMetadata, doc.meta)) else: raise ValueError("invalid meta analysis") -def render_rules(ostream, doc: rd.ResultDocument): +def render_rules(console: Console, doc: rd.ResultDocument): """ like: @@ -254,11 +270,15 @@ def render_rules(ostream, doc: rd.ResultDocument): if count == 1: capability = rutils.bold(rule.meta.name) else: - capability = f"{rutils.bold(rule.meta.name)} ({count} matches)" + capability = Text.assemble(rutils.bold(rule.meta.name), f" ({count} matches)") - ostream.writeln(capability) + console.print(capability) had_match = True + table = Table.grid(padding=(0, 2)) + table.add_column(style="dim") + table.add_column() + rows = [] ns = rule.meta.namespace @@ -310,23 +330,26 @@ def render_rules(ostream, doc: rd.ResultDocument): rows.append(("matches", "\n".join(lines))) - ostream.writeln(tabulate.tabulate(rows, tablefmt="plain")) - ostream.write("\n") + for row in rows: + table.add_row(*row) + + console.print(table) + console.print() if not had_match: - ostream.writeln(rutils.bold("no capabilities found")) + console.print(rutils.bold("no capabilities found")) def render_verbose(doc: rd.ResultDocument): - ostream = rutils.StringIO() + console = Console(highlight=False) - render_meta(ostream, doc) - ostream.write("\n") + with console.capture() as capture: + render_meta(console, doc) + console.print() + render_rules(console, doc) + console.print() - render_rules(ostream, doc) - ostream.write("\n") - - return ostream.getvalue() + return capture.get() def render(meta, rules: RuleSet, capabilities: MatchResults) -> str: diff --git a/capa/render/vverbose.py b/capa/render/vverbose.py index 2ee2af49..9c45119e 100644 --- a/capa/render/vverbose.py +++ b/capa/render/vverbose.py @@ -9,7 +9,8 @@ import logging import textwrap from typing import Dict, Iterable, Optional -import tabulate +from rich.text import Text +from rich.table import Table import capa.rules import capa.helpers @@ -22,6 +23,7 @@ import capa.render.result_document as rd import capa.features.freeze.features as frzf from capa.rules import RuleSet from capa.engine import MatchResults +from capa.render.utils import Console logger = logging.getLogger(__name__) @@ -45,7 +47,7 @@ def hanging_indent(s: str, indent: int) -> str: return textwrap.indent(s, prefix=prefix)[len(prefix) :] -def render_locations(ostream, layout: rd.Layout, locations: Iterable[frz.Address], indent: int): +def render_locations(console: Console, layout: rd.Layout, locations: Iterable[frz.Address], indent: int): import capa.render.verbose as v # it's possible to have an empty locations array here, @@ -56,7 +58,7 @@ def render_locations(ostream, layout: rd.Layout, locations: Iterable[frz.Address if len(locations) == 0: return - ostream.write(" @ ") + console.write(" @ ") location0 = locations[0] if len(locations) == 1: @@ -64,58 +66,58 @@ def render_locations(ostream, layout: rd.Layout, locations: Iterable[frz.Address if location.type == frz.AddressType.CALL: assert isinstance(layout, rd.DynamicLayout) - ostream.write(hanging_indent(v.render_call(layout, location), indent + 1)) + console.write(hanging_indent(v.render_call(layout, location), indent + 1)) else: - ostream.write(v.format_address(locations[0])) + console.write(v.format_address(locations[0])) elif location0.type == frz.AddressType.CALL and len(locations) > 1: location = locations[0] assert isinstance(layout, rd.DynamicLayout) s = f"{v.render_call(layout, location)}\nand {(len(locations) - 1)} more..." - ostream.write(hanging_indent(s, indent + 1)) + console.write(hanging_indent(s, indent + 1)) elif len(locations) > 4: # don't display too many locations, because it becomes very noisy. # probably only the first handful of locations will be useful for inspection. - ostream.write(", ".join(map(v.format_address, locations[0:4]))) - ostream.write(f", and {(len(locations) - 4)} more...") + console.write(", ".join(map(v.format_address, locations[0:4]))) + console.write(f", and {(len(locations) - 4)} more...") elif len(locations) > 1: - ostream.write(", ".join(map(v.format_address, locations))) + console.write(", ".join(map(v.format_address, locations))) else: raise RuntimeError("unreachable") -def render_statement(ostream, layout: rd.Layout, match: rd.Match, statement: rd.Statement, indent: int): - ostream.write(" " * indent) +def render_statement(console: Console, layout: rd.Layout, match: rd.Match, statement: rd.Statement, indent: int): + console.write(" " * indent) if isinstance(statement, rd.SubscopeStatement): # emit `basic block:` # rather than `subscope:` - ostream.write(statement.scope) + console.write(statement.scope) - ostream.write(":") + console.write(":") if statement.description: - ostream.write(f" = {statement.description}") - ostream.writeln("") + console.write(f" = {statement.description}") + console.writeln() elif isinstance(statement, (rd.CompoundStatement)): # emit `and:` `or:` `optional:` `not:` - ostream.write(statement.type) + console.write(statement.type) - ostream.write(":") + console.write(":") if statement.description: - ostream.write(f" = {statement.description}") - ostream.writeln("") + console.write(f" = {statement.description}") + console.writeln() elif isinstance(statement, rd.SomeStatement): - ostream.write(f"{statement.count} or more:") + console.write(f"{statement.count} or more:") if statement.description: - ostream.write(f" = {statement.description}") - ostream.writeln("") + console.write(f" = {statement.description}") + console.writeln() elif isinstance(statement, rd.RangeStatement): # `range` is a weird node, its almost a hybrid of statement+feature. @@ -133,25 +135,25 @@ def render_statement(ostream, layout: rd.Layout, match: rd.Match, statement: rd. value = rutils.bold2(value) if child.description: - ostream.write(f"count({child.type}({value} = {child.description})): ") + console.write(f"count({child.type}({value} = {child.description})): ") else: - ostream.write(f"count({child.type}({value})): ") + console.write(f"count({child.type}({value})): ") else: - ostream.write(f"count({child.type}): ") + console.write(f"count({child.type}): ") if statement.max == statement.min: - ostream.write(f"{statement.min}") + console.write(f"{statement.min}") elif statement.min == 0: - ostream.write(f"{statement.max} or fewer") + console.write(f"{statement.max} or fewer") elif statement.max == (1 << 64 - 1): - ostream.write(f"{statement.min} or more") + console.write(f"{statement.min} or more") else: - ostream.write(f"between {statement.min} and {statement.max}") + console.write(f"between {statement.min} and {statement.max}") if statement.description: - ostream.write(f" = {statement.description}") - render_locations(ostream, layout, match.locations, indent) - ostream.writeln("") + console.write(f" = {statement.description}") + render_locations(console, layout, match.locations, indent) + console.writeln() else: raise RuntimeError("unexpected match statement type: " + str(statement)) @@ -162,9 +164,9 @@ def render_string_value(s: str) -> str: def render_feature( - ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, feature: frzf.Feature, indent: int + console: Console, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, feature: frzf.Feature, indent: int ): - ostream.write(" " * indent) + console.write(" " * indent) key = feature.type value: Optional[str] @@ -205,14 +207,14 @@ def render_feature( elif isinstance(feature, frzf.OperandOffsetFeature): key = f"operand[{feature.index}].offset" - ostream.write(f"{key}: ") + console.write(f"{key}: ") if value: - ostream.write(rutils.bold2(value)) + console.write(rutils.bold2(value)) if feature.description: - ostream.write(capa.rules.DESCRIPTION_SEPARATOR) - ostream.write(feature.description) + console.write(capa.rules.DESCRIPTION_SEPARATOR) + console.write(feature.description) if isinstance(feature, (frzf.OSFeature, frzf.ArchFeature, frzf.FormatFeature)): # don't show the location of these global features @@ -224,35 +226,32 @@ def render_feature( elif isinstance(feature, (frzf.OSFeature, frzf.ArchFeature, frzf.FormatFeature)): pass else: - render_locations(ostream, layout, match.locations, indent) - ostream.write("\n") + render_locations(console, layout, match.locations, indent) + console.writeln() else: # like: # regex: /blah/ = SOME_CONSTANT # - "foo blah baz" @ 0x401000 # - "aaa blah bbb" @ 0x402000, 0x403400 - ostream.write(key) - ostream.write(": ") - ostream.write(value) - ostream.write("\n") + console.writeln(f"{key}: {value}") for capture, locations in sorted(match.captures.items()): - ostream.write(" " * (indent + 1)) - ostream.write("- ") - ostream.write(rutils.bold2(render_string_value(capture))) + console.write(" " * (indent + 1)) + console.write("- ") + console.write(rutils.bold2(render_string_value(capture))) if isinstance(layout, rd.DynamicLayout) and rule.meta.scopes.dynamic == capa.rules.Scope.CALL: # like above, don't re-render calls when in call scope. pass else: - render_locations(ostream, layout, locations, indent=indent) - ostream.write("\n") + render_locations(console, layout, locations, indent=indent) + console.writeln() -def render_node(ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, node: rd.Node, indent: int): +def render_node(console: Console, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, node: rd.Node, indent: int): if isinstance(node, rd.StatementNode): - render_statement(ostream, layout, match, node.statement, indent=indent) + render_statement(console, layout, match, node.statement, indent=indent) elif isinstance(node, rd.FeatureNode): - render_feature(ostream, layout, rule, match, node.feature, indent=indent) + render_feature(console, layout, rule, match, node.feature, indent=indent) else: raise RuntimeError("unexpected node type: " + str(node)) @@ -265,7 +264,9 @@ MODE_SUCCESS = "success" MODE_FAILURE = "failure" -def render_match(ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, indent=0, mode=MODE_SUCCESS): +def render_match( + console: Console, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Match, indent=0, mode=MODE_SUCCESS +): child_mode = mode if mode == MODE_SUCCESS: # display only nodes that evaluated successfully. @@ -297,13 +298,13 @@ def render_match(ostream, layout: rd.Layout, rule: rd.RuleMatches, match: rd.Mat else: raise RuntimeError("unexpected mode: " + mode) - render_node(ostream, layout, rule, match, match.node, indent=indent) + render_node(console, layout, rule, match, match.node, indent=indent) for child in match.children: - render_match(ostream, layout, rule, child, indent=indent + 1, mode=child_mode) + render_match(console, layout, rule, child, indent=indent + 1, mode=child_mode) -def render_rules(ostream, doc: rd.ResultDocument): +def render_rules(console: Console, doc: rd.ResultDocument): """ like: @@ -350,13 +351,13 @@ def render_rules(ostream, doc: rd.ResultDocument): if count == 1: if rule.meta.lib: lib_info = " (library rule)" - capability = f"{rutils.bold(rule.meta.name)}{lib_info}" + capability = Text.assemble(rutils.bold(rule.meta.name), f"{lib_info}") else: if rule.meta.lib: lib_info = ", only showing first match of library rule" - capability = f"{rutils.bold(rule.meta.name)} ({count} matches{lib_info})" + capability = Text.assemble(rutils.bold(rule.meta.name), f" ({count} matches{lib_info})") - ostream.writeln(capability) + console.writeln(capability) had_match = True rows = [] @@ -402,7 +403,14 @@ def render_rules(ostream, doc: rd.ResultDocument): if rule.meta.description: rows.append(("description", rule.meta.description)) - ostream.writeln(tabulate.tabulate(rows, tablefmt="plain")) + grid = Table.grid(padding=(0, 2)) + grid.add_column(style="dim") + grid.add_column() + + for row in rows: + grid.add_row(*row) + + console.writeln(grid) if capa.rules.Scope.FILE in rule.meta.scopes: matches = doc.rules[rule.meta.name].matches @@ -413,61 +421,58 @@ def render_rules(ostream, doc: rd.ResultDocument): # so, lets be explicit about our assumptions and raise an exception if they fail. raise RuntimeError(f"unexpected file scope match count: {len(matches)}") _, first_match = matches[0] - render_match(ostream, doc.meta.analysis.layout, rule, first_match, indent=0) + render_match(console, doc.meta.analysis.layout, rule, first_match, indent=0) else: for location, match in sorted(doc.rules[rule.meta.name].matches): if doc.meta.flavor == rd.Flavor.STATIC: assert rule.meta.scopes.static is not None - ostream.write(rule.meta.scopes.static.value) - ostream.write(" @ ") - ostream.write(capa.render.verbose.format_address(location)) + console.write(rule.meta.scopes.static.value + " @ ") + console.write(capa.render.verbose.format_address(location)) if rule.meta.scopes.static == capa.rules.Scope.BASIC_BLOCK: func = frz.Address.from_capa(functions_by_bb[location.to_capa()]) - ostream.write(f" in function {capa.render.verbose.format_address(func)}") + console.write(f" in function {capa.render.verbose.format_address(func)}") elif doc.meta.flavor == rd.Flavor.DYNAMIC: assert rule.meta.scopes.dynamic is not None assert isinstance(doc.meta.analysis.layout, rd.DynamicLayout) - ostream.write(rule.meta.scopes.dynamic.value) - - ostream.write(" @ ") + console.write(rule.meta.scopes.dynamic.value + " @ ") if rule.meta.scopes.dynamic == capa.rules.Scope.PROCESS: - ostream.write(v.render_process(doc.meta.analysis.layout, location)) + console.write(v.render_process(doc.meta.analysis.layout, location)) elif rule.meta.scopes.dynamic == capa.rules.Scope.THREAD: - ostream.write(v.render_thread(doc.meta.analysis.layout, location)) + console.write(v.render_thread(doc.meta.analysis.layout, location)) elif rule.meta.scopes.dynamic == capa.rules.Scope.CALL: - ostream.write(hanging_indent(v.render_call(doc.meta.analysis.layout, location), indent=1)) + console.write(hanging_indent(v.render_call(doc.meta.analysis.layout, location), indent=1)) else: capa.helpers.assert_never(rule.meta.scopes.dynamic) else: capa.helpers.assert_never(doc.meta.flavor) - ostream.write("\n") - render_match(ostream, doc.meta.analysis.layout, rule, match, indent=1) + console.writeln() + render_match(console, doc.meta.analysis.layout, rule, match, indent=1) if rule.meta.lib: # only show first match break - ostream.write("\n") + console.writeln() if not had_match: - ostream.writeln(rutils.bold("no capabilities found")) + console.writeln(rutils.bold("no capabilities found")) def render_vverbose(doc: rd.ResultDocument): - ostream = rutils.StringIO() + console = Console(highlight=False) - capa.render.verbose.render_meta(ostream, doc) - ostream.write("\n") + with console.capture() as capture: + capa.render.verbose.render_meta(console, doc) + console.writeln() + render_rules(console, doc) + console.writeln() - render_rules(ostream, doc) - ostream.write("\n") - - return ostream.getvalue() + return capture.get() def render(meta, rules: RuleSet, capabilities: MatchResults) -> str: diff --git a/pyproject.toml b/pyproject.toml index 0305c046..dc658420 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,12 +65,8 @@ dependencies = [ # or minor otherwise). # As specific constraints are identified, please provide # comments and context. - "tqdm>=4", "pyyaml>=6", - "tabulate>=0.9", "colorama>=0.4", - "termcolor>=2", - "wcwidth>=0.2", "ida-settings>=2", "ruamel.yaml>=0.18", "pefile>=2023.2.7", @@ -146,8 +142,6 @@ dev = [ "types-backports==0.1.3", "types-colorama==0.4.15.11", "types-PyYAML==6.0.8", - "types-tabulate==0.9.0.20240106", - "types-termcolor==1.1.4", "types-psutil==6.0.0.20240901", "types_requests==2.32.0.20240712", "types-protobuf==5.27.0.20240920", @@ -236,10 +230,7 @@ DEP002 = [ "types-protobuf", "types-psutil", "types-PyYAML", - "types-tabulate", - "types-termcolor", "types_requests", - "wcwidth" ] # dependencies imported but missing from definitions diff --git a/requirements.txt b/requirements.txt index 37142bc7..b63558ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,10 +39,6 @@ ruamel-yaml-clib==0.2.8 setuptools==75.1.0 six==1.16.0 sortedcontainers==2.4.0 -tabulate==0.9.0 -termcolor==2.4.0 -tqdm==4.66.5 viv-utils==0.7.11 vivisect==1.2.1 -wcwidth==0.2.13 msgspec==0.18.6 diff --git a/scripts/lint.py b/scripts/lint.py index ad9b7990..e96604e6 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -31,11 +31,9 @@ from typing import Set, Dict, List from pathlib import Path from dataclasses import field, dataclass -import tqdm import pydantic -import termcolor import ruamel.yaml -import tqdm.contrib.logging +from rich import print import capa.main import capa.rules @@ -51,18 +49,6 @@ from capa.render.result_document import RuleMetadata logger = logging.getLogger("lint") -def red(s): - return termcolor.colored(s, "red") - - -def orange(s): - return termcolor.colored(s, "yellow") - - -def green(s): - return termcolor.colored(s, "green") - - @dataclass class Context: """ @@ -80,8 +66,8 @@ class Context: class Lint: - WARN = orange("WARN") - FAIL = red("FAIL") + WARN = "[yellow]WARN[/yellow]" + FAIL = "[red]FAIL[/red]" name = "lint" level = FAIL @@ -896,7 +882,7 @@ def lint_rule(ctx: Context, rule: Rule): if (not lints_failed) and (not lints_warned) and has_examples: print("") print(f'{" (nursery) " if is_nursery_rule(rule) else ""} {rule.name}') - print(f" {Lint.WARN}: {green('no lint failures')}: Graduate the rule") + print(f" {Lint.WARN}: '[green]no lint failures[/green]': Graduate the rule") print("") else: lints_failed = len(tuple(filter(lambda v: v.level == Lint.FAIL, violations))) @@ -921,12 +907,15 @@ def lint(ctx: Context): ret = {} source_rules = [rule for rule in ctx.rules.rules.values() if not rule.is_subscope_rule()] - with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule", leave=False) as pbar: - with capa.helpers.redirecting_print_to_tqdm(False): - for rule in pbar: - name = rule.name - pbar.set_description(width(f"linting rule: {name}", 48)) - ret[name] = lint_rule(ctx, rule) + n_rules: int = len(source_rules) + + with capa.helpers.CapaProgressBar(transient=True, console=capa.helpers.log_console) as pbar: + task = pbar.add_task(description="linting", total=n_rules, unit="rule") + for rule in source_rules: + name = rule.name + pbar.update(task, description=width(f"linting rule: {name}", 48)) + ret[name] = lint_rule(ctx, rule) + pbar.advance(task) return ret @@ -1020,18 +1009,18 @@ def main(argv=None): logger.debug("lints ran for ~ %02d:%02dm", min, sec) if warned_rules: - print(orange("rules with WARN:")) + print("[yellow]rules with WARN:[/yellow]") for warned_rule in sorted(warned_rules): print(" - " + warned_rule) print() if failed_rules: - print(red("rules with FAIL:")) + print("[red]rules with FAIL:[/red]") for failed_rule in sorted(failed_rules): print(" - " + failed_rule) return 1 else: - logger.info(green("no lints failed, nice!")) + logger.info("[green]no lints failed, nice![/green]") return 0 diff --git a/scripts/profile-time.py b/scripts/profile-time.py index 34123a73..4ba66cc1 100644 --- a/scripts/profile-time.py +++ b/scripts/profile-time.py @@ -42,9 +42,10 @@ import logging import argparse import subprocess -import tqdm import humanize -import tabulate +from rich import box +from rich.table import Table +from rich.console import Console import capa.main import capa.perf @@ -92,51 +93,61 @@ def main(argv=None): except capa.main.ShouldExitError as e: return e.status_code - with tqdm.tqdm(total=args.number * args.repeat, leave=False) as pbar: + with capa.helpers.CapaProgressBar(console=capa.helpers.log_console) as progress: + total_iterations = args.number * args.repeat + task = progress.add_task("profiling", total=total_iterations) def do_iteration(): capa.perf.reset() capa.capabilities.common.find_capabilities(rules, extractor, disable_progress=True) - pbar.update(1) + + progress.advance(task) samples = timeit.repeat(do_iteration, number=args.number, repeat=args.repeat) logger.debug("perf: find capabilities: min: %0.2fs", (min(samples) / float(args.number))) - logger.debug("perf: find capabilities: avg: %0.2fs", (sum(samples) / float(args.repeat) / float(args.number))) + logger.debug( + "perf: find capabilities: avg: %0.2fs", + (sum(samples) / float(args.repeat) / float(args.number)), + ) logger.debug("perf: find capabilities: max: %0.2fs", (max(samples) / float(args.number))) for counter, count in capa.perf.counters.most_common(): logger.debug("perf: counter: %s: %s", counter, count) - print( - tabulate.tabulate( - [(counter, humanize.intcomma(count)) for counter, count in capa.perf.counters.most_common()], - headers=["feature class", "evaluation count"], - tablefmt="github", - ) - ) - print() + console = Console() - print( - tabulate.tabulate( - [ - ( - args.label, - "{:,}".format(capa.perf.counters["evaluate.feature"]), - # python documentation indicates that min(samples) should be preferred, - # so lets put that first. - # - # https://docs.python.org/3/library/timeit.html#timeit.Timer.repeat - f"{(min(samples) / float(args.number)):.2f}s", - f"{(sum(samples) / float(args.repeat) / float(args.number)):.2f}s", - f"{(max(samples) / float(args.number)):.2f}s", - ) - ], - headers=["label", "count(evaluations)", "min(time)", "avg(time)", "max(time)"], - tablefmt="github", - ) + table1 = Table(box=box.MARKDOWN) + table1.add_column("feature class") + table1.add_column("evaluation count") + + for counter, count in capa.perf.counters.most_common(): + table1.add_row(counter, humanize.intcomma(count)) + + console.print(table1) + console.print() + + table2 = Table(box=box.MARKDOWN) + table2.add_column("label") + table2.add_column("count(evaluations)", style="magenta") + table2.add_column("min(time)", style="green") + table2.add_column("avg(time)", style="yellow") + table2.add_column("max(time)", style="red") + + table2.add_row( + args.label, + # python documentation indicates that min(samples) should be preferred, + # so lets put that first. + # + # https://docs.python.org/3/library/timeit.html#timeit.Timer.repeat + "{:,}".format(capa.perf.counters["evaluate.feature"]), + f"{(min(samples) / float(args.number)):.2f}s", + f"{(sum(samples) / float(args.repeat) / float(args.number)):.2f}s", + f"{(max(samples) / float(args.number)):.2f}s", ) + console.print(table2) + return 0 diff --git a/scripts/show-unused-features.py b/scripts/show-unused-features.py index 0390cd64..be850e92 100644 --- a/scripts/show-unused-features.py +++ b/scripts/show-unused-features.py @@ -12,11 +12,12 @@ import sys import typing import logging import argparse -from typing import Set, Tuple +from typing import Set, List, Tuple from collections import Counter -import tabulate -from termcolor import colored +from rich import print +from rich.text import Text +from rich.table import Table import capa.main import capa.rules @@ -77,23 +78,30 @@ def get_file_features( return feature_map -def get_colored(s: str): +def get_colored(s: str) -> Text: if "(" in s and ")" in s: s_split = s.split("(", 1) - s_color = colored(s_split[1][:-1], "cyan") - return f"{s_split[0]}({s_color})" + return Text.assemble(s_split[0], "(", (s_split[1][:-1], "cyan"), ")") else: - return colored(s, "cyan") + return Text(s, style="cyan") def print_unused_features(feature_map: typing.Counter[Feature], rules_feature_set: Set[Feature]): - unused_features = [] + unused_features: List[Tuple[str, Text]] = [] for feature, count in reversed(feature_map.most_common()): if feature in rules_feature_set: continue unused_features.append((str(count), get_colored(str(feature)))) + + table = Table(title="Unused Features", box=None) + table.add_column("Count", style="dim") + table.add_column("Feature") + + for count_str, feature_text in unused_features: + table.add_row(count_str, feature_text) + print("\n") - print(tabulate.tabulate(unused_features, headers=["Count", "Feature"], tablefmt="plain")) + print(table) print("\n") diff --git a/tests/test_render.py b/tests/test_render.py index 7f896949..16b0e643 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -10,7 +10,6 @@ import textwrap from unittest.mock import Mock import fixtures -import rich.console import capa.rules import capa.render.utils @@ -24,6 +23,7 @@ import capa.features.basicblock import capa.render.result_document import capa.render.result_document as rd import capa.features.freeze.features +from capa.render.utils import Console def test_render_number(): @@ -154,7 +154,7 @@ def test_render_meta_maec(): # capture the output of render_maec f = io.StringIO() - console = rich.console.Console(file=f) + console = Console(file=f) capa.render.default.render_maec(mock_rd, console) output = f.getvalue() @@ -198,7 +198,7 @@ def test_render_meta_maec(): ], ) def test_render_vverbose_feature(feature, expected): - ostream = capa.render.utils.StringIO() + console = Console(highlight=False) addr = capa.features.freeze.Address.from_capa(capa.features.address.AbsoluteVirtualAddress(0x401000)) feature = capa.features.freeze.features.feature_from_capa(feature) @@ -240,6 +240,8 @@ def test_render_vverbose_feature(feature, expected): matches=(), ) - capa.render.vverbose.render_feature(ostream, layout, rm, matches, feature, indent=0) + with console.capture() as capture: + capa.render.vverbose.render_feature(console, layout, rm, matches, feature, indent=0) - assert ostream.getvalue().strip() == expected + output = capture.get().strip() + assert output == expected