diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 520e0894..002a7095 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Python uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0 with: - python-version: '3.7' + python-version: '3.8' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 64475f65..b6db661b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,7 +69,7 @@ jobs: matrix: os: [ubuntu-20.04, windows-2019, macos-11] # across all operating systems - python-version: ["3.7", "3.11"] + python-version: ["3.8", "3.11"] include: # on Ubuntu run these as well - os: ubuntu-20.04 @@ -104,7 +104,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.11"] + python-version: ["3.8", "3.11"] steps: - name: Checkout capa with submodules # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118 diff --git a/CHANGELOG.md b/CHANGELOG.md index 422f80c3..12e09f53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,14 @@ ### New Features - Utility script to detect feature overlap between new and existing CAPA rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04) +- use fancy box drawing characters for default output #1586 @williballenthin ### Breaking Changes - Update Metadata type in capa main [#1411](https://github.com/mandiant/capa/issues/1411) [@Aayush-Goel-04](https://github.com/aayush-goel-04) @manasghandat -- Updated file paths to use pathlib.Path for improved path handling and compatibility [#1534](https://github.com/mandiant/capa/issues/1534) [@Aayush-Goel-04] +- Python 3.8 is now the minimum supported Python version #1578 @williballenthin +- Updated file paths to use pathlib.Path for improved path handling and compatibility [#1534](https://github.com/mandiant/capa/issues/1534) [@Aayush-Goel-04](https://github.com/aayush-goel-04) -### New Rules (20) +### New Rules (22) - load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com - nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com @@ -31,6 +33,8 @@ - persistence/office/act-as-office-com-add-in jakub.jozwiak@mandiant.com - persistence/office/act-as-word-wll-add-in jakub.jozwiak@mandiant.com - anti-analysis/anti-debugging/debugger-evasion/hide-thread-from-debugger michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com +- host-interaction/memory/create-new-application-domain-in-dotnet jakub.jozwiak@mandiant.com +- host-interaction/gui/switch-active-desktop jakub.jozwiak@mandiant.com - ### Bug Fixes @@ -46,6 +50,8 @@ - Add logging and print redirect to tqdm for capa main [#749](https://github.com/mandiant/capa/issues/749) [@Aayush-Goel-04](https://github.com/aayush-goel-04) - extractor: fix binja installation path detection does not work with Python 3.11 - tests: refine the IDA test runner script #1513 @williballenthin +- output: don't leave behind traces of progress bar @williballenthin +- import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin ### capa explorer IDA Pro plugin diff --git a/README.md b/README.md index 2458b9b5..723671a4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-808-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-810-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/capa/helpers.py b/capa/helpers.py index e4dc850f..bd55e2b4 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -156,7 +156,7 @@ def log_unsupported_runtime_error(): logger.error("-" * 80) logger.error(" Unsupported runtime or Python interpreter.") logger.error(" ") - logger.error(" capa supports running under Python 3.7 and higher.") + logger.error(" capa supports running under Python 3.8 and higher.") logger.error(" ") logger.error( " If you're seeing this message on the command line, please ensure you're running a supported Python version." diff --git a/capa/ida/plugin/README.md b/capa/ida/plugin/README.md index 6dd07002..4bf3616c 100644 --- a/capa/ida/plugin/README.md +++ b/capa/ida/plugin/README.md @@ -95,7 +95,7 @@ can update using the `Settings` button. ### Requirements -capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested: +capa explorer supports Python versions >= 3.8.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested: * IDA 7.4 * IDA 7.5 @@ -105,7 +105,7 @@ capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The * IDA 8.1 * IDA 8.2 -capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x). +capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.8.x). If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues). diff --git a/capa/main.py b/capa/main.py index bbe1bda1..74557715 100644 --- a/capa/main.py +++ b/capa/main.py @@ -8,6 +8,7 @@ Unless required by applicable law or agreed to in writing, software distributed is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import io import os import sys import time @@ -262,7 +263,7 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro functions = list(extractor.get_functions()) n_funcs = len(functions) - pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions") + pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False) for f in pb: if extractor.is_library_function(f.address): function_name = extractor.get_function_name(f.address) @@ -976,12 +977,20 @@ def handle_common_args(args): # disable vivisect-related logging, it's verbose and not relevant for capa users set_vivisect_log_level(logging.CRITICAL) - # Since Python 3.8 cp65001 is an alias to utf_8, but not for Python < 3.8 - # TODO: remove this code when only supporting Python 3.8+ - # https://stackoverflow.com/a/3259271/87207 - import codecs - - codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None) + if isinstance(sys.stdout, io.TextIOWrapper) or hasattr(sys.stdout, "reconfigure"): + # from sys.stdout type hint: + # + # TextIO is used instead of more specific types for the standard streams, + # since they are often monkeypatched at runtime. At startup, the objects + # are initialized to instances of TextIOWrapper. + # + # To use methods from TextIOWrapper, use an isinstance check to ensure that + # the streams have not been overridden: + # + # if isinstance(sys.stdout, io.TextIOWrapper): + # sys.stdout.reconfigure(...) + sys.stdout.reconfigure(encoding="utf-8") + colorama.just_fix_windows_console() if args.color == "always": colorama.init(strip=False) @@ -1058,8 +1067,8 @@ def handle_common_args(args): def main(argv=None): - if sys.version_info < (3, 7): - raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.7+") + if sys.version_info < (3, 8): + raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+") if argv is None: argv = sys.argv[1:] diff --git a/capa/render/default.py b/capa/render/default.py index 76659252..15e2a5e8 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -40,7 +40,7 @@ def render_meta(doc: rd.ResultDocument, ostream: StringIO): ("path", doc.meta.sample.path), ] - ostream.write(tabulate.tabulate(rows, tablefmt="psql")) + ostream.write(tabulate.tabulate(rows, tablefmt="mixed_outline")) ostream.write("\n") @@ -102,7 +102,7 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): if rows: ostream.write( - tabulate.tabulate(rows, headers=[width("CAPABILITY", 50), width("NAMESPACE", 50)], tablefmt="psql") + tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline") ) ostream.write("\n") else: @@ -148,7 +148,7 @@ def render_attack(doc: rd.ResultDocument, ostream: StringIO): if rows: ostream.write( tabulate.tabulate( - rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="psql" + rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="mixed_grid" ) ) ostream.write("\n") @@ -190,7 +190,9 @@ def render_mbc(doc: rd.ResultDocument, ostream: StringIO): if rows: ostream.write( - tabulate.tabulate(rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="psql") + tabulate.tabulate( + rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="mixed_grid" + ) ) ostream.write("\n") diff --git a/rules b/rules index 76eccb54..a2989e6b 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 76eccb548b502f83522d885c93256bfcd91ccc79 +Subproject commit a2989e6ba5e145617d2aa3a23d365bff6f752284 diff --git a/scripts/import-to-ida.py b/scripts/import-to-ida.py index 058c2553..42c56445 100644 --- a/scripts/import-to-ida.py +++ b/scripts/import-to-ida.py @@ -28,13 +28,17 @@ Unless required by applicable law or agreed to in writing, software distributed is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import json import logging +import binascii import ida_nalt import ida_funcs import ida_kernwin +import capa.rules +import capa.features.freeze +import capa.render.result_document + logger = logging.getLogger("capa") @@ -64,37 +68,37 @@ def main(): if not path: return 0 - with open(path, "rb") as f: - doc = json.loads(f.read().decode("utf-8")) - - if "meta" not in doc or "rules" not in doc: - logger.error("doesn't appear to be a capa report") - return -1 + result_doc = capa.render.result_document.ResultDocument.parse_file(path) + meta, capabilities = result_doc.to_capa() # in IDA 7.4, the MD5 hash may be truncated, for example: # wanted: 84882c9d43e23d63b82004fae74ebb61 # found: b'84882C9D43E23D63B82004FAE74EBB6\x00' # # see: https://github.com/idapython/bin/issues/11 - a = doc["meta"]["sample"]["md5"].lower() - b = ida_nalt.retrieve_input_file_md5().lower() + a = meta.sample.md5.lower() + b = binascii.hexlify(ida_nalt.retrieve_input_file_md5()).decode("ascii").lower() if not a.startswith(b): logger.error("sample mismatch") return -2 rows = [] - for rule in doc["rules"].values(): - if rule["meta"].get("lib"): + for name in capabilities.keys(): + rule = result_doc.rules[name] + if rule.meta.lib: continue - if rule["meta"].get("capa/subscope"): + if rule.meta.is_subscope_rule: continue - if rule["meta"]["scope"] != "function": + if rule.meta.scope != capa.rules.Scope.FUNCTION: continue - name = rule["meta"]["name"] - ns = rule["meta"].get("namespace", "") - for va in rule["matches"].keys(): - va = int(va) + ns = rule.meta.namespace + + for address, _ in rule.matches: + if address.type != capa.features.freeze.AddressType.ABSOLUTE: + continue + + va = address.value rows.append((ns, name, va)) # order by (namespace, name) so that like things show up together diff --git a/scripts/lint.py b/scripts/lint.py index f505bd40..b34d0ccd 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -874,7 +874,7 @@ def lint(ctx: Context): ret = {} source_rules = [rule for rule in ctx.rules.rules.values() if not rule.is_subscope_rule()] - with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule") as pbar: + with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule", leave=False) as pbar: with capa.helpers.redirecting_print_to_tqdm(False): for rule in pbar: name = rule.name diff --git a/scripts/profile-time.py b/scripts/profile-time.py index 8efb664f..523497a8 100644 --- a/scripts/profile-time.py +++ b/scripts/profile-time.py @@ -110,7 +110,7 @@ def main(argv=None): args.sample, args.format, args.os, capa.main.BACKEND_VIV, sig_paths, should_save_workspace=False ) - with tqdm.tqdm(total=args.number * args.repeat) as pbar: + with tqdm.tqdm(total=args.number * args.repeat, leave=False) as pbar: def do_iteration(): capa.perf.reset() diff --git a/setup.py b/setup.py index a2fb0f5d..2ea59d69 100644 --- a/setup.py +++ b/setup.py @@ -15,13 +15,13 @@ requirements = [ "tqdm==4.65.0", "pyyaml==6.0", "tabulate==0.9.0", - "colorama==0.4.5", + "colorama==0.4.6", "termcolor==2.3.0", "wcwidth==0.2.6", "ida-settings==2.1.0", "viv-utils[flirt]==0.7.9", "halo==0.0.31", - "networkx==2.5.1", # newer versions no longer support py3.7. + "networkx==3.1", "ruamel.yaml==0.17.32", "vivisect==1.1.1", "pefile==2023.2.7", @@ -82,7 +82,7 @@ setuptools.setup( "mypy-protobuf==3.4.0", # type stubs for mypy "types-backports==0.1.3", - "types-colorama==0.4.15", + "types-colorama==0.4.15.11", "types-PyYAML==6.0.8", "types-tabulate==0.9.0.1", "types-termcolor==1.1.4", @@ -105,5 +105,5 @@ setuptools.setup( "Programming Language :: Python :: 3", "Topic :: Security", ], - python_requires=">=3.7", + python_requires=">=3.8", ) diff --git a/tests/data b/tests/data index 76810b63..8ff7e34c 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit 76810b63f8bdf829d9b36133e961ea6c14967e8a +Subproject commit 8ff7e34ce00bad26b3199d49f14d260f17da2d48