diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 520e0894..002a7095 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Python uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0 with: - python-version: '3.7' + python-version: '3.8' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 92ffcca8..aeab74ea 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,7 +69,7 @@ jobs: matrix: os: [ubuntu-20.04, windows-2019, macos-11] # across all operating systems - python-version: ["3.7", "3.11"] + python-version: ["3.8", "3.11"] include: # on Ubuntu run these as well - os: ubuntu-20.04 @@ -104,7 +104,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.11"] + python-version: ["3.8", "3.11"] steps: - name: Checkout capa with submodules # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118 diff --git a/CHANGELOG.md b/CHANGELOG.md index b4f0c324..91ad47d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,11 @@ ### Breaking Changes - Update Metadata type in capa main [#1411](https://github.com/mandiant/capa/issues/1411) [@Aayush-Goel-04](https://github.com/aayush-goel-04) @manasghandat +- Python 3.8 is now the minimum supported Python version #1578 @williballenthin - Change the old FeatureExtractor class' name into StaticFeatureExtractor, and make the former an alias for both the StaticFeatureExtractor and DynamicFeatureExtractor classes @yelhamer [#1567](https://github.com/mandiant/capa/issues/1567) +- use fancy box drawing characters for default output #1586 @williballenthin -### New Rules (11) +### New Rules (22) - load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com - nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com @@ -28,6 +30,16 @@ - host-interaction/hardware/enumerate-devices-by-category @mr-tz - host-interaction/service/continue-service @mr-tz - host-interaction/service/pause-service @mr-tz +- persistence/exchange/act-as-exchange-transport-agent jakub.jozwiak@mandiant.com +- host-interaction/file-system/create-virtual-file-system-in-dotnet jakub.jozwiak@mandiant.com +- compiler/cx_freeze/compiled-with-cx_freeze @mr-tz jakub.jozwiak@mandiant.com +- communication/socket/create-vmci-socket jakub.jozwiak@mandiant.com +- persistence/office/act-as-excel-xll-add-in jakub.jozwiak@mandiant.com +- persistence/office/act-as-office-com-add-in jakub.jozwiak@mandiant.com +- persistence/office/act-as-word-wll-add-in jakub.jozwiak@mandiant.com +- anti-analysis/anti-debugging/debugger-evasion/hide-thread-from-debugger michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com +- host-interaction/memory/create-new-application-domain-in-dotnet jakub.jozwiak@mandiant.com +- host-interaction/gui/switch-active-desktop jakub.jozwiak@mandiant.com - ### Bug Fixes @@ -43,10 +55,13 @@ - Add logging and print redirect to tqdm for capa main [#749](https://github.com/mandiant/capa/issues/749) [@Aayush-Goel-04](https://github.com/aayush-goel-04) - extractor: fix binja installation path detection does not work with Python 3.11 - tests: refine the IDA test runner script #1513 @williballenthin +- output: don't leave behind traces of progress bar @williballenthin +- import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin ### capa explorer IDA Pro plugin ### Development +- update ATT&CK/MBC data for linting #1568 @mr-tz ### Raw diffs - [capa v5.1.0...master](https://github.com/mandiant/capa/compare/v5.1.0...master) diff --git a/README.md b/README.md index 15a5e096..723671a4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa) [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases) -[![Number of rules](https://img.shields.io/badge/rules-802-blue.svg)](https://github.com/mandiant/capa-rules) +[![Number of rules](https://img.shields.io/badge/rules-810-blue.svg)](https://github.com/mandiant/capa-rules) [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster) [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt) diff --git a/capa/helpers.py b/capa/helpers.py index c8b42f85..5407416a 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -168,7 +168,7 @@ def log_unsupported_runtime_error(): logger.error("-" * 80) logger.error(" Unsupported runtime or Python interpreter.") logger.error(" ") - logger.error(" capa supports running under Python 3.7 and higher.") + logger.error(" capa supports running under Python 3.8 and higher.") logger.error(" ") logger.error( " If you're seeing this message on the command line, please ensure you're running a supported Python version." diff --git a/capa/ida/plugin/README.md b/capa/ida/plugin/README.md index 6dd07002..4bf3616c 100644 --- a/capa/ida/plugin/README.md +++ b/capa/ida/plugin/README.md @@ -95,7 +95,7 @@ can update using the `Settings` button. ### Requirements -capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested: +capa explorer supports Python versions >= 3.8.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested: * IDA 7.4 * IDA 7.5 @@ -105,7 +105,7 @@ capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The * IDA 8.1 * IDA 8.2 -capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x). +capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.8.x). If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues). diff --git a/capa/main.py b/capa/main.py index 80a6036d..4d286422 100644 --- a/capa/main.py +++ b/capa/main.py @@ -8,6 +8,7 @@ Unless required by applicable law or agreed to in writing, software distributed is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import io import os import sys import json @@ -274,7 +275,7 @@ def find_static_capabilities( functions = list(extractor.get_functions()) n_funcs = len(functions) - pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions") + pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False) for f in pb: if extractor.is_library_function(f.address): function_name = extractor.get_function_name(f.address) @@ -1028,12 +1029,20 @@ def handle_common_args(args): # disable vivisect-related logging, it's verbose and not relevant for capa users set_vivisect_log_level(logging.CRITICAL) - # Since Python 3.8 cp65001 is an alias to utf_8, but not for Python < 3.8 - # TODO: remove this code when only supporting Python 3.8+ - # https://stackoverflow.com/a/3259271/87207 - import codecs - - codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None) + if isinstance(sys.stdout, io.TextIOWrapper) or hasattr(sys.stdout, "reconfigure"): + # from sys.stdout type hint: + # + # TextIO is used instead of more specific types for the standard streams, + # since they are often monkeypatched at runtime. At startup, the objects + # are initialized to instances of TextIOWrapper. + # + # To use methods from TextIOWrapper, use an isinstance check to ensure that + # the streams have not been overridden: + # + # if isinstance(sys.stdout, io.TextIOWrapper): + # sys.stdout.reconfigure(...) + sys.stdout.reconfigure(encoding="utf-8") + colorama.just_fix_windows_console() if args.color == "always": colorama.init(strip=False) @@ -1110,8 +1119,8 @@ def handle_common_args(args): def main(argv=None): - if sys.version_info < (3, 7): - raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.7+") + if sys.version_info < (3, 8): + raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+") if argv is None: argv = sys.argv[1:] diff --git a/capa/render/default.py b/capa/render/default.py index 76659252..15e2a5e8 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -40,7 +40,7 @@ def render_meta(doc: rd.ResultDocument, ostream: StringIO): ("path", doc.meta.sample.path), ] - ostream.write(tabulate.tabulate(rows, tablefmt="psql")) + ostream.write(tabulate.tabulate(rows, tablefmt="mixed_outline")) ostream.write("\n") @@ -102,7 +102,7 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): if rows: ostream.write( - tabulate.tabulate(rows, headers=[width("CAPABILITY", 50), width("NAMESPACE", 50)], tablefmt="psql") + tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline") ) ostream.write("\n") else: @@ -148,7 +148,7 @@ def render_attack(doc: rd.ResultDocument, ostream: StringIO): if rows: ostream.write( tabulate.tabulate( - rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="psql" + rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="mixed_grid" ) ) ostream.write("\n") @@ -190,7 +190,9 @@ def render_mbc(doc: rd.ResultDocument, ostream: StringIO): if rows: ostream.write( - tabulate.tabulate(rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="psql") + tabulate.tabulate( + rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="mixed_grid" + ) ) ostream.write("\n") diff --git a/rules b/rules index 58ac3d72..a2989e6b 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 58ac3d724bb3ec74b2d0030827d474d97adbf364 +Subproject commit a2989e6ba5e145617d2aa3a23d365bff6f752284 diff --git a/scripts/import-to-ida.py b/scripts/import-to-ida.py index 058c2553..42c56445 100644 --- a/scripts/import-to-ida.py +++ b/scripts/import-to-ida.py @@ -28,13 +28,17 @@ Unless required by applicable law or agreed to in writing, software distributed is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import json import logging +import binascii import ida_nalt import ida_funcs import ida_kernwin +import capa.rules +import capa.features.freeze +import capa.render.result_document + logger = logging.getLogger("capa") @@ -64,37 +68,37 @@ def main(): if not path: return 0 - with open(path, "rb") as f: - doc = json.loads(f.read().decode("utf-8")) - - if "meta" not in doc or "rules" not in doc: - logger.error("doesn't appear to be a capa report") - return -1 + result_doc = capa.render.result_document.ResultDocument.parse_file(path) + meta, capabilities = result_doc.to_capa() # in IDA 7.4, the MD5 hash may be truncated, for example: # wanted: 84882c9d43e23d63b82004fae74ebb61 # found: b'84882C9D43E23D63B82004FAE74EBB6\x00' # # see: https://github.com/idapython/bin/issues/11 - a = doc["meta"]["sample"]["md5"].lower() - b = ida_nalt.retrieve_input_file_md5().lower() + a = meta.sample.md5.lower() + b = binascii.hexlify(ida_nalt.retrieve_input_file_md5()).decode("ascii").lower() if not a.startswith(b): logger.error("sample mismatch") return -2 rows = [] - for rule in doc["rules"].values(): - if rule["meta"].get("lib"): + for name in capabilities.keys(): + rule = result_doc.rules[name] + if rule.meta.lib: continue - if rule["meta"].get("capa/subscope"): + if rule.meta.is_subscope_rule: continue - if rule["meta"]["scope"] != "function": + if rule.meta.scope != capa.rules.Scope.FUNCTION: continue - name = rule["meta"]["name"] - ns = rule["meta"].get("namespace", "") - for va in rule["matches"].keys(): - va = int(va) + ns = rule.meta.namespace + + for address, _ in rule.matches: + if address.type != capa.features.freeze.AddressType.ABSOLUTE: + continue + + va = address.value rows.append((ns, name, va)) # order by (namespace, name) so that like things show up together diff --git a/scripts/lint.py b/scripts/lint.py index a80d3e12..8348cdea 100644 --- a/scripts/lint.py +++ b/scripts/lint.py @@ -873,7 +873,7 @@ def lint(ctx: Context): ret = {} source_rules = [rule for rule in ctx.rules.rules.values() if not rule.is_subscope_rule()] - with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule") as pbar: + with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule", leave=False) as pbar: with capa.helpers.redirecting_print_to_tqdm(False): for rule in pbar: name = rule.name diff --git a/scripts/linter-data.json b/scripts/linter-data.json index 5b9eb2ab..3be54c62 100644 --- a/scripts/linter-data.json +++ b/scripts/linter-data.json @@ -54,6 +54,7 @@ "T1583.005": "Acquire Infrastructure::Botnet", "T1583.006": "Acquire Infrastructure::Web Services", "T1583.007": "Acquire Infrastructure::Serverless", + "T1583.008": "Acquire Infrastructure::Malvertising", "T1584": "Compromise Infrastructure", "T1584.001": "Compromise Infrastructure::Domains", "T1584.002": "Compromise Infrastructure::DNS Server", @@ -88,7 +89,8 @@ "T1608.003": "Stage Capabilities::Install Digital Certificate", "T1608.004": "Stage Capabilities::Drive-by Target", "T1608.005": "Stage Capabilities::Link Target", - "T1608.006": "Stage Capabilities::SEO Poisoning" + "T1608.006": "Stage Capabilities::SEO Poisoning", + "T1650": "Acquire Access" }, "Initial Access": { "T1078": "Valid Accounts", @@ -128,6 +130,7 @@ "T1059.006": "Command and Scripting Interpreter::Python", "T1059.007": "Command and Scripting Interpreter::JavaScript", "T1059.008": "Command and Scripting Interpreter::Network Device CLI", + "T1059.009": "Command and Scripting Interpreter::Cloud API", "T1072": "Software Deployment Tools", "T1106": "Native API", "T1129": "Shared Modules", @@ -145,7 +148,8 @@ "T1569.002": "System Services::Service Execution", "T1609": "Container Administration Command", "T1610": "Deploy Container", - "T1648": "Serverless Execution" + "T1648": "Serverless Execution", + "T1651": "Cloud Administration Command" }, "Persistence": { "T1037": "Boot or Logon Initialization Scripts", @@ -247,6 +251,7 @@ "T1556.005": "Modify Authentication Process::Reversible Encryption", "T1556.006": "Modify Authentication Process::Multi-Factor Authentication", "T1556.007": "Modify Authentication Process::Hybrid Identity", + "T1556.008": "Modify Authentication Process::Network Provider DLL", "T1574": "Hijack Execution Flow", "T1574.001": "Hijack Execution Flow::DLL Search Order Hijacking", "T1574.002": "Hijack Execution Flow::DLL Side-Loading", @@ -372,6 +377,8 @@ "T1027.007": "Obfuscated Files or Information::Dynamic API Resolution", "T1027.008": "Obfuscated Files or Information::Stripped Payloads", "T1027.009": "Obfuscated Files or Information::Embedded Payloads", + "T1027.010": "Obfuscated Files or Information::Command Obfuscation", + "T1027.011": "Obfuscated Files or Information::Fileless Storage", "T1036": "Masquerading", "T1036.001": "Masquerading::Invalid Code Signature", "T1036.002": "Masquerading::Right-to-Left Override", @@ -380,6 +387,7 @@ "T1036.005": "Masquerading::Match Legitimate Name or Location", "T1036.006": "Masquerading::Space after Filename", "T1036.007": "Masquerading::Double File Extension", + "T1036.008": "Masquerading::Masquerade File Type", "T1055": "Process Injection", "T1055.001": "Process Injection::Dynamic-link Library Injection", "T1055.002": "Process Injection::Portable Executable Injection", @@ -487,6 +495,7 @@ "T1556.005": "Modify Authentication Process::Reversible Encryption", "T1556.006": "Modify Authentication Process::Multi-Factor Authentication", "T1556.007": "Modify Authentication Process::Hybrid Identity", + "T1556.008": "Modify Authentication Process::Network Provider DLL", "T1562": "Impair Defenses", "T1562.001": "Impair Defenses::Disable or Modify Tools", "T1562.002": "Impair Defenses::Disable Windows Event Logging", @@ -497,6 +506,7 @@ "T1562.008": "Impair Defenses::Disable Cloud Logs", "T1562.009": "Impair Defenses::Safe Mode Boot", "T1562.010": "Impair Defenses::Downgrade Attack", + "T1562.011": "Impair Defenses::Spoof Security Alerting", "T1564": "Hide Artifacts", "T1564.001": "Hide Artifacts::Hidden Files and Directories", "T1564.002": "Hide Artifacts::Hidden Users", @@ -574,6 +584,7 @@ "T1552.005": "Unsecured Credentials::Cloud Instance Metadata API", "T1552.006": "Unsecured Credentials::Group Policy Preferences", "T1552.007": "Unsecured Credentials::Container API", + "T1552.008": "Unsecured Credentials::Chat Messages", "T1555": "Credentials from Password Stores", "T1555.001": "Credentials from Password Stores::Keychain", "T1555.002": "Credentials from Password Stores::Securityd Memory", @@ -588,6 +599,7 @@ "T1556.005": "Modify Authentication Process::Reversible Encryption", "T1556.006": "Modify Authentication Process::Multi-Factor Authentication", "T1556.007": "Modify Authentication Process::Hybrid Identity", + "T1556.008": "Modify Authentication Process::Network Provider DLL", "T1557": "Adversary-in-the-Middle", "T1557.001": "Adversary-in-the-Middle::LLMNR/NBT-NS Poisoning and SMB Relay", "T1557.002": "Adversary-in-the-Middle::ARP Cache Poisoning", @@ -630,7 +642,7 @@ "T1124": "System Time Discovery", "T1135": "Network Share Discovery", "T1201": "Password Policy Discovery", - "T1217": "Browser Bookmark Discovery", + "T1217": "Browser Information Discovery", "T1482": "Domain Trust Discovery", "T1497": "Virtualization/Sandbox Evasion", "T1497.001": "Virtualization/Sandbox Evasion::System Checks", @@ -646,7 +658,8 @@ "T1614.001": "System Location Discovery::System Language Discovery", "T1615": "Group Policy Discovery", "T1619": "Cloud Storage Object Discovery", - "T1622": "Debugger Evasion" + "T1622": "Debugger Evasion", + "T1652": "Device Driver Discovery" }, "Lateral Movement": { "T1021": "Remote Services", @@ -656,6 +669,7 @@ "T1021.004": "Remote Services::SSH", "T1021.005": "Remote Services::VNC", "T1021.006": "Remote Services::Windows Remote Management", + "T1021.007": "Remote Services::Cloud Services", "T1072": "Software Deployment Tools", "T1080": "Taint Shared Content", "T1091": "Replication Through Removable Media", @@ -768,7 +782,8 @@ "T1537": "Transfer Data to Cloud Account", "T1567": "Exfiltration Over Web Service", "T1567.001": "Exfiltration Over Web Service::Exfiltration to Code Repository", - "T1567.002": "Exfiltration Over Web Service::Exfiltration to Cloud Storage" + "T1567.002": "Exfiltration Over Web Service::Exfiltration to Cloud Storage", + "T1567.003": "Exfiltration Over Web Service::Exfiltration to Text Storage Sites" }, "Impact": { "T1485": "Data Destruction", diff --git a/scripts/profile-time.py b/scripts/profile-time.py index 32aa31f7..be6109e3 100644 --- a/scripts/profile-time.py +++ b/scripts/profile-time.py @@ -112,7 +112,7 @@ def main(argv=None): ) assert isinstance(extractor, StaticFeatureExtractor) - with tqdm.tqdm(total=args.number * args.repeat) as pbar: + with tqdm.tqdm(total=args.number * args.repeat, leave=False) as pbar: def do_iteration(): capa.perf.reset() diff --git a/setup.py b/setup.py index 4a67c68c..3b9342ed 100644 --- a/setup.py +++ b/setup.py @@ -14,13 +14,13 @@ requirements = [ "tqdm==4.65.0", "pyyaml==6.0", "tabulate==0.9.0", - "colorama==0.4.5", + "colorama==0.4.6", "termcolor==2.3.0", "wcwidth==0.2.6", "ida-settings==2.1.0", "viv-utils[flirt]==0.7.9", "halo==0.0.31", - "networkx==2.5.1", # newer versions no longer support py3.7. + "networkx==3.1", "ruamel.yaml==0.17.32", "vivisect==1.1.1", "pefile==2023.2.7", @@ -84,7 +84,7 @@ setuptools.setup( "mypy-protobuf==3.4.0", # type stubs for mypy "types-backports==0.1.3", - "types-colorama==0.4.15", + "types-colorama==0.4.15.11", "types-PyYAML==6.0.8", "types-tabulate==0.9.0.1", "types-termcolor==1.1.4", @@ -107,5 +107,5 @@ setuptools.setup( "Programming Language :: Python :: 3", "Topic :: Security", ], - python_requires=">=3.7", + python_requires=">=3.8", ) diff --git a/tests/test_binja_features.py b/tests/test_binja_features.py index 04c8a49e..2e9e6697 100644 --- a/tests/test_binja_features.py +++ b/tests/test_binja_features.py @@ -37,6 +37,8 @@ except ImportError: indirect=["sample", "scope"], ) def test_binja_features(sample, scope, feature, expected): + if feature == capa.features.common.Characteristic("stack string"): + pytest.xfail("skip failing Binja stack string detection temporarily, see #1473") fixtures.do_test_feature_presence(fixtures.get_binja_extractor, sample, scope, feature, expected)