Merge branch 'master' into Aayush-Goel-04/Issue#1534

.github/workflows/publish.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
-          python-version: '3.7'
+          python-version: '3.8'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/tests.yml

@@ -69,7 +69,7 @@ jobs:
       matrix:
         os: [ubuntu-20.04, windows-2019, macos-11]
         # across all operating systems
-        python-version: ["3.7", "3.11"]
+        python-version: ["3.8", "3.11"]
         include:
           # on Ubuntu run these as well
           - os: ubuntu-20.04
@@ -104,7 +104,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.11"]
+        python-version: ["3.8", "3.11"]
     steps:
     - name: Checkout capa with submodules
       # do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118

CHANGELOG.md

@@ -5,12 +5,14 @@
 
 ### New Features
 - Utility script to detect feature overlap between new and existing CAPA rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
+- use fancy box drawing characters for default output #1586 @williballenthin
 
 ### Breaking Changes
 - Update Metadata type in capa main [#1411](https://github.com/mandiant/capa/issues/1411) [@Aayush-Goel-04](https://github.com/aayush-goel-04) @manasghandat
-- Updated file paths to use pathlib.Path for improved path handling and compatibility [#1534](https://github.com/mandiant/capa/issues/1534) [@Aayush-Goel-04]
+- Python 3.8 is now the minimum supported Python version #1578 @williballenthin
+- Updated file paths to use pathlib.Path for improved path handling and compatibility [#1534](https://github.com/mandiant/capa/issues/1534) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 
-### New Rules (20)
+### New Rules (22)
 
 - load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
 - nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
@@ -31,6 +33,8 @@
 - persistence/office/act-as-office-com-add-in jakub.jozwiak@mandiant.com
 - persistence/office/act-as-word-wll-add-in jakub.jozwiak@mandiant.com
 - anti-analysis/anti-debugging/debugger-evasion/hide-thread-from-debugger michael.hunhoff@mandiant.com jakub.jozwiak@mandiant.com
+- host-interaction/memory/create-new-application-domain-in-dotnet jakub.jozwiak@mandiant.com
+- host-interaction/gui/switch-active-desktop jakub.jozwiak@mandiant.com
 -
 
 ### Bug Fixes
@@ -46,6 +50,8 @@
 - Add logging and print redirect to tqdm for capa main [#749](https://github.com/mandiant/capa/issues/749) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 - extractor: fix binja installation path detection does not work with Python 3.11
 - tests: refine the IDA test runner script #1513 @williballenthin
+- output: don't leave behind traces of progress bar @williballenthin
+- import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin
 
 ### capa explorer IDA Pro plugin
 

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-808-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-810-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)

capa/helpers.py

@@ -156,7 +156,7 @@ def log_unsupported_runtime_error():
     logger.error("-" * 80)
     logger.error(" Unsupported runtime or Python interpreter.")
     logger.error(" ")
-    logger.error(" capa supports running under Python 3.7 and higher.")
+    logger.error(" capa supports running under Python 3.8 and higher.")
     logger.error(" ")
     logger.error(
         " If you're seeing this message on the command line, please ensure you're running a supported Python version."

capa/ida/plugin/README.md

@@ -95,7 +95,7 @@ can update using the `Settings` button.
 
 ### Requirements
 
-capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:
+capa explorer supports Python versions >= 3.8.x and IDA Pro versions >= 7.4. The following IDA Pro versions have been tested:
 
 * IDA 7.4
 * IDA 7.5
@@ -105,7 +105,7 @@ capa explorer supports Python versions >= 3.7.x and IDA Pro versions >= 7.4. The
 * IDA 8.1
 * IDA 8.2
 
-capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.7.x).
+capa explorer is however limited to the Python versions supported by your IDA installation (which may not include all Python versions >= 3.8.x).
 
 If you encounter issues with your specific setup, please open a new [Issue](https://github.com/mandiant/capa/issues).
 

capa/main.py

@@ -8,6 +8,7 @@ Unless required by applicable law or agreed to in writing, software distributed
  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
+import io
 import os
 import sys
 import time
@@ -262,7 +263,7 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
             functions = list(extractor.get_functions())
             n_funcs = len(functions)
 
-            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions")
+            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False)
             for f in pb:
                 if extractor.is_library_function(f.address):
                     function_name = extractor.get_function_name(f.address)
@@ -976,12 +977,20 @@ def handle_common_args(args):
     # disable vivisect-related logging, it's verbose and not relevant for capa users
     set_vivisect_log_level(logging.CRITICAL)
 
-    # Since Python 3.8 cp65001 is an alias to utf_8, but not for Python < 3.8
+    if isinstance(sys.stdout, io.TextIOWrapper) or hasattr(sys.stdout, "reconfigure"):
-    # TODO: remove this code when only supporting Python 3.8+
+        # from sys.stdout type hint:
-    # https://stackoverflow.com/a/3259271/87207
+        #
-    import codecs
+        # TextIO is used instead of more specific types for the standard streams,
-
+        # since they are often monkeypatched at runtime. At startup, the objects
-    codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None)
+        # are initialized to instances of TextIOWrapper.
+        #
+        # To use methods from TextIOWrapper, use an isinstance check to ensure that
+        # the streams have not been overridden:
+        #
+        # if isinstance(sys.stdout, io.TextIOWrapper):
+        #    sys.stdout.reconfigure(...)
+        sys.stdout.reconfigure(encoding="utf-8")
+    colorama.just_fix_windows_console()
 
     if args.color == "always":
         colorama.init(strip=False)
@@ -1058,8 +1067,8 @@ def handle_common_args(args):
 
 
 def main(argv=None):
-    if sys.version_info < (3, 7):
+    if sys.version_info < (3, 8):
-        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.7+")
+        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+")
 
     if argv is None:
         argv = sys.argv[1:]

capa/render/default.py

@@ -40,7 +40,7 @@ def render_meta(doc: rd.ResultDocument, ostream: StringIO):
         ("path", doc.meta.sample.path),
     ]
 
-    ostream.write(tabulate.tabulate(rows, tablefmt="psql"))
+    ostream.write(tabulate.tabulate(rows, tablefmt="mixed_outline"))
     ostream.write("\n")
 
 
@@ -102,7 +102,7 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
 
     if rows:
         ostream.write(
-            tabulate.tabulate(rows, headers=[width("CAPABILITY", 50), width("NAMESPACE", 50)], tablefmt="psql")
+            tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline")
         )
         ostream.write("\n")
     else:
@@ -148,7 +148,7 @@ def render_attack(doc: rd.ResultDocument, ostream: StringIO):
     if rows:
         ostream.write(
             tabulate.tabulate(
-                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="psql"
+                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="mixed_grid"
             )
         )
         ostream.write("\n")
@@ -190,7 +190,9 @@ def render_mbc(doc: rd.ResultDocument, ostream: StringIO):
 
     if rows:
         ostream.write(
-            tabulate.tabulate(rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="psql")
+            tabulate.tabulate(
+                rows, headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], tablefmt="mixed_grid"
+            )
         )
         ostream.write("\n")
 

scripts/import-to-ida.py

@@ -28,13 +28,17 @@ Unless required by applicable law or agreed to in writing, software distributed
  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and limitations under the License.
 """
-import json
 import logging
+import binascii
 
 import ida_nalt
 import ida_funcs
 import ida_kernwin
 
+import capa.rules
+import capa.features.freeze
+import capa.render.result_document
+
 logger = logging.getLogger("capa")
 
 
@@ -64,37 +68,37 @@ def main():
     if not path:
         return 0
 
-    with open(path, "rb") as f:
+    result_doc = capa.render.result_document.ResultDocument.parse_file(path)
-        doc = json.loads(f.read().decode("utf-8"))
+    meta, capabilities = result_doc.to_capa()
-
-    if "meta" not in doc or "rules" not in doc:
-        logger.error("doesn't appear to be a capa report")
-        return -1
 
     # in IDA 7.4, the MD5 hash may be truncated, for example:
     # wanted: 84882c9d43e23d63b82004fae74ebb61
     # found: b'84882C9D43E23D63B82004FAE74EBB6\x00'
     #
     # see: https://github.com/idapython/bin/issues/11
-    a = doc["meta"]["sample"]["md5"].lower()
+    a = meta.sample.md5.lower()
-    b = ida_nalt.retrieve_input_file_md5().lower()
+    b = binascii.hexlify(ida_nalt.retrieve_input_file_md5()).decode("ascii").lower()
     if not a.startswith(b):
         logger.error("sample mismatch")
         return -2
 
     rows = []
-    for rule in doc["rules"].values():
+    for name in capabilities.keys():
-        if rule["meta"].get("lib"):
+        rule = result_doc.rules[name]
+        if rule.meta.lib:
             continue
-        if rule["meta"].get("capa/subscope"):
+        if rule.meta.is_subscope_rule:
             continue
-        if rule["meta"]["scope"] != "function":
+        if rule.meta.scope != capa.rules.Scope.FUNCTION:
             continue
 
-        name = rule["meta"]["name"]
+        ns = rule.meta.namespace
-        ns = rule["meta"].get("namespace", "")
+
-        for va in rule["matches"].keys():
+        for address, _ in rule.matches:
-            va = int(va)
+            if address.type != capa.features.freeze.AddressType.ABSOLUTE:
+                continue
+
+            va = address.value
             rows.append((ns, name, va))
 
     # order by (namespace, name) so that like things show up together

scripts/lint.py

@@ -874,7 +874,7 @@ def lint(ctx: Context):
     ret = {}
 
     source_rules = [rule for rule in ctx.rules.rules.values() if not rule.is_subscope_rule()]
-    with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule") as pbar:
+    with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule", leave=False) as pbar:
         with capa.helpers.redirecting_print_to_tqdm(False):
             for rule in pbar:
                 name = rule.name

scripts/profile-time.py

@@ -110,7 +110,7 @@ def main(argv=None):
             args.sample, args.format, args.os, capa.main.BACKEND_VIV, sig_paths, should_save_workspace=False
         )
 
-    with tqdm.tqdm(total=args.number * args.repeat) as pbar:
+    with tqdm.tqdm(total=args.number * args.repeat, leave=False) as pbar:
 
         def do_iteration():
             capa.perf.reset()

setup.py

@@ -15,13 +15,13 @@ requirements = [
     "tqdm==4.65.0",
     "pyyaml==6.0",
     "tabulate==0.9.0",
-    "colorama==0.4.5",
+    "colorama==0.4.6",
     "termcolor==2.3.0",
     "wcwidth==0.2.6",
     "ida-settings==2.1.0",
     "viv-utils[flirt]==0.7.9",
     "halo==0.0.31",
-    "networkx==2.5.1",  # newer versions no longer support py3.7.
+    "networkx==3.1",
     "ruamel.yaml==0.17.32",
     "vivisect==1.1.1",
     "pefile==2023.2.7",
@@ -82,7 +82,7 @@ setuptools.setup(
             "mypy-protobuf==3.4.0",
             # type stubs for mypy
             "types-backports==0.1.3",
-            "types-colorama==0.4.15",
+            "types-colorama==0.4.15.11",
             "types-PyYAML==6.0.8",
             "types-tabulate==0.9.0.1",
             "types-termcolor==1.1.4",
@@ -105,5 +105,5 @@ setuptools.setup(
         "Programming Language :: Python :: 3",
         "Topic :: Security",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
 )