Release v4.0.1 (#1143)

* update scripts/lint.py to validate rule metadata using pydantic (#1141)

* doc: v401

Co-authored-by: Mike Hunhoff <mike.hunhoff@gmail.com>

.github/pyinstaller/pyinstaller.spec

@@ -6,51 +6,59 @@ import subprocess
 import wcwidth
 
 
-# when invoking pyinstaller from the project root,
-# this gets run from the project root.
-with open('./capa/version.py', 'wb') as f:
-    # git output will look like:
-    #
-    #     tags/v1.0.0-0-g3af38dc
-    #         ------- tag
-    #                 - commits since
-    #                   g------- git hash fragment
-    version = (subprocess.check_output(["git", "describe", "--always", "--tags", "--long"])
-               .decode("utf-8")
-               .strip()
-               .replace("tags/", ""))
-    f.write(("__version__ = '%s'" % version).encode("utf-8"))
+# git output will look like:
+#
+#     tags/v1.0.0-0-g3af38dc
+#         ------- tag
+#                 - commits since
+#                   g------- git hash fragment
+version = (
+    subprocess.check_output(["git", "describe", "--always", "--tags", "--long"])
+    .decode("utf-8")
+    .strip()
+    .replace("tags/", "")
+)
+# when invoking pyinstaller from the project root, this gets run from the project root.
+with open("./capa/version.py", "r", encoding="utf-8") as f:
+    lines = f.read()
+# version.py contains the version string and other helper functions
+# here we manually replace the version value substring with the result of the above git output
+VERSION_DEF = "__version__ = "
+s = lines.index(VERSION_DEF)
+e = s + len(VERSION_DEF)
+off_rest_file = e + lines[e:].index("\n")
+lines = lines[s:e] + f'"{version}"' + lines[off_rest_file:]
+with open("./capa/version.py", "w", encoding="utf-8") as f:
+    f.write(lines)
 
 a = Analysis(
     # when invoking pyinstaller from the project root,
     # this gets invoked from the directory of the spec file,
     # i.e. ./.github/pyinstaller
-    ['../../capa/main.py'],
-    pathex=['capa'],
+    ["../../capa/main.py"],
+    pathex=["capa"],
     binaries=None,
     datas=[
         # when invoking pyinstaller from the project root,
         # this gets invoked from the directory of the spec file,
         # i.e. ./.github/pyinstaller
-        ('../../rules', 'rules'),
-        ('../../sigs', 'sigs'),
-
+        ("../../rules", "rules"),
+        ("../../sigs", "sigs"),
         # capa.render.default uses tabulate that depends on wcwidth.
         # it seems wcwidth uses a json file `version.json`
         # and this doesn't get picked up by pyinstaller automatically.
         # so we manually embed the wcwidth resources here.
         #
         # ref: https://stackoverflow.com/a/62278462/87207
-        (os.path.dirname(wcwidth.__file__), 'wcwidth')
+        (os.path.dirname(wcwidth.__file__), "wcwidth"),
     ],
     # when invoking pyinstaller from the project root,
     # this gets run from the project root.
-    hookspath=['.github/pyinstaller/hooks'],
+    hookspath=[".github/pyinstaller/hooks"],
     runtime_hooks=None,
     excludes=[
         # ignore packages that would otherwise be bundled with the .exe.
         # review: build/pyinstaller/xref-pyinstaller.html
-
         # we don't do any GUI stuff, so ignore these modules
         "tkinter",
         "_tkinter",
@@ -60,7 +68,6 @@ a = Analysis(
         # since we don't spawn a notebook, we can safely remove these.
         "IPython",
         "ipywidgets",
-
         # these are pulled in by networkx
         # but we don't need to compute the strongly connected components.
         "numpy",
@@ -68,7 +75,6 @@ a = Analysis(
         "matplotlib",
         "pandas",
         "pytest",
-
         # deps from viv that we don't use.
         # this duplicates the entries in `hook-vivisect`,
         # but works better this way.
@@ -78,32 +84,32 @@ a = Analysis(
         "PyQt5",
         "qt5",
         "pyqtwebengine",
-        "pyasn1"
-    ])
+        "pyasn1",
+    ],
+)
 
-a.binaries = a.binaries - TOC([
- ('tcl85.dll', None, None),
- ('tk85.dll', None, None),
- ('_tkinter', None, None)])
+a.binaries = a.binaries - TOC([("tcl85.dll", None, None), ("tk85.dll", None, None), ("_tkinter", None, None)])
 
 pyz = PYZ(a.pure, a.zipped_data)
 
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          exclude_binaries=False,
-          name='capa',
-          icon='logo.ico',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )
+exe = EXE(
+    pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    exclude_binaries=False,
+    name="capa",
+    icon="logo.ico",
+    debug=False,
+    strip=None,
+    upx=True,
+    console=True,
+)
 
 # enable the following to debug the contents of the .exe
 #
-#coll = COLLECT(exe,
+# coll = COLLECT(exe,
 #               a.binaries,
 #               a.zipfiles,
 #               a.datas,

CHANGELOG.md

@@ -12,15 +12,33 @@
 
 ### Bug Fixes
 
+
 ### capa explorer IDA Pro plugin
 
 ### Development
 
 ### Raw diffs
-- [capa v4.0.0...master](https://github.com/mandiant/capa/compare/v4.0.0...master)
-- [capa-rules v4.0.0...master](https://github.com/mandiant/capa-rules/compare/v4.0.0...master)
+- [capa v4.0.1...master](https://github.com/mandiant/capa/compare/v4.0.1...master)
+- [capa-rules v4.0.1...master](https://github.com/mandiant/capa-rules/compare/v4.0.1...master)
 
-## v4.0.0 (2022-07-XX)
+
+## v4.0.1 (2022-08-15)
+Some rules contained invalid metadata fields that caused an error when rendering rule hits. We've updated all rules and enhanced the rule linter to catch such issues.
+
+### New Rules (1)
+
+- anti-analysis/obfuscation/obfuscated-with-vs-obfuscation jakub.jozwiak@mandiant.com
+
+
+### Bug Fixes
+- linter: use pydantic to validate rule metadata #1141 @mike-hunhoff
+- build binaries using PyInstaller no longer overwrites functions in version.py #1136 @mr-tz
+
+### Raw diffs
+- [capa v4.0.0...v4.0.1](https://github.com/mandiant/capa/compare/v4.0.0...v4.0.1)
+- [capa-rules v4.0.0...v4.0.1](https://github.com/mandiant/capa-rules/compare/v4.0.0...v4.0.1)
+
+## v4.0.0 (2022-08-10)
 Version 4 adds support for analyzing .NET executables. capa will autodetect .NET modules, or you can explicitly invoke the new feature extractor via `--format dotnet`. We've also extended the rule syntax for .NET features including `namespace` and `class`.
 
 Additionally, new `instruction` scope and `operand` features enable users to create more explicit rules. These features are not backwards compatible. We removed the previously used `/x32` and `/x64` flavors of number and operand features.

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-702-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-703-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
@@ -15,7 +15,7 @@ Check out:
 - the overview in our first [capa blog post](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities)
 - the major version 2.0 updates described in our [second blog post](https://www.mandiant.com/resources/capa-2-better-stronger-faster)
 - the major version 3.0 (ELF support) described in the [third blog post](https://www.mandiant.com/resources/elfant-in-the-room-capa-v3)
-- the major version 4.0 (.NET support) described in the TODO
+- the major version 4.0 (.NET support) described in the [fourth blog post](https://www.mandiant.com/resources/blog/capa-v4-casting-wider-net)
 
 ```
 $ capa.exe suspicious.exe

capa/version.py

@@ -1,4 +1,4 @@
-__version__ = "4.0.0"
+__version__ = "4.0.1"
 
 
 def get_major_version():

scripts/lint.py

@@ -34,6 +34,7 @@ from pathlib import Path
 from dataclasses import field, dataclass
 
 import tqdm
+import pydantic
 import termcolor
 import ruamel.yaml
 import tqdm.contrib.logging
@@ -45,6 +46,7 @@ import capa.helpers
 import capa.features.insn
 from capa.rules import Rule, RuleSet
 from capa.features.common import FORMAT_PE, FORMAT_DOTNET, String, Feature, Substring
+from capa.render.result_document import RuleMetadata
 
 logger = logging.getLogger("lint")
 
@@ -226,19 +228,13 @@ class ExampleFileDNE(Lint):
 class IncorrectValueType(Lint):
     name = "incorrect value type"
     recommendation = "Change value type"
-    recommendation_template = 'Change type of "{:s}" from "{:s}" to "{:s}'
-    types = {
-        "references": list,
-        "authors": list,
-    }
 
     def check_rule(self, ctx: Context, rule: Rule):
-        for k, expected in self.types.items():
-            value = rule.meta.get(k)
-            found = type(value)
-            if value and found != expected:
-                self.recommendation = self.recommendation_template.format(k, str(found), str(expected))
-                return True
+        try:
+            _ = RuleMetadata.from_capa(rule)
+        except pydantic.ValidationError as e:
+            self.recommendation = str(e).strip()
+            return True
         return False
 
 
@@ -722,11 +718,11 @@ META_LINTS = (
     MissingExamples(),
     MissingExampleOffset(),
     ExampleFileDNE(),
-    IncorrectValueType(),
     UnusualMetaField(),
     LibRuleNotInLibDirectory(),
     LibRuleHasNamespace(),
     InvalidAttckOrMbcTechnique(),
+    IncorrectValueType(),
 )
 
 
@@ -1003,8 +999,9 @@ def main(argv=None):
 
     try:
         rules = capa.main.get_rules(args.rules, disable_progress=True)
+        rule_count = len(rules)
         rules = capa.rules.RuleSet(rules)
-        logger.info("successfully loaded %s rules", len(rules))
+        logger.info("successfully loaded %s rules", rule_count)
         if args.tag:
             rules = rules.filter_rules_by_meta(args.tag)
             logger.debug("selected %s rules", len(rules))