update to v7-beta (#1942)

* update to v7-beta

.github/pyinstaller/pyinstaller.spec

@@ -18,7 +18,7 @@ a = Analysis(
         # this gets invoked from the directory of the spec file,
         # i.e. ./.github/pyinstaller
         ("../../rules", "rules"),
-        ("../../capa/sigs", "sigs"),
+        ("../../sigs", "sigs"),
         ("../../cache", "cache"),
         # capa.render.default uses tabulate that depends on wcwidth.
         # it seems wcwidth uses a json file `version.json`

.github/workflows/build.yml

@@ -57,15 +57,15 @@ jobs:
       - name: Build standalone executable
         run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
       - name: Does it run (PE)?
-        run: dist/capa "tests/data/Practical Malware Analysis Lab 01-01.dll_"
+        run: dist/capa -d "tests/data/Practical Malware Analysis Lab 01-01.dll_"
       - name: Does it run (Shellcode)?
-        run: dist/capa "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
+        run: dist/capa -d "tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32"
       - name: Does it run (ELF)?
-        run: dist/capa "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
+        run: dist/capa -d "tests/data/7351f8a40c5450557b24622417fc478d.elf_"
       - name: Does it run (CAPE)?
         run: |
           7z e "tests/data/dynamic/cape/v2.2/d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json.gz"
-          dist/capa "d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json"
+          dist/capa -d "d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json"
       - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: ${{ matrix.asset_name }}

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 ## master (unreleased)
 
+### New Features
+
+### Breaking Changes
+
+### New Rules (0)
+
+-
+
+### Bug Fixes
+
+### capa explorer IDA Pro plugin
+
+### Development
+
+### Raw diffs
+- [capa v7.0.0-beta...master](https://github.com/mandiant/capa/compare/v7.0.0-beta...master)
+- [capa-rules v7.0.0-beta...master](https://github.com/mandiant/capa-rules/compare/v7.0.0-beta...master)
+
+## v7.0.0-beta
+This is the beta release of capa v7.0 which was mainly worked on during the Google Summer of Code (GSoC) 2023. A huge
+shoutout to @colton-gabertan and @yelhamer for their amazing work.
+
+Also a big thanks to the other contributors: @aaronatp, @Aayush-Goel-04, @bkojusner, @doomedraven, @ruppde, and @xusheng6.
 ### New Features
 - add Ghidra backend #1770 #1767 @colton-gabertan @mike-hunhoff
 - add dynamic analysis via CAPE sandbox reports #48 #1535 @yelhamer
@@ -23,7 +46,7 @@
 - update freeze format to v3, adding support for dynamic analysis @williballenthin
 - extractor: ignore DLL name for api features #1815 @mr-tz
 
-### New Rules (39)
+### New Rules (41)
 
 - nursery/get-ntoskrnl-base-address @mr-tz
 - host-interaction/network/connectivity/set-tcp-connection-state @johnk3r
@@ -63,7 +86,8 @@
 - nursery/get-current-process-command-line william.ballenthin@mandiant.com
 - nursery/get-current-process-file-path william.ballenthin@mandiant.com
 - nursery/hook-routines-via-dlsym-rtld_next william.ballenthin@mandiant.com
--
+- nursery/linked-against-hp-socket still@teamt5.org
+- host-interaction/process/inject/process-ghostly-hollowing sara.rincon@mandiant.com
 
 ### Bug Fixes
 - ghidra: fix `ints_to_bytes` performance #1761 @mike-hunhoff
@@ -71,15 +95,28 @@
 - binja: use `binaryninja.load` to open files @xusheng6
 - binja: bump binja version to 3.5 #1789 @xusheng6
 - elf: better detect ELF OS via GCC .ident directives #1928 @williballenthin
+- fix setuptools package discovery #1886 @gmacon @mr-tz
 
 ### capa explorer IDA Pro plugin
 
 ### Development
 - update ATT&CK/MBC data for linting #1932 @mr-tz
 
+#### Developer Notes
+With this new release, many classes and concepts have been split up into static (mostly identical to the
+prior implementations) and dynamic ones. For example, the legacy FeatureExtractor class has been renamed to
+StaticFeatureExtractor and the DynamicFeatureExtractor has been added.
+
+Starting from version 7.0, we have moved the component responsible for feature extractor from main to a new
+capabilities' module. Now, users wishing to utilize capa’s feature extraction abilities should use that module instead
+of importing the relevant logic from the main file.
+
+For sandbox-based feature extractors, we are using Pydantic models. Contributions of more models for other sandboxes
+are very welcome!
+
 ### Raw diffs
-- [capa v6.1.0...master](https://github.com/mandiant/capa/compare/v6.1.0...master)
-- [capa-rules v6.1.0...master](https://github.com/mandiant/capa-rules/compare/v6.1.0...master)
+- [capa v6.1.0...v7.0.0-beta](https://github.com/mandiant/capa/compare/v6.1.0...v7.0.0-beta)
+- [capa-rules v6.1.0...v7.0.0-beta](https://github.com/mandiant/capa-rules/compare/v6.1.0...v7.0.0-beta)
 
 ## v6.1.0
 

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-864-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-866-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)

capa/features/extractors/cape/extractor.py

@@ -128,6 +128,14 @@ class CapeExtractor(DynamicFeatureExtractor):
         if cr.info.version not in TESTED_VERSIONS:
             logger.warning("CAPE version '%s' not tested/supported yet", cr.info.version)
 
+        # TODO(mr-tz): support more file types
+        # https://github.com/mandiant/capa/issues/1933
+        if "PE" not in cr.target.file.type:
+            logger.error(
+                "capa currently only supports PE target files, this target file's type is: '%s'.\nPlease report this at: https://github.com/mandiant/capa/issues/1933",
+                cr.target.file.type,
+            )
+
         # observed in 2.4-CAPE reports from capesandbox.com
         if cr.static is None and cr.target.file.pe is not None:
             cr.static = Static()

capa/helpers.py

@@ -156,7 +156,7 @@ def log_unsupported_format_error():
 
 def log_unsupported_cape_report_error(error: str):
     logger.error("-" * 80)
-    logger.error("Input file is not a valid CAPE report: %s", error)
+    logger.error(" Input file is not a valid CAPE report: %s", error)
     logger.error(" ")
     logger.error(" capa currently only supports analyzing standard CAPE reports in JSON format.")
     logger.error(

capa/ida/plugin/view.py

@@ -194,7 +194,9 @@ class CapaExplorerRulegenPreview(QtWidgets.QTextEdit):
             "    namespace: <insert_namespace>",
             "    authors:",
             f"      - {author}",
-            f"    scope: {scope}",
+            "    scopes:",
+            f"      static: {scope}",
+            "      dynamic: unspecified",
             "    references:",
             "      - <insert_references>",
             "    examples:",

capa/main.py

@@ -214,7 +214,7 @@ def get_default_signatures() -> List[Path]:
     """
     compute a list of file system paths to the default FLIRT signatures.
     """
-    sigs_path = get_default_root() / "capa" / "sigs"
+    sigs_path = get_default_root() / "sigs"
     logger.debug("signatures path: %s", sigs_path)
 
     ret = []
@@ -962,7 +962,7 @@ def handle_common_args(args):
             )
             logger.debug("-" * 80)
 
-            sigs_path = get_default_root() / "capa" / "sigs"
+            sigs_path = get_default_root() / "sigs"
 
             if not sigs_path.exists():
                 logger.error(

capa/version.py

@@ -5,7 +5,7 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-__version__ = "6.1.0"
+__version__ = "7.0.0-beta"
 
 
 def get_major_version():

doc/installation.md

@@ -35,6 +35,12 @@ $ unzip v4.0.0.zip
 $ capa -r /path/to/capa-rules suspicious.exe
 ```
 
+This technique also doesn't set up the default library identification [signatures](https://github.com/mandiant/capa/tree/master/sigs). You can pass the signature directory using the `-s` argument.
+For example, to run capa with both a rule path and a signature path:
+```console
+$ capa -s /path/to/capa-sigs suspicious.exe
+```
+
 Alternatively, see Method 3 below.
 
 ### 2. Use capa

pyproject.toml

@@ -56,8 +56,9 @@ dynamic = ["version"]
 [tool.setuptools.dynamic]
 version = {attr = "capa.version.__version__"}
 
-[tool.setuptools]
-packages = ["capa"]
+[tool.setuptools.packages.find]
+include = ["capa*"]
+namespaces = false
 
 [project.optional-dependencies]
 dev = [
@@ -77,7 +78,7 @@ dev = [
     "flake8-simplify==0.21.0",
     "flake8-use-pathlib==0.3.0",
     "flake8-copyright==0.2.4",
-    "ruff==0.1.13",
+    "ruff==0.1.14",
     "black==23.12.1",
     "isort==5.13.2",
     "mypy==1.8.0",

sigs/README.md

@@ -1,4 +1,4 @@
-# capa FLIRT signatures
+# capa/sigs
 
 This directory contains FLIRT signatures that capa uses to identify library functions.
 Typically, capa will ignore library functions, which reduces false positives and improves runtime.

tests/fixtures.py

@@ -100,9 +100,9 @@ def get_viv_extractor(path: Path):
     sigpaths = [
         CD / "data" / "sigs" / "test_aulldiv.pat",
         CD / "data" / "sigs" / "test_aullrem.pat.gz",
-        CD.parent / "capa" / "sigs" / "1_flare_msvc_rtf_32_64.sig",
-        CD.parent / "capa" / "sigs" / "2_flare_msvc_atlmfc_32_64.sig",
-        CD.parent / "capa" / "sigs" / "3_flare_common_libs.sig",
+        CD.parent / "sigs" / "1_flare_msvc_rtf_32_64.sig",
+        CD.parent / "sigs" / "2_flare_msvc_atlmfc_32_64.sig",
+        CD.parent / "sigs" / "3_flare_common_libs.sig",
     ]
 
     if "raw32" in path.name:

tests/test_cape_model.py

@@ -6,10 +6,13 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import gzip
+from typing import Type
 from pathlib import Path
 
+import pytest
 import fixtures
 
+from capa.exceptions import EmptyReportError, UnsupportedFormatError
 from capa.features.extractors.cape.models import Call, CapeReport
 
 CD = Path(__file__).resolve().parent
@@ -41,6 +44,35 @@ def test_cape_model_can_load(version: str, filename: str):
     assert report is not None
 
 
+@fixtures.parametrize(
+    "version,filename,exception",
+    [
+        ("v2.2", "0000a65749f5902c4d82ffa701198038f0b4870b00a27cfca109f8f933476d82.json.gz", None),
+        ("v2.2", "55dcd38773f4104b95589acc87d93bf8b4a264b4a6d823b73fb6a7ab8144c08b.json.gz", None),
+        ("v2.2", "77c961050aa252d6d595ec5120981abf02068c968f4a5be5958d10e87aa6f0e8.json.gz", EmptyReportError),
+        ("v2.2", "d46900384c78863420fb3e297d0a2f743cd2b6b3f7f82bf64059a168e07aceb7.json.gz", None),
+        ("v2.4", "36d218f384010cce9f58b8193b7d8cc855d1dff23f80d16e13a883e152d07921.json.gz", UnsupportedFormatError),
+        ("v2.4", "41ce492f04accef7931b84b8548a6ca717ffabb9bedc4f624de2d37a5345036c.json.gz", UnsupportedFormatError),
+        ("v2.4", "515a6269965ccdf1005008e017ec87fafb97fd2464af1c393ad93b438f6f33fe.json.gz", UnsupportedFormatError),
+        ("v2.4", "5d61700feabba201e1ba98df3c8210a3090c8c9f9adbf16cb3d1da3aaa2a9d96.json.gz", UnsupportedFormatError),
+        ("v2.4", "5effaf6795932d8b36755f89f99ce7436421ea2bd1ed5bc55476530c1a22009f.json.gz", UnsupportedFormatError),
+        ("v2.4", "873275144af88e9b95ea2c59ece39b8ce5a9d7fe09774b683050098ac965054d.json.gz", UnsupportedFormatError),
+        ("v2.4", "8b9aaf4fad227cde7a7dabce7ba187b0b923301718d9d40de04bdd15c9b22905.json.gz", UnsupportedFormatError),
+        ("v2.4", "b1c4aa078880c579961dc5ec899b2c2e08ae5db80b4263e4ca9607a68e2faef9.json.gz", UnsupportedFormatError),
+        ("v2.4", "fb7ade52dc5a1d6128b9c217114a46d0089147610f99f5122face29e429a1e74.json.gz", None),
+    ],
+)
+def test_cape_extractor(version: str, filename: str, exception: Type[BaseException]):
+    path = CAPE_DIR / version / filename
+
+    if exception:
+        with pytest.raises(exception):
+            _ = fixtures.get_cape_extractor(path)
+    else:
+        cr = fixtures.get_cape_extractor(path)
+        assert cr is not None
+
+
 def test_cape_model_argument():
     call = Call.model_validate_json(
         """