gitea-mirror/capa
1
0
Fork 0
mirror of https://github.com/mandiant/capa.git synced 2026-02-04 19:12:01 -08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into vmray_extractor

Browse Source
This commit is contained in:
r-sm2024
2024-06-11 08:35:48 -05:00
committed by GitHub
parent 3141e940de 1888d0e7e3
commit bdc94c13ac
10 changed files with 534 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/build.yml vendored
View File

@@ -51,7 +51,9 @@ jobs:
- name: Upgrade pip, setuptools
run: python -m pip install --upgrade pip setuptools
- name: Install capa with build requirements
run: pip install -e .[build]
run: |
pip install -r requirements.txt
pip install -e .[build]
- name: Build standalone executable
run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
- name: Does it run (PE)?

1
.github/workflows/publish.yml vendored
View File

@@ -25,6 +25,7 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
pip install -e .[build]
- name: build package
run: |

20
.github/workflows/tests.yml vendored
View File

@@ -35,7 +35,9 @@ jobs:
with:
python-version: "3.11"
- name: Install dependencies
run: pip install -e .[dev]
run: |
pip install -r requirements.txt
pip install -e .[dev]
- name: Lint with ruff
run: pre-commit run ruff
- name: Lint with isort
@@ -61,7 +63,9 @@ jobs:
with:
python-version: "3.11"
- name: Install capa
run: pip install -e .[dev]
run: |
pip install -r requirements.txt
pip install -e .[dev]
- name: Run rule linter
run: python scripts/lint.py rules/
@@ -96,7 +100,9 @@ jobs:
if: matrix.os == 'ubuntu-20.04'
run: sudo apt-get install -y libyaml-dev
- name: Install capa
run: pip install -e .[dev]
run: |
pip install -r requirements.txt
pip install -e .[dev]
- name: Run tests (fast)
# this set of tests runs about 80% of the cases in 20% of the time,
# and should catch most errors quickly.
@@ -131,7 +137,9 @@ jobs:
run: sudo apt-get install -y libyaml-dev
- name: Install capa
if: ${{ env.BN_SERIAL != 0 }}
run: pip install -e .[dev]
run: |
pip install -r requirements.txt
pip install -e .[dev]
- name: install Binary Ninja
if: ${{ env.BN_SERIAL != 0 }}
run: |
@@ -188,7 +196,9 @@ jobs:
- name: Install pyyaml
run: sudo apt-get install -y libyaml-dev
- name: Install capa
run: pip install -e .[dev]
run: |
pip install -r requirements.txt
pip install -e .[dev]
- name: Run tests
run: |
mkdir ./.github/ghidra/project

3
CHANGELOG.md
View File

@@ -7,9 +7,12 @@
- add function in capa/helpers to load plain and compressed JSON reports #1883 @Rohit1123
- document Antivirus warnings and VirusTotal false positive detections #2028 @RionEV @mr-tz
- Add json to sarif conversion script @reversingwithme
- render maec/* fields #843 @s-ff
- replace Halo spinner with Rich #2086 @s-ff
- optimize rule matching #2080 @williballenthin
- add aarch64 as a valid architecture #2144 mehunhoff@google.com @williballenthin
- relax dependency version requirements for the capa library #2053 @williballenthin
### Breaking Changes

3
capa/features/common.py
View File

@@ -409,9 +409,10 @@ class Bytes(Feature):
# other candidates here: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types
ARCH_I386 = "i386"
ARCH_AMD64 = "amd64"
ARCH_AARCH64 = "aarch64"
# dotnet
ARCH_ANY = "any"
VALID_ARCH = (ARCH_I386, ARCH_AMD64, ARCH_ANY)
VALID_ARCH = (ARCH_I386, ARCH_AMD64, ARCH_AARCH64, ARCH_ANY)
class Arch(Feature):

6
doc/installation.md
View File

@@ -91,6 +91,12 @@ For more details about creating and using virtual environments, check out the [v
##### Install development dependencies
When developing capa, please use the pinned dependencies found in `requirements.txt`.
This ensures that everyone has the exact same, reproducible environment.
Please install these dependencies before install capa (from source or from PyPI):
`$ pip install -r requirements.txt`
We use the following tools to ensure consistent code style and formatting:
- [black](https://github.com/psf/black) code formatter
- [isort](https://pypi.org/project/isort/) code formatter

103
pyproject.toml
View File

@@ -32,25 +32,78 @@ classifiers = [
"Topic :: Security",
]
dependencies = [
"tqdm==4.66.4",
"pyyaml==6.0.1",
"tabulate==0.9.0",
"colorama==0.4.6",
"termcolor==2.4.0",
"wcwidth==0.2.13",
"ida-settings==2.1.0",
"viv-utils[flirt]==0.7.9",
"networkx==3.1",
"ruamel.yaml==0.18.6",
"vivisect==1.1.1",
"pefile==2023.2.7",
"pyelftools==0.31",
"dnfile==0.14.1",
"dncil==1.0.2",
"pydantic==2.7.1",
"rich==13.7.1",
"humanize==4.9.0",
"protobuf==5.27.0",
# ---------------------------------------
# As a library, capa uses lower version bounds
# when specifying its dependencies. This lets
# other programs that use capa (and other libraries)
# to find a compatible set of dependency versions.
#
# We can optionally pin to specific versions or
# limit the upper bound when there's a good reason;
# but the default is to assume all greater versions
# probably work with capa until proven otherwise.
#
# The following link provides good background:
# https://iscinumpy.dev/post/bound-version-constraints/
#
# When we develop capa, and when we distribute it as
# a standalone binary, we'll use specific versions
# that are pinned in requirements.txt.
# But the requirements for a library are specified here
# and are looser.
#
# Related discussions:
#
# - https://github.com/mandiant/capa/issues/2053
# - https://github.com/mandiant/capa/pull/2059
# - https://github.com/mandiant/capa/pull/2079
#
# ---------------------------------------
# The following dependency versions were imported
# during June 2024 by truncating specific versions to
# their major-most version (major version when possible,
# or minor otherwise).
# As specific constraints are identified, please provide
# comments and context.
"tqdm>=4",
"pyyaml>=6",
"tabulate>=0.9",
"colorama>=0.4",
"termcolor>=2",
"wcwidth>=0.2",
"ida-settings>=2",
"ruamel.yaml>=0.18",
"pefile>=2023.2.7",
"pyelftools>=0.31",
"pydantic>=2",
"rich>=13",
"humanize>=4",
"protobuf>=5",
# ---------------------------------------
# Dependencies that we develop
#
# These dependencies are often actively influenced by capa,
# so we provide a minimum patch version that includes the
# latest bug fixes we need here.
"viv-utils[flirt]>=0.7.9",
"vivisect>=1.1.1",
"dncil>=1.0.2",
# ---------------------------------------
# Dependencies with version caps
#
# These dependencies must not exceed the version cap,
# typically due to dropping support for python releases
# we still support.
# TODO(williballenthin): networkx 3.2 doesn't support python 3.8 while capa does.
# https://github.com/mandiant/capa/issues/1966
"networkx>=3,<3.2",
# TODO(williballenthin): dnfile 0.15 changes UserString API and we havent updated yet.
# https://github.com/mandiant/capa/pull/2037
"dnfile>=0.14.1,<0.15",
]
dynamic = ["version"]
@@ -63,6 +116,10 @@ namespaces = false
[project.optional-dependencies]
dev = [
# Dev and build dependencies are not relaxed because
# we want all developer environments to be consistent.
# These dependencies are not used in production environments
# and should not conflict with other libraries/tooling.
"pre-commit==3.5.0",
"pytest==8.0.0",
"pytest-sugar==1.0.0",
@@ -99,8 +156,12 @@ dev = [
"deptry==0.16.1"
]
build = [
"pyinstaller==6.7.0",
"setuptools==69.5.1",
# Dev and build dependencies are not relaxed because
# we want all developer environments to be consistent.
# These dependencies are not used in production environments
# and should not conflict with other libraries/tooling.
"pyinstaller==6.8.0",
"setuptools==70.0.0",
"build==1.2.1"
]

46
requirements.txt Normal file
View File

@@ -0,0 +1,46 @@
# Dependencies with specific version constraints
# used during development and building the standalone executables.
# For these environments, use `pip install -r requirements.txt`
# before installing capa from source/pypi. This will ensure
# the following specific versions are used.
#
# Initially generated via: pip freeze | grep -v -- "-e"
# Kept up to date by dependabot.
annotated-types==0.7.0
colorama==0.4.6
cxxfilt==0.2.2
dncil==1.0.2
dnfile==0.15.0
funcy==2.0
humanize==4.9.0
ida-netnode==3.0
ida-settings==2.1.0
intervaltree==3.1.0
markdown-it-py==3.0.0
mdurl==0.1.2
msgpack==1.0.8
networkx==3.1
pefile==2023.2.7
pip==24.0
protobuf==5.27.1
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycparser==2.22
pydantic==2.7.3
pydantic-core==2.18.4
pyelftools==0.31
pygments==2.18.0
python-flirt==0.8.6
pyyaml==6.0.1
rich==13.7.1
ruamel-yaml==0.18.6
ruamel-yaml-clib==0.2.8
setuptools==70.0.0
six==1.16.0
sortedcontainers==2.4.0
tabulate==0.9.0
termcolor==2.4.0
tqdm==4.66.4
viv-utils==0.7.9
vivisect==1.1.1
wcwidth==0.2.13

375
scripts/capa2sarif.py Normal file
View File

@@ -0,0 +1,375 @@
# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
"""
Convert capa json output to sarif schema
usage: capa2sarif.py [-h] [-g] [-r] [-t TAG] [--version] capa_output
Capa to SARIF analysis file
positional arguments:
capa_output Path to capa JSON output file
optional arguments:
-h, --help show this help message and exit
--version show program's version number and exit
-t TAG, --tag TAG filter on rule meta field values (ruleid)
Requires:
- sarif_om 1.0.4
- jschema_to_python 1.2.3
"""
import sys
import json
import logging
import argparse
from typing import List, Optional
from pathlib import Path
from capa.version import __version__
logger = logging.getLogger("capa2sarif")
# Dependencies
try:
from sarif_om import Run, Tool, SarifLog, ToolComponent
except ImportError as e:
logger.error(
"Required import `sarif_om` is not installed. This is solved by installing `python3 -m pip install sarif_om>=1.0.4`. %s",
e,
)
exit(-4)
try:
from jschema_to_python.to_json import to_json
except ImportError as e:
logger.error(
"Required import `jschema_to_python` is not installed. This is solved by installing `python3 -m pip install jschema_to_python>=1.2.3`, %s",
e,
)
exit(-4)
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Capa to SARIF analysis file")
# Positional argument
parser.add_argument("capa_output", help="Path to capa JSON output file")
# Optional arguments
parser.add_argument(
"-g",
"--ghidra-compat",
action="store_true",
help="Compatibility for Ghidra 11.0.X",
)
parser.add_argument(
"-r",
"--radare-compat",
action="store_true",
help="Compatibility for Radare r2sarif plugin v2.0",
)
parser.add_argument("-t", "--tag", help="Filter on rule meta field values (ruleid)")
parser.add_argument(
"--version", action="version", version=f"%(prog)s {__version__}"
)
return parser.parse_args()
def main() -> int:
logging.basicConfig(level=logging.INFO)
logging.getLogger().setLevel(logging.INFO)
args = _parse_args()
try:
with Path(args.capa_output).open() as capa_output:
json_data = json.load(capa_output)
except ValueError:
logger.error(
"Input data was not valid JSON, input should be a capa json output file."
)
return -1
except json.JSONDecodeError:
# An exception has occured
logger.error(
"Input data was not valid JSON, input should be a capa json output file."
)
return -2
# Marshall json into Sarif
# Create baseline sarif structure to be populated from json data
sarif_structure: Optional[dict] = _sarif_boilerplate(
json_data["meta"], json_data["rules"]
)
if sarif_structure is None:
logger.errort("An Error has occured creating default sarif structure.")
return -3
_populate_artifact(sarif_structure, json_data["meta"])
_populate_invocations(sarif_structure, json_data["meta"])
_populate_results(sarif_structure, json_data["rules"], args.ghidra_compat)
if args.ghidra_compat:
# Ghidra can't handle this structure as of 11.0.x
if "invocations" in sarif_structure["runs"][0]:
del sarif_structure["runs"][0]["invocations"]
# artifacts must include a description as well with a text field.
if "artifacts" in sarif_structure["runs"][0]:
sarif_structure["runs"][0]["artifacts"][0]["description"] = {
"text": "placeholder"
}
# For better compliance with Ghidra table. Iteraction through properties['additionalProperties']
"""
"additionalProperties": {
"to": "<hex addr>",
"offset": 0,
"primary": true,
"index": <>"",
"kind": "<kind column value>",
"opIndex": 0,
"sourceType": ""
}
"""
if args.radare_compat:
# Add just enough for passing tests
_add_filler_optional(json_data, sarif_structure)
print(json.dumps(sarif_structure, indent=4)) # noqa: T201
return 0
def _sarif_boilerplate(data_meta: dict, data_rules: dict) -> Optional[dict]:
# Only track rules that appear in this log, not full 1k
rules = []
# Parse rules from parsed sarif structure
for key in data_rules:
# Use attack as default, if both exist then only use attack, if neither exist use the name of rule for ruleID
# this is not good practice to use long name for ruleID
attack_length = len(data_rules[key]["meta"]["attack"])
mbc_length = len(data_rules[key]["meta"]["mbc"])
if attack_length or mbc_length:
id = (
data_rules[key]["meta"]["attack"][0]["id"]
if attack_length > 0
else data_rules[key]["meta"]["mbc"][0]["id"]
)
else:
id = data_rules[key]["meta"]["name"]
# Append current rule
rules.append(
{
# Default to attack identifier, fall back to MBC, mainly relevant if both are present
"id": id,
"name": data_rules[key]["meta"]["name"],
"shortDescription": {"text": data_rules[key]["meta"]["name"]},
"messageStrings": {
"default": {"text": data_rules[key]["meta"]["name"]}
},
"properties": {
"namespace": data_rules[key]["meta"]["namespace"]
if "namespace" in data_rules[key]["meta"]
else [],
"scopes": data_rules[key]["meta"]["scopes"],
"references": data_rules[key]["meta"]["references"],
"lib": data_rules[key]["meta"]["lib"],
},
}
)
tool = Tool(
driver=ToolComponent(
name="Capa",
version=__version__,
information_uri="https://github.com/mandiant/capa",
rules=rules,
)
)
# Create a SARIF Log object, populate with a single run
sarif_log = SarifLog(
version="2.1.0",
schema_uri="https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json",
runs=[Run(tool=tool, results=[], artifacts=[], invocations=[])],
)
# Convert the SARIF log to a dictionary and then to a JSON string
try:
sarif_outline = json.loads(to_json(sarif_log))
except json.JSONDecodeError:
# An exception has occured
return None
return sarif_outline
def _populate_artifact(sarif_log: dict, meta_data: dict) -> None:
"""
@param sarif_log: dict - sarif data structure including runs
@param meta_data: dict - Capa meta output
@returns None, updates sarif_log via side-effects
"""
sample = meta_data["sample"]
artifact = {
"location": {"uri": sample["path"]},
"roles": ["analysisTarget"],
"hashes": {
"md5": sample["md5"],
"sha-1": sample["sha1"],
"sha-256": sample["sha256"],
},
}
sarif_log["runs"][0]["artifacts"].append(artifact)
def _populate_invocations(sarif_log: dict, meta_data: dict) -> None:
"""
@param sarif_log: dict - sarif data structure including runs
@param meta_data: dict - Capa meta output
@returns None, updates sarif_log via side-effects
"""
analysis_time = meta_data["timestamp"]
argv = meta_data["argv"]
analysis = meta_data["analysis"]
invoke = {
"commandLine": "capa " + " ".join(argv),
"arguments": argv if len(argv) > 0 else [],
# Format in Zulu time, this may require a conversion from local timezone
"endTimeUtc": f"{analysis_time}Z",
"executionSuccessful": True,
"properties": {
"format": analysis["format"],
"arch": analysis["arch"],
"os": analysis["os"],
"extractor": analysis["extractor"],
"rule_location": analysis["rules"],
"base_address": analysis["base_address"],
},
}
sarif_log["runs"][0]["invocations"].append(invoke)
def _enumerate_evidence(node: dict, related_count: int) -> List[dict]:
related_locations = []
if node.get("success") and node.get("node").get("type") != "statement":
label = ""
if node.get("node").get("type") == "feature":
if node.get("node").get("feature").get("type") == "api":
label = "api: " + node.get("node").get("feature").get("api")
elif node.get("node").get("feature").get("type") == "match":
label = "match: " + node.get("node").get("feature").get("match")
elif node.get("node").get("feature").get("type") == "number":
label = f"number: {node.get('node').get('feature').get('description')} ({node.get('node').get('feature').get('number')})"
elif node.get("node").get("feature").get("type") == "offset":
label = f"offset: {node.get('node').get('feature').get('description')} ({node.get('node').get('feature').get('offset')})"
elif node.get("node").get("feature").get("type") == "mnemonic":
label = f"mnemonic: {node.get('node').get('feature').get('mnemonic')}"
elif node.get("node").get("feature").get("type") == "characteristic":
label = f"characteristic: {node.get('node').get('feature').get('characteristic')}"
elif node.get("node").get("feature").get("type") == "os":
label = f"os: {node.get('node').get('feature').get('os')}"
elif node.get("node").get("feature").get("type") == "operand number":
label = f"operand: ({node.get('node').get('feature').get('index')} ) {node.get('node').get('feature').get('description')} ({node.get('node').get('feature').get('operand_number')})"
else:
logger.error(
"Not implemented %s",
node.get("node").get("feature").get("type"),
file=sys.stderr,
)
return []
else:
logger.error(
"Not implemented %s", node.get("node").get("type"), file=sys.stderr
)
return []
for loc in node.get("locations"):
if loc["type"] != "absolute":
continue
related_locations.append(
{
"id": related_count,
"message": {"text": label},
"physicalLocation": {"address": {"absoluteAddress": loc["value"]}},
}
)
related_count += 1
if node.get("success") and node.get("node").get("type") == "statement":
for child in node.get("children"):
related_locations += _enumerate_evidence(child, related_count)
return related_locations
def _populate_results(sarif_log: dict, data_rules: dict, ghidra_compat: bool) -> None:
"""
@param sarif_log: dict - sarif data structure including runs
@param meta_data: dict - Capa meta output
@returns None, updates sarif_log via side-effects
"""
results = sarif_log["runs"][0]["results"]
# Parse rules from parsed sarif structure
for key in data_rules:
# Use attack as default, if both exist then only use attack, if neither exist use the name of rule for ruleID
# this is not good practice to use long name for ruleID.
attack_length = len(data_rules[key]["meta"]["attack"])
mbc_length = len(data_rules[key]["meta"]["mbc"])
if attack_length or mbc_length:
id = (
data_rules[key]["meta"]["attack"][0]["id"]
if attack_length > 0
else data_rules[key]["meta"]["mbc"][0]["id"]
)
else:
id = data_rules[key]["meta"]["name"]
for address, details in data_rules[key]["matches"]:
related_cnt = 0
related_locations = _enumerate_evidence(details, related_cnt)
res = {
"ruleId": id,
"level": "none" if not ghidra_compat else "NONE",
"message": {"text": data_rules[key]["meta"]["name"]},
"kind": "informational" if not ghidra_compat else "INFORMATIONAL",
"locations": [
{
"physicalLocation": {
"address": {
"absoluteAddress": address["value"],
}
},
}
],
}
if not ghidra_compat:
res["relatedLocations"] = related_locations
results.append(res)
def _add_filler_optional(capa_result: dict, sarif_log: dict) -> None:
"""Update sarif file with just enough fields to pass radare tests"""
base_address = capa_result["meta"]["analysis"]["base_address"]["value"]
# Assume there is only one run, and one binary artifact
artifact = sarif_log["runs"][0]["artifacts"][0]
if "properties" not in artifact:
artifact["properties"] = {}
if "additionalProperties" not in artifact["properties"]:
artifact["properties"]["additionalProperties"] = {}
if "imageBase" not in artifact["properties"]["additionalProperties"]:
artifact["properties"]["additionalProperties"]["imageBase"] = base_address
if __name__ == "__main__":
sys.exit(main())

1
tests/test_scripts.py
View File

@@ -40,6 +40,7 @@ def get_rule_path():
[
pytest.param("capa2yara.py", [get_rules_path()]),
pytest.param("capafmt.py", [get_rule_path()]),
pytest.param("capa2sarif.py", [Path(__file__).resolve().parent / "data" / "rd" / "Practical Malware Analysis Lab 01-01.dll_.json"]),
# testing some variations of linter script
pytest.param("lint.py", ["-t", "create directory", get_rules_path()]),
# `create directory` rule has native and .NET example PEs