Compare commits

..

1 Commits

Author SHA1 Message Date
Willi Ballenthin
febe082ed4 init pages 2024-06-20 10:15:51 +00:00
247 changed files with 2204 additions and 19942 deletions

View File

@@ -1,6 +1,6 @@
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3/.devcontainer/base.Dockerfile
# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3-bullseye, 3.10-bullseye, 3-buster, 3.10-buster, etc.
# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
ARG VARIANT="3.10-bullseye"
FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}

View File

@@ -6,7 +6,7 @@
"dockerfile": "Dockerfile",
"context": "..",
"args": {
// Update 'VARIANT' to pick a Python version: 3, 3.10, etc.
// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
// Append -bullseye or -buster to pin to an OS version.
// Use -bullseye variants on local on arm64/Apple Silicon.
"VARIANT": "3.10",

View File

@@ -4,6 +4,3 @@ updates:
directory: "/"
schedule:
interval: "weekly"
ignore:
- dependency-name: "*"
update-types: ["version-update:semver-patch"]

View File

@@ -1,5 +1,8 @@
[mypy]
[mypy-tqdm.*]
ignore_missing_imports = True
[mypy-ruamel.*]
ignore_missing_imports = True

View File

@@ -2,6 +2,7 @@
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
import sys
import wcwidth
import capa.rules.cache
from pathlib import Path
@@ -28,6 +29,13 @@ a = Analysis(
("../../rules", "rules"),
("../../sigs", "sigs"),
("../../cache", "cache"),
# capa.render.default uses tabulate that depends on wcwidth.
# it seems wcwidth uses a json file `version.json`
# and this doesn't get picked up by pyinstaller automatically.
# so we manually embed the wcwidth resources here.
#
# ref: https://stackoverflow.com/a/62278462/87207
(Path(wcwidth.__file__).parent, "wcwidth"),
],
# when invoking pyinstaller from the project root,
# this gets run from the project root.
@@ -40,6 +48,11 @@ a = Analysis(
"tkinter",
"_tkinter",
"Tkinter",
# tqdm provides renderers for ipython,
# however, this drags in a lot of dependencies.
# since we don't spawn a notebook, we can safely remove these.
"IPython",
"ipywidgets",
# these are pulled in by networkx
# but we don't need to compute the strongly connected components.
"numpy",
@@ -57,10 +70,7 @@ a = Analysis(
"qt5",
"pyqtwebengine",
"pyasn1",
# don't pull in Binary Ninja/IDA bindings that should
# only be installed locally.
"binaryninja",
"ida",
],
)

8
.github/ruff.toml vendored
View File

@@ -1,16 +1,16 @@
# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
# McCabe complexity (`C901`) by default.
lint.select = ["E", "F"]
select = ["E", "F"]
# Allow autofix for all enabled rules (when `--fix`) is provided.
lint.fixable = ["ALL"]
lint.unfixable = []
fixable = ["ALL"]
unfixable = []
# E402 module level import not at top of file
# E722 do not use bare 'except'
# E501 line too long
lint.ignore = ["E402", "E722", "E501"]
ignore = ["E402", "E722", "E501"]
line-length = 120

View File

@@ -3,10 +3,6 @@ name: build
on:
pull_request:
branches: [ master ]
paths-ignore:
- 'web/**'
- 'doc/**'
- '**.md'
release:
types: [edited, published]
@@ -21,25 +17,26 @@ jobs:
# set to false for debugging
fail-fast: true
matrix:
# using Python 3.8 to support running across multiple operating systems including Windows 7
include:
- os: ubuntu-20.04
# use old linux so that the shared library versioning is more portable
artifact_name: capa
asset_name: linux
python_version: '3.10'
python_version: 3.8
- os: ubuntu-20.04
artifact_name: capa
asset_name: linux-py312
python_version: '3.12'
asset_name: linux-py311
python_version: 3.11
- os: windows-2019
artifact_name: capa.exe
asset_name: windows
python_version: '3.10'
- os: macos-13
python_version: 3.8
- os: macos-11
# use older macOS for assumed better portability
artifact_name: capa
asset_name: macos
python_version: '3.10'
python_version: 3.8
steps:
- name: Checkout capa
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -87,7 +84,7 @@ jobs:
asset_name: linux
- os: ubuntu-22.04
artifact_name: capa
asset_name: linux-py312
asset_name: linux-py311
- os: windows-2022
artifact_name: capa.exe
asset_name: windows
@@ -106,14 +103,14 @@ jobs:
# upload zipped binaries to Release page
if: github.event_name == 'release'
name: zip and upload ${{ matrix.asset_name }}
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
needs: [build]
strategy:
matrix:
include:
- asset_name: linux
artifact_name: capa
- asset_name: linux-py312
- asset_name: linux-py311
artifact_name: capa
- asset_name: windows
artifact_name: capa.exe

View File

@@ -13,11 +13,8 @@ permissions:
jobs:
check_changelog:
# no need to check for dependency updates via dependabot
# github.event.pull_request.user.login refers to PR author
if: |
github.event.pull_request.user.login != 'dependabot[bot]' &&
github.event.pull_request.user.login != 'dependabot-preview[bot]'
runs-on: ubuntu-latest
if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]'
runs-on: ubuntu-20.04
env:
NO_CHANGELOG: '[x] No CHANGELOG update needed'
steps:

View File

@@ -21,7 +21,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: '3.10'
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip

View File

@@ -9,7 +9,7 @@ permissions: read-all
jobs:
tag:
name: Tag capa rules
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- name: Checkout capa-rules
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

View File

@@ -1,22 +1,10 @@
name: CI
# tests.yml workflow will run for all changes except:
# any file or directory under web/ or doc/
# any Markdown (.md) file anywhere in the repository
on:
push:
branches: [ master ]
paths-ignore:
- 'web/**'
- 'doc/**'
- '**.md'
pull_request:
branches: [ master ]
paths-ignore:
- 'web/**'
- 'doc/**'
- '**.md'
permissions: read-all
@@ -26,7 +14,7 @@ env:
jobs:
changelog_format:
runs-on: ubuntu-22.04
runs-on: ubuntu-20.04
steps:
- name: Checkout capa
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -37,15 +25,15 @@ jobs:
if [ $number != 1 ]; then exit 1; fi
code_style:
runs-on: ubuntu-22.04
runs-on: ubuntu-20.04
steps:
- name: Checkout capa
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
# use latest available python to take advantage of best performance
- name: Set up Python 3.12
- name: Set up Python 3.11
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: "3.12"
python-version: "3.11"
- name: Install dependencies
run: |
pip install -r requirements.txt
@@ -64,16 +52,16 @@ jobs:
run: pre-commit run deptry --hook-stage manual
rule_linter:
runs-on: ubuntu-22.04
runs-on: ubuntu-20.04
steps:
- name: Checkout capa with submodules
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
submodules: recursive
- name: Set up Python 3.12
- name: Set up Python 3.11
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: "3.12"
python-version: "3.11"
- name: Install capa
run: |
pip install -r requirements.txt
@@ -88,17 +76,17 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, windows-2019, macos-13]
os: [ubuntu-20.04, windows-2019, macos-11]
# across all operating systems
python-version: ["3.10", "3.11"]
python-version: ["3.8", "3.11"]
include:
# on Ubuntu run these as well
- os: ubuntu-20.04
python-version: "3.8"
- os: ubuntu-20.04
python-version: "3.9"
- os: ubuntu-20.04
python-version: "3.10"
- os: ubuntu-20.04
python-version: "3.11"
- os: ubuntu-20.04
python-version: "3.12"
steps:
- name: Checkout capa with submodules
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -131,7 +119,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11"]
python-version: ["3.8", "3.11"]
steps:
- name: Checkout capa with submodules
# do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
@@ -173,7 +161,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11"]
python-version: ["3.8", "3.11"]
java-version: ["17"]
ghidra-version: ["11.0.1"]
public-version: ["PUBLIC_20240130"] # for ghidra releases

View File

@@ -1,134 +0,0 @@
name: deploy web to GitHub Pages
on:
push:
branches: [ master ]
paths:
- 'web/**'
# Allows to run this workflow manually from the Actions tab
workflow_dispatch:
# Sets the GITHUB_TOKEN permissions to allow deployment to GitHub Pages
permissions:
contents: read
pages: write
id-token: write
# Allow one concurrent deployment
concurrency:
group: 'pages'
cancel-in-progress: true
jobs:
build-landing-page:
name: Build landing page
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/upload-artifact@v4
with:
name: landing-page
path: './web/public'
build-explorer:
name: Build capa Explorer Web
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 1
show-progress: true
- name: Set up Node
uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
with:
node-version: 20
cache: 'npm'
cache-dependency-path: './web/explorer/package-lock.json'
- name: Install dependencies
run: npm ci
working-directory: ./web/explorer
- name: Generate release bundle
run: npm run build:bundle
working-directory: ./web/explorer
- name: Zip release bundle
run: zip -r public/capa-explorer-web.zip capa-explorer-web
working-directory: ./web/explorer
- name: Build
run: npm run build
working-directory: ./web/explorer
- uses: actions/upload-artifact@v4
with:
name: explorer
path: './web/explorer/dist'
build-rules:
name: Build rules site
runs-on: ubuntu-latest
steps:
- name: Check out the repository
uses: actions/checkout@v4
with:
submodules: 'recursive'
# full depth so that capa-rules has a full history
# and we can construct a timeline of rule updates.
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: '3.12'
- uses: extractions/setup-just@v2
- name: Install pagefind
uses: supplypike/setup-bin@v4
with:
uri: "https://github.com/CloudCannon/pagefind/releases/download/v1.1.0/pagefind-v1.1.0-x86_64-unknown-linux-musl.tar.gz"
name: "pagefind"
version: "1.1.0"
- name: Install dependencies
working-directory: ./web/rules
run: pip install -r requirements.txt
- name: Build the website
working-directory: ./web/rules
run: just build
- name: Index the website
working-directory: ./web/rules
run: pagefind --site "public"
# upload the build website to artifacts
# so that we can download and inspect, if desired.
- uses: actions/upload-artifact@v4
with:
name: rules
path: './web/rules/public'
deploy:
name: Deploy site to GitHub Pages
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
runs-on: ubuntu-latest
needs: [build-landing-page, build-explorer, build-rules]
steps:
- uses: actions/download-artifact@v4
with:
name: landing-page
path: './public/'
- uses: actions/download-artifact@v4
with:
name: explorer
path: './public/explorer'
- uses: actions/download-artifact@v4
with:
name: rules
path: './public/rules'
- name: Setup Pages
uses: actions/configure-pages@v4
- name: Upload artifact
uses: actions/upload-pages-artifact@v3
with:
path: './public'
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4

View File

@@ -1,103 +0,0 @@
name: create web release
on:
workflow_dispatch:
inputs:
version:
description: 'Version number for the release (x.x.x)'
required: true
type: string
jobs:
run-tests:
uses: ./.github/workflows/web-tests.yml
build-and-release:
needs: run-tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set release name
run: echo "RELEASE_NAME=capa-explorer-web-v${{ github.event.inputs.version }}-${GITHUB_SHA::7}" >> $GITHUB_ENV
- name: Check if release already exists
run: |
if ls web/explorer/releases/capa-explorer-web-v${{ github.event.inputs.version }}-* 1> /dev/null 2>&1; then
echo "::error:: A release with version ${{ github.event.inputs.version }} already exists"
exit 1
fi
- name: Set up Node.js
uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
with:
node-version: 20
cache: 'npm'
cache-dependency-path: 'web/explorer/package-lock.json'
- name: Install dependencies
run: npm ci
working-directory: web/explorer
- name: Build offline bundle
run: npm run build:bundle
working-directory: web/explorer
- name: Compress bundle
run: zip -r ${{ env.RELEASE_NAME }}.zip capa-explorer-web
working-directory: web/explorer
- name: Create releases directory
run: mkdir -vp web/explorer/releases
- name: Move release to releases folder
run: mv web/explorer/${{ env.RELEASE_NAME }}.zip web/explorer/releases
- name: Compute release SHA256 hash
run: |
echo "RELEASE_SHA256=$(sha256sum web/explorer/releases/${{ env.RELEASE_NAME }}.zip | awk '{print $1}')" >> $GITHUB_ENV
- name: Update CHANGELOG.md
run: |
echo "## ${{ env.RELEASE_NAME }}" >> web/explorer/releases/CHANGELOG.md
echo "- Release Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> web/explorer/releases/CHANGELOG.md
echo "- SHA256: ${{ env.RELEASE_SHA256 }}" >> web/explorer/releases/CHANGELOG.md
echo "" >> web/explorer/releases/CHANGELOG.md
cat web/explorer/releases/CHANGELOG.md
- name: Remove older releases
# keep only the latest 3 releases
run: ls -t capa-explorer-web-v*.zip | tail -n +4 | xargs -r rm --
working-directory: web/explorer/releases
- name: Stage release files
run: |
git config --local user.email "capa-dev@mandiant.com"
git config --local user.name "Capa Bot"
git add -f web/explorer/releases/${{ env.RELEASE_NAME }}.zip web/explorer/releases/CHANGELOG.md
git add -u web/explorer/releases/
- name: Create Pull Request
uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
with:
token: ${{ secrets.GITHUB_TOKEN }}
title: "explorer web: add release v${{ github.event.inputs.version }}"
body: |
This PR adds a new capa Explorer Web release v${{ github.event.inputs.version }}.
Release details:
- Name: ${{ env.RELEASE_NAME }}
- SHA256: ${{ env.RELEASE_SHA256 }}
This release is generated by the [web release](https://github.com/mandiant/capa/actions/workflows/web-release.yml) workflow.
- [x] No CHANGELOG update needed
- [x] No new tests needed
- [x] No documentation update needed
commit-message: ":robot: explorer web: add release ${{ env.RELEASE_NAME }}"
branch: release/web-v${{ github.event.inputs.version }}
add-paths: web/explorer/releases/${{ env.RELEASE_NAME }}.zip
base: master
labels: webui
delete-branch: true
committer: Capa Bot <capa-dev@mandiant.com>
author: Capa Bot <capa-dev@mandiant.com>

View File

@@ -1,43 +0,0 @@
name: capa Explorer Web tests
on:
pull_request:
branches: [ master ]
paths:
- 'web/explorer/**'
workflow_call: # this allows the workflow to be called by other workflows
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 1
show-progress: true
- name: Set up Node
uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
with:
node-version: 20
cache: 'npm'
cache-dependency-path: 'web/explorer/package-lock.json'
- name: Install dependencies
run: npm ci
working-directory: web/explorer
- name: Lint
run: npm run lint
working-directory: web/explorer
- name: Format
run: npm run format:check
working-directory: web/explorer
- name: Run unit tests
run: npm run test
working-directory: web/explorer

2
.gitignore vendored
View File

@@ -126,5 +126,3 @@ Pipfile.lock
.github/binja/binaryninja
.github/binja/download_headless.py
.github/binja/BinaryNinja-headless.zip
justfile
data/

4
.gitmodules vendored
View File

@@ -1,6 +1,6 @@
[submodule "rules"]
path = rules
url = ../../mandiant/capa-rules.git
url = ../capa-rules.git
[submodule "tests/data"]
path = tests/data
url = ../../mandiant/capa-testfiles.git
url = ../capa-testfiles.git

View File

@@ -1,25 +0,0 @@
@isort:
pre-commit run isort --show-diff-on-failure --all-files
@black:
pre-commit run black --show-diff-on-failure --all-files
@ruff:
pre-commit run ruff --all-files
@flake8:
pre-commit run flake8 --hook-stage manual --all-files
@mypy:
pre-commit run mypy --hook-stage manual --all-files
@deptry:
pre-commit run deptry --hook-stage manual --all-files
@lint:
-just isort
-just black
-just ruff
-just flake8
-just mypy
-just deptry

View File

@@ -25,7 +25,7 @@ repos:
hooks:
- id: isort
name: isort
stages: [pre-commit, pre-push, manual]
stages: [commit, push, manual]
language: system
entry: isort
args:
@@ -38,7 +38,6 @@ repos:
- "capa/"
- "scripts/"
- "tests/"
- "web/rules/scripts/"
always_run: true
pass_filenames: false
@@ -46,7 +45,7 @@ repos:
hooks:
- id: black
name: black
stages: [pre-commit, pre-push, manual]
stages: [commit, push, manual]
language: system
entry: black
args:
@@ -56,7 +55,6 @@ repos:
- "capa/"
- "scripts/"
- "tests/"
- "web/rules/scripts/"
always_run: true
pass_filenames: false
@@ -64,7 +62,7 @@ repos:
hooks:
- id: ruff
name: ruff
stages: [pre-commit, pre-push, manual]
stages: [commit, push, manual]
language: system
entry: ruff
args:
@@ -74,7 +72,6 @@ repos:
- "capa/"
- "scripts/"
- "tests/"
- "web/rules/scripts/"
always_run: true
pass_filenames: false
@@ -82,18 +79,17 @@ repos:
hooks:
- id: flake8
name: flake8
stages: [pre-push, manual]
stages: [push, manual]
language: system
entry: flake8
args:
- "--config"
- ".github/flake8.ini"
- "--extend-exclude"
- "capa/render/proto/capa_pb2.py,capa/features/extractors/binexport2/binexport2_pb2.py"
- "capa/render/proto/capa_pb2.py"
- "capa/"
- "scripts/"
- "tests/"
- "web/rules/scripts/"
always_run: true
pass_filenames: false
@@ -101,7 +97,7 @@ repos:
hooks:
- id: mypy
name: mypy
stages: [pre-push, manual]
stages: [push, manual]
language: system
entry: mypy
args:
@@ -111,7 +107,6 @@ repos:
- "capa/"
- "scripts/"
- "tests/"
- "web/rules/scripts/"
always_run: true
pass_filenames: false
@@ -119,7 +114,7 @@ repos:
hooks:
- id: deptry
name: deptry
stages: [pre-push, manual]
stages: [push, manual]
language: system
entry: deptry .
always_run: true

File diff suppressed because it is too large Load Diff

View File

@@ -1,16 +1,4 @@
<br />
<div align="center">
<a href="https://mandiant.github.io/capa/" target="_blank">
<img src="https://github.com/mandiant/capa/blob/master/.github/logo.png">
</a>
<p align="center">
<a href="https://mandiant.github.io/capa/" target="_blank">Website</a>
|
<a href="https://github.com/mandiant/capa/releases/latest" target="_blank">Download</a>
|
<a href="https://mandiant.github.io/capa/explorer/" target="_blank">Web Interface</a>
</p>
<div align="center">
![capa](https://github.com/mandiant/capa/blob/master/.github/logo.png)
[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
[![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
@@ -19,22 +7,17 @@
[![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
[![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
</div>
</div>
---
capa detects capabilities in executable files.
You run it against a PE, ELF, .NET module, shellcode file, or a sandbox report and it tells you what it thinks the program can do.
For example, it might suggest that the file is a backdoor, is capable of installing services, or relies on HTTP to communicate.
To interactively inspect capa results in your browser use the [capa Explorer Web](https://mandiant.github.io/capa/explorer/).
Check out our capa blog posts:
- [Dynamic capa: Exploring Executable Run-Time Behavior with the CAPE Sandbox](https://www.mandiant.com/resources/blog/dynamic-capa-executable-behavior-cape-sandbox)
- [capa v4: casting a wider .NET](https://www.mandiant.com/resources/blog/capa-v4-casting-wider-net) (.NET support)
- [ELFant in the Room capa v3](https://www.mandiant.com/resources/elfant-in-the-room-capa-v3) (ELF support)
- [capa 2.0: Better, Stronger, Faster](https://www.mandiant.com/resources/capa-2-better-stronger-faster)
- [capa: Automatically Identify Malware Capabilities](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities)
If you want to inspect or write capa rules, head on over to the [capa-rules repository](https://github.com/mandiant/capa-rules). Otherwise, keep reading.
Below you find a list of [our capa blog posts with more details.](#blog-posts)
# example capa output
```
$ capa.exe suspicious.exe
@@ -89,23 +72,16 @@ Download stable releases of the standalone capa binaries [here](https://github.c
To use capa as a library or integrate with another tool, see [doc/installation.md](https://github.com/mandiant/capa/blob/master/doc/installation.md) for further setup instructions.
# capa Explorer Web
The [capa Explorer Web](https://mandiant.github.io/capa/explorer/) enables you to interactively explore capa results in your web browser. Besides the online version you can download a standalone HTML file for local offline usage.
![capa Explorer Web screenshot](https://github.com/mandiant/capa/blob/master/doc/img/capa_web_explorer.png)
More details on the web UI is available in the [capa Explorer Web README](https://github.com/mandiant/capa/blob/master/web/explorer/README.md).
For more information about how to use capa, see [doc/usage.md](https://github.com/mandiant/capa/blob/master/doc/usage.md).
# example
In the above sample output, we run capa against an unknown binary (`suspicious.exe`),
and the tool reports that the program can send HTTP requests, decode data via XOR and Base64,
In the above sample output, we ran capa against an unknown binary (`suspicious.exe`),
and the tool reported that the program can send HTTP requests, decode data via XOR and Base64,
install services, and spawn new processes.
Taken together, this makes us think that `suspicious.exe` could be a persistent backdoor.
Therefore, our next analysis step might be to run `suspicious.exe` in a sandbox and try to recover the command and control server.
## detailed results
By passing the `-vv` flag (for very verbose), capa reports exactly where it found evidence of these capabilities.
This is useful for at least two reasons:
@@ -150,15 +126,10 @@ function @ 0x4011C0
...
```
capa also supports dynamic capabilities detection for multiple sandboxes including:
* [CAPE](https://github.com/kevoreilly/CAPEv2) (supported report formats: `.json`, `.json_`, `.json.gz`)
* [DRAKVUF](https://github.com/CERT-Polska/drakvuf-sandbox/) (supported report formats: `.log`, `.log.gz`)
* [VMRay](https://www.vmray.com/) (supported report formats: analysis archive `.zip`)
Additionally, capa also supports analyzing [CAPE](https://github.com/kevoreilly/CAPEv2) sandbox reports for dynamic capability extraction.
In order to use this, you first submit your sample to CAPE for analysis, and then run capa against the generated report (JSON).
To use this feature, submit your file to a supported sandbox and then download and run capa against the generated report file. This feature enables capa to match capabilities against dynamic and static features that the sandbox captured during execution.
Here's an example of running capa against a packed file, and then running capa against the CAPE report generated for the same packed file:
Here's an example of running capa against a packed binary, and then running capa against the CAPE report of that binary:
```yaml
$ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.exe
@@ -245,7 +216,6 @@ $ capa 05be49819139a3fdcdbddbdefd298398779521f3d68daa25275cc77508e42310.json
┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙
```
# capa rules
capa uses a collection of rules to identify capabilities within a program.
These rules are easy to write, even for those new to reverse engineering.
By authoring rules, you can extend the capabilities that capa recognizes.
@@ -282,28 +252,18 @@ rule:
- property/read: System.Net.Sockets.TcpClient::Client
```
The [github.com/mandiant/capa-rules](https://github.com/mandiant/capa-rules) repository contains hundreds of standard rules that are distributed with capa.
The [github.com/mandiant/capa-rules](https://github.com/mandiant/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
Please learn to write rules and contribute new entries as you find interesting techniques in malware.
# IDA Pro plugin: capa explorer
If you use IDA Pro, then you can use the [capa explorer](https://github.com/mandiant/capa/tree/master/capa/ida/plugin) plugin.
capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted directly from your IDA Pro database.
It also uses your local changes to the .idb to extract better features, such as when you rename a global variable that contains a dynamically resolved API address.
![capa + IDA Pro integration](https://github.com/mandiant/capa/blob/master/doc/img/explorer_expanded.png)
# Ghidra integration
If you use Ghidra, then you can use the [capa + Ghidra integration](/capa/ghidra/) to run capa's analysis directly on your Ghidra database and render the results in Ghidra's user interface.
<img src="https://github.com/mandiant/capa/assets/66766340/eeae33f4-99d4-42dc-a5e8-4c1b8c661492" width=300>
# blog posts
- [Dynamic capa: Exploring Executable Run-Time Behavior with the CAPE Sandbox](https://www.mandiant.com/resources/blog/dynamic-capa-executable-behavior-cape-sandbox)
- [capa v4: casting a wider .NET](https://www.mandiant.com/resources/blog/capa-v4-casting-wider-net) (.NET support)
- [ELFant in the Room capa v3](https://www.mandiant.com/resources/elfant-in-the-room-capa-v3) (ELF support)
- [capa 2.0: Better, Stronger, Faster](https://www.mandiant.com/resources/capa-2-better-stronger-faster)
- [capa: Automatically Identify Malware Capabilities](https://www.mandiant.com/resources/capa-automatically-identify-malware-capabilities)
# further information
## capa
- [Installation](https://github.com/mandiant/capa/blob/master/doc/installation.md)

View File

@@ -9,7 +9,7 @@
import logging
import itertools
import collections
from typing import Any
from typing import Any, Tuple
from capa.rules import Scope, RuleSet
from capa.engine import FeatureSet, MatchResults
@@ -64,7 +64,7 @@ def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalon
def find_capabilities(
ruleset: RuleSet, extractor: FeatureExtractor, disable_progress=None, **kwargs
) -> tuple[MatchResults, Any]:
) -> Tuple[MatchResults, Any]:
from capa.capabilities.static import find_static_capabilities
from capa.capabilities.dynamic import find_dynamic_capabilities

View File

@@ -6,16 +6,20 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import sys
import logging
import itertools
import collections
from typing import Any
from typing import Any, Tuple
import tqdm
import capa.perf
import capa.features.freeze as frz
import capa.render.result_document as rdoc
from capa.rules import Scope, RuleSet
from capa.engine import FeatureSet, MatchResults
from capa.helpers import redirecting_print_to_tqdm
from capa.capabilities.common import find_file_capabilities
from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle, DynamicFeatureExtractor
@@ -24,7 +28,7 @@ logger = logging.getLogger(__name__)
def find_call_capabilities(
ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
) -> tuple[FeatureSet, MatchResults]:
) -> Tuple[FeatureSet, MatchResults]:
"""
find matches for the given rules for the given call.
@@ -51,7 +55,7 @@ def find_call_capabilities(
def find_thread_capabilities(
ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle
) -> tuple[FeatureSet, MatchResults, MatchResults]:
) -> Tuple[FeatureSet, MatchResults, MatchResults]:
"""
find matches for the given rules within the given thread.
@@ -89,7 +93,7 @@ def find_thread_capabilities(
def find_process_capabilities(
ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle
) -> tuple[MatchResults, MatchResults, MatchResults, int]:
) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
"""
find matches for the given rules within the given process.
@@ -127,7 +131,7 @@ def find_process_capabilities(
def find_dynamic_capabilities(
ruleset: RuleSet, extractor: DynamicFeatureExtractor, disable_progress=None
) -> tuple[MatchResults, Any]:
) -> Tuple[MatchResults, Any]:
all_process_matches: MatchResults = collections.defaultdict(list)
all_thread_matches: MatchResults = collections.defaultdict(list)
all_call_matches: MatchResults = collections.defaultdict(list)
@@ -135,30 +139,38 @@ def find_dynamic_capabilities(
feature_counts = rdoc.DynamicFeatureCounts(file=0, processes=())
assert isinstance(extractor, DynamicFeatureExtractor)
processes: list[ProcessHandle] = list(extractor.get_processes())
n_processes: int = len(processes)
with redirecting_print_to_tqdm(disable_progress):
with tqdm.contrib.logging.logging_redirect_tqdm():
pbar = tqdm.tqdm
if disable_progress:
# do not use tqdm to avoid unnecessary side effects when caller intends
# to disable progress completely
def pbar(s, *args, **kwargs):
return s
with capa.helpers.CapaProgressBar(
console=capa.helpers.log_console, transient=True, disable=disable_progress
) as pbar:
task = pbar.add_task("matching", total=n_processes, unit="processes")
for p in processes:
process_matches, thread_matches, call_matches, feature_count = find_process_capabilities(
ruleset, extractor, p
)
feature_counts.processes += (
rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count),
)
logger.debug("analyzed %s and extracted %d features", p.address, feature_count)
elif not sys.stderr.isatty():
# don't display progress bar when stderr is redirected to a file
def pbar(s, *args, **kwargs):
return s
for rule_name, res in process_matches.items():
all_process_matches[rule_name].extend(res)
for rule_name, res in thread_matches.items():
all_thread_matches[rule_name].extend(res)
for rule_name, res in call_matches.items():
all_call_matches[rule_name].extend(res)
processes = list(extractor.get_processes())
pbar.advance(task)
pb = pbar(processes, desc="matching", unit=" processes", leave=False)
for p in pb:
process_matches, thread_matches, call_matches, feature_count = find_process_capabilities(
ruleset, extractor, p
)
feature_counts.processes += (
rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count),
)
logger.debug("analyzed %s and extracted %d features", p.address, feature_count)
for rule_name, res in process_matches.items():
all_process_matches[rule_name].extend(res)
for rule_name, res in thread_matches.items():
all_thread_matches[rule_name].extend(res)
for rule_name, res in call_matches.items():
all_call_matches[rule_name].extend(res)
# collection of features that captures the rule matches within process and thread scopes.
# mapping from feature (matched rule) to set of addresses at which it matched.

View File

@@ -6,18 +6,21 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import sys
import time
import logging
import itertools
import collections
from typing import Any
from typing import Any, Tuple
import tqdm.contrib.logging
import capa.perf
import capa.helpers
import capa.features.freeze as frz
import capa.render.result_document as rdoc
from capa.rules import Scope, RuleSet
from capa.engine import FeatureSet, MatchResults
from capa.helpers import redirecting_print_to_tqdm
from capa.capabilities.common import find_file_capabilities
from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, StaticFeatureExtractor
@@ -26,7 +29,7 @@ logger = logging.getLogger(__name__)
def find_instruction_capabilities(
ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
) -> tuple[FeatureSet, MatchResults]:
) -> Tuple[FeatureSet, MatchResults]:
"""
find matches for the given rules for the given instruction.
@@ -53,7 +56,7 @@ def find_instruction_capabilities(
def find_basic_block_capabilities(
ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle
) -> tuple[FeatureSet, MatchResults, MatchResults]:
) -> Tuple[FeatureSet, MatchResults, MatchResults]:
"""
find matches for the given rules within the given basic block.
@@ -93,7 +96,7 @@ def find_basic_block_capabilities(
def find_code_capabilities(
ruleset: RuleSet, extractor: StaticFeatureExtractor, fh: FunctionHandle
) -> tuple[MatchResults, MatchResults, MatchResults, int]:
) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
"""
find matches for the given rules within the given function.
@@ -131,67 +134,84 @@ def find_code_capabilities(
def find_static_capabilities(
ruleset: RuleSet, extractor: StaticFeatureExtractor, disable_progress=None
) -> tuple[MatchResults, Any]:
) -> Tuple[MatchResults, Any]:
all_function_matches: MatchResults = collections.defaultdict(list)
all_bb_matches: MatchResults = collections.defaultdict(list)
all_insn_matches: MatchResults = collections.defaultdict(list)
feature_counts = rdoc.StaticFeatureCounts(file=0, functions=())
library_functions: tuple[rdoc.LibraryFunction, ...] = ()
library_functions: Tuple[rdoc.LibraryFunction, ...] = ()
assert isinstance(extractor, StaticFeatureExtractor)
functions: list[FunctionHandle] = list(extractor.get_functions())
n_funcs: int = len(functions)
n_libs: int = 0
percentage: float = 0
with redirecting_print_to_tqdm(disable_progress):
with tqdm.contrib.logging.logging_redirect_tqdm():
pbar = tqdm.tqdm
if capa.helpers.is_runtime_ghidra():
# Ghidrathon interpreter cannot properly handle
# the TMonitor thread that is created via a monitor_interval
# > 0
pbar.monitor_interval = 0
if disable_progress:
# do not use tqdm to avoid unnecessary side effects when caller intends
# to disable progress completely
def pbar(s, *args, **kwargs):
return s
with capa.helpers.CapaProgressBar(
console=capa.helpers.log_console, transient=True, disable=disable_progress
) as pbar:
task = pbar.add_task(
"matching", total=n_funcs, unit="functions", postfix=f"skipped {n_libs} library functions, {percentage}%"
)
for f in functions:
t0 = time.time()
if extractor.is_library_function(f.address):
function_name = extractor.get_function_name(f.address)
logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
library_functions += (
rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name),
elif not sys.stderr.isatty():
# don't display progress bar when stderr is redirected to a file
def pbar(s, *args, **kwargs):
return s
functions = list(extractor.get_functions())
n_funcs = len(functions)
pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions", leave=False)
for f in pb:
t0 = time.time()
if extractor.is_library_function(f.address):
function_name = extractor.get_function_name(f.address)
logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
library_functions += (
rdoc.LibraryFunction(address=frz.Address.from_capa(f.address), name=function_name),
)
n_libs = len(library_functions)
percentage = round(100 * (n_libs / n_funcs))
if isinstance(pb, tqdm.tqdm):
pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
continue
function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(
ruleset, extractor, f
)
n_libs = len(library_functions)
percentage = round(100 * (n_libs / n_funcs))
pbar.update(task, postfix=f"skipped {n_libs} library functions, {percentage}%")
pbar.advance(task)
continue
feature_counts.functions += (
rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count),
)
t1 = time.time()
function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(ruleset, extractor, f)
feature_counts.functions += (
rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count),
)
t1 = time.time()
match_count = 0
for name, matches_ in itertools.chain(
function_matches.items(), bb_matches.items(), insn_matches.items()
):
# in practice, most matches are derived rules,
# like "check OS version/5bf4c7f39fd4492cbed0f6dc7d596d49"
# but when we log to the human, they really care about "real" rules.
if not ruleset.rules[name].is_subscope_rule():
match_count += len(matches_)
match_count = 0
for name, matches_ in itertools.chain(function_matches.items(), bb_matches.items(), insn_matches.items()):
if not ruleset.rules[name].is_subscope_rule():
match_count += len(matches_)
logger.debug(
"analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
f.address,
feature_count,
match_count,
t1 - t0,
)
logger.debug(
"analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
f.address,
feature_count,
match_count,
t1 - t0,
)
for rule_name, res in function_matches.items():
all_function_matches[rule_name].extend(res)
for rule_name, res in bb_matches.items():
all_bb_matches[rule_name].extend(res)
for rule_name, res in insn_matches.items():
all_insn_matches[rule_name].extend(res)
pbar.advance(task)
for rule_name, res in function_matches.items():
all_function_matches[rule_name].extend(res)
for rule_name, res in bb_matches.items():
all_bb_matches[rule_name].extend(res)
for rule_name, res in insn_matches.items():
all_insn_matches[rule_name].extend(res)
# collection of features that captures the rule matches within function, BB, and instruction scopes.
# mapping from feature (matched rule) to set of addresses at which it matched.

View File

@@ -8,7 +8,7 @@
import copy
import collections
from typing import TYPE_CHECKING, Union, Mapping, Iterable, Iterator
from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator
import capa.perf
import capa.features.common
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
# to collect the locations of a feature, do: `features[Number(0x10)]`
#
# aliased here so that the type can be documented and xref'd.
FeatureSet = dict[Feature, set[Address]]
FeatureSet = Dict[Feature, Set[Address]]
class Statement:
@@ -94,7 +94,7 @@ class And(Statement):
match if all of the children evaluate to True.
the order of evaluation is dictated by the property
`And.children` (type: list[Statement|Feature]).
`And.children` (type: List[Statement|Feature]).
a query optimizer may safely manipulate the order of these children.
"""
@@ -127,7 +127,7 @@ class Or(Statement):
match if any of the children evaluate to True.
the order of evaluation is dictated by the property
`Or.children` (type: list[Statement|Feature]).
`Or.children` (type: List[Statement|Feature]).
a query optimizer may safely manipulate the order of these children.
"""
@@ -176,7 +176,7 @@ class Some(Statement):
match if at least N of the children evaluate to True.
the order of evaluation is dictated by the property
`Some.children` (type: list[Statement|Feature]).
`Some.children` (type: List[Statement|Feature]).
a query optimizer may safely manipulate the order of these children.
"""
@@ -267,7 +267,7 @@ class Subscope(Statement):
# inspect(match_details)
#
# aliased here so that the type can be documented and xref'd.
MatchResults = Mapping[str, list[tuple[Address, Result]]]
MatchResults = Mapping[str, List[Tuple[Address, Result]]]
def get_rule_namespaces(rule: "capa.rules.Rule") -> Iterator[str]:
@@ -292,7 +292,7 @@ def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations:
features[capa.features.common.MatchedRule(namespace)].update(locations)
def match(rules: list["capa.rules.Rule"], features: FeatureSet, addr: Address) -> tuple[FeatureSet, MatchResults]:
def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -> Tuple[FeatureSet, MatchResults]:
"""
match the given rules against the given features,
returning an updated set of features and the matches.

View File

@@ -23,15 +23,3 @@ class UnsupportedOSError(ValueError):
class EmptyReportError(ValueError):
pass
class InvalidArgument(ValueError):
pass
class NonExistantFunctionError(ValueError):
pass
class NonExistantProcessError(ValueError):
pass

View File

@@ -6,6 +6,7 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from enum import Enum
from typing import Dict, List
from capa.helpers import assert_never
@@ -21,7 +22,7 @@ COM_PREFIXES = {
}
def load_com_database(com_type: ComType) -> dict[str, list[str]]:
def load_com_database(com_type: ComType) -> Dict[str, List[str]]:
# lazy load these python files since they are so large.
# that is, don't load them unless a COM feature is being handled.
import capa.features.com.classes

View File

@@ -5,8 +5,9 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Dict, List
COM_CLASSES: dict[str, list[str]] = {
COM_CLASSES: Dict[str, List[str]] = {
"ClusAppWiz": ["24F97150-6689-11D1-9AA7-00C04FB93A80"],
"ClusCfgAddNodesWizard": ["BB8D141E-C00A-469F-BC5C-ECD814F0BD74"],
"ClusCfgCreateClusterWizard": ["B929818E-F5B0-44DC-8A00-1B5F5F5AA1F0"],

View File

@@ -5,8 +5,9 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Dict, List
COM_INTERFACES: dict[str, list[str]] = {
COM_INTERFACES: Dict[str, List[str]] = {
"IClusterApplicationWizard": ["24F97151-6689-11D1-9AA7-00C04FB93A80"],
"IWEExtendWizard97": ["97DEDE68-FC6B-11CF-B5F5-00A0C90AB505"],
"IWCWizard97Callback": ["97DEDE67-FC6B-11CF-B5F5-00A0C90AB505"],
@@ -16333,7 +16334,7 @@ COM_INTERFACES: dict[str, list[str]] = {
"IRcsServiceDescription": ["416437de-e78b-44c9-990f-7ede1f2a0c91"],
"IRcsServiceKindSupportedChangedEventArgs": ["f47ea244-e783-4866-b3a7-4e5ccf023070"],
"IRcsServiceStatusChangedArgs": ["661ae45a-412a-460d-bdd4-dd8ea3c15583"],
"IRcsServicetuple": ["ce17a39b-2e8b-41af-b5a9-5cb072cc373c"],
"IRcsServiceTuple": ["ce17a39b-2e8b-41af-b5a9-5cb072cc373c"],
"IRcsSubscriptionReceivedArgs": ["04eaf06d-42bc-46cc-a637-eeb3a8723fe4"],
"IRcsTransport": ["fea34759-f37c-4319-8546-ec84d21d30ff"],
"IRcsTransportConfiguration": ["1fccb102-2472-4bb9-9988-c1211c83e8a9"],

View File

@@ -9,9 +9,10 @@
import re
import abc
import codecs
import typing
import logging
import collections
from typing import TYPE_CHECKING, Union, Optional
from typing import TYPE_CHECKING, Set, Dict, List, Union, Optional
if TYPE_CHECKING:
# circular import, otherwise
@@ -78,8 +79,8 @@ class Result:
self,
success: bool,
statement: Union["capa.engine.Statement", "Feature"],
children: list["Result"],
locations: Optional[set[Address]] = None,
children: List["Result"],
locations: Optional[Set[Address]] = None,
):
super().__init__()
self.success = success
@@ -212,7 +213,7 @@ class Substring(String):
# mapping from string value to list of locations.
# will unique the locations later on.
matches: collections.defaultdict[str, set[Address]] = collections.defaultdict(set)
matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
assert isinstance(self.value, str)
for feature, locations in features.items():
@@ -260,7 +261,7 @@ class _MatchedSubstring(Substring):
note: this type should only ever be constructed by `Substring.evaluate()`. it is not part of the public API.
"""
def __init__(self, substring: Substring, matches: dict[str, set[Address]]):
def __init__(self, substring: Substring, matches: Dict[str, Set[Address]]):
"""
args:
substring: the substring feature that matches.
@@ -304,7 +305,7 @@ class Regex(String):
# mapping from string value to list of locations.
# will unique the locations later on.
matches: collections.defaultdict[str, set[Address]] = collections.defaultdict(set)
matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
for feature, locations in features.items():
if not isinstance(feature, (String,)):
@@ -352,7 +353,7 @@ class _MatchedRegex(Regex):
note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
"""
def __init__(self, regex: Regex, matches: dict[str, set[Address]]):
def __init__(self, regex: Regex, matches: Dict[str, Set[Address]]):
"""
args:
regex: the regex feature that matches.
@@ -423,11 +424,10 @@ class Arch(Feature):
OS_WINDOWS = "windows"
OS_LINUX = "linux"
OS_MACOS = "macos"
OS_ANDROID = "android"
# dotnet
OS_ANY = "any"
VALID_OS = {os.value for os in capa.features.extractors.elf.OS}
VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS, OS_ANY, OS_ANDROID})
VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS, OS_ANY})
# internal only, not to be used in rules
OS_AUTO = "auto"
@@ -461,12 +461,8 @@ FORMAT_AUTO = "auto"
FORMAT_SC32 = "sc32"
FORMAT_SC64 = "sc64"
FORMAT_CAPE = "cape"
FORMAT_DRAKVUF = "drakvuf"
FORMAT_VMRAY = "vmray"
FORMAT_BINEXPORT2 = "binexport2"
FORMAT_FREEZE = "freeze"
FORMAT_RESULT = "result"
FORMAT_BINJA_DB = "binja_database"
STATIC_FORMATS = {
FORMAT_SC32,
FORMAT_SC64,
@@ -475,13 +471,9 @@ STATIC_FORMATS = {
FORMAT_DOTNET,
FORMAT_FREEZE,
FORMAT_RESULT,
FORMAT_BINEXPORT2,
FORMAT_BINJA_DB,
}
DYNAMIC_FORMATS = {
FORMAT_CAPE,
FORMAT_DRAKVUF,
FORMAT_VMRAY,
FORMAT_FREEZE,
FORMAT_RESULT,
}

View File

@@ -9,11 +9,13 @@
import abc
import hashlib
import dataclasses
from copy import copy
from types import MethodType
from typing import Any, Union, Iterator, TypeAlias
from typing import Any, Dict, Tuple, Union, Iterator
from dataclasses import dataclass
# TODO(williballenthin): use typing.TypeAlias directly when Python 3.9 is deprecated
# https://github.com/mandiant/capa/issues/1699
from typing_extensions import TypeAlias
import capa.features.address
from capa.features.common import Feature
from capa.features.address import Address, ThreadAddress, ProcessAddress, DynamicCallAddress, AbsoluteVirtualAddress
@@ -55,7 +57,7 @@ class FunctionHandle:
address: Address
inner: Any
ctx: dict[str, Any] = dataclasses.field(default_factory=dict)
ctx: Dict[str, Any] = dataclasses.field(default_factory=dict)
@dataclass
@@ -131,7 +133,7 @@ class StaticFeatureExtractor:
return self._sample_hashes
@abc.abstractmethod
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
"""
extract features found at every scope ("global").
@@ -142,12 +144,12 @@ class StaticFeatureExtractor:
print('0x%x: %s', va, feature)
yields:
tuple[Feature, Address]: feature and its location
Tuple[Feature, Address]: feature and its location
"""
raise NotImplementedError()
@abc.abstractmethod
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
"""
extract file-scope features.
@@ -158,7 +160,7 @@ class StaticFeatureExtractor:
print('0x%x: %s', va, feature)
yields:
tuple[Feature, Address]: feature and its location
Tuple[Feature, Address]: feature and its location
"""
raise NotImplementedError()
@@ -207,7 +209,7 @@ class StaticFeatureExtractor:
raise KeyError(addr)
@abc.abstractmethod
def extract_function_features(self, f: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
def extract_function_features(self, f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
"""
extract function-scope features.
the arguments are opaque values previously provided by `.get_functions()`, etc.
@@ -223,7 +225,7 @@ class StaticFeatureExtractor:
f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
yields:
tuple[Feature, Address]: feature and its location
Tuple[Feature, Address]: feature and its location
"""
raise NotImplementedError()
@@ -236,7 +238,7 @@ class StaticFeatureExtractor:
raise NotImplementedError()
@abc.abstractmethod
def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""
extract basic block-scope features.
the arguments are opaque values previously provided by `.get_functions()`, etc.
@@ -254,7 +256,7 @@ class StaticFeatureExtractor:
bb [BBHandle]: an opaque value previously fetched from `.get_basic_blocks()`.
yields:
tuple[Feature, Address]: feature and its location
Tuple[Feature, Address]: feature and its location
"""
raise NotImplementedError()
@@ -269,7 +271,7 @@ class StaticFeatureExtractor:
@abc.abstractmethod
def extract_insn_features(
self, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
extract instruction-scope features.
the arguments are opaque values previously provided by `.get_functions()`, etc.
@@ -289,27 +291,11 @@ class StaticFeatureExtractor:
insn [InsnHandle]: an opaque value previously fetched from `.get_instructions()`.
yields:
tuple[Feature, Address]: feature and its location
Tuple[Feature, Address]: feature and its location
"""
raise NotImplementedError()
def FunctionFilter(extractor: StaticFeatureExtractor, functions: set) -> StaticFeatureExtractor:
original_get_functions = extractor.get_functions
def filtered_get_functions(self):
yield from (f for f in original_get_functions() if f.address in functions)
# we make a copy of the original extractor object and then update its get_functions() method with the decorated filter one.
# this is in order to preserve the original extractor object's get_functions() method, in case it is used elsewhere in the code.
# an example where this is important is in our testfiles where we may use the same extractor object with different tests,
# with some of these tests needing to install a functions filter on the extractor object.
new_extractor = copy(extractor)
new_extractor.get_functions = MethodType(filtered_get_functions, extractor) # type: ignore
return new_extractor
@dataclass
class ProcessHandle:
"""
@@ -383,7 +369,7 @@ class DynamicFeatureExtractor:
return self._sample_hashes
@abc.abstractmethod
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
"""
extract features found at every scope ("global").
@@ -394,12 +380,12 @@ class DynamicFeatureExtractor:
print(addr, feature)
yields:
tuple[Feature, Address]: feature and its location
Tuple[Feature, Address]: feature and its location
"""
raise NotImplementedError()
@abc.abstractmethod
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
"""
extract file-scope features.
@@ -410,7 +396,7 @@ class DynamicFeatureExtractor:
print(addr, feature)
yields:
tuple[Feature, Address]: feature and its location
Tuple[Feature, Address]: feature and its location
"""
raise NotImplementedError()
@@ -422,7 +408,7 @@ class DynamicFeatureExtractor:
raise NotImplementedError()
@abc.abstractmethod
def extract_process_features(self, ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
"""
Yields all the features of a process. These include:
- file features of the process' image
@@ -445,7 +431,7 @@ class DynamicFeatureExtractor:
raise NotImplementedError()
@abc.abstractmethod
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[tuple[Feature, Address]]:
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
"""
Yields all the features of a thread. These include:
- sequenced api traces
@@ -462,7 +448,7 @@ class DynamicFeatureExtractor:
@abc.abstractmethod
def extract_call_features(
self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
Yields all features of a call. These include:
- api name
@@ -481,20 +467,4 @@ class DynamicFeatureExtractor:
raise NotImplementedError()
def ProcessFilter(extractor: DynamicFeatureExtractor, processes: set) -> DynamicFeatureExtractor:
original_get_processes = extractor.get_processes
def filtered_get_processes(self):
yield from (f for f in original_get_processes() if f.address.pid in processes)
# we make a copy of the original extractor object and then update its get_processes() method with the decorated filter one.
# this is in order to preserve the original extractor object's get_processes() method, in case it is used elsewhere in the code.
# an example where this is important is in our testfiles where we may use the same extractor object with different tests,
# with some of these tests needing to install a processes filter on the extractor object.
new_extractor = copy(extractor)
new_extractor.get_processes = MethodType(filtered_get_processes, extractor) # type: ignore
return new_extractor
FeatureExtractor: TypeAlias = Union[StaticFeatureExtractor, DynamicFeatureExtractor]

View File

@@ -1,418 +0,0 @@
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
"""
Proto files generated via protobuf v24.4:
protoc --python_out=. --mypy_out=. binexport2.proto
from BinExport2 at 6916731d5f6693c4a4f0a052501fd3bd92cfd08b
https://github.com/google/binexport/blob/6916731/binexport2.proto
"""
import io
import hashlib
import logging
import contextlib
from typing import Iterator
from pathlib import Path
from collections import defaultdict
from dataclasses import dataclass
from pefile import PE
from elftools.elf.elffile import ELFFile
import capa.features.common
import capa.features.extractors.common
import capa.features.extractors.binexport2.helpers
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
logger = logging.getLogger(__name__)
def get_binexport2(sample: Path) -> BinExport2:
be2: BinExport2 = BinExport2()
be2.ParseFromString(sample.read_bytes())
return be2
def compute_common_prefix_length(m: str, n: str) -> int:
# ensure #m < #n
if len(n) < len(m):
m, n = n, m
for i, c in enumerate(m):
if n[i] != c:
return i
return len(m)
def get_sample_from_binexport2(input_file: Path, be2: BinExport2, search_paths: list[Path]) -> Path:
"""attempt to find the sample file, given a BinExport2 file.
searches in the same directory as the BinExport2 file, and then in search_paths.
"""
def filename_similarity_key(p: Path) -> tuple[int, str]:
# note closure over input_file.
# sort first by length of common prefix, then by name (for stability)
return (compute_common_prefix_length(p.name, input_file.name), p.name)
wanted_sha256: str = be2.meta_information.executable_id.lower()
input_directory: Path = input_file.parent
siblings: list[Path] = [p for p in input_directory.iterdir() if p.is_file()]
siblings.sort(key=filename_similarity_key, reverse=True)
for sibling in siblings:
# e.g. with open IDA files in the same directory on Windows
with contextlib.suppress(PermissionError):
if hashlib.sha256(sibling.read_bytes()).hexdigest().lower() == wanted_sha256:
return sibling
for search_path in search_paths:
candidates: list[Path] = [p for p in search_path.iterdir() if p.is_file()]
candidates.sort(key=filename_similarity_key, reverse=True)
for candidate in candidates:
with contextlib.suppress(PermissionError):
if hashlib.sha256(candidate.read_bytes()).hexdigest().lower() == wanted_sha256:
return candidate
raise ValueError("cannot find sample, you may specify the path using the CAPA_SAMPLES_DIR environment variable")
class BinExport2Index:
def __init__(self, be2: BinExport2):
self.be2: BinExport2 = be2
self.callers_by_vertex_index: dict[int, list[int]] = defaultdict(list)
self.callees_by_vertex_index: dict[int, list[int]] = defaultdict(list)
# note: flow graph != call graph (vertex)
self.flow_graph_index_by_address: dict[int, int] = {}
self.flow_graph_address_by_index: dict[int, int] = {}
# edges that come from the given basic block
self.source_edges_by_basic_block_index: dict[int, list[BinExport2.FlowGraph.Edge]] = defaultdict(list)
# edges that end up at the given basic block
self.target_edges_by_basic_block_index: dict[int, list[BinExport2.FlowGraph.Edge]] = defaultdict(list)
self.vertex_index_by_address: dict[int, int] = {}
self.data_reference_index_by_source_instruction_index: dict[int, list[int]] = defaultdict(list)
self.data_reference_index_by_target_address: dict[int, list[int]] = defaultdict(list)
self.string_reference_index_by_source_instruction_index: dict[int, list[int]] = defaultdict(list)
self.insn_address_by_index: dict[int, int] = {}
self.insn_index_by_address: dict[int, int] = {}
self.insn_by_address: dict[int, BinExport2.Instruction] = {}
# must index instructions first
self._index_insn_addresses()
self._index_vertex_edges()
self._index_flow_graph_nodes()
self._index_flow_graph_edges()
self._index_call_graph_vertices()
self._index_data_references()
self._index_string_references()
def get_insn_address(self, insn_index: int) -> int:
assert insn_index in self.insn_address_by_index, f"insn must be indexed, missing {insn_index}"
return self.insn_address_by_index[insn_index]
def get_basic_block_address(self, basic_block_index: int) -> int:
basic_block: BinExport2.BasicBlock = self.be2.basic_block[basic_block_index]
first_instruction_index: int = next(self.instruction_indices(basic_block))
return self.get_insn_address(first_instruction_index)
def _index_vertex_edges(self):
for edge in self.be2.call_graph.edge:
if not edge.source_vertex_index:
continue
if not edge.target_vertex_index:
continue
self.callers_by_vertex_index[edge.target_vertex_index].append(edge.source_vertex_index)
self.callees_by_vertex_index[edge.source_vertex_index].append(edge.target_vertex_index)
def _index_flow_graph_nodes(self):
for flow_graph_index, flow_graph in enumerate(self.be2.flow_graph):
function_address: int = self.get_basic_block_address(flow_graph.entry_basic_block_index)
self.flow_graph_index_by_address[function_address] = flow_graph_index
self.flow_graph_address_by_index[flow_graph_index] = function_address
def _index_flow_graph_edges(self):
for flow_graph in self.be2.flow_graph:
for edge in flow_graph.edge:
if not edge.HasField("source_basic_block_index") or not edge.HasField("target_basic_block_index"):
continue
self.source_edges_by_basic_block_index[edge.source_basic_block_index].append(edge)
self.target_edges_by_basic_block_index[edge.target_basic_block_index].append(edge)
def _index_call_graph_vertices(self):
for vertex_index, vertex in enumerate(self.be2.call_graph.vertex):
if not vertex.HasField("address"):
continue
vertex_address: int = vertex.address
self.vertex_index_by_address[vertex_address] = vertex_index
def _index_data_references(self):
for data_reference_index, data_reference in enumerate(self.be2.data_reference):
self.data_reference_index_by_source_instruction_index[data_reference.instruction_index].append(
data_reference_index
)
self.data_reference_index_by_target_address[data_reference.address].append(data_reference_index)
def _index_string_references(self):
for string_reference_index, string_reference in enumerate(self.be2.string_reference):
self.string_reference_index_by_source_instruction_index[string_reference.instruction_index].append(
string_reference_index
)
def _index_insn_addresses(self):
# see https://github.com/google/binexport/blob/39f6445c232bb5caf5c4a2a996de91dfa20c48e8/binexport.cc#L45
if len(self.be2.instruction) == 0:
return
assert self.be2.instruction[0].HasField("address"), "first insn must have explicit address"
addr: int = 0
next_addr: int = 0
for idx, insn in enumerate(self.be2.instruction):
if insn.HasField("address"):
addr = insn.address
next_addr = addr + len(insn.raw_bytes)
else:
addr = next_addr
next_addr += len(insn.raw_bytes)
self.insn_address_by_index[idx] = addr
self.insn_index_by_address[addr] = idx
self.insn_by_address[addr] = insn
@staticmethod
def instruction_indices(basic_block: BinExport2.BasicBlock) -> Iterator[int]:
"""
For a given basic block, enumerate the instruction indices.
"""
for index_range in basic_block.instruction_index:
if not index_range.HasField("end_index"):
yield index_range.begin_index
continue
else:
yield from range(index_range.begin_index, index_range.end_index)
def basic_block_instructions(
self, basic_block: BinExport2.BasicBlock
) -> Iterator[tuple[int, BinExport2.Instruction, int]]:
"""
For a given basic block, enumerate the instruction indices,
the instruction instances, and their addresses.
"""
for instruction_index in self.instruction_indices(basic_block):
instruction: BinExport2.Instruction = self.be2.instruction[instruction_index]
instruction_address: int = self.get_insn_address(instruction_index)
yield instruction_index, instruction, instruction_address
def get_function_name_by_vertex(self, vertex_index: int) -> str:
vertex: BinExport2.CallGraph.Vertex = self.be2.call_graph.vertex[vertex_index]
name: str = f"sub_{vertex.address:x}"
if vertex.HasField("mangled_name"):
name = vertex.mangled_name
if vertex.HasField("demangled_name"):
name = vertex.demangled_name
if vertex.HasField("library_index"):
library: BinExport2.Library = self.be2.library[vertex.library_index]
if library.HasField("name"):
name = f"{library.name}!{name}"
return name
def get_function_name_by_address(self, address: int) -> str:
if address not in self.vertex_index_by_address:
return ""
vertex_index: int = self.vertex_index_by_address[address]
return self.get_function_name_by_vertex(vertex_index)
def get_instruction_by_address(self, address: int) -> BinExport2.Instruction:
assert address in self.insn_by_address, f"address must be indexed, missing {address:x}"
return self.insn_by_address[address]
class BinExport2Analysis:
def __init__(self, be2: BinExport2, idx: BinExport2Index, buf: bytes):
self.be2: BinExport2 = be2
self.idx: BinExport2Index = idx
self.buf: bytes = buf
self.base_address: int = 0
self.thunks: dict[int, int] = {}
self._find_base_address()
self._compute_thunks()
def _find_base_address(self):
sections_with_perms: Iterator[BinExport2.Section] = filter(
lambda s: s.flag_r or s.flag_w or s.flag_x, self.be2.section
)
# assume the lowest address is the base address.
# this works as long as BinExport doesn't record other
# libraries mapped into memory.
self.base_address = min(s.address for s in sections_with_perms)
logger.debug("found base address: %x", self.base_address)
def _compute_thunks(self):
for addr, idx in self.idx.vertex_index_by_address.items():
vertex: BinExport2.CallGraph.Vertex = self.be2.call_graph.vertex[idx]
if not capa.features.extractors.binexport2.helpers.is_vertex_type(
vertex, BinExport2.CallGraph.Vertex.Type.THUNK
):
continue
curr_idx: int = idx
for _ in range(capa.features.common.THUNK_CHAIN_DEPTH_DELTA):
thunk_callees: list[int] = self.idx.callees_by_vertex_index[curr_idx]
# If this doesn't hold, then it doesn't seem like this is a thunk,
# because either, len is:
# 0 and the thunk doesn't point to anything or is indirect, like `call eax`, or
# >1 and the thunk may end up at many functions.
# In any case, this doesn't appear to be the sort of thunk we're looking for.
if len(thunk_callees) != 1:
break
thunked_idx: int = thunk_callees[0]
thunked_vertex: BinExport2.CallGraph.Vertex = self.be2.call_graph.vertex[thunked_idx]
if not capa.features.extractors.binexport2.helpers.is_vertex_type(
thunked_vertex, BinExport2.CallGraph.Vertex.Type.THUNK
):
assert thunked_vertex.HasField("address")
self.thunks[addr] = thunked_vertex.address
break
curr_idx = thunked_idx
@dataclass
class MemoryRegion:
# location of the bytes, potentially relative to a base address
address: int
buf: bytes
@property
def end(self) -> int:
return self.address + len(self.buf)
def contains(self, address: int) -> bool:
# note: address must be relative to any base address
return self.address <= address < self.end
class ReadMemoryError(ValueError): ...
class AddressNotMappedError(ReadMemoryError): ...
@dataclass
class AddressSpace:
base_address: int
memory_regions: tuple[MemoryRegion, ...]
def read_memory(self, address: int, length: int) -> bytes:
rva: int = address - self.base_address
for region in self.memory_regions:
if region.contains(rva):
offset: int = rva - region.address
return region.buf[offset : offset + length]
raise AddressNotMappedError(address)
@classmethod
def from_pe(cls, pe: PE, base_address: int):
regions: list[MemoryRegion] = []
for section in pe.sections:
address: int = section.VirtualAddress
size: int = section.Misc_VirtualSize
buf: bytes = section.get_data()
if len(buf) != size:
# pad the section with NULLs
# assume page alignment is already handled.
# might need more hardening here.
buf += b"\x00" * (size - len(buf))
regions.append(MemoryRegion(address, buf))
return cls(base_address, tuple(regions))
@classmethod
def from_elf(cls, elf: ELFFile, base_address: int):
regions: list[MemoryRegion] = []
# ELF segments are for runtime data,
# ELF sections are for link-time data.
for segment in elf.iter_segments():
# assume p_align is consistent with addresses here.
# otherwise, should harden this loader.
segment_rva: int = segment.header.p_vaddr
segment_size: int = segment.header.p_memsz
segment_data: bytes = segment.data()
if len(segment_data) < segment_size:
# pad the section with NULLs
# assume page alignment is already handled.
# might need more hardening here.
segment_data += b"\x00" * (segment_size - len(segment_data))
regions.append(MemoryRegion(segment_rva, segment_data))
return cls(base_address, tuple(regions))
@classmethod
def from_buf(cls, buf: bytes, base_address: int):
if buf.startswith(capa.features.extractors.common.MATCH_PE):
pe: PE = PE(data=buf)
return cls.from_pe(pe, base_address)
elif buf.startswith(capa.features.extractors.common.MATCH_ELF):
elf: ELFFile = ELFFile(io.BytesIO(buf))
return cls.from_elf(elf, base_address)
else:
raise NotImplementedError("file format address space")
@dataclass
class AnalysisContext:
sample_bytes: bytes
be2: BinExport2
idx: BinExport2Index
analysis: BinExport2Analysis
address_space: AddressSpace
@dataclass
class FunctionContext:
ctx: AnalysisContext
flow_graph_index: int
format: set[str]
os: set[str]
arch: set[str]
@dataclass
class BasicBlockContext:
basic_block_index: int
@dataclass
class InstructionContext:
instruction_index: int

View File

@@ -1,15 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
def is_stack_register_expression(be2: BinExport2, expression: BinExport2.Expression) -> bool:
return bool(
expression and expression.type == BinExport2.Expression.REGISTER and expression.symbol.lower().endswith("sp")
)

View File

@@ -1,155 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator, Optional
import capa.features.extractors.binexport2.helpers
from capa.features.insn import MAX_STRUCTURE_SIZE, Number, Offset, OperandNumber, OperandOffset
from capa.features.common import Feature, Characteristic
from capa.features.address import Address
from capa.features.extractors.binexport2 import FunctionContext, InstructionContext
from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
from capa.features.extractors.binexport2.helpers import (
BinExport2InstructionPatternMatcher,
mask_immediate,
is_address_mapped,
get_instruction_mnemonic,
get_operand_register_expression,
get_operand_immediate_expression,
)
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
from capa.features.extractors.binexport2.arch.arm.helpers import is_stack_register_expression
logger = logging.getLogger(__name__)
def extract_insn_number_features(
fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
instruction_index: int = ii.instruction_index
instruction: BinExport2.Instruction = be2.instruction[instruction_index]
if len(instruction.operand_index) == 0:
# skip things like:
# .text:0040116e leave
return
mnemonic: str = get_instruction_mnemonic(be2, instruction)
if mnemonic in ("add", "sub"):
assert len(instruction.operand_index) == 3
operand1_expression: Optional[BinExport2.Expression] = get_operand_register_expression(
be2, be2.operand[instruction.operand_index[1]]
)
if operand1_expression and is_stack_register_expression(be2, operand1_expression):
# skip things like:
# add x0,sp,#0x8
return
for i, operand_index in enumerate(instruction.operand_index):
operand: BinExport2.Operand = be2.operand[operand_index]
immediate_expression: Optional[BinExport2.Expression] = get_operand_immediate_expression(be2, operand)
if not immediate_expression:
continue
value: int = mask_immediate(fhi.arch, immediate_expression.immediate)
if is_address_mapped(be2, value):
continue
yield Number(value), ih.address
yield OperandNumber(i, value), ih.address
if mnemonic == "add" and i == 2:
if 0 < value < MAX_STRUCTURE_SIZE:
yield Offset(value), ih.address
yield OperandOffset(i, value), ih.address
OFFSET_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
"""
ldr|ldrb|ldrh|ldrsb|ldrsh|ldrex|ldrd|str|strb|strh|strex|strd reg, [reg(not-stack), #int] ; capture #int
ldr|ldrb|ldrh|ldrsb|ldrsh|ldrex|ldrd|str|strb|strh|strex|strd reg, [reg(not-stack), #int]! ; capture #int
ldr|ldrb|ldrh|ldrsb|ldrsh|ldrex|ldrd|str|strb|strh|strex|strd reg, [reg(not-stack)], #int ; capture #int
ldp|ldpd|stp|stpd reg, reg, [reg(not-stack), #int] ; capture #int
ldp|ldpd|stp|stpd reg, reg, [reg(not-stack), #int]! ; capture #int
ldp|ldpd|stp|stpd reg, reg, [reg(not-stack)], #int ; capture #int
"""
)
def extract_insn_offset_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
match = OFFSET_PATTERNS.match_with_be2(be2, ii.instruction_index)
if not match:
return
value = match.expression.immediate
value = mask_immediate(fhi.arch, value)
if not is_address_mapped(be2, value):
value = capa.features.extractors.binexport2.helpers.twos_complement(fhi.arch, value)
yield Offset(value), ih.address
yield OperandOffset(match.operand_index, value), ih.address
NZXOR_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
"""
eor reg, reg, reg
eor reg, reg, #int
"""
)
def extract_insn_nzxor_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
if NZXOR_PATTERNS.match_with_be2(be2, ii.instruction_index) is None:
return
instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
# guaranteed to be simple int/reg operands
# so we don't have to realize the tree/list.
operands: list[BinExport2.Operand] = [be2.operand[operand_index] for operand_index in instruction.operand_index]
if operands[1] != operands[2]:
yield Characteristic("nzxor"), ih.address
INDIRECT_CALL_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
"""
blx|bx|blr reg
"""
)
def extract_function_indirect_call_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
if INDIRECT_CALL_PATTERNS.match_with_be2(be2, ii.instruction_index) is not None:
yield Characteristic("indirect call"), ih.address

View File

@@ -1,135 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Optional
from dataclasses import dataclass
from capa.features.extractors.binexport2.helpers import get_operand_expressions
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
# security cookie checks may perform non-zeroing XORs, these are expected within a certain
# byte range within the first and returning basic blocks, this helps to reduce FP features
SECURITY_COOKIE_BYTES_DELTA: int = 0x40
@dataclass
class OperandPhraseInfo:
scale: Optional[BinExport2.Expression] = None
index: Optional[BinExport2.Expression] = None
base: Optional[BinExport2.Expression] = None
displacement: Optional[BinExport2.Expression] = None
def get_operand_phrase_info(be2: BinExport2, operand: BinExport2.Operand) -> Optional[OperandPhraseInfo]:
# assume the following (see https://blog.yossarian.net/2020/06/13/How-x86_64-addresses-memory):
#
# Scale: A 2-bit constant factor
# Index: Any general purpose register
# Base: Any general purpose register
# Displacement: An integral offset
expressions: list[BinExport2.Expression] = get_operand_expressions(be2, operand)
# skip expression up to and including BinExport2.Expression.DEREFERENCE, assume caller
# has checked for BinExport2.Expression.DEREFERENCE
for i, expression in enumerate(expressions):
if expression.type == BinExport2.Expression.DEREFERENCE:
expressions = expressions[i + 1 :]
break
expression0: BinExport2.Expression
expression1: BinExport2.Expression
expression2: BinExport2.Expression
expression3: BinExport2.Expression
expression4: BinExport2.Expression
if len(expressions) == 1:
expression0 = expressions[0]
assert (
expression0.type == BinExport2.Expression.IMMEDIATE_INT
or expression0.type == BinExport2.Expression.REGISTER
)
if expression0.type == BinExport2.Expression.IMMEDIATE_INT:
# Displacement
return OperandPhraseInfo(displacement=expression0)
elif expression0.type == BinExport2.Expression.REGISTER:
# Base
return OperandPhraseInfo(base=expression0)
elif len(expressions) == 3:
expression0 = expressions[0]
expression1 = expressions[1]
expression2 = expressions[2]
assert expression0.type == BinExport2.Expression.REGISTER
assert expression1.type == BinExport2.Expression.OPERATOR
assert (
expression2.type == BinExport2.Expression.IMMEDIATE_INT
or expression2.type == BinExport2.Expression.REGISTER
)
if expression2.type == BinExport2.Expression.REGISTER:
# Base + Index
return OperandPhraseInfo(base=expression0, index=expression2)
elif expression2.type == BinExport2.Expression.IMMEDIATE_INT:
# Base + Displacement
return OperandPhraseInfo(base=expression0, displacement=expression2)
elif len(expressions) == 5:
expression0 = expressions[0]
expression1 = expressions[1]
expression2 = expressions[2]
expression3 = expressions[3]
expression4 = expressions[4]
assert expression0.type == BinExport2.Expression.REGISTER
assert expression1.type == BinExport2.Expression.OPERATOR
assert (
expression2.type == BinExport2.Expression.REGISTER
or expression2.type == BinExport2.Expression.IMMEDIATE_INT
)
assert expression3.type == BinExport2.Expression.OPERATOR
assert expression4.type == BinExport2.Expression.IMMEDIATE_INT
if expression1.symbol == "+" and expression3.symbol == "+":
# Base + Index + Displacement
return OperandPhraseInfo(base=expression0, index=expression2, displacement=expression4)
elif expression1.symbol == "+" and expression3.symbol == "*":
# Base + (Index * Scale)
return OperandPhraseInfo(base=expression0, index=expression2, scale=expression3)
elif expression1.symbol == "*" and expression3.symbol == "+":
# (Index * Scale) + Displacement
return OperandPhraseInfo(index=expression0, scale=expression2, displacement=expression3)
else:
raise NotImplementedError(expression1.symbol, expression3.symbol)
elif len(expressions) == 7:
expression0 = expressions[0]
expression1 = expressions[1]
expression2 = expressions[2]
expression3 = expressions[3]
expression4 = expressions[4]
expression5 = expressions[5]
expression6 = expressions[6]
assert expression0.type == BinExport2.Expression.REGISTER
assert expression1.type == BinExport2.Expression.OPERATOR
assert expression2.type == BinExport2.Expression.REGISTER
assert expression3.type == BinExport2.Expression.OPERATOR
assert expression4.type == BinExport2.Expression.IMMEDIATE_INT
assert expression5.type == BinExport2.Expression.OPERATOR
assert expression6.type == BinExport2.Expression.IMMEDIATE_INT
# Base + (Index * Scale) + Displacement
return OperandPhraseInfo(base=expression0, index=expression2, scale=expression4, displacement=expression6)
else:
raise NotImplementedError(len(expressions))
return None

View File

@@ -1,248 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
import capa.features.extractors.strings
import capa.features.extractors.binexport2.helpers
from capa.features.insn import MAX_STRUCTURE_SIZE, Number, Offset, OperandNumber, OperandOffset
from capa.features.common import Feature, Characteristic
from capa.features.address import Address
from capa.features.extractors.binexport2 import BinExport2Index, FunctionContext, BasicBlockContext, InstructionContext
from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
from capa.features.extractors.binexport2.helpers import (
BinExport2InstructionPatternMatcher,
mask_immediate,
is_address_mapped,
get_instruction_mnemonic,
)
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
from capa.features.extractors.binexport2.arch.intel.helpers import SECURITY_COOKIE_BYTES_DELTA
logger = logging.getLogger(__name__)
IGNORE_NUMBER_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
"""
ret #int
retn #int
add reg(stack), #int
sub reg(stack), #int
"""
)
NUMBER_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
"""
push #int0 ; capture #int0
# its a little tedious to enumerate all the address forms
# but at least we are explicit
cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar reg, #int0 ; capture #int0
cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg], #int0 ; capture #int0
cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [#int], #int0 ; capture #int0
cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg + #int], #int0 ; capture #int0
cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg + reg + #int], #int0 ; capture #int0
cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg + reg * #int], #int0 ; capture #int0
cmp|and|or|test|mov|add|adc|sub|shl|shr|sal|sar [reg + reg * #int + #int], #int0 ; capture #int0
imul reg, reg, #int ; capture #int
# note that int is first
cmp|test #int0, reg ; capture #int0
# imagine reg is zero'd out, then this is like `mov reg, #int`
# which is not uncommon.
lea reg, [reg + #int] ; capture #int
"""
)
def extract_insn_number_features(
fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
if IGNORE_NUMBER_PATTERNS.match_with_be2(be2, ii.instruction_index):
return
match = NUMBER_PATTERNS.match_with_be2(be2, ii.instruction_index)
if not match:
return
value: int = mask_immediate(fhi.arch, match.expression.immediate)
if is_address_mapped(be2, value):
return
yield Number(value), ih.address
yield OperandNumber(match.operand_index, value), ih.address
instruction_index: int = ii.instruction_index
instruction: BinExport2.Instruction = be2.instruction[instruction_index]
mnemonic: str = get_instruction_mnemonic(be2, instruction)
if mnemonic.startswith("add"):
if 0 < value < MAX_STRUCTURE_SIZE:
yield Offset(value), ih.address
yield OperandOffset(match.operand_index, value), ih.address
OFFSET_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
"""
mov|movzx|movsb|cmp [reg + reg * #int + #int0], #int ; capture #int0
mov|movzx|movsb|cmp [reg * #int + #int0], #int ; capture #int0
mov|movzx|movsb|cmp [reg + reg + #int0], #int ; capture #int0
mov|movzx|movsb|cmp [reg(not-stack) + #int0], #int ; capture #int0
mov|movzx|movsb|cmp [reg + reg * #int + #int0], reg ; capture #int0
mov|movzx|movsb|cmp [reg * #int + #int0], reg ; capture #int0
mov|movzx|movsb|cmp [reg + reg + #int0], reg ; capture #int0
mov|movzx|movsb|cmp [reg(not-stack) + #int0], reg ; capture #int0
mov|movzx|movsb|cmp|lea reg, [reg + reg * #int + #int0] ; capture #int0
mov|movzx|movsb|cmp|lea reg, [reg * #int + #int0] ; capture #int0
mov|movzx|movsb|cmp|lea reg, [reg + reg + #int0] ; capture #int0
mov|movzx|movsb|cmp|lea reg, [reg(not-stack) + #int0] ; capture #int0
"""
)
# these are patterns that access offset 0 from some pointer
# (pointer is not the stack pointer).
OFFSET_ZERO_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
"""
mov|movzx|movsb [reg(not-stack)], reg
mov|movzx|movsb [reg(not-stack)], #int
lea reg, [reg(not-stack)]
"""
)
def extract_insn_offset_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
match = OFFSET_PATTERNS.match_with_be2(be2, ii.instruction_index)
if not match:
match = OFFSET_ZERO_PATTERNS.match_with_be2(be2, ii.instruction_index)
if not match:
return
yield Offset(0), ih.address
yield OperandOffset(match.operand_index, 0), ih.address
value = mask_immediate(fhi.arch, match.expression.immediate)
if is_address_mapped(be2, value):
return
value = capa.features.extractors.binexport2.helpers.twos_complement(fhi.arch, value, 32)
yield Offset(value), ih.address
yield OperandOffset(match.operand_index, value), ih.address
def is_security_cookie(
fhi: FunctionContext,
bbi: BasicBlockContext,
instruction_address: int,
instruction: BinExport2.Instruction,
) -> bool:
"""
check if an instruction is related to security cookie checks.
"""
be2: BinExport2 = fhi.ctx.be2
idx: BinExport2Index = fhi.ctx.idx
# security cookie check should use SP or BP
op1: BinExport2.Operand = be2.operand[instruction.operand_index[1]]
op1_exprs: list[BinExport2.Expression] = [be2.expression[expr_i] for expr_i in op1.expression_index]
if all(expr.symbol.lower() not in ("bp", "esp", "ebp", "rbp", "rsp") for expr in op1_exprs):
return False
# check_nzxor_security_cookie_delta
# if insn falls at the start of first entry block of the parent function.
flow_graph: BinExport2.FlowGraph = be2.flow_graph[fhi.flow_graph_index]
basic_block_index: int = bbi.basic_block_index
bb: BinExport2.BasicBlock = be2.basic_block[basic_block_index]
if flow_graph.entry_basic_block_index == basic_block_index:
first_addr: int = min((idx.insn_address_by_index[ir.begin_index] for ir in bb.instruction_index))
if instruction_address < first_addr + SECURITY_COOKIE_BYTES_DELTA:
return True
# or insn falls at the end before return in a terminal basic block.
if basic_block_index not in (e.source_basic_block_index for e in flow_graph.edge):
last_addr: int = max((idx.insn_address_by_index[ir.end_index - 1] for ir in bb.instruction_index))
if instruction_address > last_addr - SECURITY_COOKIE_BYTES_DELTA:
return True
return False
NZXOR_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
"""
xor|xorpd|xorps|pxor reg, reg
xor|xorpd|xorps|pxor reg, #int
"""
)
def extract_insn_nzxor_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
"""
parse non-zeroing XOR instruction from the given instruction.
ignore expected non-zeroing XORs, e.g. security cookies.
"""
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
idx: BinExport2Index = fhi.ctx.idx
if NZXOR_PATTERNS.match_with_be2(be2, ii.instruction_index) is None:
return
instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
# guaranteed to be simple int/reg operands
# so we don't have to realize the tree/list.
operands: list[BinExport2.Operand] = [be2.operand[operand_index] for operand_index in instruction.operand_index]
if operands[0] == operands[1]:
return
instruction_address: int = idx.insn_address_by_index[ii.instruction_index]
if is_security_cookie(fhi, bbh.inner, instruction_address, instruction):
return
yield Characteristic("nzxor"), ih.address
INDIRECT_CALL_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
"""
call|jmp reg0
call|jmp [reg + reg * #int + #int]
call|jmp [reg + reg * #int]
call|jmp [reg * #int + #int]
call|jmp [reg + reg + #int]
call|jmp [reg + #int]
call|jmp [reg]
"""
)
def extract_function_indirect_call_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
match = INDIRECT_CALL_PATTERNS.match_with_be2(be2, ii.instruction_index)
if match is None:
return
yield Characteristic("indirect call"), ih.address

View File

@@ -1,40 +0,0 @@
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
from capa.features.common import Feature, Characteristic
from capa.features.address import Address, AbsoluteVirtualAddress
from capa.features.basicblock import BasicBlock
from capa.features.extractors.binexport2 import FunctionContext, BasicBlockContext
from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
bbi: BasicBlockContext = bbh.inner
idx = fhi.ctx.idx
basic_block_index: int = bbi.basic_block_index
target_edges: list[BinExport2.FlowGraph.Edge] = idx.target_edges_by_basic_block_index[basic_block_index]
if basic_block_index in (e.source_basic_block_index for e in target_edges):
basic_block_address: int = idx.get_basic_block_address(basic_block_index)
yield Characteristic("tight loop"), AbsoluteVirtualAddress(basic_block_address)
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
"""extract basic block features"""
for bb_handler in BASIC_BLOCK_HANDLERS:
for feature, addr in bb_handler(fh, bbh):
yield feature, addr
yield BasicBlock(), bbh.address
BASIC_BLOCK_HANDLERS = (extract_bb_tight_loop,)

File diff suppressed because one or more lines are too long

View File

@@ -1,784 +0,0 @@
"""
@generated by mypy-protobuf. Do not edit manually!
isort:skip_file
The representation is generic to accommodate various source architectures.
In particular 32 and 64 bit versions of x86, ARM, PowerPC and MIPS have been
tested.
Multiple levels of deduping have been applied to make the format more compact
and avoid redundant data duplication. Some of this due to hard-earned
experience trying to cope with intentionally obfuscated malicious binaries.
Note in particular that the same instruction may occur in multiple basic
blocks and the same basic block in multiple functions (instruction and basic
block sharing). Implemented naively, malware can use this to cause
combinatorial explosion in memory usage, DOSing the analyst. This format
should store every unique expression, mnemonic, operand, instruction and
basic block only once instead of duplicating the information for every
instance of it.
This format does _not_ try to be 100% backwards compatible with the old
version. In particular, we do not store IDA's comment types, making lossless
porting of IDA comments impossible. We do however, store comments and
expression substitutions, so porting the actual data is possible, just not
the exact IDA type.
While it would be more natural to use addresses when defining call graph and
flow graph edges and other such references, it is more efficient to employ
one more level of indirection and use indices into the basic block or
function arrays instead. This is because addresses will usually use most of
the available 64 bit space while indices will be much smaller and compress
much better (less randomly distributed).
We omit all fields that are set to their default value anyways. Note that
this has two side effects:
- changing the defaults in this proto file will, in effect, change what's
read from disk
- the generated code has_* methods are somewhat less useful
WARNING: We omit the defaults manually in the code writing the data. Do not
change the defaults here without changing the code!
TODO(cblichmann): Link flow graphs to call graph nodes. The connection is
there via the address, but tricky to extract.
"""
import builtins
import collections.abc
import google.protobuf.descriptor
import google.protobuf.internal.containers
import google.protobuf.internal.enum_type_wrapper
import google.protobuf.message
import sys
import typing
if sys.version_info >= (3, 10):
import typing as typing_extensions
else:
import typing_extensions
DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
@typing_extensions.final
class BinExport2(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
@typing_extensions.final
class Meta(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
EXECUTABLE_NAME_FIELD_NUMBER: builtins.int
EXECUTABLE_ID_FIELD_NUMBER: builtins.int
ARCHITECTURE_NAME_FIELD_NUMBER: builtins.int
TIMESTAMP_FIELD_NUMBER: builtins.int
executable_name: builtins.str
"""Input binary filename including file extension but excluding file path.
example: "insider_gcc.exe"
"""
executable_id: builtins.str
"""Application defined executable id. Often the SHA256 hash of the input
binary.
"""
architecture_name: builtins.str
"""Input architecture name, e.g. x86-32."""
timestamp: builtins.int
"""When did this file get created? Unix time. This may be used for some
primitive versioning in case the file format ever changes.
"""
def __init__(
self,
*,
executable_name: builtins.str | None = ...,
executable_id: builtins.str | None = ...,
architecture_name: builtins.str | None = ...,
timestamp: builtins.int | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["architecture_name", b"architecture_name", "executable_id", b"executable_id", "executable_name", b"executable_name", "timestamp", b"timestamp"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["architecture_name", b"architecture_name", "executable_id", b"executable_id", "executable_name", b"executable_name", "timestamp", b"timestamp"]) -> None: ...
@typing_extensions.final
class CallGraph(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
@typing_extensions.final
class Vertex(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
class _Type:
ValueType = typing.NewType("ValueType", builtins.int)
V: typing_extensions.TypeAlias = ValueType
class _TypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[BinExport2.CallGraph.Vertex._Type.ValueType], builtins.type):
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
NORMAL: BinExport2.CallGraph.Vertex._Type.ValueType # 0
"""Regular function with full disassembly."""
LIBRARY: BinExport2.CallGraph.Vertex._Type.ValueType # 1
"""This function is a well known library function."""
IMPORTED: BinExport2.CallGraph.Vertex._Type.ValueType # 2
"""Imported from a dynamic link library (e.g. dll)."""
THUNK: BinExport2.CallGraph.Vertex._Type.ValueType # 3
"""A thunk function, forwarding its work via an unconditional jump."""
INVALID: BinExport2.CallGraph.Vertex._Type.ValueType # 4
"""An invalid function (a function that contained invalid code or was
considered invalid by some heuristics).
"""
class Type(_Type, metaclass=_TypeEnumTypeWrapper): ...
NORMAL: BinExport2.CallGraph.Vertex.Type.ValueType # 0
"""Regular function with full disassembly."""
LIBRARY: BinExport2.CallGraph.Vertex.Type.ValueType # 1
"""This function is a well known library function."""
IMPORTED: BinExport2.CallGraph.Vertex.Type.ValueType # 2
"""Imported from a dynamic link library (e.g. dll)."""
THUNK: BinExport2.CallGraph.Vertex.Type.ValueType # 3
"""A thunk function, forwarding its work via an unconditional jump."""
INVALID: BinExport2.CallGraph.Vertex.Type.ValueType # 4
"""An invalid function (a function that contained invalid code or was
considered invalid by some heuristics).
"""
ADDRESS_FIELD_NUMBER: builtins.int
TYPE_FIELD_NUMBER: builtins.int
MANGLED_NAME_FIELD_NUMBER: builtins.int
DEMANGLED_NAME_FIELD_NUMBER: builtins.int
LIBRARY_INDEX_FIELD_NUMBER: builtins.int
MODULE_INDEX_FIELD_NUMBER: builtins.int
address: builtins.int
"""The function's entry point address. Messages need to be sorted, see
comment below on `vertex`.
"""
type: global___BinExport2.CallGraph.Vertex.Type.ValueType
mangled_name: builtins.str
"""If the function has a user defined, real name it will be given here.
main() is a proper name, sub_BAADF00D is not (auto generated dummy
name).
"""
demangled_name: builtins.str
"""Demangled name if the function is a mangled C++ function and we could
demangle it.
"""
library_index: builtins.int
"""If this is a library function, what is its index in library arrays."""
module_index: builtins.int
"""If module name, such as class name for DEX files, is present - index in
module table.
"""
def __init__(
self,
*,
address: builtins.int | None = ...,
type: global___BinExport2.CallGraph.Vertex.Type.ValueType | None = ...,
mangled_name: builtins.str | None = ...,
demangled_name: builtins.str | None = ...,
library_index: builtins.int | None = ...,
module_index: builtins.int | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["address", b"address", "demangled_name", b"demangled_name", "library_index", b"library_index", "mangled_name", b"mangled_name", "module_index", b"module_index", "type", b"type"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "demangled_name", b"demangled_name", "library_index", b"library_index", "mangled_name", b"mangled_name", "module_index", b"module_index", "type", b"type"]) -> None: ...
@typing_extensions.final
class Edge(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
SOURCE_VERTEX_INDEX_FIELD_NUMBER: builtins.int
TARGET_VERTEX_INDEX_FIELD_NUMBER: builtins.int
source_vertex_index: builtins.int
"""source and target index into the vertex repeated field."""
target_vertex_index: builtins.int
def __init__(
self,
*,
source_vertex_index: builtins.int | None = ...,
target_vertex_index: builtins.int | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["source_vertex_index", b"source_vertex_index", "target_vertex_index", b"target_vertex_index"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["source_vertex_index", b"source_vertex_index", "target_vertex_index", b"target_vertex_index"]) -> None: ...
VERTEX_FIELD_NUMBER: builtins.int
EDGE_FIELD_NUMBER: builtins.int
@property
def vertex(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.CallGraph.Vertex]:
"""vertices == functions in the call graph.
Important: Most downstream tooling (notably BinDiff), need these to be
sorted by `Vertex::address` (ascending). For C++, the
`BinExport2Writer` class enforces this invariant.
"""
@property
def edge(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.CallGraph.Edge]:
"""edges == calls in the call graph."""
def __init__(
self,
*,
vertex: collections.abc.Iterable[global___BinExport2.CallGraph.Vertex] | None = ...,
edge: collections.abc.Iterable[global___BinExport2.CallGraph.Edge] | None = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["edge", b"edge", "vertex", b"vertex"]) -> None: ...
@typing_extensions.final
class Expression(google.protobuf.message.Message):
"""An operand consists of 1 or more expressions, linked together as a tree."""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
class _Type:
ValueType = typing.NewType("ValueType", builtins.int)
V: typing_extensions.TypeAlias = ValueType
class _TypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[BinExport2.Expression._Type.ValueType], builtins.type):
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
SYMBOL: BinExport2.Expression._Type.ValueType # 1
IMMEDIATE_INT: BinExport2.Expression._Type.ValueType # 2
IMMEDIATE_FLOAT: BinExport2.Expression._Type.ValueType # 3
OPERATOR: BinExport2.Expression._Type.ValueType # 4
REGISTER: BinExport2.Expression._Type.ValueType # 5
SIZE_PREFIX: BinExport2.Expression._Type.ValueType # 6
DEREFERENCE: BinExport2.Expression._Type.ValueType # 7
class Type(_Type, metaclass=_TypeEnumTypeWrapper): ...
SYMBOL: BinExport2.Expression.Type.ValueType # 1
IMMEDIATE_INT: BinExport2.Expression.Type.ValueType # 2
IMMEDIATE_FLOAT: BinExport2.Expression.Type.ValueType # 3
OPERATOR: BinExport2.Expression.Type.ValueType # 4
REGISTER: BinExport2.Expression.Type.ValueType # 5
SIZE_PREFIX: BinExport2.Expression.Type.ValueType # 6
DEREFERENCE: BinExport2.Expression.Type.ValueType # 7
TYPE_FIELD_NUMBER: builtins.int
SYMBOL_FIELD_NUMBER: builtins.int
IMMEDIATE_FIELD_NUMBER: builtins.int
PARENT_INDEX_FIELD_NUMBER: builtins.int
IS_RELOCATION_FIELD_NUMBER: builtins.int
type: global___BinExport2.Expression.Type.ValueType
"""IMMEDIATE_INT is by far the most common type and thus we can save some
space by omitting it as the default.
"""
symbol: builtins.str
"""Symbol for this expression. Interpretation depends on type. Examples
include: "eax", "[", "+"
"""
immediate: builtins.int
"""If the expression can be interpreted as an integer value (IMMEDIATE_INT)
the value is given here.
"""
parent_index: builtins.int
"""The parent expression. Example expression tree for the second operand of:
mov eax, b4 [ebx + 12]
"b4" --- "[" --- "+" --- "ebx"
\\ "12"
"""
is_relocation: builtins.bool
"""true if the expression has entry in relocation table"""
def __init__(
self,
*,
type: global___BinExport2.Expression.Type.ValueType | None = ...,
symbol: builtins.str | None = ...,
immediate: builtins.int | None = ...,
parent_index: builtins.int | None = ...,
is_relocation: builtins.bool | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["immediate", b"immediate", "is_relocation", b"is_relocation", "parent_index", b"parent_index", "symbol", b"symbol", "type", b"type"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["immediate", b"immediate", "is_relocation", b"is_relocation", "parent_index", b"parent_index", "symbol", b"symbol", "type", b"type"]) -> None: ...
@typing_extensions.final
class Operand(google.protobuf.message.Message):
"""An instruction may have 0 or more operands."""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
EXPRESSION_INDEX_FIELD_NUMBER: builtins.int
@property
def expression_index(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
"""Contains all expressions constituting this operand. All expressions
should be linked into a single tree, i.e. there should only be one
expression in this list with parent_index == NULL and all others should
descend from that. Rendering order for expressions on the same tree level
(siblings) is implicitly given by the order they are referenced in this
repeated field.
Implicit: expression sequence
"""
def __init__(
self,
*,
expression_index: collections.abc.Iterable[builtins.int] | None = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["expression_index", b"expression_index"]) -> None: ...
@typing_extensions.final
class Mnemonic(google.protobuf.message.Message):
"""An instruction has exactly 1 mnemonic."""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
NAME_FIELD_NUMBER: builtins.int
name: builtins.str
"""Literal representation of the mnemonic, e.g.: "mov"."""
def __init__(
self,
*,
name: builtins.str | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["name", b"name"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["name", b"name"]) -> None: ...
@typing_extensions.final
class Instruction(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
ADDRESS_FIELD_NUMBER: builtins.int
CALL_TARGET_FIELD_NUMBER: builtins.int
MNEMONIC_INDEX_FIELD_NUMBER: builtins.int
OPERAND_INDEX_FIELD_NUMBER: builtins.int
RAW_BYTES_FIELD_NUMBER: builtins.int
COMMENT_INDEX_FIELD_NUMBER: builtins.int
address: builtins.int
"""This will only be filled for instructions that do not just flow from the
immediately preceding instruction. Regular instructions will have to
calculate their own address by adding raw_bytes.size() to the previous
instruction's address.
"""
@property
def call_target(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
"""If this is a call instruction and call targets could be determined
they'll be given here. Note that we may or may not have a flow graph for
the target and thus cannot use an index into the flow graph table here.
We could potentially use call graph nodes, but linking instructions to
the call graph directly does not seem a good choice.
"""
mnemonic_index: builtins.int
"""Index into the mnemonic array of strings. Used for de-duping the data.
The default value is used for the most common mnemonic in the executable.
"""
@property
def operand_index(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
"""Indices into the operand tree. On X86 this can be 0, 1 or 2 elements
long, 3 elements with VEX/EVEX.
Implicit: operand sequence
"""
raw_bytes: builtins.bytes
"""The unmodified input bytes corresponding to this instruction."""
@property
def comment_index(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
"""Implicit: comment sequence"""
def __init__(
self,
*,
address: builtins.int | None = ...,
call_target: collections.abc.Iterable[builtins.int] | None = ...,
mnemonic_index: builtins.int | None = ...,
operand_index: collections.abc.Iterable[builtins.int] | None = ...,
raw_bytes: builtins.bytes | None = ...,
comment_index: collections.abc.Iterable[builtins.int] | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["address", b"address", "mnemonic_index", b"mnemonic_index", "raw_bytes", b"raw_bytes"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "call_target", b"call_target", "comment_index", b"comment_index", "mnemonic_index", b"mnemonic_index", "operand_index", b"operand_index", "raw_bytes", b"raw_bytes"]) -> None: ...
@typing_extensions.final
class BasicBlock(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
@typing_extensions.final
class IndexRange(google.protobuf.message.Message):
"""This is a space optimization. The instructions for an individual basic
block will usually be in a continuous index range. Thus it is more
efficient to store the range instead of individual indices. However, this
does not hold true for all basic blocks, so we need to be able to store
multiple index ranges per block.
"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
BEGIN_INDEX_FIELD_NUMBER: builtins.int
END_INDEX_FIELD_NUMBER: builtins.int
begin_index: builtins.int
"""These work like begin and end iterators, i.e. the sequence is
[begin_index, end_index). If the sequence only contains a single
element end_index will be omitted.
"""
end_index: builtins.int
def __init__(
self,
*,
begin_index: builtins.int | None = ...,
end_index: builtins.int | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["begin_index", b"begin_index", "end_index", b"end_index"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["begin_index", b"begin_index", "end_index", b"end_index"]) -> None: ...
INSTRUCTION_INDEX_FIELD_NUMBER: builtins.int
@property
def instruction_index(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.BasicBlock.IndexRange]:
"""Implicit: instruction sequence"""
def __init__(
self,
*,
instruction_index: collections.abc.Iterable[global___BinExport2.BasicBlock.IndexRange] | None = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index"]) -> None: ...
@typing_extensions.final
class FlowGraph(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
@typing_extensions.final
class Edge(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
class _Type:
ValueType = typing.NewType("ValueType", builtins.int)
V: typing_extensions.TypeAlias = ValueType
class _TypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[BinExport2.FlowGraph.Edge._Type.ValueType], builtins.type):
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
CONDITION_TRUE: BinExport2.FlowGraph.Edge._Type.ValueType # 1
CONDITION_FALSE: BinExport2.FlowGraph.Edge._Type.ValueType # 2
UNCONDITIONAL: BinExport2.FlowGraph.Edge._Type.ValueType # 3
SWITCH: BinExport2.FlowGraph.Edge._Type.ValueType # 4
class Type(_Type, metaclass=_TypeEnumTypeWrapper): ...
CONDITION_TRUE: BinExport2.FlowGraph.Edge.Type.ValueType # 1
CONDITION_FALSE: BinExport2.FlowGraph.Edge.Type.ValueType # 2
UNCONDITIONAL: BinExport2.FlowGraph.Edge.Type.ValueType # 3
SWITCH: BinExport2.FlowGraph.Edge.Type.ValueType # 4
SOURCE_BASIC_BLOCK_INDEX_FIELD_NUMBER: builtins.int
TARGET_BASIC_BLOCK_INDEX_FIELD_NUMBER: builtins.int
TYPE_FIELD_NUMBER: builtins.int
IS_BACK_EDGE_FIELD_NUMBER: builtins.int
source_basic_block_index: builtins.int
"""Source instruction will always be the last instruction of the source
basic block, target instruction the first instruction of the target
basic block.
"""
target_basic_block_index: builtins.int
type: global___BinExport2.FlowGraph.Edge.Type.ValueType
is_back_edge: builtins.bool
"""Indicates whether this is a loop edge as determined by Lengauer-Tarjan."""
def __init__(
self,
*,
source_basic_block_index: builtins.int | None = ...,
target_basic_block_index: builtins.int | None = ...,
type: global___BinExport2.FlowGraph.Edge.Type.ValueType | None = ...,
is_back_edge: builtins.bool | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["is_back_edge", b"is_back_edge", "source_basic_block_index", b"source_basic_block_index", "target_basic_block_index", b"target_basic_block_index", "type", b"type"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["is_back_edge", b"is_back_edge", "source_basic_block_index", b"source_basic_block_index", "target_basic_block_index", b"target_basic_block_index", "type", b"type"]) -> None: ...
BASIC_BLOCK_INDEX_FIELD_NUMBER: builtins.int
ENTRY_BASIC_BLOCK_INDEX_FIELD_NUMBER: builtins.int
EDGE_FIELD_NUMBER: builtins.int
@property
def basic_block_index(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]:
"""Basic blocks are sorted by address."""
entry_basic_block_index: builtins.int
"""The flow graph's entry point address is the first instruction of the
entry_basic_block.
"""
@property
def edge(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.FlowGraph.Edge]: ...
def __init__(
self,
*,
basic_block_index: collections.abc.Iterable[builtins.int] | None = ...,
entry_basic_block_index: builtins.int | None = ...,
edge: collections.abc.Iterable[global___BinExport2.FlowGraph.Edge] | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["entry_basic_block_index", b"entry_basic_block_index"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["basic_block_index", b"basic_block_index", "edge", b"edge", "entry_basic_block_index", b"entry_basic_block_index"]) -> None: ...
@typing_extensions.final
class Reference(google.protobuf.message.Message):
"""Generic reference class used for address comments (deprecated), string
references and expression substitutions. It allows referencing from an
instruction, operand, expression subtree tuple to a de-duped string in the
string table.
"""
DESCRIPTOR: google.protobuf.descriptor.Descriptor
INSTRUCTION_INDEX_FIELD_NUMBER: builtins.int
INSTRUCTION_OPERAND_INDEX_FIELD_NUMBER: builtins.int
OPERAND_EXPRESSION_INDEX_FIELD_NUMBER: builtins.int
STRING_TABLE_INDEX_FIELD_NUMBER: builtins.int
instruction_index: builtins.int
"""Index into the global instruction table."""
instruction_operand_index: builtins.int
"""Index into the operand array local to an instruction."""
operand_expression_index: builtins.int
"""Index into the expression array local to an operand."""
string_table_index: builtins.int
"""Index into the global string table."""
def __init__(
self,
*,
instruction_index: builtins.int | None = ...,
instruction_operand_index: builtins.int | None = ...,
operand_expression_index: builtins.int | None = ...,
string_table_index: builtins.int | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index", "instruction_operand_index", b"instruction_operand_index", "operand_expression_index", b"operand_expression_index", "string_table_index", b"string_table_index"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index", "instruction_operand_index", b"instruction_operand_index", "operand_expression_index", b"operand_expression_index", "string_table_index", b"string_table_index"]) -> None: ...
@typing_extensions.final
class DataReference(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
INSTRUCTION_INDEX_FIELD_NUMBER: builtins.int
ADDRESS_FIELD_NUMBER: builtins.int
instruction_index: builtins.int
"""Index into the global instruction table."""
address: builtins.int
"""Address being referred."""
def __init__(
self,
*,
instruction_index: builtins.int | None = ...,
address: builtins.int | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["address", b"address", "instruction_index", b"instruction_index"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "instruction_index", b"instruction_index"]) -> None: ...
@typing_extensions.final
class Comment(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
class _Type:
ValueType = typing.NewType("ValueType", builtins.int)
V: typing_extensions.TypeAlias = ValueType
class _TypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[BinExport2.Comment._Type.ValueType], builtins.type):
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
DEFAULT: BinExport2.Comment._Type.ValueType # 0
"""A regular instruction comment. Typically displayed next to the
instruction disassembly.
"""
ANTERIOR: BinExport2.Comment._Type.ValueType # 1
"""A comment line that is typically displayed before (above) the
instruction it refers to.
"""
POSTERIOR: BinExport2.Comment._Type.ValueType # 2
"""Like ANTERIOR, but a typically displayed after (below)."""
FUNCTION: BinExport2.Comment._Type.ValueType # 3
"""Similar to an ANTERIOR comment, but applies to the beginning of an
identified function. Programs displaying the proto may choose to render
these differently (e.g. above an inferred function signature).
"""
ENUM: BinExport2.Comment._Type.ValueType # 4
"""Named constants, bitfields and similar."""
LOCATION: BinExport2.Comment._Type.ValueType # 5
"""Named locations, usually the target of a jump."""
GLOBAL_REFERENCE: BinExport2.Comment._Type.ValueType # 6
"""Data cross references."""
LOCAL_REFERENCE: BinExport2.Comment._Type.ValueType # 7
"""Local/stack variables."""
class Type(_Type, metaclass=_TypeEnumTypeWrapper): ...
DEFAULT: BinExport2.Comment.Type.ValueType # 0
"""A regular instruction comment. Typically displayed next to the
instruction disassembly.
"""
ANTERIOR: BinExport2.Comment.Type.ValueType # 1
"""A comment line that is typically displayed before (above) the
instruction it refers to.
"""
POSTERIOR: BinExport2.Comment.Type.ValueType # 2
"""Like ANTERIOR, but a typically displayed after (below)."""
FUNCTION: BinExport2.Comment.Type.ValueType # 3
"""Similar to an ANTERIOR comment, but applies to the beginning of an
identified function. Programs displaying the proto may choose to render
these differently (e.g. above an inferred function signature).
"""
ENUM: BinExport2.Comment.Type.ValueType # 4
"""Named constants, bitfields and similar."""
LOCATION: BinExport2.Comment.Type.ValueType # 5
"""Named locations, usually the target of a jump."""
GLOBAL_REFERENCE: BinExport2.Comment.Type.ValueType # 6
"""Data cross references."""
LOCAL_REFERENCE: BinExport2.Comment.Type.ValueType # 7
"""Local/stack variables."""
INSTRUCTION_INDEX_FIELD_NUMBER: builtins.int
INSTRUCTION_OPERAND_INDEX_FIELD_NUMBER: builtins.int
OPERAND_EXPRESSION_INDEX_FIELD_NUMBER: builtins.int
STRING_TABLE_INDEX_FIELD_NUMBER: builtins.int
REPEATABLE_FIELD_NUMBER: builtins.int
TYPE_FIELD_NUMBER: builtins.int
instruction_index: builtins.int
"""Index into the global instruction table. This is here to enable
comment processing without having to iterate over all instructions.
There is an N:M mapping of instructions to comments.
"""
instruction_operand_index: builtins.int
"""Index into the operand array local to an instruction."""
operand_expression_index: builtins.int
"""Index into the expression array local to an operand, like in Reference.
This is not currently used, but allows to implement expression
substitutions.
"""
string_table_index: builtins.int
"""Index into the global string table."""
repeatable: builtins.bool
"""Comment is propagated to all locations that reference the original
location.
"""
type: global___BinExport2.Comment.Type.ValueType
def __init__(
self,
*,
instruction_index: builtins.int | None = ...,
instruction_operand_index: builtins.int | None = ...,
operand_expression_index: builtins.int | None = ...,
string_table_index: builtins.int | None = ...,
repeatable: builtins.bool | None = ...,
type: global___BinExport2.Comment.Type.ValueType | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index", "instruction_operand_index", b"instruction_operand_index", "operand_expression_index", b"operand_expression_index", "repeatable", b"repeatable", "string_table_index", b"string_table_index", "type", b"type"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["instruction_index", b"instruction_index", "instruction_operand_index", b"instruction_operand_index", "operand_expression_index", b"operand_expression_index", "repeatable", b"repeatable", "string_table_index", b"string_table_index", "type", b"type"]) -> None: ...
@typing_extensions.final
class Section(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
ADDRESS_FIELD_NUMBER: builtins.int
SIZE_FIELD_NUMBER: builtins.int
FLAG_R_FIELD_NUMBER: builtins.int
FLAG_W_FIELD_NUMBER: builtins.int
FLAG_X_FIELD_NUMBER: builtins.int
address: builtins.int
"""Section start address."""
size: builtins.int
"""Section size."""
flag_r: builtins.bool
"""Read flag of the section, True when section is readable."""
flag_w: builtins.bool
"""Write flag of the section, True when section is writable."""
flag_x: builtins.bool
"""Execute flag of the section, True when section is executable."""
def __init__(
self,
*,
address: builtins.int | None = ...,
size: builtins.int | None = ...,
flag_r: builtins.bool | None = ...,
flag_w: builtins.bool | None = ...,
flag_x: builtins.bool | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["address", b"address", "flag_r", b"flag_r", "flag_w", b"flag_w", "flag_x", b"flag_x", "size", b"size"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["address", b"address", "flag_r", b"flag_r", "flag_w", b"flag_w", "flag_x", b"flag_x", "size", b"size"]) -> None: ...
@typing_extensions.final
class Library(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
IS_STATIC_FIELD_NUMBER: builtins.int
LOAD_ADDRESS_FIELD_NUMBER: builtins.int
NAME_FIELD_NUMBER: builtins.int
is_static: builtins.bool
"""If this library is statically linked."""
load_address: builtins.int
"""Address where this library was loaded, 0 if unknown."""
name: builtins.str
"""Name of the library (format is platform-dependent)."""
def __init__(
self,
*,
is_static: builtins.bool | None = ...,
load_address: builtins.int | None = ...,
name: builtins.str | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["is_static", b"is_static", "load_address", b"load_address", "name", b"name"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["is_static", b"is_static", "load_address", b"load_address", "name", b"name"]) -> None: ...
@typing_extensions.final
class Module(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
NAME_FIELD_NUMBER: builtins.int
name: builtins.str
"""Name, such as Java class name. Platform-dependent."""
def __init__(
self,
*,
name: builtins.str | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["name", b"name"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["name", b"name"]) -> None: ...
META_INFORMATION_FIELD_NUMBER: builtins.int
EXPRESSION_FIELD_NUMBER: builtins.int
OPERAND_FIELD_NUMBER: builtins.int
MNEMONIC_FIELD_NUMBER: builtins.int
INSTRUCTION_FIELD_NUMBER: builtins.int
BASIC_BLOCK_FIELD_NUMBER: builtins.int
FLOW_GRAPH_FIELD_NUMBER: builtins.int
CALL_GRAPH_FIELD_NUMBER: builtins.int
STRING_TABLE_FIELD_NUMBER: builtins.int
ADDRESS_COMMENT_FIELD_NUMBER: builtins.int
COMMENT_FIELD_NUMBER: builtins.int
STRING_REFERENCE_FIELD_NUMBER: builtins.int
EXPRESSION_SUBSTITUTION_FIELD_NUMBER: builtins.int
SECTION_FIELD_NUMBER: builtins.int
LIBRARY_FIELD_NUMBER: builtins.int
DATA_REFERENCE_FIELD_NUMBER: builtins.int
MODULE_FIELD_NUMBER: builtins.int
@property
def meta_information(self) -> global___BinExport2.Meta: ...
@property
def expression(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Expression]: ...
@property
def operand(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Operand]: ...
@property
def mnemonic(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Mnemonic]: ...
@property
def instruction(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Instruction]: ...
@property
def basic_block(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.BasicBlock]: ...
@property
def flow_graph(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.FlowGraph]: ...
@property
def call_graph(self) -> global___BinExport2.CallGraph: ...
@property
def string_table(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
@property
def address_comment(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Reference]:
"""No longer written. This is here so that BinDiff can work with older
BinExport files.
"""
@property
def comment(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Comment]:
"""Rich comment index used for BinDiff's comment porting."""
@property
def string_reference(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Reference]: ...
@property
def expression_substitution(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Reference]: ...
@property
def section(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Section]: ...
@property
def library(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Library]: ...
@property
def data_reference(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.DataReference]: ...
@property
def module(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___BinExport2.Module]: ...
def __init__(
self,
*,
meta_information: global___BinExport2.Meta | None = ...,
expression: collections.abc.Iterable[global___BinExport2.Expression] | None = ...,
operand: collections.abc.Iterable[global___BinExport2.Operand] | None = ...,
mnemonic: collections.abc.Iterable[global___BinExport2.Mnemonic] | None = ...,
instruction: collections.abc.Iterable[global___BinExport2.Instruction] | None = ...,
basic_block: collections.abc.Iterable[global___BinExport2.BasicBlock] | None = ...,
flow_graph: collections.abc.Iterable[global___BinExport2.FlowGraph] | None = ...,
call_graph: global___BinExport2.CallGraph | None = ...,
string_table: collections.abc.Iterable[builtins.str] | None = ...,
address_comment: collections.abc.Iterable[global___BinExport2.Reference] | None = ...,
comment: collections.abc.Iterable[global___BinExport2.Comment] | None = ...,
string_reference: collections.abc.Iterable[global___BinExport2.Reference] | None = ...,
expression_substitution: collections.abc.Iterable[global___BinExport2.Reference] | None = ...,
section: collections.abc.Iterable[global___BinExport2.Section] | None = ...,
library: collections.abc.Iterable[global___BinExport2.Library] | None = ...,
data_reference: collections.abc.Iterable[global___BinExport2.DataReference] | None = ...,
module: collections.abc.Iterable[global___BinExport2.Module] | None = ...,
) -> None: ...
def HasField(self, field_name: typing_extensions.Literal["call_graph", b"call_graph", "meta_information", b"meta_information"]) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["address_comment", b"address_comment", "basic_block", b"basic_block", "call_graph", b"call_graph", "comment", b"comment", "data_reference", b"data_reference", "expression", b"expression", "expression_substitution", b"expression_substitution", "flow_graph", b"flow_graph", "instruction", b"instruction", "library", b"library", "meta_information", b"meta_information", "mnemonic", b"mnemonic", "module", b"module", "operand", b"operand", "section", b"section", "string_reference", b"string_reference", "string_table", b"string_table"]) -> None: ...
global___BinExport2 = BinExport2

View File

@@ -1,130 +0,0 @@
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
import capa.features.extractors.elf
import capa.features.extractors.common
import capa.features.extractors.binexport2.file
import capa.features.extractors.binexport2.insn
import capa.features.extractors.binexport2.helpers
import capa.features.extractors.binexport2.function
import capa.features.extractors.binexport2.basicblock
from capa.features.common import OS, Arch, Format, Feature
from capa.features.address import Address, AbsoluteVirtualAddress
from capa.features.extractors.binexport2 import (
AddressSpace,
AnalysisContext,
BinExport2Index,
FunctionContext,
BasicBlockContext,
BinExport2Analysis,
InstructionContext,
)
from capa.features.extractors.base_extractor import (
BBHandle,
InsnHandle,
SampleHashes,
FunctionHandle,
StaticFeatureExtractor,
)
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
logger = logging.getLogger(__name__)
class BinExport2FeatureExtractor(StaticFeatureExtractor):
def __init__(self, be2: BinExport2, buf: bytes):
super().__init__(hashes=SampleHashes.from_bytes(buf))
self.be2: BinExport2 = be2
self.buf: bytes = buf
self.idx: BinExport2Index = BinExport2Index(self.be2)
self.analysis: BinExport2Analysis = BinExport2Analysis(self.be2, self.idx, self.buf)
address_space: AddressSpace = AddressSpace.from_buf(buf, self.analysis.base_address)
self.ctx: AnalysisContext = AnalysisContext(self.buf, self.be2, self.idx, self.analysis, address_space)
self.global_features: list[tuple[Feature, Address]] = []
self.global_features.extend(list(capa.features.extractors.common.extract_format(self.buf)))
self.global_features.extend(list(capa.features.extractors.common.extract_os(self.buf)))
self.global_features.extend(list(capa.features.extractors.common.extract_arch(self.buf)))
self.format: set[str] = set()
self.os: set[str] = set()
self.arch: set[str] = set()
for feature, _ in self.global_features:
assert isinstance(feature.value, str)
if isinstance(feature, Format):
self.format.add(feature.value)
elif isinstance(feature, OS):
self.os.add(feature.value)
elif isinstance(feature, Arch):
self.arch.add(feature.value)
else:
raise ValueError("unexpected global feature: %s", feature)
def get_base_address(self) -> AbsoluteVirtualAddress:
return AbsoluteVirtualAddress(self.analysis.base_address)
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
yield from self.global_features
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
yield from capa.features.extractors.binexport2.file.extract_features(self.be2, self.buf)
def get_functions(self) -> Iterator[FunctionHandle]:
for flow_graph_index, flow_graph in enumerate(self.be2.flow_graph):
entry_basic_block_index: int = flow_graph.entry_basic_block_index
flow_graph_address: int = self.idx.get_basic_block_address(entry_basic_block_index)
vertex_idx: int = self.idx.vertex_index_by_address[flow_graph_address]
be2_vertex: BinExport2.CallGraph.Vertex = self.be2.call_graph.vertex[vertex_idx]
# skip thunks
if capa.features.extractors.binexport2.helpers.is_vertex_type(
be2_vertex, BinExport2.CallGraph.Vertex.Type.THUNK
):
continue
yield FunctionHandle(
AbsoluteVirtualAddress(flow_graph_address),
inner=FunctionContext(self.ctx, flow_graph_index, self.format, self.os, self.arch),
)
def extract_function_features(self, fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
yield from capa.features.extractors.binexport2.function.extract_features(fh)
def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
fhi: FunctionContext = fh.inner
flow_graph_index: int = fhi.flow_graph_index
flow_graph: BinExport2.FlowGraph = self.be2.flow_graph[flow_graph_index]
for basic_block_index in flow_graph.basic_block_index:
basic_block_address: int = self.idx.get_basic_block_address(basic_block_index)
yield BBHandle(
address=AbsoluteVirtualAddress(basic_block_address),
inner=BasicBlockContext(basic_block_index),
)
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
yield from capa.features.extractors.binexport2.basicblock.extract_features(fh, bbh)
def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
bbi: BasicBlockContext = bbh.inner
basic_block: BinExport2.BasicBlock = self.be2.basic_block[bbi.basic_block_index]
for instruction_index, _, instruction_address in self.idx.basic_block_instructions(basic_block):
yield InsnHandle(
address=AbsoluteVirtualAddress(instruction_address),
inner=InstructionContext(instruction_index),
)
def extract_insn_features(
self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
yield from capa.features.extractors.binexport2.insn.extract_features(fh, bbh, ih)

View File

@@ -1,80 +0,0 @@
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import io
import logging
from typing import Iterator
import pefile
from elftools.elf.elffile import ELFFile
import capa.features.common
import capa.features.extractors.common
import capa.features.extractors.pefile
import capa.features.extractors.elffile
from capa.features.common import Feature
from capa.features.address import Address
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
logger = logging.getLogger(__name__)
def extract_file_export_names(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
if buf.startswith(capa.features.extractors.common.MATCH_PE):
pe: pefile.PE = pefile.PE(data=buf)
yield from capa.features.extractors.pefile.extract_file_export_names(pe)
elif buf.startswith(capa.features.extractors.common.MATCH_ELF):
elf: ELFFile = ELFFile(io.BytesIO(buf))
yield from capa.features.extractors.elffile.extract_file_export_names(elf)
else:
logger.warning("unsupported format")
def extract_file_import_names(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
if buf.startswith(capa.features.extractors.common.MATCH_PE):
pe: pefile.PE = pefile.PE(data=buf)
yield from capa.features.extractors.pefile.extract_file_import_names(pe)
elif buf.startswith(capa.features.extractors.common.MATCH_ELF):
elf: ELFFile = ELFFile(io.BytesIO(buf))
yield from capa.features.extractors.elffile.extract_file_import_names(elf)
else:
logger.warning("unsupported format")
def extract_file_section_names(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
if buf.startswith(capa.features.extractors.common.MATCH_PE):
pe: pefile.PE = pefile.PE(data=buf)
yield from capa.features.extractors.pefile.extract_file_section_names(pe)
elif buf.startswith(capa.features.extractors.common.MATCH_ELF):
elf: ELFFile = ELFFile(io.BytesIO(buf))
yield from capa.features.extractors.elffile.extract_file_section_names(elf)
else:
logger.warning("unsupported format")
def extract_file_strings(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
yield from capa.features.extractors.common.extract_file_strings(buf)
def extract_file_format(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
yield from capa.features.extractors.common.extract_format(buf)
def extract_features(be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
"""extract file features"""
for file_handler in FILE_HANDLERS:
for feature, addr in file_handler(be2, buf):
yield feature, addr
FILE_HANDLERS = (
extract_file_export_names,
extract_file_import_names,
extract_file_strings,
extract_file_section_names,
extract_file_format,
)

View File

@@ -1,72 +0,0 @@
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
from capa.features.file import FunctionName
from capa.features.common import Feature, Characteristic
from capa.features.address import Address, AbsoluteVirtualAddress
from capa.features.extractors import loops
from capa.features.extractors.binexport2 import BinExport2Index, FunctionContext
from capa.features.extractors.base_extractor import FunctionHandle
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
def extract_function_calls_to(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
be2: BinExport2 = fhi.ctx.be2
idx: BinExport2Index = fhi.ctx.idx
flow_graph_index: int = fhi.flow_graph_index
flow_graph_address: int = idx.flow_graph_address_by_index[flow_graph_index]
vertex_index: int = idx.vertex_index_by_address[flow_graph_address]
for caller_index in idx.callers_by_vertex_index[vertex_index]:
caller: BinExport2.CallGraph.Vertex = be2.call_graph.vertex[caller_index]
caller_address: int = caller.address
yield Characteristic("calls to"), AbsoluteVirtualAddress(caller_address)
def extract_function_loop(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
be2: BinExport2 = fhi.ctx.be2
flow_graph_index: int = fhi.flow_graph_index
flow_graph: BinExport2.FlowGraph = be2.flow_graph[flow_graph_index]
edges: list[tuple[int, int]] = []
for edge in flow_graph.edge:
edges.append((edge.source_basic_block_index, edge.target_basic_block_index))
if loops.has_loop(edges):
yield Characteristic("loop"), fh.address
def extract_function_name(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
be2: BinExport2 = fhi.ctx.be2
idx: BinExport2Index = fhi.ctx.idx
flow_graph_index: int = fhi.flow_graph_index
flow_graph_address: int = idx.flow_graph_address_by_index[flow_graph_index]
vertex_index: int = idx.vertex_index_by_address[flow_graph_address]
vertex: BinExport2.CallGraph.Vertex = be2.call_graph.vertex[vertex_index]
if vertex.HasField("mangled_name"):
yield FunctionName(vertex.mangled_name), fh.address
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
for func_handler in FUNCTION_HANDLERS:
for feature, addr in func_handler(fh):
yield feature, addr
FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_function_name)

View File

@@ -1,692 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import re
from typing import Union, Iterator, Optional
from collections import defaultdict
from dataclasses import dataclass
import capa.features.extractors.helpers
import capa.features.extractors.binexport2.helpers
from capa.features.common import ARCH_I386, ARCH_AMD64, ARCH_AARCH64
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
HAS_ARCH32 = {ARCH_I386}
HAS_ARCH64 = {ARCH_AARCH64, ARCH_AMD64}
HAS_ARCH_INTEL = {ARCH_I386, ARCH_AMD64}
HAS_ARCH_ARM = {ARCH_AARCH64}
def mask_immediate(arch: set[str], immediate: int) -> int:
if arch & HAS_ARCH64:
immediate &= 0xFFFFFFFFFFFFFFFF
elif arch & HAS_ARCH32:
immediate &= 0xFFFFFFFF
return immediate
def twos_complement(arch: set[str], immediate: int, default: Optional[int] = None) -> int:
if default is not None:
return capa.features.extractors.helpers.twos_complement(immediate, default)
elif arch & HAS_ARCH64:
return capa.features.extractors.helpers.twos_complement(immediate, 64)
elif arch & HAS_ARCH32:
return capa.features.extractors.helpers.twos_complement(immediate, 32)
return immediate
def is_address_mapped(be2: BinExport2, address: int) -> bool:
"""return True if the given address is mapped"""
sections_with_perms: Iterator[BinExport2.Section] = filter(lambda s: s.flag_r or s.flag_w or s.flag_x, be2.section)
return any(section.address <= address < section.address + section.size for section in sections_with_perms)
def is_vertex_type(vertex: BinExport2.CallGraph.Vertex, type_: BinExport2.CallGraph.Vertex.Type.ValueType) -> bool:
return vertex.HasField("type") and vertex.type == type_
# internal to `build_expression_tree`
# this is unstable: it is subject to change, so don't rely on it!
def _prune_expression_tree_references_to_tree_index(
expression_tree: list[list[int]],
tree_index: int,
):
# `i` is the index of the tree node that we'll search for `tree_index`
# if we remove `tree_index` from it, and it is now empty,
# then we'll need to prune references to `i`.
for i, tree_node in enumerate(expression_tree):
if tree_index in tree_node:
tree_node.remove(tree_index)
if len(tree_node) == 0:
# if the parent node is now empty,
# remove references to that parent node.
_prune_expression_tree_references_to_tree_index(expression_tree, i)
# internal to `build_expression_tree`
# this is unstable: it is subject to change, so don't rely on it!
def _prune_expression_tree_empty_shifts(
be2: BinExport2,
operand: BinExport2.Operand,
expression_tree: list[list[int]],
tree_index: int,
):
expression_index = operand.expression_index[tree_index]
expression = be2.expression[expression_index]
children_tree_indexes: list[int] = expression_tree[tree_index]
if expression.type == BinExport2.Expression.OPERATOR:
if len(children_tree_indexes) == 0 and expression.symbol in ("lsl", "lsr"):
# Ghidra may emit superfluous lsl nodes with no children.
# https://github.com/mandiant/capa/pull/2340/files#r1750003919
# Which is maybe: https://github.com/NationalSecurityAgency/ghidra/issues/6821#issuecomment-2295394697
#
# Which seems to be as if the shift wasn't there (shift of #0)
# so we want to remove references to this node from any parent nodes.
_prune_expression_tree_references_to_tree_index(expression_tree, tree_index)
return
for child_tree_index in children_tree_indexes:
_prune_expression_tree_empty_shifts(be2, operand, expression_tree, child_tree_index)
# internal to `build_expression_tree`
# this is unstable: it is subject to change, so don't rely on it!
def _fixup_expression_tree_references_to_tree_index(
expression_tree: list[list[int]],
existing_index: int,
new_index: int,
):
for tree_node in expression_tree:
for i, index in enumerate(tree_node):
if index == existing_index:
tree_node[i] = new_index
# internal to `build_expression_tree`
# this is unstable: it is subject to change, so don't rely on it!
def _fixup_expression_tree_lonely_commas(
be2: BinExport2,
operand: BinExport2.Operand,
expression_tree: list[list[int]],
tree_index: int,
):
expression_index = operand.expression_index[tree_index]
expression = be2.expression[expression_index]
children_tree_indexes: list[int] = expression_tree[tree_index]
if expression.type == BinExport2.Expression.OPERATOR:
if len(children_tree_indexes) == 1 and expression.symbol == ",":
existing_index = tree_index
new_index = children_tree_indexes[0]
_fixup_expression_tree_references_to_tree_index(expression_tree, existing_index, new_index)
for child_tree_index in children_tree_indexes:
_fixup_expression_tree_lonely_commas(be2, operand, expression_tree, child_tree_index)
# internal to `build_expression_tree`
# this is unstable: it is subject to change, so don't rely on it!
def _prune_expression_tree(
be2: BinExport2,
operand: BinExport2.Operand,
expression_tree: list[list[int]],
):
_prune_expression_tree_empty_shifts(be2, operand, expression_tree, 0)
_fixup_expression_tree_lonely_commas(be2, operand, expression_tree, 0)
# this is unstable: it is subject to change, so don't rely on it!
def _build_expression_tree(
be2: BinExport2,
operand: BinExport2.Operand,
) -> list[list[int]]:
# The reconstructed expression tree layout, linking parent nodes to their children.
#
# There is one list of integers for each expression in the operand.
# These integers are indexes of other expressions in the same operand,
# which are the children of that expression.
#
# So:
#
# [ [1, 3], [2], [], [4], [5], []]
#
# means the first expression has two children, at index 1 and 3,
# and the tree looks like:
#
# 0
# / \
# 1 3
# | |
# 2 4
# |
# 5
#
# Remember, these are the indices into the entries in operand.expression_index.
if len(operand.expression_index) == 0:
# Ghidra bug where empty operands (no expressions) may
# exist (see https://github.com/NationalSecurityAgency/ghidra/issues/6817)
return []
tree: list[list[int]] = []
for i, expression_index in enumerate(operand.expression_index):
children = []
# scan all subsequent expressions, looking for those that have parent_index == current.expression_index
for j, candidate_index in enumerate(operand.expression_index[i + 1 :]):
candidate = be2.expression[candidate_index]
if candidate.parent_index == expression_index:
children.append(i + j + 1)
tree.append(children)
_prune_expression_tree(be2, operand, tree)
return tree
def _fill_operand_expression_list(
be2: BinExport2,
operand: BinExport2.Operand,
expression_tree: list[list[int]],
tree_index: int,
expression_list: list[BinExport2.Expression],
):
"""
Walk the given expression tree and collect the expression nodes in-order.
"""
expression_index = operand.expression_index[tree_index]
expression = be2.expression[expression_index]
children_tree_indexes: list[int] = expression_tree[tree_index]
if expression.type == BinExport2.Expression.REGISTER:
assert len(children_tree_indexes) <= 1
expression_list.append(expression)
if len(children_tree_indexes) == 0:
return
elif len(children_tree_indexes) == 1:
# like for aarch64 with vector instructions, indicating vector data size:
#
# FADD V0.4S, V1.4S, V2.4S
#
# see: https://github.com/mandiant/capa/issues/2528
child_index = children_tree_indexes[0]
_fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
return
else:
raise NotImplementedError(len(children_tree_indexes))
elif expression.type == BinExport2.Expression.SYMBOL:
assert len(children_tree_indexes) <= 1
expression_list.append(expression)
if len(children_tree_indexes) == 0:
return
elif len(children_tree_indexes) == 1:
# like: v
# from: mov v0.D[0x1], x9
# |
# 0
# .
# |
# D
child_index = children_tree_indexes[0]
_fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
return
else:
raise NotImplementedError(len(children_tree_indexes))
elif expression.type == BinExport2.Expression.IMMEDIATE_INT:
assert len(children_tree_indexes) <= 1
expression_list.append(expression)
if len(children_tree_indexes) == 0:
return
elif len(children_tree_indexes) == 1:
# the ghidra exporter can produce some weird expressions,
# particularly for MSRs, like for:
#
# sreg(3, 0, c.0, c.4, 4)
#
# see: https://github.com/mandiant/capa/issues/2530
child_index = children_tree_indexes[0]
_fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
return
else:
raise NotImplementedError(len(children_tree_indexes))
elif expression.type == BinExport2.Expression.SIZE_PREFIX:
# like: b4
#
# We might want to use this occasionally, such as to disambiguate the
# size of MOVs into/out of memory. But I'm not sure when/where we need that yet.
#
# IDA spams this size prefix hint *everywhere*, so we can't rely on the exporter
# to provide it only when necessary.
assert len(children_tree_indexes) == 1
child_index = children_tree_indexes[0]
_fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
return
elif expression.type == BinExport2.Expression.OPERATOR:
if len(children_tree_indexes) == 1:
# prefix operator, like "ds:"
expression_list.append(expression)
child_index = children_tree_indexes[0]
_fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
return
elif len(children_tree_indexes) == 2:
# infix operator: like "+" in "ebp+10"
child_a = children_tree_indexes[0]
child_b = children_tree_indexes[1]
_fill_operand_expression_list(be2, operand, expression_tree, child_a, expression_list)
expression_list.append(expression)
_fill_operand_expression_list(be2, operand, expression_tree, child_b, expression_list)
return
elif len(children_tree_indexes) == 3:
# infix operator: like "+" in "ebp+ecx+10"
child_a = children_tree_indexes[0]
child_b = children_tree_indexes[1]
child_c = children_tree_indexes[2]
_fill_operand_expression_list(be2, operand, expression_tree, child_a, expression_list)
expression_list.append(expression)
_fill_operand_expression_list(be2, operand, expression_tree, child_b, expression_list)
expression_list.append(expression)
_fill_operand_expression_list(be2, operand, expression_tree, child_c, expression_list)
return
else:
raise NotImplementedError(len(children_tree_indexes))
elif expression.type == BinExport2.Expression.DEREFERENCE:
assert len(children_tree_indexes) == 1
expression_list.append(expression)
child_index = children_tree_indexes[0]
_fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
return
elif expression.type == BinExport2.Expression.IMMEDIATE_FLOAT:
raise NotImplementedError(expression.type)
else:
raise NotImplementedError(expression.type)
def get_operand_expressions(be2: BinExport2, op: BinExport2.Operand) -> list[BinExport2.Expression]:
tree = _build_expression_tree(be2, op)
expressions: list[BinExport2.Expression] = []
_fill_operand_expression_list(be2, op, tree, 0, expressions)
return expressions
def get_operand_register_expression(be2: BinExport2, operand: BinExport2.Operand) -> Optional[BinExport2.Expression]:
if len(operand.expression_index) == 1:
expression: BinExport2.Expression = be2.expression[operand.expression_index[0]]
if expression.type == BinExport2.Expression.REGISTER:
return expression
return None
def get_operand_immediate_expression(be2: BinExport2, operand: BinExport2.Operand) -> Optional[BinExport2.Expression]:
if len(operand.expression_index) == 1:
# - type: IMMEDIATE_INT
# immediate: 20588728364
# parent_index: 0
expression: BinExport2.Expression = be2.expression[operand.expression_index[0]]
if expression.type == BinExport2.Expression.IMMEDIATE_INT:
return expression
elif len(operand.expression_index) == 2:
# from IDA, which provides a size hint for every operand,
# we get the following pattern for immediate constants:
#
# - type: SIZE_PREFIX
# symbol: "b8"
# - type: IMMEDIATE_INT
# immediate: 20588728364
# parent_index: 0
expression0: BinExport2.Expression = be2.expression[operand.expression_index[0]]
expression1: BinExport2.Expression = be2.expression[operand.expression_index[1]]
if expression0.type == BinExport2.Expression.SIZE_PREFIX:
if expression1.type == BinExport2.Expression.IMMEDIATE_INT:
return expression1
return None
def get_instruction_mnemonic(be2: BinExport2, instruction: BinExport2.Instruction) -> str:
return be2.mnemonic[instruction.mnemonic_index].name.lower()
def get_instruction_operands(be2: BinExport2, instruction: BinExport2.Instruction) -> list[BinExport2.Operand]:
return [be2.operand[operand_index] for operand_index in instruction.operand_index]
def split_with_delimiters(s: str, delimiters: tuple[str, ...]) -> Iterator[str]:
"""
Splits a string by any of the provided delimiter characters,
including the delimiters in the results.
Args:
string: The string to split.
delimiters: A string containing the characters to use as delimiters.
"""
start = 0
for i, char in enumerate(s):
if char in delimiters:
yield s[start:i]
yield char
start = i + 1
if start < len(s):
yield s[start:]
BinExport2OperandPattern = Union[str, tuple[str, ...]]
@dataclass
class BinExport2InstructionPattern:
"""
This describes a way to match disassembled instructions, with mnemonics and operands.
You can specify constraints on the instruction, via:
- the mnemonics, like "mov",
- number of operands, and
- format of each operand, "[reg, reg, #int]".
During matching, you can also capture a single element, to see its concrete value.
For example, given the pattern:
mov reg0, #int0 ; capture int0
and the instruction:
mov eax, 1
Then the capture will contain the immediate integer 1.
This matcher uses the BinExport2 data layout under the hood.
"""
mnemonics: tuple[str, ...]
operands: tuple[Union[str, BinExport2OperandPattern], ...]
capture: Optional[str]
@classmethod
def from_str(cls, query: str):
"""
Parse a pattern string into a Pattern instance.
The supported syntax is like this:
br reg
br reg ; capture reg
br reg(stack) ; capture reg
br reg(not-stack) ; capture reg
mov reg0, reg1 ; capture reg0
adrp reg, #int ; capture #int
add reg, reg, #int ; capture #int
ldr reg0, [reg1] ; capture reg1
ldr|str reg, [reg, #int] ; capture #int
ldr|str reg, [reg(stack), #int] ; capture #int
ldr|str reg, [reg(not-stack), #int] ; capture #int
ldr|str reg, [reg, #int]! ; capture #int
ldr|str reg, [reg], #int ; capture #int
ldp|stp reg, reg, [reg, #int] ; capture #int
ldp|stp reg, reg, [reg, #int]! ; capture #int
ldp|stp reg, reg, [reg], #int ; capture #int
"""
#
# The implementation of the parser here is obviously ugly.
# Its handwritten and probably fragile. But since we don't
# expect this to be widely used, its probably ok.
# Don't hesitate to rewrite this if it becomes more important.
#
# Note that this doesn't have to be very performant.
# We expect these patterns to be parsed once upfront and then reused
# (globally at the module level?) rather than within any loop.
#
pattern, _, comment = query.strip().partition(";")
# we don't support fs: yet
assert ":" not in pattern
# from "capture #int" to "#int"
if comment:
comment = comment.strip()
assert comment.startswith("capture ")
capture = comment[len("capture ") :]
else:
capture = None
# from "ldr|str ..." to ["ldr", "str"]
pattern = pattern.strip()
mnemonic, _, rest = pattern.partition(" ")
mnemonics = mnemonic.split("|")
operands: list[Union[str, tuple[str, ...]]] = []
while rest:
rest = rest.strip()
if not rest.startswith("["):
# If its not a dereference, which looks like `[op, op, op, ...]`,
# then its a simple operand, which we can split by the next comma.
operand, _, rest = rest.partition(", ")
rest = rest.strip()
operands.append(operand)
else:
# This looks like a dereference, something like `[op, op, op, ...]`.
# Since these can't be nested, look for the next ] and then parse backwards.
deref_end = rest.index("]")
try:
deref_end = rest.index(", ", deref_end)
deref_end += len(", ")
except ValueError:
deref = rest
rest = ""
else:
deref = rest[:deref_end]
rest = rest[deref_end:]
rest = rest.strip()
deref = deref.rstrip(" ")
deref = deref.rstrip(",")
# like: [reg, #int]!
has_postindex_writeback = deref.endswith("!")
deref = deref.rstrip("!")
deref = deref.rstrip("]")
deref = deref.lstrip("[")
parts = tuple(split_with_delimiters(deref, (",", "+", "*")))
parts = tuple(s.strip() for s in parts)
# emit operands in this order to match
# how BinExport2 expressions are flatted
# by get_operand_expressions
if has_postindex_writeback:
operands.append(("!", "[") + parts)
else:
operands.append(("[",) + parts)
for operand in operands: # type: ignore
# Try to ensure we've parsed the operands correctly.
# This is just sanity checking.
for o in (operand,) if isinstance(operand, str) else operand:
# operands can look like:
# - reg
# - reg0
# - reg(stack)
# - reg0(stack)
# - reg(not-stack)
# - reg0(not-stack)
# - #int
# - #int0
# and a limited set of supported operators.
# use an inline regex so that its easy to read. not perf critical.
assert re.match(r"^(reg|#int)[0-9]?(\(stack\)|\(not-stack\))?$", o) or o in ("[", ",", "!", "+", "*")
return cls(tuple(mnemonics), tuple(operands), capture)
@dataclass
class MatchResult:
operand_index: int
expression_index: int
expression: BinExport2.Expression
def match(
self, mnemonic: str, operand_expressions: list[list[BinExport2.Expression]]
) -> Optional["BinExport2InstructionPattern.MatchResult"]:
"""
Match the given BinExport2 data against this pattern.
The BinExport2 expression tree must have been flattened, such as with
capa.features.extractors.binexport2.helpers.get_operand_expressions.
If there's a match, the captured Expression instance is returned.
Otherwise, you get None back.
"""
if mnemonic not in self.mnemonics:
return None
if len(self.operands) != len(operand_expressions):
return None
captured = None
for operand_index, found_expressions in enumerate(operand_expressions):
wanted_expressions = self.operands[operand_index]
# from `"reg"` to `("reg", )`
if isinstance(wanted_expressions, str):
wanted_expressions = (wanted_expressions,)
assert isinstance(wanted_expressions, tuple)
if len(wanted_expressions) != len(found_expressions):
return None
for expression_index, (wanted_expression, found_expression) in enumerate(
zip(wanted_expressions, found_expressions)
):
if wanted_expression.startswith("reg"):
if found_expression.type != BinExport2.Expression.REGISTER:
return None
if wanted_expression.endswith(")"):
if wanted_expression.endswith("(not-stack)"):
# intel 64: rsp, esp, sp,
# intel 32: ebp, ebp, bp
# arm: sp
register_name = found_expression.symbol.lower()
if register_name in ("rsp", "esp", "sp", "rbp", "ebp", "bp"):
return None
elif wanted_expression.endswith("(stack)"):
register_name = found_expression.symbol.lower()
if register_name not in ("rsp", "esp", "sp", "rbp", "ebp", "bp"):
return None
else:
raise ValueError("unexpected expression suffix", wanted_expression)
if self.capture == wanted_expression:
captured = BinExport2InstructionPattern.MatchResult(
operand_index, expression_index, found_expression
)
elif wanted_expression.startswith("#int"):
if found_expression.type != BinExport2.Expression.IMMEDIATE_INT:
return None
if self.capture == wanted_expression:
captured = BinExport2InstructionPattern.MatchResult(
operand_index, expression_index, found_expression
)
elif wanted_expression == "[":
if found_expression.type != BinExport2.Expression.DEREFERENCE:
return None
elif wanted_expression in (",", "!", "+", "*"):
if found_expression.type != BinExport2.Expression.OPERATOR:
return None
if found_expression.symbol != wanted_expression:
return None
else:
raise ValueError(found_expression)
if captured:
return captured
else:
# There were no captures, so
# return arbitrary non-None expression
return BinExport2InstructionPattern.MatchResult(operand_index, expression_index, found_expression)
class BinExport2InstructionPatternMatcher:
"""Index and match a collection of instruction patterns."""
def __init__(self, queries: list[BinExport2InstructionPattern]):
self.queries = queries
# shard the patterns by (mnemonic, #operands)
self._index: dict[tuple[str, int], list[BinExport2InstructionPattern]] = defaultdict(list)
for query in queries:
for mnemonic in query.mnemonics:
self._index[(mnemonic.lower(), len(query.operands))].append(query)
@classmethod
def from_str(cls, patterns: str):
return cls(
[
BinExport2InstructionPattern.from_str(line)
for line in filter(
lambda line: not line.startswith("#"), (line.strip() for line in patterns.split("\n"))
)
]
)
def match(
self, mnemonic: str, operand_expressions: list[list[BinExport2.Expression]]
) -> Optional[BinExport2InstructionPattern.MatchResult]:
queries = self._index.get((mnemonic.lower(), len(operand_expressions)), [])
for query in queries:
captured = query.match(mnemonic.lower(), operand_expressions)
if captured:
return captured
return None
def match_with_be2(
self, be2: BinExport2, instruction_index: int
) -> Optional[BinExport2InstructionPattern.MatchResult]:
instruction: BinExport2.Instruction = be2.instruction[instruction_index]
mnemonic: str = get_instruction_mnemonic(be2, instruction)
if (mnemonic.lower(), len(instruction.operand_index)) not in self._index:
# verify that we might have a hit before we realize the operand expression list
return None
operands = []
for operand_index in instruction.operand_index:
operands.append(get_operand_expressions(be2, be2.operand[operand_index]))
return self.match(mnemonic, operands)

View File

@@ -1,254 +0,0 @@
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
import capa.features.extractors.helpers
import capa.features.extractors.strings
import capa.features.extractors.binexport2.helpers
import capa.features.extractors.binexport2.arch.arm.insn
import capa.features.extractors.binexport2.arch.intel.insn
from capa.features.insn import API, Mnemonic
from capa.features.common import Bytes, String, Feature, Characteristic
from capa.features.address import Address, AbsoluteVirtualAddress
from capa.features.extractors.binexport2 import (
AddressSpace,
AnalysisContext,
BinExport2Index,
FunctionContext,
ReadMemoryError,
BinExport2Analysis,
InstructionContext,
)
from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
from capa.features.extractors.binexport2.helpers import HAS_ARCH_ARM, HAS_ARCH_INTEL
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
logger = logging.getLogger(__name__)
def extract_insn_api_features(fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
be2_index: BinExport2Index = fhi.ctx.idx
be2_analysis: BinExport2Analysis = fhi.ctx.analysis
insn: BinExport2.Instruction = be2.instruction[ii.instruction_index]
for addr in insn.call_target:
addr = be2_analysis.thunks.get(addr, addr)
if addr not in be2_index.vertex_index_by_address:
# disassembler did not define function at address
logger.debug("0x%x is not a vertex", addr)
continue
vertex_idx: int = be2_index.vertex_index_by_address[addr]
vertex: BinExport2.CallGraph.Vertex = be2.call_graph.vertex[vertex_idx]
if not capa.features.extractors.binexport2.helpers.is_vertex_type(
vertex, BinExport2.CallGraph.Vertex.Type.IMPORTED
):
continue
if not vertex.HasField("mangled_name"):
logger.debug("vertex %d does not have mangled_name", vertex_idx)
continue
api_name: str = vertex.mangled_name
for name in capa.features.extractors.helpers.generate_symbols("", api_name):
yield API(name), ih.address
def extract_insn_number_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
if fhi.arch & HAS_ARCH_INTEL:
yield from capa.features.extractors.binexport2.arch.intel.insn.extract_insn_number_features(fh, bbh, ih)
elif fhi.arch & HAS_ARCH_ARM:
yield from capa.features.extractors.binexport2.arch.arm.insn.extract_insn_number_features(fh, bbh, ih)
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
ctx: AnalysisContext = fhi.ctx
be2: BinExport2 = ctx.be2
idx: BinExport2Index = ctx.idx
address_space: AddressSpace = ctx.address_space
instruction_index: int = ii.instruction_index
if instruction_index in idx.string_reference_index_by_source_instruction_index:
# disassembler already identified string reference from instruction
return
reference_addresses: list[int] = []
if instruction_index in idx.data_reference_index_by_source_instruction_index:
for data_reference_index in idx.data_reference_index_by_source_instruction_index[instruction_index]:
data_reference: BinExport2.DataReference = be2.data_reference[data_reference_index]
data_reference_address: int = data_reference.address
if data_reference_address in idx.insn_address_by_index:
# appears to be code
continue
reference_addresses.append(data_reference_address)
for reference_address in reference_addresses:
try:
# if at end of segment then there might be an overrun here.
buf: bytes = address_space.read_memory(reference_address, 0x100)
except ReadMemoryError:
logger.debug("failed to read memory: 0x%x", reference_address)
continue
if capa.features.extractors.helpers.all_zeros(buf):
continue
is_string: bool = False
# note: we *always* break after the first iteration
for s in capa.features.extractors.strings.extract_ascii_strings(buf):
if s.offset != 0:
break
yield String(s.s), ih.address
is_string = True
break
# note: we *always* break after the first iteration
for s in capa.features.extractors.strings.extract_unicode_strings(buf):
if s.offset != 0:
break
yield String(s.s), ih.address
is_string = True
break
if not is_string:
yield Bytes(buf), ih.address
def extract_insn_string_features(
fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
idx: BinExport2Index = fhi.ctx.idx
instruction_index: int = ii.instruction_index
if instruction_index in idx.string_reference_index_by_source_instruction_index:
for string_reference_index in idx.string_reference_index_by_source_instruction_index[instruction_index]:
string_reference: BinExport2.Reference = be2.string_reference[string_reference_index]
string_index: int = string_reference.string_table_index
string: str = be2.string_table[string_index]
yield String(string), ih.address
def extract_insn_offset_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
if fhi.arch & HAS_ARCH_INTEL:
yield from capa.features.extractors.binexport2.arch.intel.insn.extract_insn_offset_features(fh, bbh, ih)
elif fhi.arch & HAS_ARCH_ARM:
yield from capa.features.extractors.binexport2.arch.arm.insn.extract_insn_offset_features(fh, bbh, ih)
def extract_insn_nzxor_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
if fhi.arch & HAS_ARCH_INTEL:
yield from capa.features.extractors.binexport2.arch.intel.insn.extract_insn_nzxor_characteristic_features(
fh, bbh, ih
)
elif fhi.arch & HAS_ARCH_ARM:
yield from capa.features.extractors.binexport2.arch.arm.insn.extract_insn_nzxor_characteristic_features(
fh, bbh, ih
)
def extract_insn_mnemonic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
mnemonic: BinExport2.Mnemonic = be2.mnemonic[instruction.mnemonic_index]
mnemonic_name: str = mnemonic.name.lower()
yield Mnemonic(mnemonic_name), ih.address
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
"""extract functions calls from features
most relevant at the function scope;
however, its most efficient to extract at the instruction scope.
"""
fhi: FunctionContext = fh.inner
ii: InstructionContext = ih.inner
be2: BinExport2 = fhi.ctx.be2
instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
for call_target_address in instruction.call_target:
addr: AbsoluteVirtualAddress = AbsoluteVirtualAddress(call_target_address)
yield Characteristic("calls from"), addr
if fh.address == addr:
yield Characteristic("recursive call"), addr
def extract_function_indirect_call_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
fhi: FunctionContext = fh.inner
if fhi.arch & HAS_ARCH_INTEL:
yield from capa.features.extractors.binexport2.arch.intel.insn.extract_function_indirect_call_characteristic_features(
fh, bbh, ih
)
elif fhi.arch & HAS_ARCH_ARM:
yield from capa.features.extractors.binexport2.arch.arm.insn.extract_function_indirect_call_characteristic_features(
fh, bbh, ih
)
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[tuple[Feature, Address]]:
"""extract instruction features"""
for inst_handler in INSTRUCTION_HANDLERS:
for feature, ea in inst_handler(f, bbh, insn):
yield feature, ea
INSTRUCTION_HANDLERS = (
extract_insn_api_features,
extract_insn_number_features,
extract_insn_bytes_features,
extract_insn_string_features,
extract_insn_offset_features,
extract_insn_nzxor_characteristic_features,
extract_insn_mnemonic_features,
extract_function_calls_from,
extract_function_indirect_call_characteristic_features,
)

View File

@@ -5,25 +5,115 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
import string
from typing import Tuple, Iterator
from binaryninja import Function
from binaryninja import BasicBlock as BinjaBasicBlock
from binaryninja import (
BinaryView,
SymbolType,
RegisterValueType,
VariableSourceType,
MediumLevelILOperation,
MediumLevelILBasicBlock,
MediumLevelILInstruction,
)
from capa.features.common import Feature, Characteristic
from capa.features.address import Address
from capa.features.basicblock import BasicBlock
from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def get_printable_len_ascii(s: bytes) -> int:
"""Return string length if all operand bytes are ascii or utf16-le printable"""
count = 0
for c in s:
if c == 0:
return count
if c < 127 and chr(c) in string.printable:
count += 1
return count
def get_printable_len_wide(s: bytes) -> int:
"""Return string length if all operand bytes are ascii or utf16-le printable"""
if all(c == 0x00 for c in s[1::2]):
return get_printable_len_ascii(s[::2])
return 0
def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
bv: BinaryView = f.view
if il.operation != MediumLevelILOperation.MLIL_CALL:
return 0
target = il.dest
if target.operation not in [MediumLevelILOperation.MLIL_CONST, MediumLevelILOperation.MLIL_CONST_PTR]:
return 0
addr = target.value.value
sym = bv.get_symbol_at(addr)
if not sym or sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.SymbolicFunctionSymbol]:
return 0
if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
return 0
if len(il.params) < 2:
return 0
dest = il.params[0]
if dest.operation in [MediumLevelILOperation.MLIL_ADDRESS_OF, MediumLevelILOperation.MLIL_VAR]:
var = dest.src
else:
return 0
if var.source_type != VariableSourceType.StackVariableSourceType:
return 0
src = il.params[1]
if src.value.type != RegisterValueType.ConstantDataAggregateValue:
return 0
s = f.get_constant_data(RegisterValueType.ConstantDataAggregateValue, src.value.value)
return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
"""check basic block for stackstring indicators
true if basic block contains enough moves of constant bytes to the stack
"""
count = 0
for il in bb:
count += get_stack_string_len(f, il)
if count > MIN_STACKSTRING_LEN:
return True
return False
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract stackstring indicators from basic block"""
bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
if bb[1] is not None and bb_contains_stackstring(fh.inner, bb[1]):
yield Characteristic("stack string"), bbh.address
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract tight loop indicators from a basic block"""
bb: BinjaBasicBlock = bbh.inner
for edge in bb.outgoing_edges:
if edge.target.start == bb.start:
bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
for edge in bb[0].outgoing_edges:
if edge.target.start == bb[0].start:
yield Characteristic("tight loop"), bbh.address
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract basic block features"""
for bb_handler in BASIC_BLOCK_HANDLERS:
for feature, addr in bb_handler(fh, bbh):
@@ -31,4 +121,7 @@ def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Featur
yield BasicBlock(), bbh.address
BASIC_BLOCK_HANDLERS = (extract_bb_tight_loop,)
BASIC_BLOCK_HANDLERS = (
extract_bb_tight_loop,
extract_bb_stackstring,
)

View File

@@ -5,7 +5,7 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
from typing import List, Tuple, Iterator
import binaryninja as binja
@@ -28,9 +28,9 @@ from capa.features.extractors.base_extractor import (
class BinjaFeatureExtractor(StaticFeatureExtractor):
def __init__(self, bv: binja.BinaryView):
super().__init__(hashes=SampleHashes.from_bytes(bv.file.raw.read(0, bv.file.raw.length)))
super().__init__(hashes=SampleHashes.from_bytes(bv.file.raw.read(0, len(bv.file.raw))))
self.bv = bv
self.global_features: list[tuple[Feature, Address]] = []
self.global_features: List[Tuple[Feature, Address]] = []
self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv))
self.global_features.extend(capa.features.extractors.binja.global_.extract_os(self.bv))
self.global_features.extend(capa.features.extractors.binja.global_.extract_arch(self.bv))
@@ -48,24 +48,31 @@ class BinjaFeatureExtractor(StaticFeatureExtractor):
for f in self.bv.functions:
yield FunctionHandle(address=AbsoluteVirtualAddress(f.start), inner=f)
def extract_function_features(self, fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.binja.function.extract_features(fh)
def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
f: binja.Function = fh.inner
for bb in f.basic_blocks:
yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=bb)
# Set up a MLIL basic block dict look up to associate the disassembly basic block with its MLIL basic block
mlil_lookup = {}
for mlil_bb in f.mlil.basic_blocks:
mlil_lookup[mlil_bb.source_block.start] = mlil_bb
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
for bb in f.basic_blocks:
mlil_bb = mlil_lookup.get(bb.start)
yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=(bb, mlil_bb))
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.binja.basicblock.extract_features(fh, bbh)
def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
import capa.features.extractors.binja.helpers as binja_helpers
bb: binja.BasicBlock = bbh.inner
addr = bb.start
bb: Tuple[binja.BasicBlock, binja.MediumLevelILBasicBlock] = bbh.inner
addr = bb[0].start
for text, length in bb:
for text, length in bb[0]:
insn = binja_helpers.DisassemblyInstruction(addr, length, text)
yield InsnHandle(address=AbsoluteVirtualAddress(addr), inner=insn)
addr += length

View File

@@ -5,7 +5,9 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
import struct
from typing import Tuple, Iterator
from binaryninja import Segment, BinaryView, SymbolType, SymbolBinding
@@ -13,42 +15,64 @@ import capa.features.extractors.common
import capa.features.extractors.helpers
import capa.features.extractors.strings
from capa.features.file import Export, Import, Section, FunctionName
from capa.features.common import (
FORMAT_PE,
FORMAT_ELF,
FORMAT_SC32,
FORMAT_SC64,
FORMAT_BINJA_DB,
Format,
String,
Feature,
Characteristic,
)
from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
from capa.features.extractors.binja.helpers import read_c_string, unmangle_c_name
def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[tuple[Feature, Address]]:
"""check segment for embedded PE"""
start = 0
if bv.view_type == "PE" and seg.start == bv.start:
# If this is the first segment of the binary, skip the first bytes.
# Otherwise, there will always be a matched PE at the start of the binaryview.
def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[int, int]]:
"""check segment for embedded PE
adapted for binja from:
https://github.com/vivisect/vivisect/blob/7be4037b1cecc4551b397f840405a1fc606f9b53/PE/carve.py#L19
"""
mz_xor = [
(
capa.features.extractors.helpers.xor_static(b"MZ", i),
capa.features.extractors.helpers.xor_static(b"PE", i),
i,
)
for i in range(256)
]
todo = []
# If this is the first segment of the binary, skip the first bytes. Otherwise, there will always be a matched
# PE at the start of the binaryview.
start = seg.start
if bv.view_type == "PE" and start == bv.start:
start += 1
buf = bv.read(seg.start, seg.length)
for mzx, pex, i in mz_xor:
for off, _ in bv.find_all_data(start, seg.end, mzx):
todo.append((off, mzx, pex, i))
for offset, _ in capa.features.extractors.helpers.carve_pe(buf, start):
yield Characteristic("embedded pe"), FileOffsetAddress(seg.start + offset)
while len(todo):
off, mzx, pex, i = todo.pop()
# The MZ header has one field we will check e_lfanew is at 0x3c
e_lfanew = off + 0x3C
if seg.end < (e_lfanew + 4):
continue
newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(bv.read(e_lfanew, 4), i))[0]
peoff = off + newoff
if seg.end < (peoff + 2):
continue
if bv.read(peoff, 2) == pex:
yield off, i
def extract_file_embedded_pe(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
def extract_file_embedded_pe(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
"""extract embedded PE features"""
for seg in bv.segments:
yield from check_segment_for_pe(bv, seg)
for ea, _ in check_segment_for_pe(bv, seg):
yield Characteristic("embedded pe"), FileOffsetAddress(ea)
def extract_file_export_names(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
"""extract function exports"""
for sym in bv.get_symbols_of_type(SymbolType.FunctionSymbol) + bv.get_symbols_of_type(SymbolType.DataSymbol):
if sym.binding in [SymbolBinding.GlobalBinding, SymbolBinding.WeakBinding]:
@@ -82,7 +106,7 @@ def extract_file_export_names(bv: BinaryView) -> Iterator[tuple[Feature, Address
yield Characteristic("forwarded export"), AbsoluteVirtualAddress(sym.address)
def extract_file_import_names(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
"""extract function imports
1. imports by ordinal:
@@ -106,19 +130,19 @@ def extract_file_import_names(bv: BinaryView) -> Iterator[tuple[Feature, Address
yield Import(name), addr
def extract_file_section_names(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
def extract_file_section_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
"""extract section names"""
for name, section in bv.sections.items():
yield Section(name), AbsoluteVirtualAddress(section.start)
def extract_file_strings(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
def extract_file_strings(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
"""extract ASCII and UTF-16 LE strings"""
for s in bv.strings:
yield String(s.value), FileOffsetAddress(s.start)
def extract_file_function_names(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
def extract_file_function_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
"""
extract the names of statically-linked library functions.
"""
@@ -137,22 +161,12 @@ def extract_file_function_names(bv: BinaryView) -> Iterator[tuple[Feature, Addre
yield FunctionName(name[1:]), sym.address
def extract_file_format(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
if bv.file.database is not None:
yield Format(FORMAT_BINJA_DB), NO_ADDRESS
def extract_file_format(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
view_type = bv.view_type
if view_type in ["PE", "COFF"]:
yield Format(FORMAT_PE), NO_ADDRESS
elif view_type == "ELF":
yield Format(FORMAT_ELF), NO_ADDRESS
elif view_type == "Mapped":
if bv.arch.name == "x86":
yield Format(FORMAT_SC32), NO_ADDRESS
elif bv.arch.name == "x86_64":
yield Format(FORMAT_SC64), NO_ADDRESS
else:
raise NotImplementedError(f"unexpected raw file with arch: {bv.arch}")
elif view_type == "Raw":
# no file type to return when processing a binary file, but we want to continue processing
return
@@ -160,7 +174,7 @@ def extract_file_format(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
raise NotImplementedError(f"unexpected file format: {view_type}")
def extract_features(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
def extract_features(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
"""extract file features"""
for file_handler in FILE_HANDLERS:
for feature, addr in file_handler(bv):

View File

@@ -5,175 +5,31 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import os
import sys
import logging
import subprocess
import importlib.util
from typing import Optional
from pathlib import Path
logger = logging.getLogger(__name__)
# When the script gets executed as a standalone executable (via PyInstaller), `import binaryninja` does not work because
# we have excluded the binaryninja module in `pyinstaller.spec`. The trick here is to call the system Python and try
# to find out the path of the binaryninja module that has been installed.
# Note, including the binaryninja module in the `pyinstaller.spec` would not work, since the binaryninja module tries to
# find the binaryninja core e.g., `libbinaryninjacore.dylib`, using a relative path. And this does not work when the
# binaryninja module is extracted by the PyInstaller.
CODE = r"""
code = r"""
from pathlib import Path
from importlib import util
spec = util.find_spec('binaryninja')
if spec is not None:
if len(spec.submodule_search_locations) > 0:
path = Path(spec.submodule_search_locations[0])
# encode the path with utf8 then convert to hex, make sure it can be read and restored properly
print(str(path.parent).encode('utf8').hex())
path = Path(spec.submodule_search_locations[0])
# encode the path with utf8 then convert to hex, make sure it can be read and restored properly
print(str(path.parent).encode('utf8').hex())
"""
def find_binaryninja_path_via_subprocess() -> Optional[Path]:
raw_output = subprocess.check_output(["python", "-c", CODE]).decode("ascii").strip()
output = bytes.fromhex(raw_output).decode("utf8")
if not output.strip():
return None
return Path(output)
def get_desktop_entry(name: str) -> Optional[Path]:
"""
Find the path for the given XDG Desktop Entry name.
Like:
>> get_desktop_entry("com.vector35.binaryninja.desktop")
Path("~/.local/share/applications/com.vector35.binaryninja.desktop")
"""
assert sys.platform in ("linux", "linux2")
assert name.endswith(".desktop")
data_dirs = os.environ.get("XDG_DATA_DIRS", "/usr/share") + f":{Path.home()}/.local/share"
for data_dir in data_dirs.split(":"):
applications = Path(data_dir) / "applications"
for application in applications.glob("*.desktop"):
if application.name == name:
return application
return None
def get_binaryninja_path(desktop_entry: Path) -> Optional[Path]:
# from: Exec=/home/wballenthin/software/binaryninja/binaryninja %u
# to: /home/wballenthin/software/binaryninja/
for line in desktop_entry.read_text(encoding="utf-8").splitlines():
if not line.startswith("Exec="):
continue
if not line.endswith("binaryninja %u"):
continue
binaryninja_path = Path(line[len("Exec=") : -len("binaryninja %u")])
if not binaryninja_path.exists():
return None
return binaryninja_path
return None
def validate_binaryninja_path(binaryninja_path: Path) -> bool:
if not binaryninja_path:
return False
module_path = binaryninja_path / "python"
if not module_path.is_dir():
return False
if not (module_path / "binaryninja" / "__init__.py").is_file():
return False
return True
def find_binaryninja() -> Optional[Path]:
binaryninja_path = find_binaryninja_path_via_subprocess()
if not binaryninja_path or not validate_binaryninja_path(binaryninja_path):
if sys.platform == "linux" or sys.platform == "linux2":
# ok
logger.debug("detected OS: linux")
elif sys.platform == "darwin":
logger.warning("unsupported platform to find Binary Ninja: %s", sys.platform)
return None
elif sys.platform == "win32":
logger.warning("unsupported platform to find Binary Ninja: %s", sys.platform)
return None
else:
logger.warning("unsupported platform to find Binary Ninja: %s", sys.platform)
return None
desktop_entry = get_desktop_entry("com.vector35.binaryninja.desktop")
if not desktop_entry:
logger.debug("failed to find Binary Ninja application")
return None
logger.debug("found Binary Ninja application: %s", desktop_entry)
binaryninja_path = get_binaryninja_path(desktop_entry)
if not binaryninja_path:
logger.debug("failed to determine Binary Ninja installation path")
return None
if not validate_binaryninja_path(binaryninja_path):
logger.debug("failed to validate Binary Ninja installation")
return None
logger.debug("found Binary Ninja installation: %s", binaryninja_path)
return binaryninja_path / "python"
def is_binaryninja_installed() -> bool:
"""Is the binaryninja module ready to import?"""
try:
return importlib.util.find_spec("binaryninja") is not None
except ModuleNotFoundError:
return False
def has_binaryninja() -> bool:
if is_binaryninja_installed():
logger.debug("found installed Binary Ninja API")
return True
logger.debug("Binary Ninja API not installed, searching...")
binaryninja_path = find_binaryninja()
if not binaryninja_path:
logger.debug("failed to find Binary Ninja installation")
logger.debug("found Binary Ninja API: %s", binaryninja_path)
return binaryninja_path is not None
def load_binaryninja() -> bool:
try:
import binaryninja
return True
except ImportError:
binaryninja_path = find_binaryninja()
if not binaryninja_path:
return False
sys.path.append(binaryninja_path.absolute().as_posix())
try:
import binaryninja # noqa: F401 unused import
return True
except ImportError:
return False
def find_binja_path() -> Path:
raw_output = subprocess.check_output(["python", "-c", code]).decode("ascii").strip()
return Path(bytes.fromhex(raw_output).decode("utf8"))
if __name__ == "__main__":
print(find_binaryninja_path_via_subprocess())
print(find_binja_path())

View File

@@ -5,28 +5,14 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import string
from typing import Iterator
from typing import Tuple, Iterator
from binaryninja import (
Function,
BinaryView,
SymbolType,
ILException,
RegisterValueType,
VariableSourceType,
LowLevelILOperation,
MediumLevelILOperation,
MediumLevelILBasicBlock,
MediumLevelILInstruction,
)
from binaryninja import Function, BinaryView, SymbolType, RegisterValueType, LowLevelILOperation
from capa.features.file import FunctionName
from capa.features.common import Feature, Characteristic
from capa.features.address import Address, AbsoluteVirtualAddress
from capa.features.extractors import loops
from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
from capa.features.extractors.binja.helpers import get_llil_instr_at_addr
from capa.features.extractors.base_extractor import FunctionHandle
@@ -38,7 +24,7 @@ def extract_function_calls_to(fh: FunctionHandle):
# Everything that is a code reference to the current function is considered a caller, which actually includes
# many other references that are NOT a caller. For example, an instruction `push function_start` will also be
# considered a caller to the function
llil = get_llil_instr_at_addr(func.view, caller.address)
llil = caller.llil
if (llil is None) or llil.operation not in [
LowLevelILOperation.LLIL_CALL,
LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
@@ -47,13 +33,14 @@ def extract_function_calls_to(fh: FunctionHandle):
]:
continue
if llil.dest.operation not in [
LowLevelILOperation.LLIL_CONST,
LowLevelILOperation.LLIL_CONST_PTR,
if llil.dest.value.type not in [
RegisterValueType.ImportedAddressValue,
RegisterValueType.ConstantValue,
RegisterValueType.ConstantPointerValue,
]:
continue
address = llil.dest.constant
address = llil.dest.value.value
if address != func.start:
continue
@@ -108,103 +95,10 @@ def extract_function_name(fh: FunctionHandle):
yield FunctionName(name[1:]), sym.address
def get_printable_len_ascii(s: bytes) -> int:
"""Return string length if all operand bytes are ascii or utf16-le printable"""
count = 0
for c in s:
if c == 0:
return count
if c < 127 and chr(c) in string.printable:
count += 1
return count
def get_printable_len_wide(s: bytes) -> int:
"""Return string length if all operand bytes are ascii or utf16-le printable"""
if all(c == 0x00 for c in s[1::2]):
return get_printable_len_ascii(s[::2])
return 0
def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
bv: BinaryView = f.view
if il.operation != MediumLevelILOperation.MLIL_CALL:
return 0
target = il.dest
if target.operation not in [MediumLevelILOperation.MLIL_CONST, MediumLevelILOperation.MLIL_CONST_PTR]:
return 0
addr = target.value.value
sym = bv.get_symbol_at(addr)
if not sym or sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.SymbolicFunctionSymbol]:
return 0
if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
return 0
if len(il.params) < 2:
return 0
dest = il.params[0]
if dest.operation in [MediumLevelILOperation.MLIL_ADDRESS_OF, MediumLevelILOperation.MLIL_VAR]:
var = dest.src
else:
return 0
if var.source_type != VariableSourceType.StackVariableSourceType:
return 0
src = il.params[1]
if src.value.type != RegisterValueType.ConstantDataAggregateValue:
return 0
s = f.get_constant_data(RegisterValueType.ConstantDataAggregateValue, src.value.value)
return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
"""check basic block for stackstring indicators
true if basic block contains enough moves of constant bytes to the stack
"""
count = 0
for il in bb:
count += get_stack_string_len(f, il)
if count > MIN_STACKSTRING_LEN:
return True
return False
def extract_stackstring(fh: FunctionHandle):
"""extract stackstring indicators"""
func: Function = fh.inner
bv: BinaryView = func.view
if bv is None:
return
try:
mlil = func.mlil
except ILException:
return
for block in mlil.basic_blocks:
if bb_contains_stackstring(func, block):
yield Characteristic("stack string"), block.source_block.start
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
for func_handler in FUNCTION_HANDLERS:
for feature, addr in func_handler(fh):
yield feature, addr
FUNCTION_HANDLERS = (
extract_function_calls_to,
extract_function_loop,
extract_recursive_call,
extract_function_name,
extract_stackstring,
)
FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call, extract_function_name)

View File

@@ -6,7 +6,7 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from typing import Tuple, Iterator
from binaryninja import BinaryView
@@ -16,7 +16,7 @@ from capa.features.address import NO_ADDRESS, Address
logger = logging.getLogger(__name__)
def extract_os(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
def extract_os(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
name = bv.platform.name
if "-" in name:
name = name.split("-")[0]
@@ -45,7 +45,7 @@ def extract_os(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
return
def extract_arch(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
def extract_arch(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
arch = bv.arch.name
if arch == "x86_64":
yield Arch(ARCH_AMD64), NO_ADDRESS

View File

@@ -6,10 +6,10 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import re
from typing import Callable, Optional
from typing import List, Callable
from dataclasses import dataclass
from binaryninja import BinaryView, LowLevelILFunction, LowLevelILInstruction
from binaryninja import BinaryView, LowLevelILInstruction
from binaryninja.architecture import InstructionTextToken
@@ -17,7 +17,7 @@ from binaryninja.architecture import InstructionTextToken
class DisassemblyInstruction:
address: int
length: int
text: list[InstructionTextToken]
text: List[InstructionTextToken]
LLIL_VISITOR = Callable[[LowLevelILInstruction, LowLevelILInstruction, int], bool]
@@ -54,7 +54,7 @@ def unmangle_c_name(name: str) -> str:
def read_c_string(bv: BinaryView, offset: int, max_len: int) -> str:
s: list[str] = []
s: List[str] = []
while len(s) < max_len:
try:
c = bv.read(offset + len(s), 1)[0]
@@ -67,13 +67,3 @@ def read_c_string(bv: BinaryView, offset: int, max_len: int) -> str:
s.append(chr(c))
return "".join(s)
def get_llil_instr_at_addr(bv: BinaryView, addr: int) -> Optional[LowLevelILInstruction]:
arch = bv.arch
buffer = bv.read(addr, arch.max_instr_length)
llil = LowLevelILFunction(arch=arch)
llil.current_address = addr
if arch.get_instruction_low_level_il(buffer, addr, llil) == 0:
return None
return llil[0]

View File

@@ -5,7 +5,7 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Any, Iterator, Optional
from typing import Any, List, Tuple, Iterator, Optional
from binaryninja import Function
from binaryninja import BasicBlock as BinjaBasicBlock
@@ -23,7 +23,7 @@ import capa.features.extractors.helpers
from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
from capa.features.address import Address, AbsoluteVirtualAddress
from capa.features.extractors.binja.helpers import DisassemblyInstruction, visit_llil_exprs, get_llil_instr_at_addr
from capa.features.extractors.binja.helpers import DisassemblyInstruction, visit_llil_exprs
from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
# security cookie checks may perform non-zeroing XORs, these are expected within a certain
@@ -36,27 +36,35 @@ SECURITY_COOKIE_BYTES_DELTA = 0x40
# 2. The function must only make one call/jump to another address
# If the function being checked is a stub function, returns the target address. Otherwise, return None.
def is_stub_function(bv: BinaryView, addr: int) -> Optional[int]:
llil = get_llil_instr_at_addr(bv, addr)
if llil is None or llil.operation not in [
LowLevelILOperation.LLIL_CALL,
LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
LowLevelILOperation.LLIL_JUMP,
LowLevelILOperation.LLIL_TAILCALL,
]:
return None
funcs = bv.get_functions_at(addr)
for func in funcs:
if len(func.basic_blocks) != 1:
continue
# The LLIL instruction retrieved by `get_llil_instr_at_addr` did not go through a full analysis, so we cannot check
# `llil.dest.value.type` here
if llil.dest.operation not in [
LowLevelILOperation.LLIL_CONST,
LowLevelILOperation.LLIL_CONST_PTR,
]:
return None
call_count = 0
call_target = None
for il in func.llil.instructions:
if il.operation in [
LowLevelILOperation.LLIL_CALL,
LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
LowLevelILOperation.LLIL_JUMP,
LowLevelILOperation.LLIL_TAILCALL,
]:
call_count += 1
if il.dest.value.type in [
RegisterValueType.ImportedAddressValue,
RegisterValueType.ConstantValue,
RegisterValueType.ConstantPointerValue,
]:
call_target = il.dest.value.value
return llil.dest.constant
if call_count == 1 and call_target is not None:
return call_target
return None
def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction API features
@@ -115,7 +123,7 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
def extract_insn_number_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction number features
example:
@@ -123,7 +131,7 @@ def extract_insn_number_features(
"""
func: Function = fh.inner
results: list[tuple[Any[Number, OperandNumber], Address]] = []
results: List[Tuple[Any[Number, OperandNumber], Address]] = []
def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
if il.operation == LowLevelILOperation.LLIL_LOAD:
@@ -154,7 +162,7 @@ def extract_insn_number_features(
yield from results
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""
parse referenced byte sequences
example:
@@ -201,7 +209,7 @@ def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
def extract_insn_string_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction string features
@@ -258,7 +266,7 @@ def extract_insn_string_features(
def extract_insn_offset_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction structure offset features
@@ -267,7 +275,7 @@ def extract_insn_offset_features(
"""
func: Function = fh.inner
results: list[tuple[Any[Offset, OperandOffset], Address]] = []
results: List[Tuple[Any[Offset, OperandOffset], Address]] = []
address_size = func.view.arch.address_size * 8
def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
@@ -345,7 +353,7 @@ def is_nzxor_stack_cookie(f: Function, bb: BinjaBasicBlock, llil: LowLevelILInst
def extract_insn_nzxor_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction non-zeroing XOR instruction
ignore expected non-zeroing XORs, e.g. security cookies
@@ -359,7 +367,7 @@ def extract_insn_nzxor_characteristic_features(
# e.g., <llil: eax = 0>, (LLIL_SET_REG). So we do not need to check whether the two operands are the same.
if il.operation == LowLevelILOperation.LLIL_XOR:
# Exclude cases related to the stack cookie
if is_nzxor_stack_cookie(fh.inner, bbh.inner, il):
if is_nzxor_stack_cookie(fh.inner, bbh.inner[0], il):
return False
results.append((Characteristic("nzxor"), ih.address))
return False
@@ -374,7 +382,7 @@ def extract_insn_nzxor_characteristic_features(
def extract_insn_mnemonic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction mnemonic features"""
insn: DisassemblyInstruction = ih.inner
yield Mnemonic(insn.text[0].text), ih.address
@@ -382,7 +390,7 @@ def extract_insn_mnemonic_features(
def extract_insn_obfs_call_plus_5_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse call $+5 instruction from the given instruction.
"""
@@ -393,7 +401,7 @@ def extract_insn_obfs_call_plus_5_characteristic_features(
def extract_insn_peb_access_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction peb access
fs:[0x30] on x86, gs:[0x60] on x64
@@ -436,7 +444,7 @@ def extract_insn_peb_access_characteristic_features(
def extract_insn_segment_access_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction fs or gs access"""
func: Function = fh.inner
@@ -463,7 +471,7 @@ def extract_insn_segment_access_features(
def extract_insn_cross_section_cflow(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""inspect the instruction for a CALL or JMP that crosses section boundaries"""
func: Function = fh.inner
bv: BinaryView = func.view
@@ -483,7 +491,7 @@ def extract_insn_cross_section_cflow(
yield Characteristic("cross section flow"), ih.address
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract functions calls from features
most relevant at the function scope, however, its most efficient to extract at the instruction scope
@@ -526,7 +534,7 @@ def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
def extract_function_indirect_call_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
does not include calls like => call ds:dword_ABD4974
@@ -554,7 +562,7 @@ def extract_function_indirect_call_characteristic_features(
yield Characteristic("indirect call"), ih.address
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract instruction features"""
for inst_handler in INSTRUCTION_HANDLERS:
for feature, ea in inst_handler(f, bbh, insn):

View File

@@ -7,9 +7,8 @@
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from typing import Tuple, Iterator
import capa.features.extractors.helpers
from capa.helpers import assert_never
from capa.features.insn import API, Number
from capa.features.common import String, Feature
@@ -20,7 +19,7 @@ from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, Pr
logger = logging.getLogger(__name__)
def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[tuple[Feature, Address]]:
def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
"""
this method extracts the given call's features (such as API name and arguments),
and returns them as API, Number, and String features.
@@ -51,11 +50,10 @@ def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -
else:
assert_never(value)
for name in capa.features.extractors.helpers.generate_symbols("", call.api):
yield API(name), ch.address
yield API(call.api), ch.address
def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
for handler in CALL_HANDLERS:
for feature, addr in handler(ph, th, ch):
yield feature, addr

View File

@@ -7,7 +7,7 @@
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Union, Iterator
from typing import Dict, Tuple, Union, Iterator
import capa.features.extractors.cape.call
import capa.features.extractors.cape.file
@@ -50,16 +50,16 @@ class CapeExtractor(DynamicFeatureExtractor):
assert self.report.static is not None and self.report.static.pe is not None
return AbsoluteVirtualAddress(self.report.static.pe.imagebase)
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
yield from self.global_features
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.cape.file.extract_features(self.report)
def get_processes(self) -> Iterator[ProcessHandle]:
yield from capa.features.extractors.cape.file.get_processes(self.report)
def extract_process_features(self, ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.cape.process.extract_features(ph)
def get_process_name(self, ph) -> str:
@@ -69,7 +69,7 @@ class CapeExtractor(DynamicFeatureExtractor):
def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
yield from capa.features.extractors.cape.process.get_threads(ph)
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[tuple[Feature, Address]]:
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
if False:
# force this routine to be a generator,
# but we don't actually have any elements to generate.
@@ -81,7 +81,7 @@ class CapeExtractor(DynamicFeatureExtractor):
def extract_call_features(
self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.cape.call.extract_features(ph, th, ch)
def get_call_name(self, ph, th, ch) -> str:
@@ -122,7 +122,7 @@ class CapeExtractor(DynamicFeatureExtractor):
return "".join(parts)
@classmethod
def from_report(cls, report: dict) -> "CapeExtractor":
def from_report(cls, report: Dict) -> "CapeExtractor":
cr = CapeReport.model_validate(report)
if cr.info.version not in TESTED_VERSIONS:

View File

@@ -7,7 +7,7 @@
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from typing import Tuple, Iterator
from capa.features.file import Export, Import, Section
from capa.features.common import String, Feature
@@ -41,7 +41,7 @@ def get_processes(report: CapeReport) -> Iterator[ProcessHandle]:
seen_processes[addr].append(process)
def extract_import_names(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_import_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
"""
extract imported function names
"""
@@ -62,57 +62,57 @@ def extract_import_names(report: CapeReport) -> Iterator[tuple[Feature, Address]
yield Import(name), AbsoluteVirtualAddress(function.address)
def extract_export_names(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_export_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
assert report.static is not None and report.static.pe is not None
for function in report.static.pe.exports:
yield Export(function.name), AbsoluteVirtualAddress(function.address)
def extract_section_names(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_section_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
assert report.static is not None and report.static.pe is not None
for section in report.static.pe.sections:
yield Section(section.name), AbsoluteVirtualAddress(section.virtual_address)
def extract_file_strings(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_file_strings(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
if report.strings is not None:
for string in report.strings:
yield String(string), NO_ADDRESS
def extract_used_regkeys(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_used_regkeys(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
for regkey in report.behavior.summary.keys:
yield String(regkey), NO_ADDRESS
def extract_used_files(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_used_files(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
for file in report.behavior.summary.files:
yield String(file), NO_ADDRESS
def extract_used_mutexes(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_used_mutexes(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
for mutex in report.behavior.summary.mutexes:
yield String(mutex), NO_ADDRESS
def extract_used_commands(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_used_commands(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
for cmd in report.behavior.summary.executed_commands:
yield String(cmd), NO_ADDRESS
def extract_used_apis(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_used_apis(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
for symbol in report.behavior.summary.resolved_apis:
yield String(symbol), NO_ADDRESS
def extract_used_services(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_used_services(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
for svc in report.behavior.summary.created_services:
yield String(svc), NO_ADDRESS
for svc in report.behavior.summary.started_services:
yield String(svc), NO_ADDRESS
def extract_features(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
for handler in FILE_HANDLERS:
for feature, addr in handler(report):
yield feature, addr

View File

@@ -7,7 +7,7 @@
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from typing import Tuple, Iterator
from capa.features.common import (
OS,
@@ -28,7 +28,7 @@ from capa.features.extractors.cape.models import CapeReport
logger = logging.getLogger(__name__)
def extract_arch(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_arch(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
if "Intel 80386" in report.target.file.type:
yield Arch(ARCH_I386), NO_ADDRESS
elif "x86-64" in report.target.file.type:
@@ -40,7 +40,7 @@ def extract_arch(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
)
def extract_format(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_format(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
if "PE" in report.target.file.type:
yield Format(FORMAT_PE), NO_ADDRESS
elif "ELF" in report.target.file.type:
@@ -48,11 +48,11 @@ def extract_format(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
else:
logger.warning("unknown file format, file command output: %s", report.target.file.type)
raise ValueError(
f"unrecognized file format from the CAPE report; output of file command: {report.target.file.type}"
"unrecognized file format from the CAPE report; output of file command: {report.target.file.type}"
)
def extract_os(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_os(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
# this variable contains the output of the file command
file_output = report.target.file.type
@@ -73,14 +73,14 @@ def extract_os(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
else:
# if the operating system information is missing from the cape report, it's likely a bug
logger.warning("unrecognized OS: %s", file_output)
raise ValueError(f"unrecognized OS from the CAPE report; output of file command: {file_output}")
raise ValueError("unrecognized OS from the CAPE report; output of file command: {file_output}")
else:
# the sample is shellcode
logger.debug("unsupported file format, file command output: %s", file_output)
yield OS(OS_ANY), NO_ADDRESS
def extract_features(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
for global_handler in GLOBAL_HANDLER:
for feature, addr in global_handler(report):
yield feature, addr

View File

@@ -6,12 +6,12 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Any
from typing import Any, Dict, List
from capa.features.extractors.base_extractor import ProcessHandle
def find_process(processes: list[dict[str, Any]], ph: ProcessHandle) -> dict[str, Any]:
def find_process(processes: List[Dict[str, Any]], ph: ProcessHandle) -> Dict[str, Any]:
"""
find a specific process identified by a process handler.

View File

@@ -6,9 +6,10 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import binascii
from typing import Any, Union, Literal, Optional, Annotated, TypeAlias
from typing import Any, Dict, List, Union, Literal, Optional
from pydantic import Field, BaseModel, ConfigDict
from typing_extensions import Annotated, TypeAlias
from pydantic.functional_validators import BeforeValidator
@@ -58,11 +59,11 @@ Skip: TypeAlias = Optional[Any]
# in a field with this type.
# then we can update the model with the discovered shape.
TODO: TypeAlias = None
ListTODO: TypeAlias = list[None]
ListTODO: TypeAlias = List[None]
DictTODO: TypeAlias = ExactModel
Emptydict: TypeAlias = BaseModel
EmptyList: TypeAlias = list[Any]
EmptyDict: TypeAlias = BaseModel
EmptyList: TypeAlias = List[Any]
class Info(FlexibleModel):
@@ -76,7 +77,7 @@ class ImportedSymbol(ExactModel):
class ImportedDll(ExactModel):
dll: str
imports: list[ImportedSymbol]
imports: List[ImportedSymbol]
class DirectoryEntry(ExactModel):
@@ -148,7 +149,7 @@ class Signer(ExactModel):
aux_valid: Optional[bool] = None
aux_error: Optional[bool] = None
aux_error_desc: Optional[str] = None
aux_signers: Optional[list[AuxSigner]] = None
aux_signers: Optional[List[AuxSigner]] = None
class Overlay(ExactModel):
@@ -177,22 +178,22 @@ class PE(ExactModel):
pdbpath: Optional[str] = None
timestamp: str
# list[ImportedDll], or dict[basename(dll), ImportedDll]
imports: Union[list[ImportedDll], dict[str, ImportedDll]]
# List[ImportedDll], or Dict[basename(dll), ImportedDll]
imports: Union[List[ImportedDll], Dict[str, ImportedDll]]
imported_dll_count: Optional[int] = None
imphash: str
exported_dll_name: Optional[str] = None
exports: list[ExportedSymbol]
exports: List[ExportedSymbol]
dirents: list[DirectoryEntry]
sections: list[Section]
dirents: List[DirectoryEntry]
sections: List[Section]
ep_bytes: Optional[HexBytes] = None
overlay: Optional[Overlay] = None
resources: list[Resource]
versioninfo: list[KV]
resources: List[Resource]
versioninfo: List[KV]
# base64 encoded data
icon: Optional[str] = None
@@ -203,7 +204,7 @@ class PE(ExactModel):
# short hex string
icon_dhash: Optional[str] = None
digital_signers: list[DigitalSigner]
digital_signers: List[DigitalSigner]
guest_signers: Signer
@@ -216,9 +217,9 @@ class File(FlexibleModel):
cape_type: Optional[str] = None
pid: Optional[Union[int, Literal[""]]] = None
name: Union[list[str], str]
name: Union[List[str], str]
path: str
guest_paths: Union[list[str], str, None]
guest_paths: Union[List[str], str, None]
timestamp: Optional[str] = None
#
@@ -243,7 +244,7 @@ class File(FlexibleModel):
ep_bytes: Optional[HexBytes] = None
entrypoint: Optional[int] = None
data: Optional[str] = None
strings: Optional[list[str]] = None
strings: Optional[List[str]] = None
#
# detections (skip)
@@ -282,7 +283,7 @@ class Call(ExactModel):
api: str
arguments: list[Argument]
arguments: List[Argument]
status: bool
return_: HexInt = Field(alias="return")
pretty_return: Optional[str] = None
@@ -297,18 +298,15 @@ class Call(ExactModel):
id: int
# FlexibleModel to account for extended fields
# refs: https://github.com/mandiant/capa/issues/2466
# https://github.com/kevoreilly/CAPEv2/pull/2199
class Process(FlexibleModel):
class Process(ExactModel):
process_id: int
process_name: str
parent_id: int
module_path: str
first_seen: str
calls: list[Call]
threads: list[int]
environ: dict[str, str]
calls: List[Call]
threads: List[int]
environ: Dict[str, str]
class ProcessTree(ExactModel):
@@ -316,25 +314,25 @@ class ProcessTree(ExactModel):
pid: int
parent_id: int
module_path: str
threads: list[int]
environ: dict[str, str]
children: list["ProcessTree"]
threads: List[int]
environ: Dict[str, str]
children: List["ProcessTree"]
class Summary(ExactModel):
files: list[str]
read_files: list[str]
write_files: list[str]
delete_files: list[str]
keys: list[str]
read_keys: list[str]
write_keys: list[str]
delete_keys: list[str]
executed_commands: list[str]
resolved_apis: list[str]
mutexes: list[str]
created_services: list[str]
started_services: list[str]
files: List[str]
read_files: List[str]
write_files: List[str]
delete_files: List[str]
keys: List[str]
read_keys: List[str]
write_keys: List[str]
delete_keys: List[str]
executed_commands: List[str]
resolved_apis: List[str]
mutexes: List[str]
created_services: List[str]
started_services: List[str]
class EncryptedBuffer(ExactModel):
@@ -351,12 +349,12 @@ class Behavior(ExactModel):
summary: Summary
# list of processes, of threads, of calls
processes: list[Process]
processes: List[Process]
# tree of processes
processtree: list[ProcessTree]
processtree: List[ProcessTree]
anomaly: list[str]
encryptedbuffers: list[EncryptedBuffer]
anomaly: List[str]
encryptedbuffers: List[EncryptedBuffer]
# these are small objects that describe atomic events,
# like file move, registry access.
# we'll detect the same with our API call analysis.
@@ -375,7 +373,7 @@ class Static(ExactModel):
class Cape(ExactModel):
payloads: list[ProcessFile]
payloads: List[ProcessFile]
configs: Skip = None
@@ -391,7 +389,7 @@ class CapeReport(FlexibleModel):
# static analysis results
#
static: Optional[Static] = None
strings: Optional[list[str]] = None
strings: Optional[List[str]] = None
#
# dynamic analysis results
@@ -400,10 +398,10 @@ class CapeReport(FlexibleModel):
behavior: Behavior
# post-processed results: payloads and extracted configs
CAPE: Optional[Union[Cape, list]] = None
dropped: Optional[list[File]] = None
procdump: Optional[list[ProcessFile]] = None
procmemory: Optional[ListTODO] = None
CAPE: Optional[Union[Cape, List]] = None
dropped: Optional[List[File]] = None
procdump: Optional[List[ProcessFile]] = None
procmemory: ListTODO
# =========================================================================
# information we won't use in capa
@@ -439,7 +437,7 @@ class CapeReport(FlexibleModel):
malfamily_tag: Optional[str] = None
malscore: float
detections: Skip = None
detections2pid: Optional[dict[int, list[str]]] = None
detections2pid: Optional[Dict[int, List[str]]] = None
# AV detections for the sample.
virustotal: Skip = None

View File

@@ -7,7 +7,7 @@
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from typing import List, Tuple, Iterator
from capa.features.common import String, Feature
from capa.features.address import Address, ThreadAddress
@@ -22,14 +22,14 @@ def get_threads(ph: ProcessHandle) -> Iterator[ThreadHandle]:
get the threads associated with a given process
"""
process: Process = ph.inner
threads: list[int] = process.threads
threads: List[int] = process.threads
for thread in threads:
address: ThreadAddress = ThreadAddress(process=ph.address, tid=thread)
yield ThreadHandle(address=address, inner={})
def extract_environ_strings(ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
def extract_environ_strings(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
"""
extract strings from a process' provided environment variables.
"""
@@ -39,7 +39,7 @@ def extract_environ_strings(ph: ProcessHandle) -> Iterator[tuple[Feature, Addres
yield String(value), ph.address
def extract_features(ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
for handler in PROCESS_HANDLERS:
for feature, addr in handler(ph):
yield feature, addr

View File

@@ -10,7 +10,7 @@ import re
import logging
import binascii
import contextlib
from typing import Iterator
from typing import Tuple, Iterator
import pefile
@@ -45,7 +45,7 @@ MATCH_RESULT = b'{"meta":'
MATCH_JSON_OBJECT = b'{"'
def extract_file_strings(buf: bytes, **kwargs) -> Iterator[tuple[String, Address]]:
def extract_file_strings(buf: bytes, **kwargs) -> Iterator[Tuple[String, Address]]:
"""
extract ASCII and UTF-16 LE strings from file
"""
@@ -56,7 +56,7 @@ def extract_file_strings(buf: bytes, **kwargs) -> Iterator[tuple[String, Address
yield String(s.s), FileOffsetAddress(s.offset)
def extract_format(buf: bytes) -> Iterator[tuple[Feature, Address]]:
def extract_format(buf: bytes) -> Iterator[Tuple[Feature, Address]]:
if buf.startswith(MATCH_PE):
yield Format(FORMAT_PE), NO_ADDRESS
elif buf.startswith(MATCH_ELF):
@@ -75,11 +75,11 @@ def extract_format(buf: bytes) -> Iterator[tuple[Feature, Address]]:
# 1. handling a file format (e.g. macho)
#
# for (1), this logic will need to be updated as the format is implemented.
logger.debug("unknown file format: %s", buf[:4].hex())
logger.debug("unsupported file format: %s", binascii.hexlify(buf[:4]).decode("ascii"))
return
def extract_arch(buf) -> Iterator[tuple[Feature, Address]]:
def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
if buf.startswith(MATCH_PE):
yield from capa.features.extractors.pefile.extract_file_arch(pe=pefile.PE(data=buf))
@@ -111,7 +111,7 @@ def extract_arch(buf) -> Iterator[tuple[Feature, Address]]:
return
def extract_os(buf, os=OS_AUTO) -> Iterator[tuple[Feature, Address]]:
def extract_os(buf, os=OS_AUTO) -> Iterator[Tuple[Feature, Address]]:
if os != OS_AUTO:
yield OS(os), NO_ADDRESS

View File

@@ -8,7 +8,7 @@
from __future__ import annotations
from typing import Union, Iterator, Optional
from typing import Dict, List, Tuple, Union, Iterator, Optional
from pathlib import Path
import dnfile
@@ -41,11 +41,11 @@ from capa.features.extractors.dnfile.helpers import (
class DnFileFeatureExtractorCache:
def __init__(self, pe: dnfile.dnPE):
self.imports: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
self.native_imports: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
self.methods: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
self.fields: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
self.types: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
self.imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
self.native_imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
self.methods: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
self.fields: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
self.types: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
for import_ in get_dotnet_managed_imports(pe):
self.imports[import_.token] = import_
@@ -84,7 +84,7 @@ class DnfileFeatureExtractor(StaticFeatureExtractor):
self.token_cache: DnFileFeatureExtractorCache = DnFileFeatureExtractorCache(self.pe)
# pre-compute these because we'll yield them at *every* scope.
self.global_features: list[tuple[Feature, Address]] = []
self.global_features: List[Tuple[Feature, Address]] = []
self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_format())
self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_os(pe=self.pe))
self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_arch(pe=self.pe))
@@ -100,7 +100,7 @@ class DnfileFeatureExtractor(StaticFeatureExtractor):
def get_functions(self) -> Iterator[FunctionHandle]:
# create a method lookup table
methods: dict[Address, FunctionHandle] = {}
methods: Dict[Address, FunctionHandle] = {}
for token, method in get_dotnet_managed_method_bodies(self.pe):
fh: FunctionHandle = FunctionHandle(
address=DNTokenAddress(token),
@@ -136,7 +136,7 @@ class DnfileFeatureExtractor(StaticFeatureExtractor):
yield from methods.values()
def extract_function_features(self, fh) -> Iterator[tuple[Feature, Address]]:
def extract_function_features(self, fh) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.dnfile.function.extract_features(fh)
def get_basic_blocks(self, f) -> Iterator[BBHandle]:
@@ -157,5 +157,5 @@ class DnfileFeatureExtractor(StaticFeatureExtractor):
inner=insn,
)
def extract_insn_features(self, fh, bbh, ih) -> Iterator[tuple[Feature, Address]]:
def extract_insn_features(self, fh, bbh, ih) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.dnfile.insn.extract_features(fh, bbh, ih)

View File

@@ -8,7 +8,7 @@
from __future__ import annotations
from typing import Iterator
from typing import Tuple, Iterator
import dnfile
@@ -18,35 +18,35 @@ from capa.features.common import Class, Format, String, Feature, Namespace, Char
from capa.features.address import Address
def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[tuple[Import, Address]]:
def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[Tuple[Import, Address]]:
yield from capa.features.extractors.dotnetfile.extract_file_import_names(pe=pe)
def extract_file_format(pe: dnfile.dnPE) -> Iterator[tuple[Format, Address]]:
def extract_file_format(pe: dnfile.dnPE) -> Iterator[Tuple[Format, Address]]:
yield from capa.features.extractors.dotnetfile.extract_file_format(pe=pe)
def extract_file_function_names(pe: dnfile.dnPE) -> Iterator[tuple[FunctionName, Address]]:
def extract_file_function_names(pe: dnfile.dnPE) -> Iterator[Tuple[FunctionName, Address]]:
yield from capa.features.extractors.dotnetfile.extract_file_function_names(pe=pe)
def extract_file_strings(pe: dnfile.dnPE) -> Iterator[tuple[String, Address]]:
def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, Address]]:
yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe)
def extract_file_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[tuple[Characteristic, Address]]:
def extract_file_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[Tuple[Characteristic, Address]]:
yield from capa.features.extractors.dotnetfile.extract_file_mixed_mode_characteristic_features(pe=pe)
def extract_file_namespace_features(pe: dnfile.dnPE) -> Iterator[tuple[Namespace, Address]]:
def extract_file_namespace_features(pe: dnfile.dnPE) -> Iterator[Tuple[Namespace, Address]]:
yield from capa.features.extractors.dotnetfile.extract_file_namespace_features(pe=pe)
def extract_file_class_features(pe: dnfile.dnPE) -> Iterator[tuple[Class, Address]]:
def extract_file_class_features(pe: dnfile.dnPE) -> Iterator[Tuple[Class, Address]]:
yield from capa.features.extractors.dotnetfile.extract_file_class_features(pe=pe)
def extract_features(pe: dnfile.dnPE) -> Iterator[tuple[Feature, Address]]:
def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
for file_handler in FILE_HANDLERS:
for feature, address in file_handler(pe):
yield feature, address

View File

@@ -9,7 +9,7 @@
from __future__ import annotations
import logging
from typing import Iterator
from typing import Tuple, Iterator
from capa.features.common import Feature, Characteristic
from capa.features.address import Address
@@ -18,30 +18,30 @@ from capa.features.extractors.base_extractor import FunctionHandle
logger = logging.getLogger(__name__)
def extract_function_calls_to(fh: FunctionHandle) -> Iterator[tuple[Characteristic, Address]]:
def extract_function_calls_to(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
"""extract callers to a function"""
for dest in fh.ctx["calls_to"]:
yield Characteristic("calls to"), dest
def extract_function_calls_from(fh: FunctionHandle) -> Iterator[tuple[Characteristic, Address]]:
def extract_function_calls_from(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
"""extract callers from a function"""
for src in fh.ctx["calls_from"]:
yield Characteristic("calls from"), src
def extract_recursive_call(fh: FunctionHandle) -> Iterator[tuple[Characteristic, Address]]:
def extract_recursive_call(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
"""extract recursive function call"""
if fh.address in fh.ctx["calls_to"]:
yield Characteristic("recursive call"), fh.address
def extract_function_loop(fh: FunctionHandle) -> Iterator[tuple[Characteristic, Address]]:
def extract_function_loop(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
"""extract loop indicators from a function"""
raise NotImplementedError()
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
for func_handler in FUNCTION_HANDLERS:
for feature, addr in func_handler(fh):
yield feature, addr

View File

@@ -9,7 +9,7 @@
from __future__ import annotations
import logging
from typing import Union, Iterator, Optional
from typing import Dict, Tuple, Union, Iterator, Optional
import dnfile
from dncil.cil.body import CilMethodBody
@@ -144,7 +144,7 @@ def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[DnType]:
)
def get_dotnet_methoddef_property_accessors(pe: dnfile.dnPE) -> Iterator[tuple[int, str]]:
def get_dotnet_methoddef_property_accessors(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
"""get MethodDef methods used to access properties
see https://www.ntcore.com/files/dotnetformat.htm
@@ -194,7 +194,7 @@ def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
"""
nested_class_table = get_dotnet_nested_class_table_index(pe)
accessor_map: dict[int, str] = {}
accessor_map: Dict[int, str] = {}
for methoddef, methoddef_access in get_dotnet_methoddef_property_accessors(pe):
accessor_map[methoddef] = methoddef_access
@@ -252,7 +252,7 @@ def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
yield DnType(token, typedefname, namespace=typedefnamespace, member=field.row.Name)
def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[tuple[int, CilMethodBody]]:
def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
"""get managed methods from MethodDef table"""
for rid, method_def in iter_dotnet_table(pe, dnfile.mdtable.MethodDef.number):
assert isinstance(method_def, dnfile.mdtable.MethodDefRow)
@@ -332,7 +332,7 @@ def get_dotnet_table_row(pe: dnfile.dnPE, table_index: int, row_index: int) -> O
def resolve_nested_typedef_name(
nested_class_table: dict, index: int, typedef: dnfile.mdtable.TypeDefRow, pe: dnfile.dnPE
) -> tuple[str, tuple[str, ...]]:
) -> Tuple[str, Tuple[str, ...]]:
"""Resolves all nested TypeDef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
if index in nested_class_table:
@@ -368,7 +368,7 @@ def resolve_nested_typedef_name(
def resolve_nested_typeref_name(
index: int, typeref: dnfile.mdtable.TypeRefRow, pe: dnfile.dnPE
) -> tuple[str, tuple[str, ...]]:
) -> Tuple[str, Tuple[str, ...]]:
"""Resolves all nested TypeRef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
# If the ResolutionScope decodes to a typeRef type then it is nested
if isinstance(typeref.ResolutionScope.table, dnfile.mdtable.TypeRef):
@@ -398,7 +398,7 @@ def resolve_nested_typeref_name(
return str(typeref.TypeNamespace), (str(typeref.TypeName),)
def get_dotnet_nested_class_table_index(pe: dnfile.dnPE) -> dict[int, int]:
def get_dotnet_nested_class_table_index(pe: dnfile.dnPE) -> Dict[int, int]:
"""Build index for EnclosingClass based off the NestedClass row index in the nestedclass table"""
nested_class_table = {}
@@ -442,7 +442,7 @@ def is_dotnet_mixed_mode(pe: dnfile.dnPE) -> bool:
return not bool(pe.net.Flags.CLR_ILONLY)
def iter_dotnet_table(pe: dnfile.dnPE, table_index: int) -> Iterator[tuple[int, dnfile.base.MDTableRow]]:
def iter_dotnet_table(pe: dnfile.dnPE, table_index: int) -> Iterator[Tuple[int, dnfile.base.MDTableRow]]:
assert pe.net is not None
assert pe.net.mdtables is not None

View File

@@ -9,7 +9,7 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Union, Iterator, Optional
from typing import TYPE_CHECKING, Tuple, Union, Iterator, Optional
if TYPE_CHECKING:
from capa.features.extractors.dnfile.extractor import DnFileFeatureExtractorCache
@@ -61,7 +61,7 @@ def get_callee(
return callee
def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction API features"""
if ih.inner.opcode not in (
OpCodes.Call,
@@ -83,7 +83,7 @@ def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterato
yield API(name), ih.address
def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction property features"""
name: Optional[str] = None
access: Optional[str] = None
@@ -118,7 +118,7 @@ def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> It
def extract_insn_namespace_class_features(
fh: FunctionHandle, bh, ih: InsnHandle
) -> Iterator[tuple[Union[Namespace, Class], Address]]:
) -> Iterator[Tuple[Union[Namespace, Class], Address]]:
"""parse instruction namespace and class features"""
type_: Optional[Union[DnType, DnUnmanagedMethod]] = None
@@ -173,13 +173,13 @@ def extract_insn_namespace_class_features(
yield Namespace(type_.namespace), ih.address
def extract_insn_number_features(fh, bh, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_number_features(fh, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction number features"""
if ih.inner.is_ldc():
yield Number(ih.inner.get_ldc()), ih.address
def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction string features"""
if not ih.inner.is_ldstr():
return
@@ -197,7 +197,7 @@ def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iter
def extract_unmanaged_call_characteristic_features(
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Characteristic, Address]]:
) -> Iterator[Tuple[Characteristic, Address]]:
if ih.inner.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
return
@@ -209,7 +209,7 @@ def extract_unmanaged_call_characteristic_features(
yield Characteristic("unmanaged call"), ih.address
def extract_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract instruction features"""
for inst_handler in INSTRUCTION_HANDLERS:
for feature, addr in inst_handler(fh, bbh, ih):

View File

@@ -6,17 +6,17 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Optional
from typing import Tuple, Optional
class DnType:
def __init__(
self, token: int, class_: tuple[str, ...], namespace: str = "", member: str = "", access: Optional[str] = None
self, token: int, class_: Tuple[str, ...], namespace: str = "", member: str = "", access: Optional[str] = None
):
self.token: int = token
self.access: Optional[str] = access
self.namespace: str = namespace
self.class_: tuple[str, ...] = class_
self.class_: Tuple[str, ...] = class_
if member == ".ctor":
member = "ctor"
@@ -44,7 +44,7 @@ class DnType:
return str(self)
@staticmethod
def format_name(class_: tuple[str, ...], namespace: str = "", member: str = ""):
def format_name(class_: Tuple[str, ...], namespace: str = "", member: str = ""):
if len(class_) > 1:
class_str = "/".join(class_) # Concat items in tuple, separated by a "/"
else:

View File

@@ -6,7 +6,7 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from typing import Tuple, Iterator
from pathlib import Path
import dnfile
@@ -48,12 +48,12 @@ from capa.features.extractors.dnfile.helpers import (
logger = logging.getLogger(__name__)
def extract_file_format(**kwargs) -> Iterator[tuple[Format, Address]]:
def extract_file_format(**kwargs) -> Iterator[Tuple[Format, Address]]:
yield Format(FORMAT_DOTNET), NO_ADDRESS
yield Format(FORMAT_PE), NO_ADDRESS
def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Import, Address]]:
def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Import, Address]]:
for method in get_dotnet_managed_imports(pe):
# like System.IO.File::OpenRead
yield Import(str(method)), DNTokenAddress(method.token)
@@ -64,12 +64,12 @@ def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Impor
yield Import(name), DNTokenAddress(imp.token)
def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[FunctionName, Address]]:
def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[FunctionName, Address]]:
for method in get_dotnet_managed_methods(pe):
yield FunctionName(str(method)), DNTokenAddress(method.token)
def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Namespace, Address]]:
def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Namespace, Address]]:
"""emit namespace features from TypeRef and TypeDef tables"""
# namespaces may be referenced multiple times, so we need to filter
@@ -93,7 +93,7 @@ def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple
yield Namespace(namespace), NO_ADDRESS
def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Class, Address]]:
def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Class, Address]]:
"""emit class features from TypeRef and TypeDef tables"""
nested_class_table = get_dotnet_nested_class_table_index(pe)
@@ -116,11 +116,11 @@ def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Cla
yield Class(DnType.format_name(typerefname, namespace=typerefnamespace)), DNTokenAddress(token)
def extract_file_os(**kwargs) -> Iterator[tuple[OS, Address]]:
def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
yield OS(OS_ANY), NO_ADDRESS
def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Arch, Address]]:
def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Arch, Address]]:
# to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
# .NET 4.5 added option: any CPU, 32-bit preferred
assert pe.net is not None
@@ -134,18 +134,18 @@ def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Arch, Address
yield Arch(ARCH_ANY), NO_ADDRESS
def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[String, Address]]:
def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[String, Address]]:
yield from capa.features.extractors.common.extract_file_strings(pe.__data__)
def extract_file_mixed_mode_characteristic_features(
pe: dnfile.dnPE, **kwargs
) -> Iterator[tuple[Characteristic, Address]]:
) -> Iterator[Tuple[Characteristic, Address]]:
if is_dotnet_mixed_mode(pe):
yield Characteristic("mixed mode"), NO_ADDRESS
def extract_file_features(pe: dnfile.dnPE) -> Iterator[tuple[Feature, Address]]:
def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
for file_handler in FILE_HANDLERS:
for feature, addr in file_handler(pe=pe): # type: ignore
yield feature, addr
@@ -162,7 +162,7 @@ FILE_HANDLERS = (
)
def extract_global_features(pe: dnfile.dnPE) -> Iterator[tuple[Feature, Address]]:
def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
for handler in GLOBAL_HANDLERS:
for feature, va in handler(pe=pe): # type: ignore
yield feature, va
@@ -204,7 +204,7 @@ class DotnetFileFeatureExtractor(StaticFeatureExtractor):
def is_mixed_mode(self) -> bool:
return is_dotnet_mixed_mode(self.pe)
def get_runtime_version(self) -> tuple[int, int]:
def get_runtime_version(self) -> Tuple[int, int]:
assert self.pe.net is not None
assert self.pe.net.struct is not None
assert self.pe.net.struct.MajorRuntimeVersion is not None

View File

@@ -1,58 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
import capa.features.extractors.helpers
from capa.features.insn import API, Number
from capa.features.common import String, Feature
from capa.features.address import Address
from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
from capa.features.extractors.drakvuf.models import Call
logger = logging.getLogger(__name__)
def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[tuple[Feature, Address]]:
"""
This method extracts the given call's features (such as API name and arguments),
and returns them as API, Number, and String features.
args:
ph: process handle (for defining the extraction scope)
th: thread handle (for defining the extraction scope)
ch: call handle (for defining the extraction scope)
yields:
Feature, address; where Feature is either: API, Number, or String.
"""
call: Call = ch.inner
# list similar to disassembly: arguments right-to-left, call
for arg_value in reversed(call.arguments.values()):
try:
yield Number(int(arg_value, 0)), ch.address
except ValueError:
# DRAKVUF automatically resolves the contents of memory addresses, (e.g. Arg1="0xc6f217efe0:\"ntdll.dll\"").
# For those cases we yield the entire string as it, since yielding the address only would
# likely not provide any matches, and yielding just the memory contentswould probably be misleading,
# but yielding the entire string would be helpful for an analyst looking at the verbose output
yield String(arg_value), ch.address
for name in capa.features.extractors.helpers.generate_symbols("", call.name):
yield API(name), ch.address
def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[tuple[Feature, Address]]:
for handler in CALL_HANDLERS:
for feature, addr in handler(ph, th, ch):
yield feature, addr
CALL_HANDLERS = (extract_call_features,)

View File

@@ -1,96 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Union, Iterator
import capa.features.extractors.drakvuf.call
import capa.features.extractors.drakvuf.file
import capa.features.extractors.drakvuf.thread
import capa.features.extractors.drakvuf.global_
import capa.features.extractors.drakvuf.process
from capa.features.common import Feature, Characteristic
from capa.features.address import NO_ADDRESS, Address, ThreadAddress, ProcessAddress, AbsoluteVirtualAddress, _NoAddress
from capa.features.extractors.base_extractor import (
CallHandle,
SampleHashes,
ThreadHandle,
ProcessHandle,
DynamicFeatureExtractor,
)
from capa.features.extractors.drakvuf.models import Call, DrakvufReport
from capa.features.extractors.drakvuf.helpers import index_calls
logger = logging.getLogger(__name__)
class DrakvufExtractor(DynamicFeatureExtractor):
def __init__(self, report: DrakvufReport):
super().__init__(
# DRAKVUF currently does not yield hash information about the sample in its output
hashes=SampleHashes(md5="", sha1="", sha256="")
)
self.report: DrakvufReport = report
# sort the api calls to prevent going through the entire list each time
self.sorted_calls: dict[ProcessAddress, dict[ThreadAddress, list[Call]]] = index_calls(report)
# pre-compute these because we'll yield them at *every* scope.
self.global_features = list(capa.features.extractors.drakvuf.global_.extract_features(self.report))
def get_base_address(self) -> Union[AbsoluteVirtualAddress, _NoAddress, None]:
# DRAKVUF currently does not yield information about the PE's address
return NO_ADDRESS
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
yield from self.global_features
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
yield from capa.features.extractors.drakvuf.file.extract_features(self.report)
def get_processes(self) -> Iterator[ProcessHandle]:
yield from capa.features.extractors.drakvuf.file.get_processes(self.sorted_calls)
def extract_process_features(self, ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
yield from capa.features.extractors.drakvuf.process.extract_features(ph)
def get_process_name(self, ph: ProcessHandle) -> str:
return ph.inner["process_name"]
def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
yield from capa.features.extractors.drakvuf.process.get_threads(self.sorted_calls, ph)
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[tuple[Feature, Address]]:
if False:
# force this routine to be a generator,
# but we don't actually have any elements to generate.
yield Characteristic("never"), NO_ADDRESS
return
def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
yield from capa.features.extractors.drakvuf.thread.get_calls(self.sorted_calls, ph, th)
def get_call_name(self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> str:
call: Call = ch.inner
call_name = "{}({}){}".format(
call.name,
", ".join(f"{arg_name}={arg_value}" for arg_name, arg_value in call.arguments.items()),
(f" -> {getattr(call, 'return_value', '')}"), # SysCalls don't have a return value, while WinApi calls do
)
return call_name
def extract_call_features(
self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
) -> Iterator[tuple[Feature, Address]]:
yield from capa.features.extractors.drakvuf.call.extract_features(ph, th, ch)
@classmethod
def from_report(cls, report: Iterator[dict]) -> "DrakvufExtractor":
dr = DrakvufReport.from_raw_report(report)
return DrakvufExtractor(report=dr)

View File

@@ -1,56 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from capa.features.file import Import
from capa.features.common import Feature
from capa.features.address import Address, ThreadAddress, ProcessAddress, AbsoluteVirtualAddress
from capa.features.extractors.helpers import generate_symbols
from capa.features.extractors.base_extractor import ProcessHandle
from capa.features.extractors.drakvuf.models import Call, DrakvufReport
logger = logging.getLogger(__name__)
def get_processes(calls: dict[ProcessAddress, dict[ThreadAddress, list[Call]]]) -> Iterator[ProcessHandle]:
"""
Get all the created processes for a sample.
"""
for proc_addr, calls_per_thread in calls.items():
sample_call = next(iter(calls_per_thread.values()))[0] # get process name
yield ProcessHandle(proc_addr, inner={"process_name": sample_call.process_name})
def extract_import_names(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
"""
Extract imported function names.
"""
if report.loaded_dlls is None:
return
dlls = report.loaded_dlls
for dll in dlls:
dll_base_name = dll.name.split("\\")[-1]
for function_name, function_address in dll.imports.items():
for name in generate_symbols(dll_base_name, function_name, include_dll=True):
yield Import(name), AbsoluteVirtualAddress(function_address)
def extract_features(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
for handler in FILE_HANDLERS:
for feature, addr in handler(report):
yield feature, addr
FILE_HANDLERS = (
# TODO(yelhamer): extract more file features from other DRAKVUF plugins
# https://github.com/mandiant/capa/issues/2169
extract_import_names,
)

View File

@@ -1,44 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from capa.features.common import OS, FORMAT_PE, ARCH_AMD64, OS_WINDOWS, Arch, Format, Feature
from capa.features.address import NO_ADDRESS, Address
from capa.features.extractors.drakvuf.models import DrakvufReport
logger = logging.getLogger(__name__)
def extract_format(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
# DRAKVUF sandbox currently supports only Windows as the guest: https://drakvuf-sandbox.readthedocs.io/en/latest/usage/getting_started.html
yield Format(FORMAT_PE), NO_ADDRESS
def extract_os(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
# DRAKVUF sandbox currently supports only PE files: https://drakvuf-sandbox.readthedocs.io/en/latest/usage/getting_started.html
yield OS(OS_WINDOWS), NO_ADDRESS
def extract_arch(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
# DRAKVUF sandbox currently supports only x64 Windows as the guest: https://drakvuf-sandbox.readthedocs.io/en/latest/usage/getting_started.html
yield Arch(ARCH_AMD64), NO_ADDRESS
def extract_features(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
for global_handler in GLOBAL_HANDLER:
for feature, addr in global_handler(report):
yield feature, addr
GLOBAL_HANDLER = (
extract_format,
extract_os,
extract_arch,
)

View File

@@ -1,38 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import itertools
from capa.features.address import ThreadAddress, ProcessAddress
from capa.features.extractors.drakvuf.models import Call, DrakvufReport
def index_calls(report: DrakvufReport) -> dict[ProcessAddress, dict[ThreadAddress, list[Call]]]:
# this method organizes calls into processes and threads, and then sorts them based on
# timestamp so that we can address individual calls per index (CallAddress requires call index)
result: dict[ProcessAddress, dict[ThreadAddress, list[Call]]] = {}
for call in itertools.chain(report.syscalls, report.apicalls):
if call.pid == 0:
# DRAKVUF captures api/native calls from all processes running on the system.
# we ignore the pid 0 since it's a system process and it's unlikely for it to
# be hijacked or so on, in addition to capa addresses not supporting null pids
continue
proc_addr = ProcessAddress(pid=call.pid, ppid=call.ppid)
thread_addr = ThreadAddress(process=proc_addr, tid=call.tid)
if proc_addr not in result:
result[proc_addr] = {}
if thread_addr not in result[proc_addr]:
result[proc_addr][thread_addr] = []
result[proc_addr][thread_addr].append(call)
for proc, threads in result.items():
for thread in threads:
result[proc][thread].sort(key=lambda call: call.timestamp)
return result

View File

@@ -1,137 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Any, Iterator
from pydantic import Field, BaseModel, ConfigDict, model_validator
logger = logging.getLogger(__name__)
REQUIRED_SYSCALL_FIELD_NAMES = {
"Plugin",
"TimeStamp",
"PID",
"PPID",
"TID",
"UserName",
"UserId",
"ProcessName",
"Method",
"EventUID",
"Module",
"vCPU",
"CR3",
"Syscall",
"NArgs",
}
class ConciseModel(BaseModel):
ConfigDict(extra="ignore")
class DiscoveredDLL(ConciseModel):
plugin_name: str = Field(alias="Plugin")
event: str = Field(alias="Event")
name: str = Field(alias="DllName")
pid: int = Field(alias="PID")
class LoadedDLL(ConciseModel):
plugin_name: str = Field(alias="Plugin")
event: str = Field(alias="Event")
name: str = Field(alias="DllName")
imports: dict[str, int] = Field(alias="Rva")
class Call(ConciseModel):
plugin_name: str = Field(alias="Plugin")
timestamp: str = Field(alias="TimeStamp")
process_name: str = Field(alias="ProcessName")
ppid: int = Field(alias="PPID")
pid: int = Field(alias="PID")
tid: int = Field(alias="TID")
name: str = Field(alias="Method")
arguments: dict[str, str]
class WinApiCall(Call):
# This class models Windows API calls captured by DRAKVUF (DLLs, etc.).
arguments: dict[str, str] = Field(alias="Arguments")
event: str = Field(alias="Event")
return_value: str = Field(alias="ReturnValue")
@model_validator(mode="before")
@classmethod
def build_arguments(cls, values: dict[str, Any]) -> dict[str, Any]:
args = values["Arguments"]
values["Arguments"] = dict(arg.split("=", 1) for arg in args)
return values
class SystemCall(Call):
# This class models native Windows API calls captured by DRAKVUF.
# Schema: {
# "Plugin": "syscall",
# "TimeStamp": "1716999134.582553",
# "PID": 3888, "PPID": 2852, "TID": 368, "UserName": "SessionID", "UserId": 2,
# "ProcessName": "\\Device\\HarddiskVolume2\\Windows\\explorer.exe",
# "Method": "NtSetIoCompletionEx",
# "EventUID": "0x27",
# "Module": "nt",
# "vCPU": 0,
# "CR3": "0x119b1002",
# "Syscall": 419,
# "NArgs": 6,
# "IoCompletionHandle": "0xffffffff80001ac0", "IoCompletionReserveHandle": "0xffffffff8000188c",
# "KeyContext": "0x0", "ApcContext": "0x2", "IoStatus": "0x7ffb00000000", "IoStatusInformation": "0x0"
# }
# The keys up until "NArgs" are common to all the native calls that DRAKVUF reports, with
# the remaining keys representing the call's specific arguments.
syscall_number: int = Field(alias="Syscall")
module: str = Field(alias="Module")
nargs: int = Field(alias="NArgs")
@model_validator(mode="before")
@classmethod
def build_extra(cls, values: dict[str, Any]) -> dict[str, Any]:
# DRAKVUF stores argument names and values as entries in the syscall's entry.
# This model validator collects those arguments into a list in the model.
values["arguments"] = {
name: value for name, value in values.items() if name not in REQUIRED_SYSCALL_FIELD_NAMES
}
return values
class DrakvufReport(ConciseModel):
syscalls: list[SystemCall] = []
apicalls: list[WinApiCall] = []
discovered_dlls: list[DiscoveredDLL] = []
loaded_dlls: list[LoadedDLL] = []
@classmethod
def from_raw_report(cls, entries: Iterator[dict]) -> "DrakvufReport":
report = cls()
for entry in entries:
plugin = entry.get("Plugin")
# TODO(yelhamer): add support for more DRAKVUF plugins
# https://github.com/mandiant/capa/issues/2181
if plugin == "syscall":
report.syscalls.append(SystemCall(**entry))
elif plugin == "apimon":
event = entry.get("Event")
if event == "api_called":
report.apicalls.append(WinApiCall(**entry))
elif event == "dll_loaded":
report.loaded_dlls.append(LoadedDLL(**entry))
elif event == "dll_discovered":
report.discovered_dlls.append(DiscoveredDLL(**entry))
return report

View File

@@ -1,40 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from capa.features.common import String, Feature
from capa.features.address import Address, ThreadAddress, ProcessAddress
from capa.features.extractors.base_extractor import ThreadHandle, ProcessHandle
from capa.features.extractors.drakvuf.models import Call
logger = logging.getLogger(__name__)
def get_threads(
calls: dict[ProcessAddress, dict[ThreadAddress, list[Call]]], ph: ProcessHandle
) -> Iterator[ThreadHandle]:
"""
Get the threads associated with a given process.
"""
for thread_addr in calls[ph.address]:
yield ThreadHandle(address=thread_addr, inner={})
def extract_process_name(ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
yield String(ph.inner["process_name"]), ph.address
def extract_features(ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
for handler in PROCESS_HANDLERS:
for feature, addr in handler(ph):
yield feature, addr
PROCESS_HANDLERS = (extract_process_name,)

View File

@@ -1,24 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Iterator
from capa.features.address import ThreadAddress, ProcessAddress, DynamicCallAddress
from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
from capa.features.extractors.drakvuf.models import Call
logger = logging.getLogger(__name__)
def get_calls(
sorted_calls: dict[ProcessAddress, dict[ThreadAddress, list[Call]]], ph: ProcessHandle, th: ThreadHandle
) -> Iterator[CallHandle]:
for i, call in enumerate(sorted_calls[ph.address][th.address]):
call_addr = DynamicCallAddress(thread=th.address, id=i)
yield CallHandle(address=call_addr, inner=call)

View File

@@ -10,7 +10,7 @@ import logging
import itertools
import collections
from enum import Enum
from typing import TYPE_CHECKING, BinaryIO, Iterator, Optional
from typing import TYPE_CHECKING, Set, Dict, List, Tuple, BinaryIO, Iterator, Optional
from dataclasses import dataclass
if TYPE_CHECKING:
@@ -310,9 +310,6 @@ class ELF:
98: "TPC",
99: "SNP1K",
100: "ST200",
# https://www.sco.com/developers/gabi/latest/ch4.eheader.html
183: "aarch64",
243: "riscv",
}
@property
@@ -394,7 +391,7 @@ class ELF:
return read_cstr(phdr.buf, 0)
@property
def versions_needed(self) -> dict[str, set[str]]:
def versions_needed(self) -> Dict[str, Set[str]]:
# symbol version requirements are stored in the .gnu.version_r section,
# which has type SHT_GNU_verneed (0x6ffffffe).
#
@@ -452,7 +449,7 @@ class ELF:
return {}
@property
def dynamic_entries(self) -> Iterator[tuple[int, int]]:
def dynamic_entries(self) -> Iterator[Tuple[int, int]]:
"""
read the entries from the dynamic section,
yielding the tag and value for each entry.
@@ -547,7 +544,7 @@ class ELF:
logger.warning("failed to read DT_NEEDED entry: %s", str(e))
@property
def symtab(self) -> Optional[tuple[Shdr, Shdr]]:
def symtab(self) -> Optional[Tuple[Shdr, Shdr]]:
"""
fetch the Shdr for the symtab and the associated strtab.
"""
@@ -682,7 +679,7 @@ class SymTab:
symtab: Shdr,
strtab: Shdr,
) -> None:
self.symbols: list[Symbol] = []
self.symbols: List[Symbol] = []
self.symtab = symtab
self.strtab = strtab

View File

@@ -7,7 +7,7 @@
# See the License for the specific language governing permissions and limitations under the License.
import io
import logging
from typing import Iterator
from typing import Tuple, Iterator
from pathlib import Path
from elftools.elf.elffile import ELFFile, DynamicSegment, SymbolTableSection
@@ -50,12 +50,7 @@ def extract_file_export_names(elf: ELFFile, **kwargs):
if not isinstance(segment, DynamicSegment):
continue
tab_ptr, tab_offset = segment.get_table_offset("DT_SYMTAB")
if tab_ptr is None or tab_offset is None:
logger.debug("Dynamic segment doesn't contain DT_SYMTAB")
continue
logger.debug("Dynamic segment contains %s symbols: ", segment.num_symbols())
logger.debug("Dynamic Segment contains %s symbols: ", segment.num_symbols())
for symbol in segment.iter_symbols():
# The following conditions are based on the following article
@@ -81,11 +76,6 @@ def extract_file_import_names(elf: ELFFile, **kwargs):
if not isinstance(segment, DynamicSegment):
continue
tab_ptr, tab_offset = segment.get_table_offset("DT_SYMTAB")
if tab_ptr is None or tab_offset is None:
logger.debug("Dynamic segment doesn't contain DT_SYMTAB")
continue
for _, symbol in enumerate(segment.iter_symbols()):
# The following conditions are based on the following article
# http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
@@ -110,16 +100,7 @@ def extract_file_import_names(elf: ELFFile, **kwargs):
logger.debug("Dynamic Segment contains %s relocation tables:", len(relocation_tables))
for relocation_table in relocation_tables.values():
relocations = []
for i in range(relocation_table.num_relocations()):
try:
relocations.append(relocation_table.get_relocation(i))
except TypeError:
# ELF is corrupt and the relocation table is invalid,
# so stop processing it.
break
for relocation in relocations:
for relocation in relocation_table.iter_relocations():
# Extract the symbol name from the symbol table using the symbol index in the relocation
if relocation["r_info_sym"] not in symbol_names:
continue
@@ -158,15 +139,11 @@ def extract_file_arch(elf: ELFFile, **kwargs):
yield Arch("i386"), NO_ADDRESS
elif arch == "x64":
yield Arch("amd64"), NO_ADDRESS
elif arch == "ARM":
yield Arch("arm"), NO_ADDRESS
elif arch == "AArch64":
yield Arch("aarch64"), NO_ADDRESS
else:
logger.warning("unsupported architecture: %s", arch)
def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[tuple[Feature, int]]:
def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
for file_handler in FILE_HANDLERS:
for feature, addr in file_handler(elf=elf, buf=buf): # type: ignore
yield feature, addr
@@ -182,7 +159,7 @@ FILE_HANDLERS = (
)
def extract_global_features(elf: ELFFile, buf: bytes) -> Iterator[tuple[Feature, int]]:
def extract_global_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
for global_handler in GLOBAL_HANDLERS:
for feature, addr in global_handler(elf=elf, buf=buf): # type: ignore
yield feature, addr

View File

@@ -8,7 +8,7 @@
import string
import struct
from typing import Iterator
from typing import Tuple, Iterator
import ghidra
from ghidra.program.model.lang import OperandType
@@ -97,7 +97,7 @@ def _bb_has_tight_loop(bb: ghidra.program.model.block.CodeBlock):
return False
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract stackstring indicators from basic block"""
bb: ghidra.program.model.block.CodeBlock = bbh.inner
@@ -105,7 +105,7 @@ def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[
yield Characteristic("stack string"), bbh.address
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""check basic block for tight loop indicators"""
bb: ghidra.program.model.block.CodeBlock = bbh.inner
@@ -119,7 +119,7 @@ BASIC_BLOCK_HANDLERS = (
)
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""
extract features from the given basic block.
@@ -127,7 +127,7 @@ def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Featur
bb: the basic block to process.
yields:
tuple[Feature, int]: the features and their location found in this basic block.
Tuple[Feature, int]: the features and their location found in this basic block.
"""
yield BasicBlock(), bbh.address
for bb_handler in BASIC_BLOCK_HANDLERS:

View File

@@ -5,7 +5,7 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
from typing import List, Tuple, Iterator
import capa.features.extractors.ghidra.file
import capa.features.extractors.ghidra.insn
@@ -40,7 +40,7 @@ class GhidraFeatureExtractor(StaticFeatureExtractor):
)
)
self.global_features: list[tuple[Feature, Address]] = []
self.global_features: List[Tuple[Feature, Address]] = []
self.global_features.extend(capa.features.extractors.ghidra.file.extract_file_format())
self.global_features.extend(capa.features.extractors.ghidra.global_.extract_os())
self.global_features.extend(capa.features.extractors.ghidra.global_.extract_arch())
@@ -73,7 +73,7 @@ class GhidraFeatureExtractor(StaticFeatureExtractor):
func = getFunctionContaining(toAddr(addr)) # type: ignore [name-defined] # noqa: F821
return FunctionHandle(address=AbsoluteVirtualAddress(func.getEntryPoint().getOffset()), inner=func)
def extract_function_features(self, fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.ghidra.function.extract_features(fh)
def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
@@ -81,7 +81,7 @@ class GhidraFeatureExtractor(StaticFeatureExtractor):
yield from ghidra_helpers.get_function_blocks(fh)
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.ghidra.basicblock.extract_features(fh, bbh)
def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:

View File

@@ -7,7 +7,7 @@
# See the License for the specific language governing permissions and limitations under the License.
import re
import struct
from typing import Iterator
from typing import List, Tuple, Iterator
from ghidra.program.model.symbol import SourceType, SymbolType
@@ -22,7 +22,7 @@ from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, Absolu
MAX_OFFSET_PE_AFTER_MZ = 0x200
def find_embedded_pe(block_bytez: bytes, mz_xor: list[tuple[bytes, bytes, int]]) -> Iterator[tuple[int, int]]:
def find_embedded_pe(block_bytez: bytes, mz_xor: List[Tuple[bytes, bytes, int]]) -> Iterator[Tuple[int, int]]:
"""check segment for embedded PE
adapted for Ghidra from:
@@ -60,11 +60,11 @@ def find_embedded_pe(block_bytez: bytes, mz_xor: list[tuple[bytes, bytes, int]])
yield off, i
def extract_file_embedded_pe() -> Iterator[tuple[Feature, Address]]:
def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
"""extract embedded PE features"""
# pre-compute XOR pairs
mz_xor: list[tuple[bytes, bytes, int]] = [
mz_xor: List[Tuple[bytes, bytes, int]] = [
(
capa.features.extractors.helpers.xor_static(b"MZ", i),
capa.features.extractors.helpers.xor_static(b"PE", i),
@@ -84,14 +84,14 @@ def extract_file_embedded_pe() -> Iterator[tuple[Feature, Address]]:
yield Characteristic("embedded pe"), FileOffsetAddress(ea)
def extract_file_export_names() -> Iterator[tuple[Feature, Address]]:
def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
"""extract function exports"""
st = currentProgram().getSymbolTable() # type: ignore [name-defined] # noqa: F821
for addr in st.getExternalEntryPointIterator():
yield Export(st.getPrimarySymbol(addr).getName()), AbsoluteVirtualAddress(addr.getOffset())
def extract_file_import_names() -> Iterator[tuple[Feature, Address]]:
def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
"""extract function imports
1. imports by ordinal:
@@ -116,14 +116,14 @@ def extract_file_import_names() -> Iterator[tuple[Feature, Address]]:
yield Import(name), AbsoluteVirtualAddress(addr)
def extract_file_section_names() -> Iterator[tuple[Feature, Address]]:
def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
"""extract section names"""
for block in currentProgram().getMemory().getBlocks(): # type: ignore [name-defined] # noqa: F821
yield Section(block.getName()), AbsoluteVirtualAddress(block.getStart().getOffset())
def extract_file_strings() -> Iterator[tuple[Feature, Address]]:
def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
"""extract ASCII and UTF-16 LE strings"""
for block in currentProgram().getMemory().getBlocks(): # type: ignore [name-defined] # noqa: F821
@@ -141,7 +141,7 @@ def extract_file_strings() -> Iterator[tuple[Feature, Address]]:
yield String(s.s), FileOffsetAddress(offset)
def extract_file_function_names() -> Iterator[tuple[Feature, Address]]:
def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
"""
extract the names of statically-linked library functions.
"""
@@ -162,7 +162,7 @@ def extract_file_function_names() -> Iterator[tuple[Feature, Address]]:
yield FunctionName(name[1:]), addr
def extract_file_format() -> Iterator[tuple[Feature, Address]]:
def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
ef = currentProgram().getExecutableFormat() # type: ignore [name-defined] # noqa: F821
if "PE" in ef:
yield Format(FORMAT_PE), NO_ADDRESS
@@ -175,7 +175,7 @@ def extract_file_format() -> Iterator[tuple[Feature, Address]]:
raise NotImplementedError(f"unexpected file format: {ef}")
def extract_features() -> Iterator[tuple[Feature, Address]]:
def extract_features() -> Iterator[Tuple[Feature, Address]]:
"""extract file features"""
for file_handler in FILE_HANDLERS:
for feature, addr in file_handler():

View File

@@ -5,7 +5,7 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
from typing import Tuple, Iterator
import ghidra
from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
@@ -49,7 +49,7 @@ def extract_recursive_call(fh: FunctionHandle):
yield Characteristic("recursive call"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
for func_handler in FUNCTION_HANDLERS:
for feature, addr in func_handler(fh):
yield feature, addr

View File

@@ -7,7 +7,7 @@
# See the License for the specific language governing permissions and limitations under the License.
import logging
import contextlib
from typing import Iterator
from typing import Tuple, Iterator
import capa.ghidra.helpers
import capa.features.extractors.elf
@@ -18,7 +18,7 @@ from capa.features.address import NO_ADDRESS, Address
logger = logging.getLogger(__name__)
def extract_os() -> Iterator[tuple[Feature, Address]]:
def extract_os() -> Iterator[Tuple[Feature, Address]]:
format_name: str = currentProgram().getExecutableFormat() # type: ignore [name-defined] # noqa: F821
if "PE" in format_name:
@@ -45,7 +45,7 @@ def extract_os() -> Iterator[tuple[Feature, Address]]:
return
def extract_arch() -> Iterator[tuple[Feature, Address]]:
def extract_arch() -> Iterator[Tuple[Feature, Address]]:
lang_id = currentProgram().getMetadata().get("Language ID") # type: ignore [name-defined] # noqa: F821
if "x86" in lang_id and "64" in lang_id:

View File

@@ -5,7 +5,7 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
from typing import Dict, List, Iterator
import ghidra
import java.lang
@@ -20,7 +20,7 @@ from capa.features.address import AbsoluteVirtualAddress
from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
def ints_to_bytes(bytez: list[int]) -> bytes:
def ints_to_bytes(bytez: List[int]) -> bytes:
"""convert Java signed ints to Python bytes
args:
@@ -83,10 +83,10 @@ def get_insn_in_range(bbh: BBHandle) -> Iterator[InsnHandle]:
yield InsnHandle(address=AbsoluteVirtualAddress(insn.getAddress().getOffset()), inner=insn)
def get_file_imports() -> dict[int, list[str]]:
def get_file_imports() -> Dict[int, List[str]]:
"""get all import names & addrs"""
import_dict: dict[int, list[str]] = {}
import_dict: Dict[int, List[str]] = {}
for f in currentProgram().getFunctionManager().getExternalFunctions(): # type: ignore [name-defined] # noqa: F821
for r in f.getSymbol().getReferences():
@@ -110,7 +110,7 @@ def get_file_imports() -> dict[int, list[str]]:
return import_dict
def get_file_externs() -> dict[int, list[str]]:
def get_file_externs() -> Dict[int, List[str]]:
"""
Gets function names & addresses of statically-linked library functions
@@ -124,7 +124,7 @@ def get_file_externs() -> dict[int, list[str]]:
- Note: See Symbol Table labels
"""
extern_dict: dict[int, list[str]] = {}
extern_dict: Dict[int, List[str]] = {}
for sym in currentProgram().getSymbolTable().getAllSymbols(True): # type: ignore [name-defined] # noqa: F821
# .isExternal() misses more than this config for the function symbols
@@ -143,7 +143,7 @@ def get_file_externs() -> dict[int, list[str]]:
return extern_dict
def map_fake_import_addrs() -> dict[int, list[int]]:
def map_fake_import_addrs() -> Dict[int, List[int]]:
"""
Map ghidra's fake import entrypoints to their
real addresses
@@ -162,7 +162,7 @@ def map_fake_import_addrs() -> dict[int, list[int]]:
- 0x473090 -> PTR_CreateServiceW_00473090
- 'EXTERNAL:00000025' -> External Address (ghidra.program.model.address.SpecialAddress)
"""
fake_dict: dict[int, list[int]] = {}
fake_dict: Dict[int, List[int]] = {}
for f in currentProgram().getFunctionManager().getExternalFunctions(): # type: ignore [name-defined] # noqa: F821
for r in f.getSymbol().getReferences():
@@ -174,9 +174,9 @@ def map_fake_import_addrs() -> dict[int, list[int]]:
def check_addr_for_api(
addr: ghidra.program.model.address.Address,
fakes: dict[int, list[int]],
imports: dict[int, list[str]],
externs: dict[int, list[str]],
fakes: Dict[int, List[int]],
imports: Dict[int, List[str]],
externs: Dict[int, List[str]],
) -> bool:
offset = addr.getOffset()

View File

@@ -5,7 +5,7 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Any, Iterator
from typing import Any, Dict, Tuple, Iterator
import ghidra
from ghidra.program.model.lang import OperandType
@@ -26,21 +26,21 @@ SECURITY_COOKIE_BYTES_DELTA = 0x40
OPERAND_TYPE_DYNAMIC_ADDRESS = OperandType.DYNAMIC | OperandType.ADDRESS
def get_imports(ctx: dict[str, Any]) -> dict[int, Any]:
def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
"""Populate the import cache for this context"""
if "imports_cache" not in ctx:
ctx["imports_cache"] = capa.features.extractors.ghidra.helpers.get_file_imports()
return ctx["imports_cache"]
def get_externs(ctx: dict[str, Any]) -> dict[int, Any]:
def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
"""Populate the externs cache for this context"""
if "externs_cache" not in ctx:
ctx["externs_cache"] = capa.features.extractors.ghidra.helpers.get_file_externs()
return ctx["externs_cache"]
def get_fakes(ctx: dict[str, Any]) -> dict[int, Any]:
def get_fakes(ctx: Dict[str, Any]) -> Dict[int, Any]:
"""Populate the fake import addrs cache for this context"""
if "fakes_cache" not in ctx:
ctx["fakes_cache"] = capa.features.extractors.ghidra.helpers.map_fake_import_addrs()
@@ -48,7 +48,7 @@ def get_fakes(ctx: dict[str, Any]) -> dict[int, Any]:
def check_for_api_call(
insn, externs: dict[int, Any], fakes: dict[int, Any], imports: dict[int, Any], imp_or_ex: bool
insn, externs: Dict[int, Any], fakes: Dict[int, Any], imports: Dict[int, Any], imp_or_ex: bool
) -> Iterator[Any]:
"""check instruction for API call
@@ -110,7 +110,7 @@ def check_for_api_call(
yield info
def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
insn: ghidra.program.database.code.InstructionDB = ih.inner
if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
@@ -131,7 +131,7 @@ def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle)
yield API(ext), ih.address
def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction number features
example:
@@ -186,7 +186,7 @@ def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandl
yield OperandOffset(i, const), addr
def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction structure offset features
@@ -219,7 +219,7 @@ def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandl
yield OperandOffset(i, op_off), ih.address
def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""
parse referenced byte sequences
@@ -234,7 +234,7 @@ def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
yield Bytes(extracted_bytes), ih.address
def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction string features
@@ -249,7 +249,7 @@ def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandl
def extract_insn_mnemonic_features(
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction mnemonic features"""
insn: ghidra.program.database.code.InstructionDB = ih.inner
@@ -258,7 +258,7 @@ def extract_insn_mnemonic_features(
def extract_insn_obfs_call_plus_5_characteristic_features(
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse call $+5 instruction from the given instruction.
"""
@@ -279,7 +279,7 @@ def extract_insn_obfs_call_plus_5_characteristic_features(
def extract_insn_segment_access_features(
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction fs or gs access"""
insn: ghidra.program.database.code.InstructionDB = ih.inner
@@ -294,7 +294,7 @@ def extract_insn_segment_access_features(
def extract_insn_peb_access_characteristic_features(
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction peb access
fs:[0x30] on x86, gs:[0x60] on x64
@@ -310,7 +310,7 @@ def extract_insn_peb_access_characteristic_features(
def extract_insn_cross_section_cflow(
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""inspect the instruction for a CALL or JMP that crosses section boundaries"""
insn: ghidra.program.database.code.InstructionDB = ih.inner
@@ -364,7 +364,7 @@ def extract_function_calls_from(
fh: FunctionHandle,
bb: BBHandle,
ih: InsnHandle,
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""extract functions calls from features
most relevant at the function scope, however, its most efficient to extract at the instruction scope
@@ -393,7 +393,7 @@ def extract_function_indirect_call_characteristic_features(
fh: FunctionHandle,
bb: BBHandle,
ih: InsnHandle,
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
does not include calls like => call ds:dword_ABD4974
@@ -442,7 +442,7 @@ def extract_insn_nzxor_characteristic_features(
fh: FunctionHandle,
bb: BBHandle,
ih: InsnHandle,
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
f: ghidra.program.database.function.FunctionDB = fh.inner
insn: ghidra.program.database.code.InstructionDB = ih.inner
@@ -461,7 +461,7 @@ def extract_features(
fh: FunctionHandle,
bb: BBHandle,
insn: InsnHandle,
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
for insn_handler in INSTRUCTION_HANDLERS:
for feature, addr in insn_handler(fh, bb, insn):
yield feature, addr

View File

@@ -8,7 +8,7 @@
import struct
import builtins
from typing import Iterator
from typing import Tuple, Iterator
MIN_STACKSTRING_LEN = 8
@@ -63,7 +63,6 @@ def generate_symbols(dll: str, symbol: str, include_dll=False) -> Iterator[str]:
# trim extensions observed in dynamic traces
dll = dll[0:-4] if dll.endswith(".dll") else dll
dll = dll[0:-4] if dll.endswith(".drv") else dll
dll = dll[0:-3] if dll.endswith(".so") else dll
if include_dll or is_ordinal(symbol):
# ws2_32.#1
@@ -119,7 +118,7 @@ def twos_complement(val: int, bits: int) -> int:
return val
def carve_pe(pbytes: bytes, offset: int = 0) -> Iterator[tuple[int, int]]:
def carve_pe(pbytes: bytes, offset: int = 0) -> Iterator[Tuple[int, int]]:
"""
Generate (offset, key) tuples of embedded PEs

View File

@@ -8,7 +8,7 @@
import string
import struct
from typing import Iterator
from typing import Tuple, Iterator
import idaapi
@@ -80,19 +80,19 @@ def bb_contains_stackstring(f: idaapi.func_t, bb: idaapi.BasicBlock) -> bool:
return False
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract stackstring indicators from basic block"""
if bb_contains_stackstring(fh.inner, bbh.inner):
yield Characteristic("stack string"), bbh.address
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract tight loop indicators from a basic block"""
if capa.features.extractors.ida.helpers.is_basic_block_tight_loop(bbh.inner):
yield Characteristic("tight loop"), bbh.address
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract basic block features"""
for bb_handler in BASIC_BLOCK_HANDLERS:
for feature, addr in bb_handler(fh, bbh):

View File

@@ -5,9 +5,10 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
from typing import List, Tuple, Iterator
import idaapi
import ida_nalt
import capa.ida.helpers
import capa.features.extractors.elf
@@ -31,12 +32,10 @@ class IdaFeatureExtractor(StaticFeatureExtractor):
def __init__(self):
super().__init__(
hashes=SampleHashes(
md5=capa.ida.helpers.retrieve_input_file_md5(),
sha1="(unknown)",
sha256=capa.ida.helpers.retrieve_input_file_sha256(),
md5=ida_nalt.retrieve_input_file_md5(), sha1="(unknown)", sha256=ida_nalt.retrieve_input_file_sha256()
)
)
self.global_features: list[tuple[Feature, Address]] = []
self.global_features: List[Tuple[Feature, Address]] = []
self.global_features.extend(capa.features.extractors.ida.file.extract_file_format())
self.global_features.extend(capa.features.extractors.ida.global_.extract_os())
self.global_features.extend(capa.features.extractors.ida.global_.extract_arch())
@@ -61,7 +60,7 @@ class IdaFeatureExtractor(StaticFeatureExtractor):
f = idaapi.get_func(ea)
return FunctionHandle(address=AbsoluteVirtualAddress(f.start_ea), inner=f)
def extract_function_features(self, fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.ida.function.extract_features(fh)
def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
@@ -70,7 +69,7 @@ class IdaFeatureExtractor(StaticFeatureExtractor):
for bb in ida_helpers.get_function_blocks(fh.inner):
yield BBHandle(address=AbsoluteVirtualAddress(bb.start_ea), inner=bb)
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.ida.basicblock.extract_features(fh, bbh)
def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:

View File

@@ -7,14 +7,13 @@
# See the License for the specific language governing permissions and limitations under the License.
import struct
from typing import Iterator
from typing import Tuple, Iterator
import idc
import idaapi
import idautils
import ida_entry
import capa.ida.helpers
import capa.features.extractors.common
import capa.features.extractors.helpers
import capa.features.extractors.strings
@@ -26,7 +25,7 @@ from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, Absolu
MAX_OFFSET_PE_AFTER_MZ = 0x200
def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[tuple[int, int]]:
def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
"""check segment for embedded PE
adapted for IDA from:
@@ -71,7 +70,7 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[tuple[int, int]]:
yield off, i
def extract_file_embedded_pe() -> Iterator[tuple[Feature, Address]]:
def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
"""extract embedded PE features
IDA must load resource sections for this to be complete
@@ -83,7 +82,7 @@ def extract_file_embedded_pe() -> Iterator[tuple[Feature, Address]]:
yield Characteristic("embedded pe"), FileOffsetAddress(ea)
def extract_file_export_names() -> Iterator[tuple[Feature, Address]]:
def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
"""extract function exports"""
for _, ordinal, ea, name in idautils.Entries():
forwarded_name = ida_entry.get_entry_forwarder(ordinal)
@@ -95,7 +94,7 @@ def extract_file_export_names() -> Iterator[tuple[Feature, Address]]:
yield Characteristic("forwarded export"), AbsoluteVirtualAddress(ea)
def extract_file_import_names() -> Iterator[tuple[Feature, Address]]:
def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
"""extract function imports
1. imports by ordinal:
@@ -131,7 +130,7 @@ def extract_file_import_names() -> Iterator[tuple[Feature, Address]]:
yield Import(info[1]), AbsoluteVirtualAddress(ea)
def extract_file_section_names() -> Iterator[tuple[Feature, Address]]:
def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
"""extract section names
IDA must load resource sections for this to be complete
@@ -142,7 +141,7 @@ def extract_file_section_names() -> Iterator[tuple[Feature, Address]]:
yield Section(idaapi.get_segm_name(seg)), AbsoluteVirtualAddress(seg.start_ea)
def extract_file_strings() -> Iterator[tuple[Feature, Address]]:
def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
"""extract ASCII and UTF-16 LE strings
IDA must load resource sections for this to be complete
@@ -160,7 +159,7 @@ def extract_file_strings() -> Iterator[tuple[Feature, Address]]:
yield String(s.s), FileOffsetAddress(seg.start_ea + s.offset)
def extract_file_function_names() -> Iterator[tuple[Feature, Address]]:
def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
"""
extract the names of statically-linked library functions.
"""
@@ -177,21 +176,21 @@ def extract_file_function_names() -> Iterator[tuple[Feature, Address]]:
yield FunctionName(name[1:]), addr
def extract_file_format() -> Iterator[tuple[Feature, Address]]:
filetype = capa.ida.helpers.get_filetype()
def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
file_info = idaapi.get_inf_structure()
if filetype in (idaapi.f_PE, idaapi.f_COFF):
if file_info.filetype in (idaapi.f_PE, idaapi.f_COFF):
yield Format(FORMAT_PE), NO_ADDRESS
elif filetype == idaapi.f_ELF:
elif file_info.filetype == idaapi.f_ELF:
yield Format(FORMAT_ELF), NO_ADDRESS
elif filetype == idaapi.f_BIN:
elif file_info.filetype == idaapi.f_BIN:
# no file type to return when processing a binary file, but we want to continue processing
return
else:
raise NotImplementedError(f"unexpected file format: {filetype}")
raise NotImplementedError(f"unexpected file format: {file_info.filetype}")
def extract_features() -> Iterator[tuple[Feature, Address]]:
def extract_features() -> Iterator[Tuple[Feature, Address]]:
"""extract file features"""
for file_handler in FILE_HANDLERS:
for feature, addr in file_handler():

View File

@@ -5,7 +5,7 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
from typing import Tuple, Iterator
import idaapi
import idautils
@@ -43,7 +43,7 @@ def extract_recursive_call(fh: FunctionHandle):
yield Characteristic("recursive call"), fh.address
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
for func_handler in FUNCTION_HANDLERS:
for feature, addr in func_handler(fh):
yield feature, addr

View File

@@ -7,8 +7,9 @@
# See the License for the specific language governing permissions and limitations under the License.
import logging
import contextlib
from typing import Iterator
from typing import Tuple, Iterator
import idaapi
import ida_loader
import capa.ida.helpers
@@ -19,7 +20,7 @@ from capa.features.address import NO_ADDRESS, Address
logger = logging.getLogger(__name__)
def extract_os() -> Iterator[tuple[Feature, Address]]:
def extract_os() -> Iterator[Tuple[Feature, Address]]:
format_name: str = ida_loader.get_file_type_name()
if "PE" in format_name:
@@ -46,13 +47,13 @@ def extract_os() -> Iterator[tuple[Feature, Address]]:
return
def extract_arch() -> Iterator[tuple[Feature, Address]]:
procname = capa.ida.helpers.get_processor_name()
if procname == "metapc" and capa.ida.helpers.is_64bit():
def extract_arch() -> Iterator[Tuple[Feature, Address]]:
info: idaapi.idainfo = idaapi.get_inf_structure()
if info.procname == "metapc" and info.is_64bit():
yield Arch(ARCH_AMD64), NO_ADDRESS
elif procname == "metapc" and capa.ida.helpers.is_32bit():
elif info.procname == "metapc" and info.is_32bit():
yield Arch(ARCH_I386), NO_ADDRESS
elif procname == "metapc":
elif info.procname == "metapc":
logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel")
return
else:
@@ -60,5 +61,5 @@ def extract_arch() -> Iterator[tuple[Feature, Address]]:
# 1. handling a new architecture (e.g. aarch64)
#
# for (1), this logic will need to be updated as the format is implemented.
logger.debug("unsupported architecture: %s", procname)
logger.debug("unsupported architecture: %s", info.procname)
return

View File

@@ -6,7 +6,7 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import functools
from typing import Any, Iterator, Optional
from typing import Any, Dict, Tuple, Iterator, Optional
import idc
import idaapi
@@ -21,57 +21,28 @@ from capa.features.extractors.base_extractor import FunctionHandle
IDA_NALT_ENCODING = ida_nalt.get_default_encoding_idx(ida_nalt.BPU_1B) # use one byte-per-character encoding
if hasattr(ida_bytes, "parse_binpat_str"):
# TODO (mr): use find_bytes
# https://github.com/mandiant/capa/issues/2339
def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
"""yield all ea of a given byte sequence
def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
"""yield all ea of a given byte sequence
args:
start: min virtual address
end: max virtual address
seq: bytes to search e.g. b"\x01\x03"
"""
patterns = ida_bytes.compiled_binpat_vec_t()
args:
start: min virtual address
end: max virtual address
seq: bytes to search e.g. b"\x01\x03"
"""
patterns = ida_bytes.compiled_binpat_vec_t()
seqstr = " ".join([f"{b:02x}" for b in seq])
err = ida_bytes.parse_binpat_str(patterns, 0, seqstr, 16, IDA_NALT_ENCODING)
seqstr = " ".join([f"{b:02x}" for b in seq])
err = ida_bytes.parse_binpat_str(patterns, 0, seqstr, 16, IDA_NALT_ENCODING)
if err:
return
if err:
return
while True:
ea = ida_bytes.bin_search(start, end, patterns, ida_bytes.BIN_SEARCH_FORWARD)
if isinstance(ea, int):
# "ea_t" in IDA 8.4, 8.3
pass
elif isinstance(ea, tuple):
# "drc_t" in IDA 9
ea = ea[0]
else:
raise NotImplementedError(f"bin_search returned unhandled type: {type(ea)}")
if ea == idaapi.BADADDR:
break
start = ea + 1
yield ea
else:
# for IDA 7.5 and older; using deprecated find_binary instead of bin_search
def find_byte_sequence(start: int, end: int, seq: bytes) -> Iterator[int]:
"""yield all ea of a given byte sequence
args:
start: min virtual address
end: max virtual address
seq: bytes to search e.g. b"\x01\x03"
"""
seqstr = " ".join([f"{b:02x}" for b in seq])
while True:
ea = idaapi.find_binary(start, end, seqstr, 0, idaapi.SEARCH_DOWN)
if ea == idaapi.BADADDR:
break
start = ea + 1
yield ea
while True:
ea = ida_bytes.bin_search(start, end, patterns, ida_bytes.BIN_SEARCH_FORWARD)
if ea == idaapi.BADADDR:
break
start = ea + 1
yield ea
def get_functions(
@@ -132,9 +103,9 @@ def inspect_import(imports, library, ea, function, ordinal):
return True
def get_file_imports() -> dict[int, tuple[str, str, int]]:
def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
"""get file imports"""
imports: dict[int, tuple[str, str, int]] = {}
imports: Dict[int, Tuple[str, str, int]] = {}
for idx in range(idaapi.get_import_module_qty()):
library = idaapi.get_import_module_name(idx)
@@ -155,7 +126,7 @@ def get_file_imports() -> dict[int, tuple[str, str, int]]:
return imports
def get_file_externs() -> dict[int, tuple[str, str, int]]:
def get_file_externs() -> Dict[int, Tuple[str, str, int]]:
externs = {}
for seg in get_segments(skip_header_segments=True):
@@ -256,7 +227,7 @@ def find_string_at(ea: int, min_: int = 4) -> str:
return ""
def get_op_phrase_info(op: idaapi.op_t) -> dict:
def get_op_phrase_info(op: idaapi.op_t) -> Dict:
"""parse phrase features from operand
Pretty much dup of sark's implementation:
@@ -331,7 +302,7 @@ def is_frame_register(reg: int) -> bool:
return reg in (idautils.procregs.sp.reg, idautils.procregs.bp.reg)
def get_insn_ops(insn: idaapi.insn_t, target_ops: Optional[tuple[Any]] = None) -> idaapi.op_t:
def get_insn_ops(insn: idaapi.insn_t, target_ops: Optional[Tuple[Any]] = None) -> idaapi.op_t:
"""yield op_t for instruction, filter on type if specified"""
for op in insn.ops:
if op.type == idaapi.o_void:

View File

@@ -1,117 +0,0 @@
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at: [package root]/LICENSE.txt
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import os
import sys
import json
import logging
import importlib.util
from typing import Optional
from pathlib import Path
logger = logging.getLogger(__name__)
def is_idalib_installed() -> bool:
try:
return importlib.util.find_spec("idapro") is not None
except ModuleNotFoundError:
return False
def get_idalib_user_config_path() -> Optional[Path]:
"""Get the path to the user's config file based on platform following IDA's user directories."""
# derived from `py-activate-idalib.py` from IDA v9.0 Beta 4
if sys.platform == "win32":
# On Windows, use the %APPDATA%\Hex-Rays\IDA Pro directory
config_dir = Path(os.getenv("APPDATA")) / "Hex-Rays" / "IDA Pro"
else:
# On macOS and Linux, use ~/.idapro
config_dir = Path.home() / ".idapro"
# Return the full path to the config file (now in JSON format)
user_config_path = config_dir / "ida-config.json"
if not user_config_path.exists():
return None
return user_config_path
def find_idalib() -> Optional[Path]:
config_path = get_idalib_user_config_path()
if not config_path:
logger.error("IDA Pro user configuration does not exist, please make sure you've installed idalib properly.")
return None
config = json.loads(config_path.read_text(encoding="utf-8"))
try:
ida_install_dir = Path(config["Paths"]["ida-install-dir"])
except KeyError:
logger.error(
"IDA Pro user configuration does not contain location of IDA Pro installation, please make sure you've installed idalib properly."
)
return None
if not ida_install_dir.exists():
return None
libname = {
"win32": "idalib.dll",
"linux": "libidalib.so",
"linux2": "libidalib.so",
"darwin": "libidalib.dylib",
}[sys.platform]
if not (ida_install_dir / "ida.hlp").is_file():
return None
if not (ida_install_dir / libname).is_file():
return None
idalib_path = ida_install_dir / "idalib" / "python"
if not idalib_path.exists():
return None
if not (idalib_path / "idapro" / "__init__.py").is_file():
return None
return idalib_path
def has_idalib() -> bool:
if is_idalib_installed():
logger.debug("found installed IDA idalib API")
return True
logger.debug("IDA idalib API not installed, searching...")
idalib_path = find_idalib()
if not idalib_path:
logger.debug("failed to find IDA idalib installation")
logger.debug("found IDA idalib API: %s", idalib_path)
return idalib_path is not None
def load_idalib() -> bool:
try:
import idapro
return True
except ImportError:
idalib_path = find_idalib()
if not idalib_path:
return False
sys.path.append(idalib_path.absolute().as_posix())
try:
import idapro # noqa: F401 unused import
return True
except ImportError:
return False

View File

@@ -5,11 +5,9 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import re
from typing import Any, Iterator, Optional
from typing import Any, Dict, Tuple, Iterator
import idc
import ida_ua
import idaapi
import idautils
@@ -25,21 +23,21 @@ from capa.features.extractors.base_extractor import BBHandle, InsnHandle, Functi
SECURITY_COOKIE_BYTES_DELTA = 0x40
def get_imports(ctx: dict[str, Any]) -> dict[int, Any]:
def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
if "imports_cache" not in ctx:
ctx["imports_cache"] = capa.features.extractors.ida.helpers.get_file_imports()
return ctx["imports_cache"]
def get_externs(ctx: dict[str, Any]) -> dict[int, Any]:
def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
if "externs_cache" not in ctx:
ctx["externs_cache"] = capa.features.extractors.ida.helpers.get_file_externs()
return ctx["externs_cache"]
def check_for_api_call(insn: idaapi.insn_t, funcs: dict[int, Any]) -> Optional[tuple[str, str]]:
def check_for_api_call(insn: idaapi.insn_t, funcs: Dict[int, Any]) -> Iterator[Any]:
"""check instruction for API call"""
info = None
info = ()
ref = insn.ea
# attempt to resolve API calls by following chained thunks to a reasonable depth
@@ -54,7 +52,7 @@ def check_for_api_call(insn: idaapi.insn_t, funcs: dict[int, Any]) -> Optional[t
except IndexError:
break
info = funcs.get(ref)
info = funcs.get(ref, ())
if info:
break
@@ -62,10 +60,11 @@ def check_for_api_call(insn: idaapi.insn_t, funcs: dict[int, Any]) -> Optional[t
if not f or not (f.flags & idaapi.FUNC_THUNK):
break
return info
if info:
yield info
def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction API features
@@ -77,39 +76,16 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
if insn.get_canon_mnem() not in ("call", "jmp"):
return
# check call to imported functions
api = check_for_api_call(insn, get_imports(fh.ctx))
if api:
# check calls to imported functions
for api in check_for_api_call(insn, get_imports(fh.ctx)):
# tuple (<module>, <function>, <ordinal>)
for name in capa.features.extractors.helpers.generate_symbols(api[0], api[1]):
yield API(name), ih.address
# a call instruction should only call one function, stop if a call to an import is extracted
return
# check call to extern functions
api = check_for_api_call(insn, get_externs(fh.ctx))
if api:
# check calls to extern functions
for api in check_for_api_call(insn, get_externs(fh.ctx)):
# tuple (<module>, <function>, <ordinal>)
yield API(api[1]), ih.address
# a call instruction should only call one function, stop if a call to an extern is extracted
return
# extract dynamically resolved APIs stored in renamed globals (renamed for example using `renimp.idc`)
# examples: `CreateProcessA`, `HttpSendRequestA`
if insn.Op1.type == ida_ua.o_mem:
op_addr = insn.Op1.addr
op_name = idaapi.get_name(op_addr)
# when renaming a global using an API name, IDA assigns it the function type
# ensure we do not extract something wrong by checking that the address has a name and a type
# we could check that the type is a function definition, but that complicates the code
if (not op_name.startswith("off_")) and idc.get_type(op_addr):
# Remove suffix used in repeated names, for example _0 in VirtualFree_0
match = re.match(r"(.+)_\d+", op_name)
if match:
op_name = match.group(1)
# the global name does not include the DLL name, so we can't extract it
for name in capa.features.extractors.helpers.generate_symbols("", op_name):
yield API(name), ih.address
# extract IDA/FLIRT recognized API functions
targets = tuple(idautils.CodeRefsFrom(insn.ea, False))
@@ -135,7 +111,7 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
def extract_insn_number_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction number features
example:
@@ -181,7 +157,7 @@ def extract_insn_number_features(
yield OperandOffset(i, const), ih.address
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""
parse referenced byte sequences
example:
@@ -203,7 +179,7 @@ def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
def extract_insn_string_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction string features
@@ -221,7 +197,7 @@ def extract_insn_string_features(
def extract_insn_offset_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction structure offset features
@@ -369,7 +345,7 @@ def is_nzxor_stack_cookie(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.
def extract_insn_nzxor_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse instruction non-zeroing XOR instruction
ignore expected non-zeroing XORs, e.g. security cookies
@@ -387,14 +363,14 @@ def extract_insn_nzxor_characteristic_features(
def extract_insn_mnemonic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction mnemonic features"""
yield Mnemonic(idc.print_insn_mnem(ih.inner.ea)), ih.address
def extract_insn_obfs_call_plus_5_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""
parse call $+5 instruction from the given instruction.
"""
@@ -409,7 +385,7 @@ def extract_insn_obfs_call_plus_5_characteristic_features(
def extract_insn_peb_access_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction peb access
fs:[0x30] on x86, gs:[0x60] on x64
@@ -437,7 +413,7 @@ def extract_insn_peb_access_characteristic_features(
def extract_insn_segment_access_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""parse instruction fs or gs access
TODO:
@@ -466,7 +442,7 @@ def extract_insn_segment_access_features(
def extract_insn_cross_section_cflow(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""inspect the instruction for a CALL or JMP that crosses section boundaries"""
insn: idaapi.insn_t = ih.inner
@@ -482,7 +458,7 @@ def extract_insn_cross_section_cflow(
yield Characteristic("cross section flow"), ih.address
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract functions calls from features
most relevant at the function scope, however, its most efficient to extract at the instruction scope
@@ -496,7 +472,7 @@ def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
def extract_function_indirect_call_characteristic_features(
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
"""extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
does not include calls like => call ds:dword_ABD4974
@@ -509,7 +485,7 @@ def extract_function_indirect_call_characteristic_features(
yield Characteristic("indirect call"), ih.address
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
"""extract instruction features"""
for inst_handler in INSTRUCTION_HANDLERS:
for feature, ea in inst_handler(f, bbh, insn):

View File

@@ -5,9 +5,11 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Union, TypeAlias
from typing import Dict, List, Tuple, Union
from dataclasses import dataclass
from typing_extensions import TypeAlias
from capa.features.common import Feature
from capa.features.address import NO_ADDRESS, Address, ThreadAddress, ProcessAddress, DynamicCallAddress
from capa.features.extractors.base_extractor import (
@@ -25,19 +27,19 @@ from capa.features.extractors.base_extractor import (
@dataclass
class InstructionFeatures:
features: list[tuple[Address, Feature]]
features: List[Tuple[Address, Feature]]
@dataclass
class BasicBlockFeatures:
features: list[tuple[Address, Feature]]
instructions: dict[Address, InstructionFeatures]
features: List[Tuple[Address, Feature]]
instructions: Dict[Address, InstructionFeatures]
@dataclass
class FunctionFeatures:
features: list[tuple[Address, Feature]]
basic_blocks: dict[Address, BasicBlockFeatures]
features: List[Tuple[Address, Feature]]
basic_blocks: Dict[Address, BasicBlockFeatures]
@dataclass
@@ -50,9 +52,9 @@ class NullStaticFeatureExtractor(StaticFeatureExtractor):
base_address: Address
sample_hashes: SampleHashes
global_features: list[Feature]
file_features: list[tuple[Address, Feature]]
functions: dict[Address, FunctionFeatures]
global_features: List[Feature]
file_features: List[Tuple[Address, Feature]]
functions: Dict[Address, FunctionFeatures]
def get_base_address(self):
return self.base_address
@@ -96,19 +98,19 @@ class NullStaticFeatureExtractor(StaticFeatureExtractor):
@dataclass
class CallFeatures:
name: str
features: list[tuple[Address, Feature]]
features: List[Tuple[Address, Feature]]
@dataclass
class ThreadFeatures:
features: list[tuple[Address, Feature]]
calls: dict[Address, CallFeatures]
features: List[Tuple[Address, Feature]]
calls: Dict[Address, CallFeatures]
@dataclass
class ProcessFeatures:
features: list[tuple[Address, Feature]]
threads: dict[Address, ThreadFeatures]
features: List[Tuple[Address, Feature]]
threads: Dict[Address, ThreadFeatures]
name: str
@@ -116,9 +118,9 @@ class ProcessFeatures:
class NullDynamicFeatureExtractor(DynamicFeatureExtractor):
base_address: Address
sample_hashes: SampleHashes
global_features: list[Feature]
file_features: list[tuple[Address, Feature]]
processes: dict[Address, ProcessFeatures]
global_features: List[Feature]
file_features: List[Tuple[Address, Feature]]
processes: Dict[Address, ProcessFeatures]
def extract_global_features(self):
for feature in self.global_features:

View File

@@ -130,13 +130,7 @@ def extract_file_arch(pe, **kwargs):
elif pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_AMD64"]:
yield Arch(ARCH_AMD64), NO_ADDRESS
else:
try:
logger.warning(
"unsupported architecture: %s",
pefile.MACHINE_TYPE[pe.FILE_HEADER.Machine],
)
except KeyError:
logger.warning("unknown architecture: %s", pe.FILE_HEADER.Machine)
logger.warning("unsupported architecture: %s", pefile.MACHINE_TYPE[pe.FILE_HEADER.Machine])
def extract_file_features(pe, buf):
@@ -148,11 +142,11 @@ def extract_file_features(pe, buf):
buf: the raw sample bytes
yields:
tuple[Feature, VA]: a feature and its location.
Tuple[Feature, VA]: a feature and its location.
"""
for file_handler in FILE_HANDLERS:
# file_handler: type: (pe, bytes) -> Iterable[tuple[Feature, Address]]
# file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
for feature, va in file_handler(pe=pe, buf=buf): # type: ignore
yield feature, va
@@ -177,10 +171,10 @@ def extract_global_features(pe, buf):
buf: the raw sample bytes
yields:
tuple[Feature, VA]: a feature and its location.
Tuple[Feature, VA]: a feature and its location.
"""
for handler in GLOBAL_HANDLERS:
# file_handler: type: (pe, bytes) -> Iterable[tuple[Feature, Address]]
# file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
for feature, va in handler(pe=pe, buf=buf): # type: ignore
yield feature, va

View File

@@ -8,7 +8,7 @@
import string
import struct
from typing import Iterator
from typing import Tuple, Iterator
import envi
import envi.archs.i386.disasm
@@ -20,7 +20,7 @@ from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
def interface_extract_basic_block_XXX(f: FunctionHandle, bb: BBHandle) -> Iterator[tuple[Feature, Address]]:
def interface_extract_basic_block_XXX(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""
parse features from the given basic block.
@@ -47,7 +47,7 @@ def _bb_has_tight_loop(f, bb):
return False
def extract_bb_tight_loop(f: FunctionHandle, bb: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_bb_tight_loop(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""check basic block for tight loop indicators"""
if _bb_has_tight_loop(f, bb.inner):
yield Characteristic("tight loop"), bb.address
@@ -70,7 +70,7 @@ def _bb_has_stackstring(f, bb):
return False
def extract_stackstring(f: FunctionHandle, bb: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_stackstring(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""check basic block for stackstring indicators"""
if _bb_has_stackstring(f, bb.inner):
yield Characteristic("stack string"), bb.address
@@ -145,7 +145,7 @@ def is_printable_utf16le(chars: bytes) -> bool:
return False
def extract_features(f: FunctionHandle, bb: BBHandle) -> Iterator[tuple[Feature, Address]]:
def extract_features(f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
"""
extract features from the given basic block.
@@ -154,7 +154,7 @@ def extract_features(f: FunctionHandle, bb: BBHandle) -> Iterator[tuple[Feature,
bb (viv_utils.BasicBlock): the basic block to process.
yields:
tuple[Feature, int]: the features and their location found in this basic block.
Tuple[Feature, int]: the features and their location found in this basic block.
"""
yield BasicBlock(), AbsoluteVirtualAddress(bb.inner.va)
for bb_handler in BASIC_BLOCK_HANDLERS:

View File

@@ -6,7 +6,7 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
import logging
from typing import Any, Iterator
from typing import Any, Dict, List, Tuple, Iterator
from pathlib import Path
import viv_utils
@@ -39,7 +39,7 @@ class VivisectFeatureExtractor(StaticFeatureExtractor):
super().__init__(hashes=SampleHashes.from_bytes(self.buf))
# pre-compute these because we'll yield them at *every* scope.
self.global_features: list[tuple[Feature, Address]] = []
self.global_features: List[Tuple[Feature, Address]] = []
self.global_features.extend(capa.features.extractors.viv.file.extract_file_format(self.buf))
self.global_features.extend(capa.features.extractors.common.extract_os(self.buf, os))
self.global_features.extend(capa.features.extractors.viv.global_.extract_arch(self.vw))
@@ -55,13 +55,13 @@ class VivisectFeatureExtractor(StaticFeatureExtractor):
yield from capa.features.extractors.viv.file.extract_features(self.vw, self.buf)
def get_functions(self) -> Iterator[FunctionHandle]:
cache: dict[str, Any] = {}
cache: Dict[str, Any] = {}
for va in sorted(self.vw.getFunctions()):
yield FunctionHandle(
address=AbsoluteVirtualAddress(va), inner=viv_utils.Function(self.vw, va), ctx={"cache": cache}
)
def extract_function_features(self, fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.viv.function.extract_features(fh)
def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
@@ -69,7 +69,7 @@ class VivisectFeatureExtractor(StaticFeatureExtractor):
for bb in f.basic_blocks:
yield BBHandle(address=AbsoluteVirtualAddress(bb.va), inner=bb)
def extract_basic_block_features(self, fh: FunctionHandle, bbh) -> Iterator[tuple[Feature, Address]]:
def extract_basic_block_features(self, fh: FunctionHandle, bbh) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.viv.basicblock.extract_features(fh, bbh)
def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
@@ -79,7 +79,7 @@ class VivisectFeatureExtractor(StaticFeatureExtractor):
def extract_insn_features(
self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
) -> Iterator[tuple[Feature, Address]]:
) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.viv.insn.extract_features(fh, bbh, ih)
def is_library_function(self, addr):

View File

@@ -5,7 +5,7 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
from typing import Iterator
from typing import Tuple, Iterator
import PE.carve as pe_carve # vivisect PE
import vivisect
@@ -21,7 +21,7 @@ from capa.features.common import Feature, Characteristic
from capa.features.address import Address, FileOffsetAddress, AbsoluteVirtualAddress
def extract_file_embedded_pe(buf, **kwargs) -> Iterator[tuple[Feature, Address]]:
def extract_file_embedded_pe(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]:
for offset, _ in pe_carve.carve(buf, 1):
yield Characteristic("embedded pe"), FileOffsetAddress(offset)
@@ -37,7 +37,7 @@ def get_first_vw_filename(vw: vivisect.VivWorkspace):
return next(iter(vw.filemeta.keys()))
def extract_file_export_names(vw: vivisect.VivWorkspace, **kwargs) -> Iterator[tuple[Feature, Address]]:
def extract_file_export_names(vw: vivisect.VivWorkspace, **kwargs) -> Iterator[Tuple[Feature, Address]]:
for va, _, name, _ in vw.getExports():
yield Export(name), AbsoluteVirtualAddress(va)
@@ -56,7 +56,7 @@ def extract_file_export_names(vw: vivisect.VivWorkspace, **kwargs) -> Iterator[t
yield Characteristic("forwarded export"), AbsoluteVirtualAddress(va)
def extract_file_import_names(vw, **kwargs) -> Iterator[tuple[Feature, Address]]:
def extract_file_import_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
"""
extract imported function names
1. imports by ordinal:
@@ -91,16 +91,16 @@ def is_viv_ord_impname(impname: str) -> bool:
return True
def extract_file_section_names(vw, **kwargs) -> Iterator[tuple[Feature, Address]]:
def extract_file_section_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
for va, _, segname, _ in vw.getSegments():
yield Section(segname), AbsoluteVirtualAddress(va)
def extract_file_strings(buf, **kwargs) -> Iterator[tuple[Feature, Address]]:
def extract_file_strings(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.common.extract_file_strings(buf)
def extract_file_function_names(vw, **kwargs) -> Iterator[tuple[Feature, Address]]:
def extract_file_function_names(vw, **kwargs) -> Iterator[Tuple[Feature, Address]]:
"""
extract the names of statically-linked library functions.
"""
@@ -117,11 +117,11 @@ def extract_file_function_names(vw, **kwargs) -> Iterator[tuple[Feature, Address
yield FunctionName(name[1:]), addr
def extract_file_format(buf, **kwargs) -> Iterator[tuple[Feature, Address]]:
def extract_file_format(buf, **kwargs) -> Iterator[Tuple[Feature, Address]]:
yield from capa.features.extractors.common.extract_format(buf)
def extract_features(vw, buf: bytes) -> Iterator[tuple[Feature, Address]]:
def extract_features(vw, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
"""
extract file features from given workspace
@@ -130,7 +130,7 @@ def extract_features(vw, buf: bytes) -> Iterator[tuple[Feature, Address]]:
buf: the raw input file bytes
yields:
tuple[Feature, Address]: a feature and its location.
Tuple[Feature, Address]: a feature and its location.
"""
for file_handler in FILE_HANDLERS:

Some files were not shown because too many files have changed in this diff Show More