mirror of
https://github.com/mandiant/capa.git
synced 2025-12-08 13:50:38 -08:00
Compare commits
1 Commits
update-ida
...
mr/library
| Author | SHA1 | Date | |
|---|---|---|---|
|
304f835a1a |
@@ -1,22 +0,0 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "9.2.1"
|
||||
|
||||
[[tool.bumpversion.files]]
|
||||
filename = "capa/version.py"
|
||||
search = '__version__ = "{current_version}"'
|
||||
replace = '__version__ = "{new_version}"'
|
||||
|
||||
[[tool.bumpversion.files]]
|
||||
filename = "capa/ida/plugin/ida-plugin.json"
|
||||
search = '"version": "{current_version}"'
|
||||
replace = '"version": "{new_version}"'
|
||||
|
||||
[[tool.bumpversion.files]]
|
||||
filename = "capa/ida/plugin/ida-plugin.json"
|
||||
search = '"flare-capa=={current_version}"'
|
||||
replace = '"flare-capa=={new_version}"'
|
||||
|
||||
[[tool.bumpversion.files]]
|
||||
filename = "CHANGELOG.md"
|
||||
search = "v{current_version}...master"
|
||||
replace = "{current_version}...{new_version}"
|
||||
@@ -1,6 +1,6 @@
|
||||
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.233.0/containers/python-3/.devcontainer/base.Dockerfile
|
||||
|
||||
# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3-bullseye, 3.10-bullseye, 3-buster, 3.10-buster, etc.
|
||||
# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
|
||||
ARG VARIANT="3.10-bullseye"
|
||||
FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"dockerfile": "Dockerfile",
|
||||
"context": "..",
|
||||
"args": {
|
||||
// Update 'VARIANT' to pick a Python version: 3, 3.10, etc.
|
||||
// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
|
||||
// Append -bullseye or -buster to pin to an OS version.
|
||||
// Use -bullseye variants on local on arm64/Apple Silicon.
|
||||
"VARIANT": "3.10",
|
||||
|
||||
46
.github/CODE_OF_CONDUCT.md
vendored
Normal file
46
.github/CODE_OF_CONDUCT.md
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to creating a positive environment include:
|
||||
|
||||
* Using welcoming and inclusive language
|
||||
* Being respectful of differing viewpoints and experiences
|
||||
* Gracefully accepting constructive criticism
|
||||
* Focusing on what is best for the community
|
||||
* Showing empathy towards other community members
|
||||
|
||||
Examples of unacceptable behavior by participants include:
|
||||
|
||||
* The use of sexualized language or imagery and unwelcome sexual attention or advances
|
||||
* Trolling, insulting/derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or electronic address, without explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a professional setting
|
||||
|
||||
## Our Responsibilities
|
||||
|
||||
Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
|
||||
|
||||
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [https://contributor-covenant.org/version/1/4][version]
|
||||
|
||||
[homepage]: https://contributor-covenant.org
|
||||
[version]: https://contributor-covenant.org/version/1/4/
|
||||
20
.github/CONTRIBUTING.md
vendored
20
.github/CONTRIBUTING.md
vendored
@@ -25,7 +25,7 @@ The following is a set of guidelines for contributing to capa and its packages,
|
||||
|
||||
## Code of Conduct
|
||||
|
||||
This project follows [Google's Open Source Community Guidelines](https://opensource.google/conduct).
|
||||
This project and everyone participating in it is governed by the [Capa Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the maintainers.
|
||||
|
||||
## What should I know before I get started?
|
||||
|
||||
@@ -168,17 +168,15 @@ While the prerequisites above must be satisfied prior to having your pull reques
|
||||
|
||||
### Contributor License Agreement
|
||||
|
||||
Contributions to this project must be accompanied by a
|
||||
[Contributor License Agreement](https://cla.developers.google.com/about) (CLA).
|
||||
You (or your employer) retain the copyright to your contribution; this simply
|
||||
gives us permission to use and redistribute your contributions as part of the
|
||||
project.
|
||||
Contributions to this project must be accompanied by a Contributor License
|
||||
Agreement. You (or your employer) retain the copyright to your contribution,
|
||||
this simply gives us permission to use and redistribute your contributions as
|
||||
part of the project. Head over to <https://cla.developers.google.com/> to see
|
||||
your current agreements on file or to sign a new one.
|
||||
|
||||
If you or your current employer have already signed the Google CLA (even if it
|
||||
was for a different project), you probably don't need to do it again.
|
||||
|
||||
Visit <https://cla.developers.google.com/> to see your current agreements or to
|
||||
sign a new one.
|
||||
You generally only need to submit a CLA once, so if you've already submitted one
|
||||
(even if it was for a different project), you probably don't need to do it
|
||||
again.
|
||||
|
||||
## Styleguides
|
||||
|
||||
|
||||
4
.github/ISSUE_TEMPLATE/bug_report.md
vendored
4
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -10,8 +10,8 @@ We use submodules to separate code, rules and test data. If your issue is relate
|
||||
# Have you checked that your issue isn't already filed?
|
||||
Please search if there is a similar issue at https://github.com/mandiant/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
|
||||
|
||||
# Have you read Google's Code of Conduct?
|
||||
By filing an issue, you are expected to comply with it, including treating everyone with respect: https://opensource.google/conduct
|
||||
# Have you read capa's Code of Conduct?
|
||||
By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/mandiant/capa/blob/master/.github/CODE_OF_CONDUCT.md
|
||||
|
||||
# Have you read capa's CONTRIBUTING guide?
|
||||
It contains helpful information about how to contribute to capa. Check https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md#reporting-bugs
|
||||
|
||||
4
.github/ISSUE_TEMPLATE/feature_request.md
vendored
4
.github/ISSUE_TEMPLATE/feature_request.md
vendored
@@ -10,8 +10,8 @@ We use submodules to separate code, rules and test data. If your issue is relate
|
||||
# Have you checked that your issue isn't already filed?
|
||||
Please search if there is a similar issue at https://github.com/mandiant/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.
|
||||
|
||||
# Have you read Google's Code of Conduct?
|
||||
By filing an issue, you are expected to comply with it, including treating everyone with respect: https://opensource.google/conduct
|
||||
# Have you read capa's Code of Conduct?
|
||||
By filing an Issue, you are expected to comply with it, including treating everyone with respect: https://github.com/mandiant/capa/blob/master/.github/CODE_OF_CONDUCT.md
|
||||
|
||||
# Have you read capa's CONTRIBUTING guide?
|
||||
It contains helpful information about how to contribute to capa. Check https://github.com/mandiant/capa/blob/master/.github/CONTRIBUTING.md#suggesting-enhancements
|
||||
|
||||
2
.github/flake8.ini
vendored
2
.github/flake8.ini
vendored
@@ -40,4 +40,4 @@ per-file-ignores =
|
||||
|
||||
copyright-check = True
|
||||
copyright-min-file-size = 1
|
||||
copyright-regexp = Copyright \d{4} Google LLC
|
||||
copyright-regexp = Copyright \(C\) \d{4} Mandiant, Inc. All Rights Reserved.
|
||||
14
.github/pyinstaller/hooks/hook-vivisect.py
vendored
14
.github/pyinstaller/hooks/hook-vivisect.py
vendored
@@ -1,16 +1,4 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
|
||||
from PyInstaller.utils.hooks import copy_metadata
|
||||
|
||||
|
||||
18
.github/pyinstaller/pyinstaller.spec
vendored
18
.github/pyinstaller/pyinstaller.spec
vendored
@@ -1,18 +1,5 @@
|
||||
# -*- mode: python -*-
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
import sys
|
||||
|
||||
import capa.rules.cache
|
||||
@@ -74,9 +61,6 @@ a = Analysis(
|
||||
# only be installed locally.
|
||||
"binaryninja",
|
||||
"ida",
|
||||
# remove once https://github.com/mandiant/capa/issues/2681 has
|
||||
# been addressed by PyInstaller
|
||||
"pkg_resources",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
92
.github/workflows/build.yml
vendored
92
.github/workflows/build.yml
vendored
@@ -9,7 +9,6 @@ on:
|
||||
- '**.md'
|
||||
release:
|
||||
types: [edited, published]
|
||||
workflow_dispatch: # manual trigger for testing
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -22,39 +21,26 @@ jobs:
|
||||
# set to false for debugging
|
||||
fail-fast: true
|
||||
matrix:
|
||||
# using Python 3.8 to support running across multiple operating systems including Windows 7
|
||||
include:
|
||||
- os: ubuntu-22.04
|
||||
- os: ubuntu-20.04
|
||||
# use old linux so that the shared library versioning is more portable
|
||||
artifact_name: capa
|
||||
asset_name: linux
|
||||
python_version: '3.10'
|
||||
- os: ubuntu-22.04-arm
|
||||
artifact_name: capa
|
||||
asset_name: linux-arm64
|
||||
python_version: '3.10'
|
||||
- os: ubuntu-22.04
|
||||
python_version: 3.8
|
||||
- os: ubuntu-20.04
|
||||
artifact_name: capa
|
||||
asset_name: linux-py312
|
||||
python_version: '3.12'
|
||||
- os: windows-2022
|
||||
python_version: 3.12
|
||||
- os: windows-2019
|
||||
artifact_name: capa.exe
|
||||
asset_name: windows
|
||||
python_version: '3.10'
|
||||
# Windows 11 ARM64 complains of conflicting package version
|
||||
# Additionally, there is no ARM64 build of Python for Python 3.10 on Windows 11 ARM: https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
|
||||
#- os: windows-11-arm
|
||||
# artifact_name: capa.exe
|
||||
# asset_name: windows-arm64
|
||||
# python_version: '3.12'
|
||||
- os: macos-13
|
||||
python_version: 3.8
|
||||
- os: macos-12
|
||||
# use older macOS for assumed better portability
|
||||
artifact_name: capa
|
||||
asset_name: macos
|
||||
python_version: '3.10'
|
||||
- os: macos-14
|
||||
artifact_name: capa
|
||||
asset_name: macos-arm64
|
||||
python_version: '3.10'
|
||||
python_version: 3.8
|
||||
steps:
|
||||
- name: Checkout capa
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
||||
@@ -64,7 +50,7 @@ jobs:
|
||||
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
|
||||
with:
|
||||
python-version: ${{ matrix.python_version }}
|
||||
- if: matrix.os == 'ubuntu-22.04' || matrix.os == 'ubuntu-22.04-arm'
|
||||
- if: matrix.os == 'ubuntu-20.04'
|
||||
run: sudo apt-get install -y libyaml-dev
|
||||
- name: Upgrade pip, setuptools
|
||||
run: python -m pip install --upgrade pip setuptools
|
||||
@@ -74,28 +60,6 @@ jobs:
|
||||
pip install -e .[build]
|
||||
- name: Build standalone executable
|
||||
run: pyinstaller --log-level DEBUG .github/pyinstaller/pyinstaller.spec
|
||||
- name: Does it run without warnings or errors?
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "${{ matrix.os }}" == "windows-2022" ]] || [[ "${{ matrix.os }}" == "windows-11-arm" ]]; then
|
||||
EXECUTABLE=".\\dist\\capa"
|
||||
else
|
||||
EXECUTABLE="./dist/capa"
|
||||
fi
|
||||
|
||||
output=$(${EXECUTABLE} --version 2>&1)
|
||||
exit_code=$?
|
||||
|
||||
echo "${output}"
|
||||
echo "${exit_code}"
|
||||
|
||||
if echo "${output}" | grep -iE 'error|warning'; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${exit_code}" -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
- name: Does it run (PE)?
|
||||
run: dist/capa -d "tests/data/Practical Malware Analysis Lab 01-01.dll_"
|
||||
- name: Does it run (Shellcode)?
|
||||
@@ -111,29 +75,51 @@ jobs:
|
||||
name: ${{ matrix.asset_name }}
|
||||
path: dist/${{ matrix.artifact_name }}
|
||||
|
||||
test_run:
|
||||
name: Test run on ${{ matrix.os }} / ${{ matrix.asset_name }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
needs: [build]
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
# OSs not already tested above
|
||||
- os: ubuntu-22.04
|
||||
artifact_name: capa
|
||||
asset_name: linux
|
||||
- os: ubuntu-22.04
|
||||
artifact_name: capa
|
||||
asset_name: linux-py312
|
||||
- os: windows-2022
|
||||
artifact_name: capa.exe
|
||||
asset_name: windows
|
||||
steps:
|
||||
- name: Download ${{ matrix.asset_name }}
|
||||
uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
|
||||
with:
|
||||
name: ${{ matrix.asset_name }}
|
||||
- name: Set executable flag
|
||||
if: matrix.os != 'windows-2022'
|
||||
run: chmod +x ${{ matrix.artifact_name }}
|
||||
- name: Run capa
|
||||
run: ./${{ matrix.artifact_name }} -h
|
||||
|
||||
zip_and_upload:
|
||||
# upload zipped binaries to Release page
|
||||
if: github.event_name == 'release'
|
||||
name: zip and upload ${{ matrix.asset_name }}
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
needs: [build]
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- asset_name: linux
|
||||
artifact_name: capa
|
||||
- asset_name: linux-arm64
|
||||
artifact_name: capa
|
||||
- asset_name: linux-py312
|
||||
artifact_name: capa
|
||||
- asset_name: windows
|
||||
artifact_name: capa.exe
|
||||
#- asset_name: windows-arm64
|
||||
# artifact_name: capa.exe
|
||||
- asset_name: macos
|
||||
artifact_name: capa
|
||||
- asset_name: macos-arm64
|
||||
artifact_name: capa
|
||||
steps:
|
||||
- name: Download ${{ matrix.asset_name }}
|
||||
uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
|
||||
|
||||
7
.github/workflows/changelog.yml
vendored
7
.github/workflows/changelog.yml
vendored
@@ -13,11 +13,8 @@ permissions:
|
||||
jobs:
|
||||
check_changelog:
|
||||
# no need to check for dependency updates via dependabot
|
||||
# github.event.pull_request.user.login refers to PR author
|
||||
if: |
|
||||
github.event.pull_request.user.login != 'dependabot[bot]' &&
|
||||
github.event.pull_request.user.login != 'dependabot-preview[bot]'
|
||||
runs-on: ubuntu-latest
|
||||
if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]'
|
||||
runs-on: ubuntu-20.04
|
||||
env:
|
||||
NO_CHANGELOG: '[x] No CHANGELOG update needed'
|
||||
steps:
|
||||
|
||||
4
.github/workflows/publish.yml
vendored
4
.github/workflows/publish.yml
vendored
@@ -21,7 +21,7 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: '3.8'
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
with:
|
||||
path: dist/*
|
||||
- name: publish package
|
||||
uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # release/v1.12.4
|
||||
uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0 # release/v1
|
||||
with:
|
||||
skip-existing: true
|
||||
verbose: true
|
||||
|
||||
2
.github/workflows/tag.yml
vendored
2
.github/workflows/tag.yml
vendored
@@ -9,7 +9,7 @@ permissions: read-all
|
||||
jobs:
|
||||
tag:
|
||||
name: Tag capa rules
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout capa-rules
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
||||
|
||||
36
.github/workflows/tests.yml
vendored
36
.github/workflows/tests.yml
vendored
@@ -26,7 +26,7 @@ env:
|
||||
|
||||
jobs:
|
||||
changelog_format:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout capa
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
||||
@@ -37,15 +37,15 @@ jobs:
|
||||
if [ $number != 1 ]; then exit 1; fi
|
||||
|
||||
code_style:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout capa
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
||||
# use latest available python to take advantage of best performance
|
||||
- name: Set up Python 3.12
|
||||
- name: Set up Python 3.11
|
||||
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
python-version: "3.11"
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
@@ -64,16 +64,16 @@ jobs:
|
||||
run: pre-commit run deptry --hook-stage manual
|
||||
|
||||
rule_linter:
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout capa with submodules
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Set up Python 3.12
|
||||
- name: Set up Python 3.11
|
||||
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
python-version: "3.11"
|
||||
- name: Install capa
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
@@ -88,17 +88,17 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-22.04, windows-2022, macos-13]
|
||||
os: [ubuntu-20.04, windows-2019, macos-12]
|
||||
# across all operating systems
|
||||
python-version: ["3.10", "3.11"]
|
||||
python-version: ["3.8", "3.11"]
|
||||
include:
|
||||
# on Ubuntu run these as well
|
||||
- os: ubuntu-22.04
|
||||
- os: ubuntu-20.04
|
||||
python-version: "3.8"
|
||||
- os: ubuntu-20.04
|
||||
python-version: "3.9"
|
||||
- os: ubuntu-20.04
|
||||
python-version: "3.10"
|
||||
- os: ubuntu-22.04
|
||||
python-version: "3.11"
|
||||
- os: ubuntu-22.04
|
||||
python-version: "3.12"
|
||||
steps:
|
||||
- name: Checkout capa with submodules
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install pyyaml
|
||||
if: matrix.os == 'ubuntu-22.04'
|
||||
if: matrix.os == 'ubuntu-20.04'
|
||||
run: sudo apt-get install -y libyaml-dev
|
||||
- name: Install capa
|
||||
run: |
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11"]
|
||||
python-version: ["3.9", "3.11"]
|
||||
steps:
|
||||
- name: Checkout capa with submodules
|
||||
# do only run if BN_SERIAL is available, have to do this in every step, see https://github.com/orgs/community/discussions/26726#discussioncomment-3253118
|
||||
@@ -168,12 +168,12 @@ jobs:
|
||||
|
||||
ghidra-tests:
|
||||
name: Ghidra tests for ${{ matrix.python-version }}
|
||||
runs-on: ubuntu-22.04
|
||||
runs-on: ubuntu-20.04
|
||||
needs: [tests]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11"]
|
||||
python-version: ["3.8", "3.11"]
|
||||
java-version: ["17"]
|
||||
ghidra-version: ["11.0.1"]
|
||||
public-version: ["PUBLIC_20240130"] # for ghidra releases
|
||||
|
||||
103
.github/workflows/web-release.yml
vendored
103
.github/workflows/web-release.yml
vendored
@@ -1,103 +0,0 @@
|
||||
name: create web release
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Version number for the release (x.x.x)'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
run-tests:
|
||||
uses: ./.github/workflows/web-tests.yml
|
||||
|
||||
build-and-release:
|
||||
needs: run-tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set release name
|
||||
run: echo "RELEASE_NAME=capa-explorer-web-v${{ github.event.inputs.version }}-${GITHUB_SHA::7}" >> $GITHUB_ENV
|
||||
|
||||
- name: Check if release already exists
|
||||
run: |
|
||||
if ls web/explorer/releases/capa-explorer-web-v${{ github.event.inputs.version }}-* 1> /dev/null 2>&1; then
|
||||
echo "::error:: A release with version ${{ github.event.inputs.version }} already exists"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
cache-dependency-path: 'web/explorer/package-lock.json'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: web/explorer
|
||||
|
||||
- name: Build offline bundle
|
||||
run: npm run build:bundle
|
||||
working-directory: web/explorer
|
||||
|
||||
- name: Compress bundle
|
||||
run: zip -r ${{ env.RELEASE_NAME }}.zip capa-explorer-web
|
||||
working-directory: web/explorer
|
||||
|
||||
- name: Create releases directory
|
||||
run: mkdir -vp web/explorer/releases
|
||||
|
||||
- name: Move release to releases folder
|
||||
run: mv web/explorer/${{ env.RELEASE_NAME }}.zip web/explorer/releases
|
||||
|
||||
- name: Compute release SHA256 hash
|
||||
run: |
|
||||
echo "RELEASE_SHA256=$(sha256sum web/explorer/releases/${{ env.RELEASE_NAME }}.zip | awk '{print $1}')" >> $GITHUB_ENV
|
||||
|
||||
- name: Update CHANGELOG.md
|
||||
run: |
|
||||
echo "## ${{ env.RELEASE_NAME }}" >> web/explorer/releases/CHANGELOG.md
|
||||
echo "- Release Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> web/explorer/releases/CHANGELOG.md
|
||||
echo "- SHA256: ${{ env.RELEASE_SHA256 }}" >> web/explorer/releases/CHANGELOG.md
|
||||
echo "" >> web/explorer/releases/CHANGELOG.md
|
||||
cat web/explorer/releases/CHANGELOG.md
|
||||
|
||||
- name: Remove older releases
|
||||
# keep only the latest 3 releases
|
||||
run: ls -t capa-explorer-web-v*.zip | tail -n +4 | xargs -r rm --
|
||||
working-directory: web/explorer/releases
|
||||
|
||||
- name: Stage release files
|
||||
run: |
|
||||
git config --local user.email "capa-dev@mandiant.com"
|
||||
git config --local user.name "Capa Bot"
|
||||
git add -f web/explorer/releases/${{ env.RELEASE_NAME }}.zip web/explorer/releases/CHANGELOG.md
|
||||
git add -u web/explorer/releases/
|
||||
|
||||
- name: Create Pull Request
|
||||
uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
title: "explorer web: add release v${{ github.event.inputs.version }}"
|
||||
body: |
|
||||
This PR adds a new capa Explorer Web release v${{ github.event.inputs.version }}.
|
||||
|
||||
Release details:
|
||||
- Name: ${{ env.RELEASE_NAME }}
|
||||
- SHA256: ${{ env.RELEASE_SHA256 }}
|
||||
|
||||
This release is generated by the [web release](https://github.com/mandiant/capa/actions/workflows/web-release.yml) workflow.
|
||||
|
||||
- [x] No CHANGELOG update needed
|
||||
- [x] No new tests needed
|
||||
- [x] No documentation update needed
|
||||
commit-message: ":robot: explorer web: add release ${{ env.RELEASE_NAME }}"
|
||||
branch: release/web-v${{ github.event.inputs.version }}
|
||||
add-paths: web/explorer/releases/${{ env.RELEASE_NAME }}.zip
|
||||
base: master
|
||||
labels: webui
|
||||
delete-branch: true
|
||||
committer: Capa Bot <capa-dev@mandiant.com>
|
||||
author: Capa Bot <capa-dev@mandiant.com>
|
||||
13
.github/workflows/web-tests.yml
vendored
13
.github/workflows/web-tests.yml
vendored
@@ -1,11 +1,10 @@
|
||||
name: capa Explorer Web tests
|
||||
name: Capa Explorer Web tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'web/explorer/**'
|
||||
workflow_call: # this allows the workflow to be called by other workflows
|
||||
|
||||
jobs:
|
||||
test:
|
||||
@@ -24,20 +23,20 @@ jobs:
|
||||
with:
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
cache-dependency-path: 'web/explorer/package-lock.json'
|
||||
cache-dependency-path: './web/explorer/package-lock.json'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: web/explorer
|
||||
working-directory: ./web/explorer
|
||||
|
||||
- name: Lint
|
||||
run: npm run lint
|
||||
working-directory: web/explorer
|
||||
working-directory: ./web/explorer
|
||||
|
||||
- name: Format
|
||||
run: npm run format:check
|
||||
working-directory: web/explorer
|
||||
working-directory: ./web/explorer
|
||||
|
||||
- name: Run unit tests
|
||||
run: npm run test
|
||||
working-directory: web/explorer
|
||||
working-directory: ./web/explorer
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -122,7 +122,6 @@ scripts/perf/*.zip
|
||||
*/.DS_Store
|
||||
Pipfile
|
||||
Pipfile.lock
|
||||
uv.lock
|
||||
/cache/
|
||||
.github/binja/binaryninja
|
||||
.github/binja/download_headless.py
|
||||
|
||||
@@ -25,7 +25,7 @@ repos:
|
||||
hooks:
|
||||
- id: isort
|
||||
name: isort
|
||||
stages: [pre-commit, pre-push, manual]
|
||||
stages: [commit, push, manual]
|
||||
language: system
|
||||
entry: isort
|
||||
args:
|
||||
@@ -46,7 +46,7 @@ repos:
|
||||
hooks:
|
||||
- id: black
|
||||
name: black
|
||||
stages: [pre-commit, pre-push, manual]
|
||||
stages: [commit, push, manual]
|
||||
language: system
|
||||
entry: black
|
||||
args:
|
||||
@@ -64,7 +64,7 @@ repos:
|
||||
hooks:
|
||||
- id: ruff
|
||||
name: ruff
|
||||
stages: [pre-commit, pre-push, manual]
|
||||
stages: [commit, push, manual]
|
||||
language: system
|
||||
entry: ruff
|
||||
args:
|
||||
@@ -82,7 +82,7 @@ repos:
|
||||
hooks:
|
||||
- id: flake8
|
||||
name: flake8
|
||||
stages: [pre-push, manual]
|
||||
stages: [push, manual]
|
||||
language: system
|
||||
entry: flake8
|
||||
args:
|
||||
@@ -101,7 +101,7 @@ repos:
|
||||
hooks:
|
||||
- id: mypy
|
||||
name: mypy
|
||||
stages: [pre-push, manual]
|
||||
stages: [push, manual]
|
||||
language: system
|
||||
entry: mypy
|
||||
args:
|
||||
@@ -119,7 +119,7 @@ repos:
|
||||
hooks:
|
||||
- id: deptry
|
||||
name: deptry
|
||||
stages: [pre-push, manual]
|
||||
stages: [push, manual]
|
||||
language: system
|
||||
entry: deptry .
|
||||
always_run: true
|
||||
|
||||
292
CHANGELOG.md
292
CHANGELOG.md
@@ -3,311 +3,27 @@
|
||||
## master (unreleased)
|
||||
|
||||
### New Features
|
||||
- ci: add support for arm64 binary releases
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
### New Rules (21)
|
||||
### New Rules (0)
|
||||
|
||||
- anti-analysis/anti-vm/vm-detection/detect-mouse-movement-via-activity-checks-on-windows tevajdr@gmail.com
|
||||
- nursery/create-executable-heap moritz.raabe@mandiant.com
|
||||
- anti-analysis/packer/dxpack/packed-with-dxpack jakubjozwiak@google.com
|
||||
- anti-analysis/anti-av/patch-bitdefender-hooking-dll-function jakubjozwiak@google.com
|
||||
- nursery/acquire-load-driver-privileges mehunhoff@google.com
|
||||
- nursery/communicate-using-ftp mehunhoff@google.com
|
||||
- linking/static/eclipse-paho-mqtt-c/linked-against-eclipse-paho-mqtt-c jakubjozwiak@google.com
|
||||
- linking/static/qmqtt/linked-against-qmqtt jakubjozwiak@google.com
|
||||
- anti-analysis/anti-forensic/disable-powershell-transcription jakubjozwiak@google.com
|
||||
- host-interaction/powershell/bypass-powershell-constrained-language-mode-via-getsystemlockdownpolicy-patch jakubjozwiak@google.com
|
||||
- linking/static/grpc/linked-against-grpc jakubjozwiak@google.com
|
||||
- linking/static/hp-socket/linked-against-hp-socket jakubjozwiak@google.com
|
||||
- load-code/execute-jscript-via-vsaengine-in-dotnet jakubjozwiak@google.com
|
||||
- linking/static/funchook/linked-against-funchook jakubjozwiak@google.com
|
||||
- linking/static/plthook/linked-against-plthook jakubjozwiak@google.com
|
||||
- host-interaction/network/enumerate-tcp-connections-via-wmi-com-api jakubjozwiak@google.com
|
||||
- host-interaction/network/routing-table/create-routing-table-entry jakubjozwiak@google.com
|
||||
- host-interaction/network/routing-table/get-routing-table michael.hunhoff@mandiant.com
|
||||
- host-interaction/file-system/use-io_uring-io-interface-on-linux jakubjozwiak@google.com
|
||||
- collection/keylog/log-keystrokes-via-direct-input zeze-zeze
|
||||
-
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- binja: fix a crash during feature extraction when the MLIL is unavailable @xusheng6 #2714
|
||||
|
||||
### capa Explorer Web
|
||||
|
||||
### capa Explorer IDA Pro plugin
|
||||
|
||||
- add `ida-plugin.json` for inclusion in the IDA Pro plugin repository @williballenthin
|
||||
- ida plugin: add Qt compatibility layer for PyQt5 and PySide6 support @williballenthin #2707
|
||||
- ida plugin: update ida-settings API @mr-tz #2736
|
||||
|
||||
### Development
|
||||
|
||||
- ci: remove redundant "test_run" action from build workflow @mike-hunhoff #2692
|
||||
- dev: add bumpmyversion to bump and sync versions across the project @mr-tz
|
||||
|
||||
### Raw diffs
|
||||
- [capa v9.2.1...master](https://github.com/mandiant/capa/compare/v9.2.1...master)
|
||||
- [capa-rules v9.2.1...master](https://github.com/mandiant/capa-rules/compare/v9.2.1...master)
|
||||
|
||||
## v9.2.1
|
||||
|
||||
This point release fixes bugs including removing an unnecessary PyInstaller warning message and enabling the standalone binary to execute on systems running older versions of glibc.
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- ci: exclude pkg_resources from PyInstaller build @mike-hunhoff #2684
|
||||
- ci: downgrade Ubuntu version to accommodate older glibc versions @mike-hunhoff #2684
|
||||
|
||||
### Development
|
||||
|
||||
- ci: upgrade Windows version to avoid deprecation @mike-hunhoff #2684
|
||||
- ci: check if build runs without warnings or errors @mike-hunhoff #2684
|
||||
|
||||
### Raw diffs
|
||||
- [capa v9.2.0...v9.2.1](https://github.com/mandiant/capa/compare/v9.2.0...v9.2.1)
|
||||
- [capa-rules v9.2.0...v9.2.1](https://github.com/mandiant/capa-rules/compare/v9.2.0...v9.2.1)
|
||||
|
||||
## v9.2.0
|
||||
|
||||
This release improves a few aspects of dynamic analysis, including relaxing our validation on fields across many CAPE versions and processing additional VMRay submission file types, for example.
|
||||
It also includes an updated rule pack containing new rules and rule fixes.
|
||||
|
||||
### New Features
|
||||
- vmray: do not restrict analysis to PE and ELF files, e.g. docx @mike-hunhoff #2672
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
### New Rules (22)
|
||||
|
||||
- communication/socket/connect-socket moritz.raabe@mandiant.com joakim@intezer.com mrhafizfarhad@gmail.com
|
||||
- communication/socket/udp/connect-udp-socket mrhafizfarhad@gmail.com
|
||||
- nursery/enter-debug-mode-in-dotnet @v1bh475u
|
||||
- nursery/decrypt-data-using-tripledes-in-dotnet 0xRavenspar
|
||||
- nursery/encrypt-data-using-tripledes-in-dotnet 0xRavenspar
|
||||
- nursery/disable-system-features-via-registry-on-windows mehunhoff@google.com
|
||||
- data-manipulation/encryption/chaskey/encrypt-data-using-chaskey still@teamt5.org
|
||||
- data-manipulation/encryption/speck/encrypt-data-using-speck still@teamt5.org
|
||||
- load-code/dotnet/load-assembly-via-iassembly still@teamt5.org
|
||||
- malware-family/donut-loader/load-shellcode-via-donut still@teamt5.org
|
||||
- nursery/disable-device-guard-features-via-registry-on-windows mehunhoff@google.com
|
||||
- nursery/disable-firewall-features-via-registry-on-windows mehunhoff@google.com
|
||||
- nursery/disable-system-restore-features-via-registry-on-windows mehunhoff@google.com
|
||||
- nursery/disable-windows-defender-features-via-registry-on-windows mehunhoff@google.com
|
||||
- host-interaction/file-system/write/clear-file-content jakeperalta7
|
||||
- host-interaction/filter/unload-minifilter-driver JakePeralta7
|
||||
- exploitation/enumeration/make-suspicious-ntquerysysteminformation-call zdw@google.com
|
||||
- exploitation/gadgets/load-ntoskrnl zdw@google.com
|
||||
- exploitation/gadgets/resolve-ntoskrnl-gadgets zdw@google.com
|
||||
- exploitation/spraying/make-suspicious-ntfscontrolfile-call zdw@google.com
|
||||
- anti-analysis/anti-forensic/unload-sysmon JakePeralta7
|
||||
|
||||
### Bug Fixes
|
||||
- cape: make some fields optional @williballenthin #2631 #2632
|
||||
- lint: add WARN for regex features that contain unescaped dot #2635
|
||||
- lint: add ERROR for incomplete registry control set regex #2643
|
||||
- binja: update unit test core version #2670
|
||||
|
||||
### Raw diffs
|
||||
- [capa v9.1.0...v9.2.0](https://github.com/mandiant/capa/compare/v9.1.0...v9.2.0)
|
||||
- [capa-rules v9.1.0...v9.2.0](https://github.com/mandiant/capa-rules/compare/v9.1.0...v9.2.0)
|
||||
|
||||
## v9.1.0
|
||||
|
||||
This release improves a few aspects of dynamic analysis, relaxing our validation on fields across many CAPE versions, for example.
|
||||
It also includes an updated rule pack in which many dynamic rules make better use of the "span of calls" scope.
|
||||
|
||||
|
||||
### New Rules (3)
|
||||
|
||||
- host-interaction/registry/change-registry-key-timestamp wballenthin@google.com
|
||||
- host-interaction/mutex/check-mutex-and-terminate-process-on-windows @_re_fox moritz.raabe@mandiant.com mehunhoff@google.com
|
||||
- anti-analysis/anti-forensic/clear-logs/clear-windows-event-logs-remotely 99.elad.levi@gmail.com
|
||||
|
||||
### Bug Fixes
|
||||
- only parse CAPE fields required for analysis @mike-hunhoff #2607
|
||||
- main: render result document without needing associated rules @williballenthin #2610
|
||||
- vmray: only verify process OS and monitor IDs match @mike-hunhoff #2613
|
||||
- render: don't assume prior matches exist within a thread @mike-hunhoff #2612
|
||||
|
||||
### Raw diffs
|
||||
- [capa v9.0.0...v9.1.0](https://github.com/mandiant/capa/compare/v9.0.0...v9.1.0)
|
||||
- [capa-rules v9.0.0...v9.1.0](https://github.com/mandiant/capa-rules/compare/v9.0.0...v9.1.0)
|
||||
|
||||
## v9.0.0
|
||||
|
||||
This release introduces a new scope for dynamic analysis, "span of calls",
|
||||
that matches features against a across a sliding window of API calls within a thread.
|
||||
Its useful for identifying behaviors that span multiple API calls,
|
||||
such as `OpenFile`/`ReadFile`/`CloseFile`, without having to analyze an entire thread, which may be very long.
|
||||
|
||||
The release also contains a number of bug fixes and enhancements by new contributors: @v1bh475u and @dhruvak001. Welcome and thank you!
|
||||
|
||||
### New Features
|
||||
|
||||
- add warning for dynamic .NET samples #1864 @v1bh475u
|
||||
- add lint for detecting duplicate features in capa-rules #2250 @v1bh475u
|
||||
- add span-of-calls scope to match features against a across a sliding window of API calls within a thread @williballenthin #2532
|
||||
- add lint to catch rules that depend on other rules with impossible scope @williballenthin #2124
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- remove `is_static_limitation` method from `capa.rules.Rule`
|
||||
- add span-of-calls scope to rule format
|
||||
- capabilities functions return dataclasses instead of tuples
|
||||
|
||||
### New Rules (3)
|
||||
|
||||
- data-manipulation/encryption/rsa/encrypt-data-using-rsa-via-embedded-library @Ana06
|
||||
- data-manipulation/encryption/use-bigint-function @Ana06
|
||||
- internal/limitation/dynamic/internal-dotnet-file-limitation @v1bh475u
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- dynamic: only check file limitations for static file formats @mr-tz
|
||||
- vmray: load more analysis archives @mr-tz
|
||||
- vmray: skip non-printable strings @mike-hunhoff
|
||||
- vmray: loosen file checks to enable processing more file types @mike-hunhoff #2571
|
||||
- strings: add type hints and fix uncovered bugs @williballenthin #2555
|
||||
- elffile: handle symbols without a name @williballenthin #2553
|
||||
- project: remove pytest-cov that wasn't used @williballenthin @2491
|
||||
- replace binascii methods with native Python methods @v1bh475u #2582
|
||||
- rules: scopes can now have subscope blocks with the same scope @williballenthin #2584
|
||||
|
||||
### capa Explorer Web
|
||||
|
||||
### capa Explorer IDA Pro plugin
|
||||
|
||||
### Development
|
||||
|
||||
- license & copyright: Correct LICENSE file and improve copyright and license information headers in the source code files @Ana06
|
||||
- documentation: Improve CLA and Code of Conduct information in CONTRIBUTING @Ana06
|
||||
|
||||
### Raw diffs
|
||||
- [capa v8.0.1...v9.0.0](https://github.com/mandiant/capa/compare/v8.0.1...v9.0.0)
|
||||
- [capa-rules v8.0.1...v9.0.0](https://github.com/mandiant/capa-rules/compare/v8.0.1...v9.0.0)
|
||||
|
||||
## v8.0.1
|
||||
|
||||
This point release fixes an issue with the IDAPython API to now handle IDA Pro 8.3, 8.4, and 9.0 correctly.
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- handle IDA 8.3/8.4 vs. 9.0 API change @mr-tz
|
||||
|
||||
### Raw diffs
|
||||
- [capa v8.0.0...v8.0.1](https://github.com/mandiant/capa/compare/v8.0.0...v8.0.1)
|
||||
- [capa-rules v8.0.0...v8.0.1](https://github.com/mandiant/capa-rules/compare/v8.0.0...v8.0.1)
|
||||
|
||||
## v8.0.0
|
||||
|
||||
capa version 8 adds support for IDA Pro 9.0 (and idalib). The release comes with various improvements and bug fixes for the Binary Ninja backend (including to load with database files) -- thanks to @xusheng6.
|
||||
|
||||
Additional bug fixes improve the dynamic and BinExport backends.
|
||||
|
||||
capa version 8 now requires Python 3.10 or newer.
|
||||
|
||||
Special thanks to @Tamir-K, @harshit-wadhwani, @jorik-utwente for their great contributions.
|
||||
|
||||
### New Features
|
||||
|
||||
- allow call as valid subscope for call scoped rules @mr-tz
|
||||
- support loading and analyzing a Binary Ninja database #2496 @xusheng6
|
||||
- vmray: record process command line details @mr-tz
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- remove support for Python 3.8 and use Python 3.10 as minimum now #1966 @mr-tz
|
||||
|
||||
### New Rules (54)
|
||||
|
||||
- nursery/get-shadow-password-file-entry-on-linux jonathanlepore@google.com
|
||||
- nursery/set-shadow-password-file-entry-on-linux jonathanlepore@google.com
|
||||
- collection/browser/get-chrome-cookiemonster still@teamt5.org
|
||||
- collection/browser/get-elevation-service-for-chromium-based-browsers still@teamt5.org
|
||||
- collection/get-steam-token still@teamt5.org
|
||||
- nursery/persist-via-application-shimming j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-bits-job j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-print-processors-registry-key j.j.vannielen@utwente.nl
|
||||
- linking/static/touchsocket/linked-against-touchsocket still@teamt5.org
|
||||
- runtime/dotnet/compiled-with-dotnet-aot still@teamt5.org
|
||||
- nursery/persist-via-errorhandler-script j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-get-variable-hijack j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-iphlpapi-dll-hijack j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-lnk-shortcut j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-powershell-profile j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-windows-accessibility-tools j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-windows-terminal-profile j.j.vannielen@utwente.nl
|
||||
- nursery/write-to-browser-extension-directory j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-aedebug-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-amsi-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-app-paths-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-appcertdlls-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-appx-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-autodialdll-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-autoplayhandlers-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-bootverificationprogram-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-code-signing-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-com-hijack j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-command-processor-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-contextmenuhandlers-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-cor_profiler_path-registry-value j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-default-file-association-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-disk-cleanup-handler-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-dotnet-dbgmanageddebugger-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-dotnet_startup_hooks-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-explorer-tools-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-filter-handlers-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-group-policy-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-hhctrl-com-hijack j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-htmlhelp-author-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-image-file-execution-options-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-lsa-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-natural-language-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-netsh-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-network-provider-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-path-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-print-monitors-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-rdp-startup-programs-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-silentprocessexit-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-telemetrycontroller-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-timeproviders-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-ts-initialprogram-registry-key j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-userinitmprlogonscript-registry-value j.j.vannielen@utwente.nl
|
||||
- nursery/persist-via-windows-error-reporting-registry-key j.j.vannielen@utwente.nl
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- extractor: fix exception when PE extractor encounters unknown architecture #2440 @Tamir-K
|
||||
- IDA Pro: rename ida to idapro module for plugin and idalib in IDA 9.0 #2453 @mr-tz
|
||||
- ghidra: fix saving of base address @mr-tz
|
||||
- binja: support loading raw x86/x86_64 shellcode #2489 @xusheng6
|
||||
- binja: fix crash when the IL of certain functions are not available. #2249 @xusheng6
|
||||
- binja: major performance improvement on the binja extractor. #1414 @xusheng6
|
||||
- cape: make Process model flexible and procmemory optional to load newest reports #2466 @mr-tz
|
||||
- binja: fix unit test failure by fixing up the analysis for file al-khaser_x64.exe_ #2507 @xusheng6
|
||||
- binja: move the stack string detection to function level #2516 @xusheng6
|
||||
- BinExport2: fix handling of incorrect thunk functions #2524 @williballenthin
|
||||
- BinExport2: more precise pruning of expressions @williballenthin
|
||||
- BinExport2: better handle weird expression trees from Ghidra #2528 #2530 @williballenthin
|
||||
|
||||
### capa Explorer Web
|
||||
|
||||
### capa Explorer IDA Pro plugin
|
||||
|
||||
- fix bug preventing saving of capa results via Save button @mr-tz
|
||||
- fix saving of base address @mr-tz
|
||||
|
||||
### Development
|
||||
- CI: use macos-13 since macos-12 is deprecated and will be removed on December 3rd, 2024 #2173 @mr-tz
|
||||
- CI: update Binary Ninja version to 4.2 #2499 @xusheng6
|
||||
|
||||
### Raw diffs
|
||||
- [capa v7.4.0...v8.0.0](https://github.com/mandiant/capa/compare/v7.4.0...v8.0.0)
|
||||
- [capa-rules v7.4.0...v8.0.0](https://github.com/mandiant/capa-rules/compare/v7.4.0...v8.0.0)
|
||||
- [capa v7.4.0...master](https://github.com/mandiant/capa/compare/v7.4.0...master)
|
||||
- [capa-rules v7.4.0...master](https://github.com/mandiant/capa-rules/compare/v7.4.0...master)
|
||||
|
||||
## v7.4.0
|
||||
|
||||
@@ -463,8 +179,6 @@ Special thanks to our repeat and new contributors:
|
||||
- CI: update Binary Ninja version to 4.1 and use Python 3.9 to test it #2211 @xusheng6
|
||||
- CI: update tests.yml workflow to exclude web and documentation files #2263 @s-ff
|
||||
- CI: update build.yml workflow to exclude web and documentation files #2270 @s-ff
|
||||
- CI: add web releases workflow #2455 @s-ff
|
||||
- CI: skip changelog.yml for dependabot PRs #2471
|
||||
|
||||
### Raw diffs
|
||||
|
||||
|
||||
@@ -187,7 +187,7 @@
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
Copyright (C) 2020 Mandiant, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
85
README.md
85
README.md
@@ -38,47 +38,49 @@ Below you find a list of [our capa blog posts with more details.](#blog-posts)
|
||||
```
|
||||
$ capa.exe suspicious.exe
|
||||
|
||||
+--------------------+------------------------------------------------------------------------+
|
||||
| ATT&CK Tactic | ATT&CK Technique |
|
||||
|--------------------+------------------------------------------------------------------------|
|
||||
| DEFENSE EVASION | Obfuscated Files or Information [T1027] |
|
||||
| DISCOVERY | Query Registry [T1012] |
|
||||
| | System Information Discovery [T1082] |
|
||||
| EXECUTION | Command and Scripting Interpreter::Windows Command Shell [T1059.003] |
|
||||
| | Shared Modules [T1129] |
|
||||
| EXFILTRATION | Exfiltration Over C2 Channel [T1041] |
|
||||
| PERSISTENCE | Create or Modify System Process::Windows Service [T1543.003] |
|
||||
+--------------------+------------------------------------------------------------------------+
|
||||
+------------------------+--------------------------------------------------------------------------------+
|
||||
| ATT&CK Tactic | ATT&CK Technique |
|
||||
|------------------------+--------------------------------------------------------------------------------|
|
||||
| DEFENSE EVASION | Obfuscated Files or Information [T1027] |
|
||||
| DISCOVERY | Query Registry [T1012] |
|
||||
| | System Information Discovery [T1082] |
|
||||
| EXECUTION | Command and Scripting Interpreter::Windows Command Shell [T1059.003] |
|
||||
| | Shared Modules [T1129] |
|
||||
| EXFILTRATION | Exfiltration Over C2 Channel [T1041] |
|
||||
| PERSISTENCE | Create or Modify System Process::Windows Service [T1543.003] |
|
||||
+------------------------+--------------------------------------------------------------------------------+
|
||||
|
||||
+-------------------------------------------+-------------------------------------------------+
|
||||
| CAPABILITY | NAMESPACE |
|
||||
|-------------------------------------------+-------------------------------------------------|
|
||||
| read and send data from client to server | c2/file-transfer |
|
||||
| execute shell command and capture output | c2/shell |
|
||||
| receive data (2 matches) | communication |
|
||||
| send data (6 matches) | communication |
|
||||
| connect to HTTP server (3 matches) | communication/http/client |
|
||||
| send HTTP request (3 matches) | communication/http/client |
|
||||
| create pipe | communication/named-pipe/create |
|
||||
| get socket status (2 matches) | communication/socket |
|
||||
| receive data on socket (2 matches) | communication/socket/receive |
|
||||
| send data on socket (3 matches) | communication/socket/send |
|
||||
| connect TCP socket | communication/socket/tcp |
|
||||
| encode data using Base64 | data-manipulation/encoding/base64 |
|
||||
| encode data using XOR (6 matches) | data-manipulation/encoding/xor |
|
||||
| run as a service | executable/pe |
|
||||
| get common file path (3 matches) | host-interaction/file-system |
|
||||
| read file | host-interaction/file-system/read |
|
||||
| write file (2 matches) | host-interaction/file-system/write |
|
||||
| print debug messages (2 matches) | host-interaction/log/debug/write-event |
|
||||
| resolve DNS | host-interaction/network/dns/resolve |
|
||||
| get hostname | host-interaction/os/hostname |
|
||||
| create process | host-interaction/process/create |
|
||||
| create registry key | host-interaction/registry/create |
|
||||
| create service | host-interaction/service/create |
|
||||
| create thread | host-interaction/thread/create |
|
||||
| persist via Windows service | persistence/service |
|
||||
+-------------------------------------------+-------------------------------------------------+
|
||||
+-------------------------------------------------------+-------------------------------------------------+
|
||||
| CAPABILITY | NAMESPACE |
|
||||
|-------------------------------------------------------+-------------------------------------------------|
|
||||
| check for OutputDebugString error | anti-analysis/anti-debugging/debugger-detection |
|
||||
| read and send data from client to server | c2/file-transfer |
|
||||
| execute shell command and capture output | c2/shell |
|
||||
| receive data (2 matches) | communication |
|
||||
| send data (6 matches) | communication |
|
||||
| connect to HTTP server (3 matches) | communication/http/client |
|
||||
| send HTTP request (3 matches) | communication/http/client |
|
||||
| create pipe | communication/named-pipe/create |
|
||||
| get socket status (2 matches) | communication/socket |
|
||||
| receive data on socket (2 matches) | communication/socket/receive |
|
||||
| send data on socket (3 matches) | communication/socket/send |
|
||||
| connect TCP socket | communication/socket/tcp |
|
||||
| encode data using Base64 | data-manipulation/encoding/base64 |
|
||||
| encode data using XOR (6 matches) | data-manipulation/encoding/xor |
|
||||
| run as a service | executable/pe |
|
||||
| get common file path (3 matches) | host-interaction/file-system |
|
||||
| read file | host-interaction/file-system/read |
|
||||
| write file (2 matches) | host-interaction/file-system/write |
|
||||
| print debug messages (2 matches) | host-interaction/log/debug/write-event |
|
||||
| resolve DNS | host-interaction/network/dns/resolve |
|
||||
| get hostname | host-interaction/os/hostname |
|
||||
| create a process with modified I/O handles and window | host-interaction/process/create |
|
||||
| create process | host-interaction/process/create |
|
||||
| create registry key | host-interaction/registry/create |
|
||||
| create service | host-interaction/service/create |
|
||||
| create thread | host-interaction/thread/create |
|
||||
| persist via Windows service | persistence/service |
|
||||
+-------------------------------------------------------+-------------------------------------------------+
|
||||
```
|
||||
|
||||
# download and usage
|
||||
@@ -315,6 +317,3 @@ If you use Ghidra, then you can use the [capa + Ghidra integration](/capa/ghidra
|
||||
|
||||
## capa testfiles
|
||||
The [capa-testfiles repository](https://github.com/mandiant/capa-testfiles) contains the data we use to test capa's code and rules
|
||||
|
||||
## mailing list
|
||||
Subscribe to the FLARE mailing list for community announcements! Email "subscribe" to [flare-external@google.com](mailto:flare-external@google.com?subject=subscribe).
|
||||
|
||||
164
capa/analysis/flirt.py
Normal file
164
capa/analysis/flirt.py
Normal file
@@ -0,0 +1,164 @@
|
||||
import io
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import rich
|
||||
from rich.console import Console
|
||||
from rich.logging import RichHandler
|
||||
|
||||
import capa.helpers
|
||||
import capa.features.extractors.ida.idalib as idalib
|
||||
|
||||
if not idalib.has_idalib():
|
||||
raise RuntimeError("cannot find IDA idalib module.")
|
||||
|
||||
if not idalib.load_idalib():
|
||||
raise RuntimeError("failed to load IDA idalib module.")
|
||||
|
||||
import idaapi
|
||||
import idapro
|
||||
import ida_auto
|
||||
import idautils
|
||||
import ida_funcs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
def colorbool(v: bool) -> str:
|
||||
if v:
|
||||
return f"[green]{str(v)}[/green]"
|
||||
else:
|
||||
return f"[red]{str(v)}[/red]"
|
||||
|
||||
|
||||
def colorname(n: str) -> str:
|
||||
if n.startswith("sub_"):
|
||||
return n
|
||||
else:
|
||||
return f"[cyan]{n}[/cyan]"
|
||||
|
||||
|
||||
class FunctionId(BaseModel):
|
||||
address: int
|
||||
is_library: bool
|
||||
is_thunk: bool
|
||||
name: str
|
||||
|
||||
def to_row(self):
|
||||
row = [hex(self.address)]
|
||||
row.append(colorbool(self.is_library))
|
||||
row.append(colorbool(self.is_thunk))
|
||||
row.append(colorname(self.name))
|
||||
return row
|
||||
|
||||
|
||||
def configure_logging(args):
|
||||
if args.quiet:
|
||||
logging.getLogger().setLevel(logging.WARNING)
|
||||
elif args.debug:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
else:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
# use [/] after the logger name to reset any styling,
|
||||
# and prevent the color from carrying over to the message
|
||||
logformat = "[dim]%(name)s[/]: %(message)s"
|
||||
|
||||
# set markup=True to allow the use of Rich's markup syntax in log messages
|
||||
rich_handler = RichHandler(markup=True, show_time=False, show_path=True, console=capa.helpers.log_console)
|
||||
rich_handler.setFormatter(logging.Formatter(logformat))
|
||||
|
||||
# use RichHandler for root logger
|
||||
logging.getLogger().addHandler(rich_handler)
|
||||
|
||||
if args.debug:
|
||||
logging.getLogger("capa").setLevel(logging.DEBUG)
|
||||
logging.getLogger("viv_utils").setLevel(logging.DEBUG)
|
||||
else:
|
||||
logging.getLogger("capa").setLevel(logging.ERROR)
|
||||
logging.getLogger("viv_utils").setLevel(logging.ERROR)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
if argv is None:
|
||||
argv = sys.argv[1:]
|
||||
|
||||
parser = argparse.ArgumentParser(description="Identify library functions using FLIRT.")
|
||||
parser.add_argument(
|
||||
"input_file",
|
||||
type=Path,
|
||||
help="path to file to analyze",
|
||||
)
|
||||
parser.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR")
|
||||
parser.add_argument("-q", "--quiet", action="store_true", help="disable all output but errors")
|
||||
args = parser.parse_args(args=argv)
|
||||
|
||||
configure_logging(args)
|
||||
|
||||
time0 = time.time()
|
||||
|
||||
# stderr=True is used here to redirect the spinner banner to stderr, so that users can redirect capa's output.
|
||||
console = Console(stderr=True, quiet=False)
|
||||
|
||||
logger.debug("idalib: opening database...")
|
||||
# idalib writes to stdout (ugh), so we have to capture that
|
||||
# so as not to screw up structured output.
|
||||
with capa.helpers.stdout_redirector(io.BytesIO()):
|
||||
with console.status("analyzing program...", spinner="dots"):
|
||||
if idapro.open_database(str(args.input_file), run_auto_analysis=True):
|
||||
raise RuntimeError("failed to analyze input file")
|
||||
|
||||
logger.debug("idalib: waiting for analysis...")
|
||||
|
||||
# TODO: add more signature (files)
|
||||
# TOOD: apply more signatures
|
||||
|
||||
ida_auto.auto_wait()
|
||||
logger.debug("idalib: opened database.")
|
||||
|
||||
table = rich.table.Table()
|
||||
table.add_column("FVA")
|
||||
table.add_column("library?")
|
||||
table.add_column("thunk?")
|
||||
table.add_column("name")
|
||||
|
||||
LIBONLY = True
|
||||
count = 0
|
||||
|
||||
for ea in idautils.Functions(start=None, end=None):
|
||||
f = idaapi.get_func(ea)
|
||||
is_thunk = bool(f.flags & idaapi.FUNC_THUNK)
|
||||
is_lib = bool(f.flags & idaapi.FUNC_LIB)
|
||||
fname = idaapi.get_func_name(ea)
|
||||
|
||||
if LIBONLY and not is_lib:
|
||||
continue
|
||||
|
||||
fid = FunctionId(address=ea, is_library=is_lib, is_thunk=is_thunk, name=fname)
|
||||
table.add_row(*fid.to_row())
|
||||
|
||||
count += 1
|
||||
if count > 50:
|
||||
break
|
||||
|
||||
rich.print(table)
|
||||
|
||||
# TODO can we include which signature matched per function?
|
||||
for index in range(0, ida_funcs.get_idasgn_qty()):
|
||||
signame, optlibs, nmatches = ida_funcs.get_idasgn_desc_with_matches(index)
|
||||
rich.print(signame, optlibs, nmatches)
|
||||
|
||||
idapro.close_database()
|
||||
|
||||
min, sec = divmod(time.time() - time0, 60)
|
||||
logger.debug("FLIRT-based library identification ran for ~ %02d:%02dm", min, sec)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,43 +1,25 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
import itertools
|
||||
import collections
|
||||
from typing import Optional
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Tuple
|
||||
|
||||
from capa.rules import Rule, Scope, RuleSet
|
||||
from capa.rules import Scope, RuleSet
|
||||
from capa.engine import FeatureSet, MatchResults
|
||||
from capa.features.address import NO_ADDRESS
|
||||
from capa.render.result_document import LibraryFunction, StaticFeatureCounts, DynamicFeatureCounts
|
||||
from capa.features.extractors.base_extractor import FeatureExtractor, StaticFeatureExtractor, DynamicFeatureExtractor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileCapabilities:
|
||||
features: FeatureSet
|
||||
matches: MatchResults
|
||||
feature_count: int
|
||||
|
||||
|
||||
def find_file_capabilities(
|
||||
ruleset: RuleSet, extractor: FeatureExtractor, function_features: FeatureSet
|
||||
) -> FileCapabilities:
|
||||
def find_file_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, function_features: FeatureSet):
|
||||
file_features: FeatureSet = collections.defaultdict(set)
|
||||
|
||||
for feature, va in itertools.chain(extractor.extract_file_features(), extractor.extract_global_features()):
|
||||
@@ -54,18 +36,35 @@ def find_file_capabilities(
|
||||
|
||||
file_features.update(function_features)
|
||||
|
||||
features, matches = ruleset.match(Scope.FILE, file_features, NO_ADDRESS)
|
||||
return FileCapabilities(features, matches, len(file_features))
|
||||
_, matches = ruleset.match(Scope.FILE, file_features, NO_ADDRESS)
|
||||
return matches, len(file_features)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Capabilities:
|
||||
matches: MatchResults
|
||||
feature_counts: StaticFeatureCounts | DynamicFeatureCounts
|
||||
library_functions: Optional[tuple[LibraryFunction, ...]] = None
|
||||
def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalone=True) -> bool:
|
||||
file_limitation_rules = list(filter(lambda r: r.is_file_limitation_rule(), rules.rules.values()))
|
||||
|
||||
for file_limitation_rule in file_limitation_rules:
|
||||
if file_limitation_rule.name not in capabilities:
|
||||
continue
|
||||
|
||||
logger.warning("-" * 80)
|
||||
for line in file_limitation_rule.meta.get("description", "").split("\n"):
|
||||
logger.warning(" %s", line)
|
||||
logger.warning(" Identified via rule: %s", file_limitation_rule.name)
|
||||
if is_standalone:
|
||||
logger.warning(" ")
|
||||
logger.warning(" Use -v or -vv if you really want to see the capabilities identified by capa.")
|
||||
logger.warning("-" * 80)
|
||||
|
||||
# bail on first file limitation
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_progress=None, **kwargs) -> Capabilities:
|
||||
def find_capabilities(
|
||||
ruleset: RuleSet, extractor: FeatureExtractor, disable_progress=None, **kwargs
|
||||
) -> Tuple[MatchResults, Any]:
|
||||
from capa.capabilities.static import find_static_capabilities
|
||||
from capa.capabilities.dynamic import find_dynamic_capabilities
|
||||
|
||||
@@ -78,40 +77,3 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
|
||||
return find_dynamic_capabilities(ruleset, extractor, disable_progress=disable_progress, **kwargs)
|
||||
|
||||
raise ValueError(f"unexpected extractor type: {extractor.__class__.__name__}")
|
||||
|
||||
|
||||
def has_limitation(rules: list, capabilities: Capabilities | FileCapabilities, is_standalone: bool) -> bool:
|
||||
|
||||
for rule in rules:
|
||||
if rule.name not in capabilities.matches:
|
||||
continue
|
||||
logger.warning("-" * 80)
|
||||
for line in rule.meta.get("description", "").split("\n"):
|
||||
logger.warning(" %s", line)
|
||||
logger.warning(" Identified via rule: %s", rule.name)
|
||||
if is_standalone:
|
||||
logger.warning(" ")
|
||||
logger.warning(" Use -v or -vv if you really want to see the capabilities identified by capa.")
|
||||
logger.warning("-" * 80)
|
||||
|
||||
# bail on first file limitation
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_static_limitation_rule(r: Rule) -> bool:
|
||||
return r.meta.get("namespace", "") == "internal/limitation/static"
|
||||
|
||||
|
||||
def has_static_limitation(rules: RuleSet, capabilities: Capabilities | FileCapabilities, is_standalone=True) -> bool:
|
||||
file_limitation_rules = list(filter(lambda r: is_static_limitation_rule(r), rules.rules.values()))
|
||||
return has_limitation(file_limitation_rules, capabilities, is_standalone)
|
||||
|
||||
|
||||
def is_dynamic_limitation_rule(r: Rule) -> bool:
|
||||
return r.meta.get("namespace", "") == "internal/limitation/dynamic"
|
||||
|
||||
|
||||
def has_dynamic_limitation(rules: RuleSet, capabilities: Capabilities | FileCapabilities, is_standalone=True) -> bool:
|
||||
dynamic_limitation_rules = list(filter(lambda r: is_dynamic_limitation_rule(r), rules.rules.values()))
|
||||
return has_limitation(dynamic_limitation_rules, capabilities, is_standalone)
|
||||
|
||||
@@ -1,55 +1,34 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
import itertools
|
||||
import collections
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, List, Tuple
|
||||
|
||||
import capa.perf
|
||||
import capa.engine
|
||||
import capa.helpers
|
||||
import capa.features.freeze as frz
|
||||
import capa.render.result_document as rdoc
|
||||
from capa.rules import Scope, RuleSet
|
||||
from capa.engine import FeatureSet, MatchResults
|
||||
from capa.features.address import _NoAddress
|
||||
from capa.capabilities.common import Capabilities, find_file_capabilities
|
||||
from capa.capabilities.common import find_file_capabilities
|
||||
from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle, DynamicFeatureExtractor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# The number of calls that make up a span of calls.
|
||||
#
|
||||
# The larger this is, the more calls are grouped together to match rule logic.
|
||||
# This means a longer chain can be recognized; however, its a bit more expensive.
|
||||
SPAN_SIZE = 20
|
||||
|
||||
|
||||
@dataclass
|
||||
class CallCapabilities:
|
||||
features: FeatureSet
|
||||
matches: MatchResults
|
||||
|
||||
|
||||
def find_call_capabilities(
|
||||
ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
|
||||
) -> CallCapabilities:
|
||||
) -> Tuple[FeatureSet, MatchResults]:
|
||||
"""
|
||||
find matches for the given rules for the given call.
|
||||
|
||||
returns: tuple containing (features for call, match results for call)
|
||||
"""
|
||||
# all features found for the call.
|
||||
features: FeatureSet = collections.defaultdict(set)
|
||||
@@ -67,105 +46,16 @@ def find_call_capabilities(
|
||||
for addr, _ in res:
|
||||
capa.engine.index_rule_matches(features, rule, [addr])
|
||||
|
||||
return CallCapabilities(features, matches)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ThreadCapabilities:
|
||||
features: FeatureSet
|
||||
thread_matches: MatchResults
|
||||
span_matches: MatchResults
|
||||
call_matches: MatchResults
|
||||
|
||||
|
||||
class SpanOfCallsMatcher:
|
||||
def __init__(self, ruleset: RuleSet):
|
||||
super().__init__()
|
||||
self.ruleset = ruleset
|
||||
|
||||
# matches found at the span scope.
|
||||
self.matches: MatchResults = collections.defaultdict(list)
|
||||
|
||||
# We match spans as the sliding window of calls with size SPAN_SIZE.
|
||||
#
|
||||
# For each call, we consider the window of SPAN_SIZE calls leading up to it,
|
||||
# merging all their features and doing a match.
|
||||
#
|
||||
# We track these features in two data structures:
|
||||
# 1. a deque of those features found in the prior calls.
|
||||
# We'll append to it, and as it grows larger than SPAN_SIZE, the oldest items are removed.
|
||||
# 2. a live set of features seen in the span.
|
||||
# As we pop from the deque, we remove features from the current set,
|
||||
# and as we push to the deque, we insert features to the current set.
|
||||
# With this approach, our algorithm performance is independent of SPAN_SIZE.
|
||||
# The naive algorithm, of merging all the trailing feature sets at each call, is dependent upon SPAN_SIZE
|
||||
# (that is, runtime gets slower the larger SPAN_SIZE is).
|
||||
self.current_feature_sets: collections.deque[FeatureSet] = collections.deque(maxlen=SPAN_SIZE)
|
||||
self.current_features: FeatureSet = collections.defaultdict(set)
|
||||
|
||||
# the names of rules matched at the last span,
|
||||
# so that we can deduplicate long strings of the same matches.
|
||||
self.last_span_matches: set[str] = set()
|
||||
|
||||
def next(self, ch: CallHandle, call_features: FeatureSet):
|
||||
# As we add items to the end of the deque, overflow and drop the oldest items (at the left end).
|
||||
# While we could rely on `deque.append` with `maxlen` set (which we provide above),
|
||||
# we want to use the dropped item first, to remove the old features, so we manually pop it here.
|
||||
if len(self.current_feature_sets) == SPAN_SIZE:
|
||||
overflowing_feature_set = self.current_feature_sets.popleft()
|
||||
|
||||
for feature, vas in overflowing_feature_set.items():
|
||||
if len(vas) == 1 and isinstance(next(iter(vas)), _NoAddress):
|
||||
# `vas == { NO_ADDRESS }` without the garbage.
|
||||
#
|
||||
# ignore the common case of global features getting added/removed/trimmed repeatedly,
|
||||
# like arch/os/format.
|
||||
continue
|
||||
|
||||
self.current_features[feature] -= vas
|
||||
if not self.current_features[feature]:
|
||||
del self.current_features[feature]
|
||||
|
||||
# update the deque and set of features with the latest call's worth of features.
|
||||
self.current_feature_sets.append(call_features)
|
||||
for feature, vas in call_features.items():
|
||||
self.current_features[feature] |= vas
|
||||
|
||||
_, matches = self.ruleset.match(Scope.SPAN_OF_CALLS, self.current_features, ch.address)
|
||||
|
||||
newly_encountered_rules = set(matches.keys()) - self.last_span_matches
|
||||
|
||||
# don't emit match results for rules seen during the immediately preceeding spans.
|
||||
#
|
||||
# This means that we won't emit duplicate matches when there are multiple spans
|
||||
# that overlap a single matching event.
|
||||
# It also handles the case of a tight loop containing matched logic;
|
||||
# only the first match will be recorded.
|
||||
#
|
||||
# In theory, this means the result document doesn't have *every* possible match location,
|
||||
# but in practice, humans will only be interested in the first handful anyways.
|
||||
suppressed_rules = set(self.last_span_matches)
|
||||
|
||||
# however, if a newly encountered rule depends on a suppressed rule,
|
||||
# don't suppress that rule match, or we won't be able to reconstruct the vverbose output.
|
||||
# see: https://github.com/mandiant/capa/pull/2532#issuecomment-2548508130
|
||||
for new_rule in newly_encountered_rules:
|
||||
suppressed_rules -= set(self.ruleset.rules[new_rule].get_dependencies(self.ruleset.rules_by_namespace))
|
||||
|
||||
for rule_name, res in matches.items():
|
||||
if rule_name in suppressed_rules:
|
||||
continue
|
||||
self.matches[rule_name].extend(res)
|
||||
|
||||
self.last_span_matches = set(matches.keys())
|
||||
return features, matches
|
||||
|
||||
|
||||
def find_thread_capabilities(
|
||||
ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle, th: ThreadHandle
|
||||
) -> ThreadCapabilities:
|
||||
) -> Tuple[FeatureSet, MatchResults, MatchResults]:
|
||||
"""
|
||||
find matches for the given rules within the given thread,
|
||||
which includes matches for all the spans and calls within it.
|
||||
find matches for the given rules within the given thread.
|
||||
|
||||
returns: tuple containing (features for thread, match results for thread, match results for calls)
|
||||
"""
|
||||
# all features found within this thread,
|
||||
# includes features found within calls.
|
||||
@@ -175,19 +65,14 @@ def find_thread_capabilities(
|
||||
# might be found at different calls, that's ok.
|
||||
call_matches: MatchResults = collections.defaultdict(list)
|
||||
|
||||
span_matcher = SpanOfCallsMatcher(ruleset)
|
||||
|
||||
call_count = 0
|
||||
for call_count, ch in enumerate(extractor.get_calls(ph, th)): # noqa: B007
|
||||
call_capabilities = find_call_capabilities(ruleset, extractor, ph, th, ch)
|
||||
for feature, vas in call_capabilities.features.items():
|
||||
for ch in extractor.get_calls(ph, th):
|
||||
ifeatures, imatches = find_call_capabilities(ruleset, extractor, ph, th, ch)
|
||||
for feature, vas in ifeatures.items():
|
||||
features[feature].update(vas)
|
||||
|
||||
for rule_name, res in call_capabilities.matches.items():
|
||||
for rule_name, res in imatches.items():
|
||||
call_matches[rule_name].extend(res)
|
||||
|
||||
span_matcher.next(ch, call_capabilities.features)
|
||||
|
||||
for feature, va in itertools.chain(extractor.extract_thread_features(ph, th), extractor.extract_global_features()):
|
||||
features[feature].add(va)
|
||||
|
||||
@@ -199,31 +84,16 @@ def find_thread_capabilities(
|
||||
for va, _ in res:
|
||||
capa.engine.index_rule_matches(features, rule, [va])
|
||||
|
||||
logger.debug(
|
||||
"analyzed thread %d[%d] with %d events, %d features, and %d matches",
|
||||
th.address.process.pid,
|
||||
th.address.tid,
|
||||
call_count,
|
||||
len(features),
|
||||
len(matches) + len(span_matcher.matches) + len(call_matches),
|
||||
)
|
||||
return ThreadCapabilities(features, matches, span_matcher.matches, call_matches)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessCapabilities:
|
||||
process_matches: MatchResults
|
||||
thread_matches: MatchResults
|
||||
span_matches: MatchResults
|
||||
call_matches: MatchResults
|
||||
feature_count: int
|
||||
return features, matches, call_matches
|
||||
|
||||
|
||||
def find_process_capabilities(
|
||||
ruleset: RuleSet, extractor: DynamicFeatureExtractor, ph: ProcessHandle
|
||||
) -> ProcessCapabilities:
|
||||
) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
|
||||
"""
|
||||
find matches for the given rules within the given process.
|
||||
|
||||
returns: tuple containing (match results for process, match results for threads, match results for calls, number of features)
|
||||
"""
|
||||
# all features found within this process,
|
||||
# includes features found within threads (and calls).
|
||||
@@ -233,54 +103,39 @@ def find_process_capabilities(
|
||||
# might be found at different threads, that's ok.
|
||||
thread_matches: MatchResults = collections.defaultdict(list)
|
||||
|
||||
# matches found at the span-of-calls scope.
|
||||
# might be found at different spans, that's ok.
|
||||
span_matches: MatchResults = collections.defaultdict(list)
|
||||
|
||||
# matches found at the call scope.
|
||||
# might be found at different calls, that's ok.
|
||||
call_matches: MatchResults = collections.defaultdict(list)
|
||||
|
||||
for th in extractor.get_threads(ph):
|
||||
thread_capabilities = find_thread_capabilities(ruleset, extractor, ph, th)
|
||||
for feature, vas in thread_capabilities.features.items():
|
||||
features, tmatches, cmatches = find_thread_capabilities(ruleset, extractor, ph, th)
|
||||
for feature, vas in features.items():
|
||||
process_features[feature].update(vas)
|
||||
|
||||
for rule_name, res in thread_capabilities.thread_matches.items():
|
||||
for rule_name, res in tmatches.items():
|
||||
thread_matches[rule_name].extend(res)
|
||||
|
||||
for rule_name, res in thread_capabilities.span_matches.items():
|
||||
span_matches[rule_name].extend(res)
|
||||
|
||||
for rule_name, res in thread_capabilities.call_matches.items():
|
||||
for rule_name, res in cmatches.items():
|
||||
call_matches[rule_name].extend(res)
|
||||
|
||||
for feature, va in itertools.chain(extractor.extract_process_features(ph), extractor.extract_global_features()):
|
||||
process_features[feature].add(va)
|
||||
|
||||
_, process_matches = ruleset.match(Scope.PROCESS, process_features, ph.address)
|
||||
|
||||
logger.debug(
|
||||
"analyzed process %d and extracted %d features with %d matches",
|
||||
ph.address.pid,
|
||||
len(process_features),
|
||||
len(process_matches),
|
||||
)
|
||||
return ProcessCapabilities(process_matches, thread_matches, span_matches, call_matches, len(process_features))
|
||||
return process_matches, thread_matches, call_matches, len(process_features)
|
||||
|
||||
|
||||
def find_dynamic_capabilities(
|
||||
ruleset: RuleSet, extractor: DynamicFeatureExtractor, disable_progress: bool = False
|
||||
) -> Capabilities:
|
||||
ruleset: RuleSet, extractor: DynamicFeatureExtractor, disable_progress=None
|
||||
) -> Tuple[MatchResults, Any]:
|
||||
all_process_matches: MatchResults = collections.defaultdict(list)
|
||||
all_thread_matches: MatchResults = collections.defaultdict(list)
|
||||
all_span_matches: MatchResults = collections.defaultdict(list)
|
||||
all_call_matches: MatchResults = collections.defaultdict(list)
|
||||
|
||||
feature_counts = rdoc.DynamicFeatureCounts(file=0, processes=())
|
||||
|
||||
assert isinstance(extractor, DynamicFeatureExtractor)
|
||||
processes: list[ProcessHandle] = list(extractor.get_processes())
|
||||
processes: List[ProcessHandle] = list(extractor.get_processes())
|
||||
n_processes: int = len(processes)
|
||||
|
||||
with capa.helpers.CapaProgressBar(
|
||||
@@ -288,20 +143,19 @@ def find_dynamic_capabilities(
|
||||
) as pbar:
|
||||
task = pbar.add_task("matching", total=n_processes, unit="processes")
|
||||
for p in processes:
|
||||
process_capabilities = find_process_capabilities(ruleset, extractor, p)
|
||||
feature_counts.processes += (
|
||||
rdoc.ProcessFeatureCount(
|
||||
address=frz.Address.from_capa(p.address), count=process_capabilities.feature_count
|
||||
),
|
||||
process_matches, thread_matches, call_matches, feature_count = find_process_capabilities(
|
||||
ruleset, extractor, p
|
||||
)
|
||||
feature_counts.processes += (
|
||||
rdoc.ProcessFeatureCount(address=frz.Address.from_capa(p.address), count=feature_count),
|
||||
)
|
||||
logger.debug("analyzed %s and extracted %d features", p.address, feature_count)
|
||||
|
||||
for rule_name, res in process_capabilities.process_matches.items():
|
||||
for rule_name, res in process_matches.items():
|
||||
all_process_matches[rule_name].extend(res)
|
||||
for rule_name, res in process_capabilities.thread_matches.items():
|
||||
for rule_name, res in thread_matches.items():
|
||||
all_thread_matches[rule_name].extend(res)
|
||||
for rule_name, res in process_capabilities.span_matches.items():
|
||||
all_span_matches[rule_name].extend(res)
|
||||
for rule_name, res in process_capabilities.call_matches.items():
|
||||
for rule_name, res in call_matches.items():
|
||||
all_call_matches[rule_name].extend(res)
|
||||
|
||||
pbar.advance(task)
|
||||
@@ -310,26 +164,29 @@ def find_dynamic_capabilities(
|
||||
# mapping from feature (matched rule) to set of addresses at which it matched.
|
||||
process_and_lower_features: FeatureSet = collections.defaultdict(set)
|
||||
for rule_name, results in itertools.chain(
|
||||
all_process_matches.items(), all_thread_matches.items(), all_span_matches.items(), all_call_matches.items()
|
||||
all_process_matches.items(), all_thread_matches.items(), all_call_matches.items()
|
||||
):
|
||||
locations = {p[0] for p in results}
|
||||
rule = ruleset[rule_name]
|
||||
capa.engine.index_rule_matches(process_and_lower_features, rule, locations)
|
||||
|
||||
all_file_capabilities = find_file_capabilities(ruleset, extractor, process_and_lower_features)
|
||||
feature_counts.file = all_file_capabilities.feature_count
|
||||
all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, process_and_lower_features)
|
||||
feature_counts.file = feature_count
|
||||
|
||||
matches = dict(
|
||||
itertools.chain(
|
||||
# each rule exists in exactly one scope,
|
||||
# so there won't be any overlap among these following MatchResults,
|
||||
# and we can merge the dictionaries naively.
|
||||
all_call_matches.items(),
|
||||
all_span_matches.items(),
|
||||
all_thread_matches.items(),
|
||||
all_process_matches.items(),
|
||||
all_file_capabilities.matches.items(),
|
||||
all_call_matches.items(),
|
||||
all_file_matches.items(),
|
||||
)
|
||||
)
|
||||
|
||||
return Capabilities(matches, feature_counts)
|
||||
meta = {
|
||||
"feature_counts": feature_counts,
|
||||
}
|
||||
|
||||
return matches, meta
|
||||
|
||||
@@ -1,23 +1,16 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import time
|
||||
import logging
|
||||
import itertools
|
||||
import collections
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, List, Tuple
|
||||
|
||||
import capa.perf
|
||||
import capa.helpers
|
||||
@@ -25,23 +18,19 @@ import capa.features.freeze as frz
|
||||
import capa.render.result_document as rdoc
|
||||
from capa.rules import Scope, RuleSet
|
||||
from capa.engine import FeatureSet, MatchResults
|
||||
from capa.capabilities.common import Capabilities, find_file_capabilities
|
||||
from capa.capabilities.common import find_file_capabilities
|
||||
from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, StaticFeatureExtractor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class InstructionCapabilities:
|
||||
features: FeatureSet
|
||||
matches: MatchResults
|
||||
|
||||
|
||||
def find_instruction_capabilities(
|
||||
ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
|
||||
) -> InstructionCapabilities:
|
||||
) -> Tuple[FeatureSet, MatchResults]:
|
||||
"""
|
||||
find matches for the given rules for the given instruction.
|
||||
|
||||
returns: tuple containing (features for instruction, match results for instruction)
|
||||
"""
|
||||
# all features found for the instruction.
|
||||
features: FeatureSet = collections.defaultdict(set)
|
||||
@@ -59,21 +48,16 @@ def find_instruction_capabilities(
|
||||
for addr, _ in res:
|
||||
capa.engine.index_rule_matches(features, rule, [addr])
|
||||
|
||||
return InstructionCapabilities(features, matches)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BasicBlockCapabilities:
|
||||
features: FeatureSet
|
||||
basic_block_matches: MatchResults
|
||||
instruction_matches: MatchResults
|
||||
return features, matches
|
||||
|
||||
|
||||
def find_basic_block_capabilities(
|
||||
ruleset: RuleSet, extractor: StaticFeatureExtractor, f: FunctionHandle, bb: BBHandle
|
||||
) -> BasicBlockCapabilities:
|
||||
) -> Tuple[FeatureSet, MatchResults, MatchResults]:
|
||||
"""
|
||||
find matches for the given rules within the given basic block.
|
||||
|
||||
returns: tuple containing (features for basic block, match results for basic block, match results for instructions)
|
||||
"""
|
||||
# all features found within this basic block,
|
||||
# includes features found within instructions.
|
||||
@@ -84,11 +68,11 @@ def find_basic_block_capabilities(
|
||||
insn_matches: MatchResults = collections.defaultdict(list)
|
||||
|
||||
for insn in extractor.get_instructions(f, bb):
|
||||
instruction_capabilities = find_instruction_capabilities(ruleset, extractor, f, bb, insn)
|
||||
for feature, vas in instruction_capabilities.features.items():
|
||||
ifeatures, imatches = find_instruction_capabilities(ruleset, extractor, f, bb, insn)
|
||||
for feature, vas in ifeatures.items():
|
||||
features[feature].update(vas)
|
||||
|
||||
for rule_name, res in instruction_capabilities.matches.items():
|
||||
for rule_name, res in imatches.items():
|
||||
insn_matches[rule_name].extend(res)
|
||||
|
||||
for feature, va in itertools.chain(
|
||||
@@ -104,20 +88,16 @@ def find_basic_block_capabilities(
|
||||
for va, _ in res:
|
||||
capa.engine.index_rule_matches(features, rule, [va])
|
||||
|
||||
return BasicBlockCapabilities(features, matches, insn_matches)
|
||||
return features, matches, insn_matches
|
||||
|
||||
|
||||
@dataclass
|
||||
class CodeCapabilities:
|
||||
function_matches: MatchResults
|
||||
basic_block_matches: MatchResults
|
||||
instruction_matches: MatchResults
|
||||
feature_count: int
|
||||
|
||||
|
||||
def find_code_capabilities(ruleset: RuleSet, extractor: StaticFeatureExtractor, fh: FunctionHandle) -> CodeCapabilities:
|
||||
def find_code_capabilities(
|
||||
ruleset: RuleSet, extractor: StaticFeatureExtractor, fh: FunctionHandle
|
||||
) -> Tuple[MatchResults, MatchResults, MatchResults, int]:
|
||||
"""
|
||||
find matches for the given rules within the given function.
|
||||
|
||||
returns: tuple containing (match results for function, match results for basic blocks, match results for instructions, number of features)
|
||||
"""
|
||||
# all features found within this function,
|
||||
# includes features found within basic blocks (and instructions).
|
||||
@@ -132,35 +112,35 @@ def find_code_capabilities(ruleset: RuleSet, extractor: StaticFeatureExtractor,
|
||||
insn_matches: MatchResults = collections.defaultdict(list)
|
||||
|
||||
for bb in extractor.get_basic_blocks(fh):
|
||||
basic_block_capabilities = find_basic_block_capabilities(ruleset, extractor, fh, bb)
|
||||
for feature, vas in basic_block_capabilities.features.items():
|
||||
features, bmatches, imatches = find_basic_block_capabilities(ruleset, extractor, fh, bb)
|
||||
for feature, vas in features.items():
|
||||
function_features[feature].update(vas)
|
||||
|
||||
for rule_name, res in basic_block_capabilities.basic_block_matches.items():
|
||||
for rule_name, res in bmatches.items():
|
||||
bb_matches[rule_name].extend(res)
|
||||
|
||||
for rule_name, res in basic_block_capabilities.instruction_matches.items():
|
||||
for rule_name, res in imatches.items():
|
||||
insn_matches[rule_name].extend(res)
|
||||
|
||||
for feature, va in itertools.chain(extractor.extract_function_features(fh), extractor.extract_global_features()):
|
||||
function_features[feature].add(va)
|
||||
|
||||
_, function_matches = ruleset.match(Scope.FUNCTION, function_features, fh.address)
|
||||
return CodeCapabilities(function_matches, bb_matches, insn_matches, len(function_features))
|
||||
return function_matches, bb_matches, insn_matches, len(function_features)
|
||||
|
||||
|
||||
def find_static_capabilities(
|
||||
ruleset: RuleSet, extractor: StaticFeatureExtractor, disable_progress=None
|
||||
) -> Capabilities:
|
||||
) -> Tuple[MatchResults, Any]:
|
||||
all_function_matches: MatchResults = collections.defaultdict(list)
|
||||
all_bb_matches: MatchResults = collections.defaultdict(list)
|
||||
all_insn_matches: MatchResults = collections.defaultdict(list)
|
||||
|
||||
feature_counts = rdoc.StaticFeatureCounts(file=0, functions=())
|
||||
library_functions: tuple[rdoc.LibraryFunction, ...] = ()
|
||||
library_functions: Tuple[rdoc.LibraryFunction, ...] = ()
|
||||
|
||||
assert isinstance(extractor, StaticFeatureExtractor)
|
||||
functions: list[FunctionHandle] = list(extractor.get_functions())
|
||||
functions: List[FunctionHandle] = list(extractor.get_functions())
|
||||
n_funcs: int = len(functions)
|
||||
n_libs: int = 0
|
||||
percentage: float = 0
|
||||
@@ -185,36 +165,30 @@ def find_static_capabilities(
|
||||
pbar.advance(task)
|
||||
continue
|
||||
|
||||
code_capabilities = find_code_capabilities(ruleset, extractor, f)
|
||||
function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(ruleset, extractor, f)
|
||||
feature_counts.functions += (
|
||||
rdoc.FunctionFeatureCount(
|
||||
address=frz.Address.from_capa(f.address), count=code_capabilities.feature_count
|
||||
),
|
||||
rdoc.FunctionFeatureCount(address=frz.Address.from_capa(f.address), count=feature_count),
|
||||
)
|
||||
t1 = time.time()
|
||||
|
||||
match_count = 0
|
||||
for name, matches_ in itertools.chain(
|
||||
code_capabilities.function_matches.items(),
|
||||
code_capabilities.basic_block_matches.items(),
|
||||
code_capabilities.instruction_matches.items(),
|
||||
):
|
||||
for name, matches_ in itertools.chain(function_matches.items(), bb_matches.items(), insn_matches.items()):
|
||||
if not ruleset.rules[name].is_subscope_rule():
|
||||
match_count += len(matches_)
|
||||
|
||||
logger.debug(
|
||||
"analyzed function 0x%x and extracted %d features, %d matches in %0.02fs",
|
||||
f.address,
|
||||
code_capabilities.feature_count,
|
||||
feature_count,
|
||||
match_count,
|
||||
t1 - t0,
|
||||
)
|
||||
|
||||
for rule_name, res in code_capabilities.function_matches.items():
|
||||
for rule_name, res in function_matches.items():
|
||||
all_function_matches[rule_name].extend(res)
|
||||
for rule_name, res in code_capabilities.basic_block_matches.items():
|
||||
for rule_name, res in bb_matches.items():
|
||||
all_bb_matches[rule_name].extend(res)
|
||||
for rule_name, res in code_capabilities.instruction_matches.items():
|
||||
for rule_name, res in insn_matches.items():
|
||||
all_insn_matches[rule_name].extend(res)
|
||||
|
||||
pbar.advance(task)
|
||||
@@ -229,8 +203,8 @@ def find_static_capabilities(
|
||||
rule = ruleset[rule_name]
|
||||
capa.engine.index_rule_matches(function_and_lower_features, rule, locations)
|
||||
|
||||
all_file_capabilities = find_file_capabilities(ruleset, extractor, function_and_lower_features)
|
||||
feature_counts.file = all_file_capabilities.feature_count
|
||||
all_file_matches, feature_count = find_file_capabilities(ruleset, extractor, function_and_lower_features)
|
||||
feature_counts.file = feature_count
|
||||
|
||||
matches: MatchResults = dict(
|
||||
itertools.chain(
|
||||
@@ -240,8 +214,13 @@ def find_static_capabilities(
|
||||
all_insn_matches.items(),
|
||||
all_bb_matches.items(),
|
||||
all_function_matches.items(),
|
||||
all_file_capabilities.matches.items(),
|
||||
all_file_matches.items(),
|
||||
)
|
||||
)
|
||||
|
||||
return Capabilities(matches, feature_counts, library_functions)
|
||||
meta = {
|
||||
"feature_counts": feature_counts,
|
||||
"library_functions": library_functions,
|
||||
}
|
||||
|
||||
return matches, meta
|
||||
|
||||
@@ -1,21 +1,14 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import copy
|
||||
import collections
|
||||
from typing import TYPE_CHECKING, Union, Mapping, Iterable, Iterator
|
||||
from typing import TYPE_CHECKING, Set, Dict, List, Tuple, Union, Mapping, Iterable, Iterator
|
||||
|
||||
import capa.perf
|
||||
import capa.features.common
|
||||
@@ -34,7 +27,7 @@ if TYPE_CHECKING:
|
||||
# to collect the locations of a feature, do: `features[Number(0x10)]`
|
||||
#
|
||||
# aliased here so that the type can be documented and xref'd.
|
||||
FeatureSet = dict[Feature, set[Address]]
|
||||
FeatureSet = Dict[Feature, Set[Address]]
|
||||
|
||||
|
||||
class Statement:
|
||||
@@ -101,7 +94,7 @@ class And(Statement):
|
||||
match if all of the children evaluate to True.
|
||||
|
||||
the order of evaluation is dictated by the property
|
||||
`And.children` (type: list[Statement|Feature]).
|
||||
`And.children` (type: List[Statement|Feature]).
|
||||
a query optimizer may safely manipulate the order of these children.
|
||||
"""
|
||||
|
||||
@@ -134,7 +127,7 @@ class Or(Statement):
|
||||
match if any of the children evaluate to True.
|
||||
|
||||
the order of evaluation is dictated by the property
|
||||
`Or.children` (type: list[Statement|Feature]).
|
||||
`Or.children` (type: List[Statement|Feature]).
|
||||
a query optimizer may safely manipulate the order of these children.
|
||||
"""
|
||||
|
||||
@@ -183,7 +176,7 @@ class Some(Statement):
|
||||
match if at least N of the children evaluate to True.
|
||||
|
||||
the order of evaluation is dictated by the property
|
||||
`Some.children` (type: list[Statement|Feature]).
|
||||
`Some.children` (type: List[Statement|Feature]).
|
||||
a query optimizer may safely manipulate the order of these children.
|
||||
"""
|
||||
|
||||
@@ -274,7 +267,7 @@ class Subscope(Statement):
|
||||
# inspect(match_details)
|
||||
#
|
||||
# aliased here so that the type can be documented and xref'd.
|
||||
MatchResults = Mapping[str, list[tuple[Address, Result]]]
|
||||
MatchResults = Mapping[str, List[Tuple[Address, Result]]]
|
||||
|
||||
|
||||
def get_rule_namespaces(rule: "capa.rules.Rule") -> Iterator[str]:
|
||||
@@ -299,7 +292,7 @@ def index_rule_matches(features: FeatureSet, rule: "capa.rules.Rule", locations:
|
||||
features[capa.features.common.MatchedRule(namespace)].update(locations)
|
||||
|
||||
|
||||
def match(rules: list["capa.rules.Rule"], features: FeatureSet, addr: Address) -> tuple[FeatureSet, MatchResults]:
|
||||
def match(rules: List["capa.rules.Rule"], features: FeatureSet, addr: Address) -> Tuple[FeatureSet, MatchResults]:
|
||||
"""
|
||||
match the given rules against the given features,
|
||||
returning an updated set of features and the matches.
|
||||
|
||||
@@ -1,18 +1,10 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
class UnsupportedRuntimeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import abc
|
||||
|
||||
|
||||
@@ -114,7 +107,8 @@ class DynamicCallAddress(Address):
|
||||
return hash((self.thread, self.id))
|
||||
|
||||
def __eq__(self, other):
|
||||
return isinstance(other, DynamicCallAddress) and (self.thread, self.id) == (other.thread, other.id)
|
||||
assert isinstance(other, DynamicCallAddress)
|
||||
return (self.thread, self.id) == (other.thread, other.id)
|
||||
|
||||
def __lt__(self, other):
|
||||
assert isinstance(other, DynamicCallAddress)
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
from capa.features.common import Feature
|
||||
|
||||
|
||||
@@ -1,18 +1,12 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from enum import Enum
|
||||
from typing import Dict, List
|
||||
|
||||
from capa.helpers import assert_never
|
||||
|
||||
@@ -28,7 +22,7 @@ COM_PREFIXES = {
|
||||
}
|
||||
|
||||
|
||||
def load_com_database(com_type: ComType) -> dict[str, list[str]]:
|
||||
def load_com_database(com_type: ComType) -> Dict[str, List[str]]:
|
||||
# lazy load these python files since they are so large.
|
||||
# that is, don't load them unless a COM feature is being handled.
|
||||
import capa.features.com.classes
|
||||
|
||||
@@ -1,19 +1,13 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
COM_CLASSES: dict[str, list[str]] = {
|
||||
COM_CLASSES: Dict[str, List[str]] = {
|
||||
"ClusAppWiz": ["24F97150-6689-11D1-9AA7-00C04FB93A80"],
|
||||
"ClusCfgAddNodesWizard": ["BB8D141E-C00A-469F-BC5C-ECD814F0BD74"],
|
||||
"ClusCfgCreateClusterWizard": ["B929818E-F5B0-44DC-8A00-1B5F5F5AA1F0"],
|
||||
|
||||
@@ -1,19 +1,13 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
COM_INTERFACES: dict[str, list[str]] = {
|
||||
COM_INTERFACES: Dict[str, List[str]] = {
|
||||
"IClusterApplicationWizard": ["24F97151-6689-11D1-9AA7-00C04FB93A80"],
|
||||
"IWEExtendWizard97": ["97DEDE68-FC6B-11CF-B5F5-00A0C90AB505"],
|
||||
"IWCWizard97Callback": ["97DEDE67-FC6B-11CF-B5F5-00A0C90AB505"],
|
||||
@@ -16340,7 +16334,7 @@ COM_INTERFACES: dict[str, list[str]] = {
|
||||
"IRcsServiceDescription": ["416437de-e78b-44c9-990f-7ede1f2a0c91"],
|
||||
"IRcsServiceKindSupportedChangedEventArgs": ["f47ea244-e783-4866-b3a7-4e5ccf023070"],
|
||||
"IRcsServiceStatusChangedArgs": ["661ae45a-412a-460d-bdd4-dd8ea3c15583"],
|
||||
"IRcsServicetuple": ["ce17a39b-2e8b-41af-b5a9-5cb072cc373c"],
|
||||
"IRcsServiceTuple": ["ce17a39b-2e8b-41af-b5a9-5cb072cc373c"],
|
||||
"IRcsSubscriptionReceivedArgs": ["04eaf06d-42bc-46cc-a637-eeb3a8723fe4"],
|
||||
"IRcsTransport": ["fea34759-f37c-4319-8546-ec84d21d30ff"],
|
||||
"IRcsTransportConfiguration": ["1fccb102-2472-4bb9-9988-c1211c83e8a9"],
|
||||
|
||||
@@ -1,24 +1,18 @@
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import re
|
||||
import abc
|
||||
import codecs
|
||||
import typing
|
||||
import logging
|
||||
import collections
|
||||
from typing import TYPE_CHECKING, Union, Optional
|
||||
from typing import TYPE_CHECKING, Set, Dict, List, Union, Optional
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# circular import, otherwise
|
||||
@@ -85,14 +79,14 @@ class Result:
|
||||
self,
|
||||
success: bool,
|
||||
statement: Union["capa.engine.Statement", "Feature"],
|
||||
children: list["Result"],
|
||||
locations: Optional[set[Address]] = None,
|
||||
children: List["Result"],
|
||||
locations: Optional[Set[Address]] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.success = success
|
||||
self.statement = statement
|
||||
self.children = children
|
||||
self.locations = frozenset(locations) if locations is not None else frozenset()
|
||||
self.locations = locations if locations is not None else set()
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, bool):
|
||||
@@ -105,25 +99,6 @@ class Result:
|
||||
def __nonzero__(self):
|
||||
return self.success
|
||||
|
||||
def __str__(self):
|
||||
# as this object isn't user facing, this formatting is just to help with debugging
|
||||
|
||||
lines: list[str] = []
|
||||
|
||||
def rec(m: "Result", indent: int):
|
||||
if isinstance(m.statement, capa.engine.Statement):
|
||||
line = (" " * indent) + str(m.statement.name) + " " + str(m.success)
|
||||
else:
|
||||
line = (" " * indent) + str(m.statement) + " " + str(m.success) + " " + str(m.locations)
|
||||
|
||||
lines.append(line)
|
||||
|
||||
for child in m.children:
|
||||
rec(child, indent + 1)
|
||||
|
||||
rec(self, 0)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class Feature(abc.ABC): # noqa: B024
|
||||
# this is an abstract class, since we don't want anyone to instantiate it directly,
|
||||
@@ -194,11 +169,7 @@ class Feature(abc.ABC): # noqa: B024
|
||||
def evaluate(self, features: "capa.engine.FeatureSet", short_circuit=True) -> Result:
|
||||
capa.perf.counters["evaluate.feature"] += 1
|
||||
capa.perf.counters["evaluate.feature." + self.name] += 1
|
||||
success = self in features
|
||||
if success:
|
||||
return Result(True, self, [], locations=features[self])
|
||||
else:
|
||||
return Result(False, self, [], locations=None)
|
||||
return Result(self in features, self, [], locations=features.get(self, set()))
|
||||
|
||||
|
||||
class MatchedRule(Feature):
|
||||
@@ -242,7 +213,7 @@ class Substring(String):
|
||||
|
||||
# mapping from string value to list of locations.
|
||||
# will unique the locations later on.
|
||||
matches: collections.defaultdict[str, set[Address]] = collections.defaultdict(set)
|
||||
matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
|
||||
|
||||
assert isinstance(self.value, str)
|
||||
for feature, locations in features.items():
|
||||
@@ -290,7 +261,7 @@ class _MatchedSubstring(Substring):
|
||||
note: this type should only ever be constructed by `Substring.evaluate()`. it is not part of the public API.
|
||||
"""
|
||||
|
||||
def __init__(self, substring: Substring, matches: dict[str, set[Address]]):
|
||||
def __init__(self, substring: Substring, matches: Dict[str, Set[Address]]):
|
||||
"""
|
||||
args:
|
||||
substring: the substring feature that matches.
|
||||
@@ -334,7 +305,7 @@ class Regex(String):
|
||||
|
||||
# mapping from string value to list of locations.
|
||||
# will unique the locations later on.
|
||||
matches: collections.defaultdict[str, set[Address]] = collections.defaultdict(set)
|
||||
matches: typing.DefaultDict[str, Set[Address]] = collections.defaultdict(set)
|
||||
|
||||
for feature, locations in features.items():
|
||||
if not isinstance(feature, (String,)):
|
||||
@@ -382,7 +353,7 @@ class _MatchedRegex(Regex):
|
||||
note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
|
||||
"""
|
||||
|
||||
def __init__(self, regex: Regex, matches: dict[str, set[Address]]):
|
||||
def __init__(self, regex: Regex, matches: Dict[str, Set[Address]]):
|
||||
"""
|
||||
args:
|
||||
regex: the regex feature that matches.
|
||||
@@ -496,7 +467,6 @@ FORMAT_VMRAY = "vmray"
|
||||
FORMAT_BINEXPORT2 = "binexport2"
|
||||
FORMAT_FREEZE = "freeze"
|
||||
FORMAT_RESULT = "result"
|
||||
FORMAT_BINJA_DB = "binja_database"
|
||||
STATIC_FORMATS = {
|
||||
FORMAT_SC32,
|
||||
FORMAT_SC64,
|
||||
@@ -506,7 +476,6 @@ STATIC_FORMATS = {
|
||||
FORMAT_FREEZE,
|
||||
FORMAT_RESULT,
|
||||
FORMAT_BINEXPORT2,
|
||||
FORMAT_BINJA_DB,
|
||||
}
|
||||
DYNAMIC_FORMATS = {
|
||||
FORMAT_CAPE,
|
||||
|
||||
@@ -1,26 +1,23 @@
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import abc
|
||||
import hashlib
|
||||
import dataclasses
|
||||
from copy import copy
|
||||
from types import MethodType
|
||||
from typing import Any, Union, Iterator, TypeAlias
|
||||
from typing import Any, Set, Dict, Tuple, Union, Iterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
# TODO(williballenthin): use typing.TypeAlias directly when Python 3.9 is deprecated
|
||||
# https://github.com/mandiant/capa/issues/1699
|
||||
from typing_extensions import TypeAlias
|
||||
|
||||
import capa.features.address
|
||||
from capa.features.common import Feature
|
||||
from capa.features.address import Address, ThreadAddress, ProcessAddress, DynamicCallAddress, AbsoluteVirtualAddress
|
||||
@@ -62,7 +59,7 @@ class FunctionHandle:
|
||||
|
||||
address: Address
|
||||
inner: Any
|
||||
ctx: dict[str, Any] = dataclasses.field(default_factory=dict)
|
||||
ctx: Dict[str, Any] = dataclasses.field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -138,7 +135,7 @@ class StaticFeatureExtractor:
|
||||
return self._sample_hashes
|
||||
|
||||
@abc.abstractmethod
|
||||
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract features found at every scope ("global").
|
||||
|
||||
@@ -149,12 +146,12 @@ class StaticFeatureExtractor:
|
||||
print('0x%x: %s', va, feature)
|
||||
|
||||
yields:
|
||||
tuple[Feature, Address]: feature and its location
|
||||
Tuple[Feature, Address]: feature and its location
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abc.abstractmethod
|
||||
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract file-scope features.
|
||||
|
||||
@@ -165,7 +162,7 @@ class StaticFeatureExtractor:
|
||||
print('0x%x: %s', va, feature)
|
||||
|
||||
yields:
|
||||
tuple[Feature, Address]: feature and its location
|
||||
Tuple[Feature, Address]: feature and its location
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -214,7 +211,7 @@ class StaticFeatureExtractor:
|
||||
raise KeyError(addr)
|
||||
|
||||
@abc.abstractmethod
|
||||
def extract_function_features(self, f: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_features(self, f: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract function-scope features.
|
||||
the arguments are opaque values previously provided by `.get_functions()`, etc.
|
||||
@@ -230,7 +227,7 @@ class StaticFeatureExtractor:
|
||||
f [FunctionHandle]: an opaque value previously fetched from `.get_functions()`.
|
||||
|
||||
yields:
|
||||
tuple[Feature, Address]: feature and its location
|
||||
Tuple[Feature, Address]: feature and its location
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -243,7 +240,7 @@ class StaticFeatureExtractor:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abc.abstractmethod
|
||||
def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_basic_block_features(self, f: FunctionHandle, bb: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract basic block-scope features.
|
||||
the arguments are opaque values previously provided by `.get_functions()`, etc.
|
||||
@@ -261,7 +258,7 @@ class StaticFeatureExtractor:
|
||||
bb [BBHandle]: an opaque value previously fetched from `.get_basic_blocks()`.
|
||||
|
||||
yields:
|
||||
tuple[Feature, Address]: feature and its location
|
||||
Tuple[Feature, Address]: feature and its location
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -276,7 +273,7 @@ class StaticFeatureExtractor:
|
||||
@abc.abstractmethod
|
||||
def extract_insn_features(
|
||||
self, f: FunctionHandle, bb: BBHandle, insn: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract instruction-scope features.
|
||||
the arguments are opaque values previously provided by `.get_functions()`, etc.
|
||||
@@ -296,12 +293,12 @@ class StaticFeatureExtractor:
|
||||
insn [InsnHandle]: an opaque value previously fetched from `.get_instructions()`.
|
||||
|
||||
yields:
|
||||
tuple[Feature, Address]: feature and its location
|
||||
Tuple[Feature, Address]: feature and its location
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def FunctionFilter(extractor: StaticFeatureExtractor, functions: set) -> StaticFeatureExtractor:
|
||||
def FunctionFilter(extractor: StaticFeatureExtractor, functions: Set) -> StaticFeatureExtractor:
|
||||
original_get_functions = extractor.get_functions
|
||||
|
||||
def filtered_get_functions(self):
|
||||
@@ -390,7 +387,7 @@ class DynamicFeatureExtractor:
|
||||
return self._sample_hashes
|
||||
|
||||
@abc.abstractmethod
|
||||
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract features found at every scope ("global").
|
||||
|
||||
@@ -401,12 +398,12 @@ class DynamicFeatureExtractor:
|
||||
print(addr, feature)
|
||||
|
||||
yields:
|
||||
tuple[Feature, Address]: feature and its location
|
||||
Tuple[Feature, Address]: feature and its location
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abc.abstractmethod
|
||||
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract file-scope features.
|
||||
|
||||
@@ -417,7 +414,7 @@ class DynamicFeatureExtractor:
|
||||
print(addr, feature)
|
||||
|
||||
yields:
|
||||
tuple[Feature, Address]: feature and its location
|
||||
Tuple[Feature, Address]: feature and its location
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@@ -429,7 +426,7 @@ class DynamicFeatureExtractor:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abc.abstractmethod
|
||||
def extract_process_features(self, ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
Yields all the features of a process. These include:
|
||||
- file features of the process' image
|
||||
@@ -452,7 +449,7 @@ class DynamicFeatureExtractor:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abc.abstractmethod
|
||||
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
Yields all the features of a thread. These include:
|
||||
- sequenced api traces
|
||||
@@ -469,7 +466,7 @@ class DynamicFeatureExtractor:
|
||||
@abc.abstractmethod
|
||||
def extract_call_features(
|
||||
self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
Yields all features of a call. These include:
|
||||
- api name
|
||||
@@ -488,11 +485,11 @@ class DynamicFeatureExtractor:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def ProcessFilter(extractor: DynamicFeatureExtractor, pids: set[int]) -> DynamicFeatureExtractor:
|
||||
def ProcessFilter(extractor: DynamicFeatureExtractor, processes: Set) -> DynamicFeatureExtractor:
|
||||
original_get_processes = extractor.get_processes
|
||||
|
||||
def filtered_get_processes(self):
|
||||
yield from (f for f in original_get_processes() if f.address.pid in pids)
|
||||
yield from (f for f in original_get_processes() if f.address.pid in processes)
|
||||
|
||||
# we make a copy of the original extractor object and then update its get_processes() method with the decorated filter one.
|
||||
# this is in order to preserve the original extractor object's get_processes() method, in case it is used elsewhere in the code.
|
||||
@@ -504,16 +501,4 @@ def ProcessFilter(extractor: DynamicFeatureExtractor, pids: set[int]) -> Dynamic
|
||||
return new_extractor
|
||||
|
||||
|
||||
def ThreadFilter(extractor: DynamicFeatureExtractor, threads: set[Address]) -> DynamicFeatureExtractor:
|
||||
original_get_threads = extractor.get_threads
|
||||
|
||||
def filtered_get_threads(self, ph: ProcessHandle):
|
||||
yield from (t for t in original_get_threads(ph) if t.address in threads)
|
||||
|
||||
new_extractor = copy(extractor)
|
||||
new_extractor.get_threads = MethodType(filtered_get_threads, extractor) # type: ignore
|
||||
|
||||
return new_extractor
|
||||
|
||||
|
||||
FeatureExtractor: TypeAlias = Union[StaticFeatureExtractor, DynamicFeatureExtractor]
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
"""
|
||||
Proto files generated via protobuf v24.4:
|
||||
|
||||
@@ -24,7 +17,7 @@ import io
|
||||
import hashlib
|
||||
import logging
|
||||
import contextlib
|
||||
from typing import Iterator
|
||||
from typing import Set, Dict, List, Tuple, Iterator
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
@@ -58,13 +51,13 @@ def compute_common_prefix_length(m: str, n: str) -> int:
|
||||
return len(m)
|
||||
|
||||
|
||||
def get_sample_from_binexport2(input_file: Path, be2: BinExport2, search_paths: list[Path]) -> Path:
|
||||
def get_sample_from_binexport2(input_file: Path, be2: BinExport2, search_paths: List[Path]) -> Path:
|
||||
"""attempt to find the sample file, given a BinExport2 file.
|
||||
|
||||
searches in the same directory as the BinExport2 file, and then in search_paths.
|
||||
"""
|
||||
|
||||
def filename_similarity_key(p: Path) -> tuple[int, str]:
|
||||
def filename_similarity_key(p: Path) -> Tuple[int, str]:
|
||||
# note closure over input_file.
|
||||
# sort first by length of common prefix, then by name (for stability)
|
||||
return (compute_common_prefix_length(p.name, input_file.name), p.name)
|
||||
@@ -72,7 +65,7 @@ def get_sample_from_binexport2(input_file: Path, be2: BinExport2, search_paths:
|
||||
wanted_sha256: str = be2.meta_information.executable_id.lower()
|
||||
|
||||
input_directory: Path = input_file.parent
|
||||
siblings: list[Path] = [p for p in input_directory.iterdir() if p.is_file()]
|
||||
siblings: List[Path] = [p for p in input_directory.iterdir() if p.is_file()]
|
||||
siblings.sort(key=filename_similarity_key, reverse=True)
|
||||
for sibling in siblings:
|
||||
# e.g. with open IDA files in the same directory on Windows
|
||||
@@ -81,7 +74,7 @@ def get_sample_from_binexport2(input_file: Path, be2: BinExport2, search_paths:
|
||||
return sibling
|
||||
|
||||
for search_path in search_paths:
|
||||
candidates: list[Path] = [p for p in search_path.iterdir() if p.is_file()]
|
||||
candidates: List[Path] = [p for p in search_path.iterdir() if p.is_file()]
|
||||
candidates.sort(key=filename_similarity_key, reverse=True)
|
||||
for candidate in candidates:
|
||||
with contextlib.suppress(PermissionError):
|
||||
@@ -95,27 +88,27 @@ class BinExport2Index:
|
||||
def __init__(self, be2: BinExport2):
|
||||
self.be2: BinExport2 = be2
|
||||
|
||||
self.callers_by_vertex_index: dict[int, list[int]] = defaultdict(list)
|
||||
self.callees_by_vertex_index: dict[int, list[int]] = defaultdict(list)
|
||||
self.callers_by_vertex_index: Dict[int, List[int]] = defaultdict(list)
|
||||
self.callees_by_vertex_index: Dict[int, List[int]] = defaultdict(list)
|
||||
|
||||
# note: flow graph != call graph (vertex)
|
||||
self.flow_graph_index_by_address: dict[int, int] = {}
|
||||
self.flow_graph_address_by_index: dict[int, int] = {}
|
||||
self.flow_graph_index_by_address: Dict[int, int] = {}
|
||||
self.flow_graph_address_by_index: Dict[int, int] = {}
|
||||
|
||||
# edges that come from the given basic block
|
||||
self.source_edges_by_basic_block_index: dict[int, list[BinExport2.FlowGraph.Edge]] = defaultdict(list)
|
||||
self.source_edges_by_basic_block_index: Dict[int, List[BinExport2.FlowGraph.Edge]] = defaultdict(list)
|
||||
# edges that end up at the given basic block
|
||||
self.target_edges_by_basic_block_index: dict[int, list[BinExport2.FlowGraph.Edge]] = defaultdict(list)
|
||||
self.target_edges_by_basic_block_index: Dict[int, List[BinExport2.FlowGraph.Edge]] = defaultdict(list)
|
||||
|
||||
self.vertex_index_by_address: dict[int, int] = {}
|
||||
self.vertex_index_by_address: Dict[int, int] = {}
|
||||
|
||||
self.data_reference_index_by_source_instruction_index: dict[int, list[int]] = defaultdict(list)
|
||||
self.data_reference_index_by_target_address: dict[int, list[int]] = defaultdict(list)
|
||||
self.string_reference_index_by_source_instruction_index: dict[int, list[int]] = defaultdict(list)
|
||||
self.data_reference_index_by_source_instruction_index: Dict[int, List[int]] = defaultdict(list)
|
||||
self.data_reference_index_by_target_address: Dict[int, List[int]] = defaultdict(list)
|
||||
self.string_reference_index_by_source_instruction_index: Dict[int, List[int]] = defaultdict(list)
|
||||
|
||||
self.insn_address_by_index: dict[int, int] = {}
|
||||
self.insn_index_by_address: dict[int, int] = {}
|
||||
self.insn_by_address: dict[int, BinExport2.Instruction] = {}
|
||||
self.insn_address_by_index: Dict[int, int] = {}
|
||||
self.insn_index_by_address: Dict[int, int] = {}
|
||||
self.insn_by_address: Dict[int, BinExport2.Instruction] = {}
|
||||
|
||||
# must index instructions first
|
||||
self._index_insn_addresses()
|
||||
@@ -215,7 +208,7 @@ class BinExport2Index:
|
||||
|
||||
def basic_block_instructions(
|
||||
self, basic_block: BinExport2.BasicBlock
|
||||
) -> Iterator[tuple[int, BinExport2.Instruction, int]]:
|
||||
) -> Iterator[Tuple[int, BinExport2.Instruction, int]]:
|
||||
"""
|
||||
For a given basic block, enumerate the instruction indices,
|
||||
the instruction instances, and their addresses.
|
||||
@@ -260,7 +253,7 @@ class BinExport2Analysis:
|
||||
self.idx: BinExport2Index = idx
|
||||
self.buf: bytes = buf
|
||||
self.base_address: int = 0
|
||||
self.thunks: dict[int, int] = {}
|
||||
self.thunks: Dict[int, int] = {}
|
||||
|
||||
self._find_base_address()
|
||||
self._compute_thunks()
|
||||
@@ -286,14 +279,12 @@ class BinExport2Analysis:
|
||||
|
||||
curr_idx: int = idx
|
||||
for _ in range(capa.features.common.THUNK_CHAIN_DEPTH_DELTA):
|
||||
thunk_callees: list[int] = self.idx.callees_by_vertex_index[curr_idx]
|
||||
# If this doesn't hold, then it doesn't seem like this is a thunk,
|
||||
thunk_callees: List[int] = self.idx.callees_by_vertex_index[curr_idx]
|
||||
# if this doesn't hold, then it doesn't seem like this is a thunk,
|
||||
# because either, len is:
|
||||
# 0 and the thunk doesn't point to anything or is indirect, like `call eax`, or
|
||||
# 0 and the thunk doesn't point to anything, or
|
||||
# >1 and the thunk may end up at many functions.
|
||||
# In any case, this doesn't appear to be the sort of thunk we're looking for.
|
||||
if len(thunk_callees) != 1:
|
||||
break
|
||||
assert len(thunk_callees) == 1, f"thunk @ {hex(addr)} failed"
|
||||
|
||||
thunked_idx: int = thunk_callees[0]
|
||||
thunked_vertex: BinExport2.CallGraph.Vertex = self.be2.call_graph.vertex[thunked_idx]
|
||||
@@ -333,7 +324,7 @@ class AddressNotMappedError(ReadMemoryError): ...
|
||||
@dataclass
|
||||
class AddressSpace:
|
||||
base_address: int
|
||||
memory_regions: tuple[MemoryRegion, ...]
|
||||
memory_regions: Tuple[MemoryRegion, ...]
|
||||
|
||||
def read_memory(self, address: int, length: int) -> bytes:
|
||||
rva: int = address - self.base_address
|
||||
@@ -346,7 +337,7 @@ class AddressSpace:
|
||||
|
||||
@classmethod
|
||||
def from_pe(cls, pe: PE, base_address: int):
|
||||
regions: list[MemoryRegion] = []
|
||||
regions: List[MemoryRegion] = []
|
||||
for section in pe.sections:
|
||||
address: int = section.VirtualAddress
|
||||
size: int = section.Misc_VirtualSize
|
||||
@@ -364,7 +355,7 @@ class AddressSpace:
|
||||
|
||||
@classmethod
|
||||
def from_elf(cls, elf: ELFFile, base_address: int):
|
||||
regions: list[MemoryRegion] = []
|
||||
regions: List[MemoryRegion] = []
|
||||
|
||||
# ELF segments are for runtime data,
|
||||
# ELF sections are for link-time data.
|
||||
@@ -410,9 +401,9 @@ class AnalysisContext:
|
||||
class FunctionContext:
|
||||
ctx: AnalysisContext
|
||||
flow_graph_index: int
|
||||
format: set[str]
|
||||
os: set[str]
|
||||
arch: set[str]
|
||||
format: Set[str]
|
||||
os: Set[str]
|
||||
arch: Set[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
|
||||
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
from typing import Iterator, Optional
|
||||
from typing import List, Tuple, Iterator, Optional
|
||||
|
||||
import capa.features.extractors.binexport2.helpers
|
||||
from capa.features.insn import MAX_STRUCTURE_SIZE, Number, Offset, OperandNumber, OperandOffset
|
||||
@@ -37,7 +30,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def extract_insn_number_features(
|
||||
fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
|
||||
@@ -98,7 +91,7 @@ OFFSET_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
|
||||
|
||||
def extract_insn_offset_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
|
||||
@@ -127,7 +120,7 @@ NZXOR_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
|
||||
|
||||
def extract_insn_nzxor_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
be2: BinExport2 = fhi.ctx.be2
|
||||
@@ -138,7 +131,7 @@ def extract_insn_nzxor_characteristic_features(
|
||||
instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
|
||||
# guaranteed to be simple int/reg operands
|
||||
# so we don't have to realize the tree/list.
|
||||
operands: list[BinExport2.Operand] = [be2.operand[operand_index] for operand_index in instruction.operand_index]
|
||||
operands: List[BinExport2.Operand] = [be2.operand[operand_index] for operand_index in instruction.operand_index]
|
||||
|
||||
if operands[1] != operands[2]:
|
||||
yield Characteristic("nzxor"), ih.address
|
||||
@@ -153,7 +146,7 @@ INDIRECT_CALL_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
|
||||
|
||||
def extract_function_indirect_call_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
be2: BinExport2 = fhi.ctx.be2
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Optional
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import List, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
from capa.features.extractors.binexport2.helpers import get_operand_expressions
|
||||
@@ -39,7 +32,7 @@ def get_operand_phrase_info(be2: BinExport2, operand: BinExport2.Operand) -> Opt
|
||||
# Base: Any general purpose register
|
||||
# Displacement: An integral offset
|
||||
|
||||
expressions: list[BinExport2.Expression] = get_operand_expressions(be2, operand)
|
||||
expressions: List[BinExport2.Expression] = get_operand_expressions(be2, operand)
|
||||
|
||||
# skip expression up to and including BinExport2.Expression.DEREFERENCE, assume caller
|
||||
# has checked for BinExport2.Expression.DEREFERENCE
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import List, Tuple, Iterator
|
||||
|
||||
import capa.features.extractors.strings
|
||||
import capa.features.extractors.binexport2.helpers
|
||||
@@ -70,7 +63,7 @@ NUMBER_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
|
||||
|
||||
def extract_insn_number_features(
|
||||
fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
|
||||
@@ -130,7 +123,7 @@ OFFSET_ZERO_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
|
||||
|
||||
def extract_insn_offset_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
|
||||
@@ -168,7 +161,7 @@ def is_security_cookie(
|
||||
|
||||
# security cookie check should use SP or BP
|
||||
op1: BinExport2.Operand = be2.operand[instruction.operand_index[1]]
|
||||
op1_exprs: list[BinExport2.Expression] = [be2.expression[expr_i] for expr_i in op1.expression_index]
|
||||
op1_exprs: List[BinExport2.Expression] = [be2.expression[expr_i] for expr_i in op1.expression_index]
|
||||
if all(expr.symbol.lower() not in ("bp", "esp", "ebp", "rbp", "rsp") for expr in op1_exprs):
|
||||
return False
|
||||
|
||||
@@ -199,7 +192,7 @@ NZXOR_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
|
||||
|
||||
def extract_insn_nzxor_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse non-zeroing XOR instruction from the given instruction.
|
||||
ignore expected non-zeroing XORs, e.g. security cookies.
|
||||
@@ -216,7 +209,7 @@ def extract_insn_nzxor_characteristic_features(
|
||||
instruction: BinExport2.Instruction = be2.instruction[ii.instruction_index]
|
||||
# guaranteed to be simple int/reg operands
|
||||
# so we don't have to realize the tree/list.
|
||||
operands: list[BinExport2.Operand] = [be2.operand[operand_index] for operand_index in instruction.operand_index]
|
||||
operands: List[BinExport2.Operand] = [be2.operand[operand_index] for operand_index in instruction.operand_index]
|
||||
|
||||
if operands[0] == operands[1]:
|
||||
return
|
||||
@@ -243,7 +236,7 @@ INDIRECT_CALL_PATTERNS = BinExport2InstructionPatternMatcher.from_str(
|
||||
|
||||
def extract_function_indirect_call_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
be2: BinExport2 = fhi.ctx.be2
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
|
||||
from typing import Iterator
|
||||
from typing import List, Tuple, Iterator
|
||||
|
||||
from capa.features.common import Feature, Characteristic
|
||||
from capa.features.address import Address, AbsoluteVirtualAddress
|
||||
@@ -23,20 +16,20 @@ from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
|
||||
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
|
||||
|
||||
|
||||
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
bbi: BasicBlockContext = bbh.inner
|
||||
|
||||
idx = fhi.ctx.idx
|
||||
|
||||
basic_block_index: int = bbi.basic_block_index
|
||||
target_edges: list[BinExport2.FlowGraph.Edge] = idx.target_edges_by_basic_block_index[basic_block_index]
|
||||
target_edges: List[BinExport2.FlowGraph.Edge] = idx.target_edges_by_basic_block_index[basic_block_index]
|
||||
if basic_block_index in (e.source_basic_block_index for e in target_edges):
|
||||
basic_block_address: int = idx.get_basic_block_address(basic_block_index)
|
||||
yield Characteristic("tight loop"), AbsoluteVirtualAddress(basic_block_address)
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract basic block features"""
|
||||
for bb_handler in BASIC_BLOCK_HANDLERS:
|
||||
for feature, addr in bb_handler(fh, bbh):
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Set, List, Tuple, Iterator
|
||||
|
||||
import capa.features.extractors.elf
|
||||
import capa.features.extractors.common
|
||||
@@ -55,14 +48,14 @@ class BinExport2FeatureExtractor(StaticFeatureExtractor):
|
||||
address_space: AddressSpace = AddressSpace.from_buf(buf, self.analysis.base_address)
|
||||
self.ctx: AnalysisContext = AnalysisContext(self.buf, self.be2, self.idx, self.analysis, address_space)
|
||||
|
||||
self.global_features: list[tuple[Feature, Address]] = []
|
||||
self.global_features: List[Tuple[Feature, Address]] = []
|
||||
self.global_features.extend(list(capa.features.extractors.common.extract_format(self.buf)))
|
||||
self.global_features.extend(list(capa.features.extractors.common.extract_os(self.buf)))
|
||||
self.global_features.extend(list(capa.features.extractors.common.extract_arch(self.buf)))
|
||||
|
||||
self.format: set[str] = set()
|
||||
self.os: set[str] = set()
|
||||
self.arch: set[str] = set()
|
||||
self.format: Set[str] = set()
|
||||
self.os: Set[str] = set()
|
||||
self.arch: Set[str] = set()
|
||||
|
||||
for feature, _ in self.global_features:
|
||||
assert isinstance(feature.value, str)
|
||||
@@ -79,10 +72,10 @@ class BinExport2FeatureExtractor(StaticFeatureExtractor):
|
||||
def get_base_address(self) -> AbsoluteVirtualAddress:
|
||||
return AbsoluteVirtualAddress(self.analysis.base_address)
|
||||
|
||||
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from self.global_features
|
||||
|
||||
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.binexport2.file.extract_features(self.be2, self.buf)
|
||||
|
||||
def get_functions(self) -> Iterator[FunctionHandle]:
|
||||
@@ -104,7 +97,7 @@ class BinExport2FeatureExtractor(StaticFeatureExtractor):
|
||||
inner=FunctionContext(self.ctx, flow_graph_index, self.format, self.os, self.arch),
|
||||
)
|
||||
|
||||
def extract_function_features(self, fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.binexport2.function.extract_features(fh)
|
||||
|
||||
def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
|
||||
@@ -119,7 +112,7 @@ class BinExport2FeatureExtractor(StaticFeatureExtractor):
|
||||
inner=BasicBlockContext(basic_block_index),
|
||||
)
|
||||
|
||||
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.binexport2.basicblock.extract_features(fh, bbh)
|
||||
|
||||
def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
|
||||
@@ -133,5 +126,5 @@ class BinExport2FeatureExtractor(StaticFeatureExtractor):
|
||||
|
||||
def extract_insn_features(
|
||||
self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.binexport2.insn.extract_features(fh, bbh, ih)
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import io
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import pefile
|
||||
from elftools.elf.elffile import ELFFile
|
||||
@@ -30,7 +23,7 @@ from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_file_export_names(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_export_names(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
|
||||
if buf.startswith(capa.features.extractors.common.MATCH_PE):
|
||||
pe: pefile.PE = pefile.PE(data=buf)
|
||||
yield from capa.features.extractors.pefile.extract_file_export_names(pe)
|
||||
@@ -41,7 +34,7 @@ def extract_file_export_names(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Fe
|
||||
logger.warning("unsupported format")
|
||||
|
||||
|
||||
def extract_file_import_names(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_import_names(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
|
||||
if buf.startswith(capa.features.extractors.common.MATCH_PE):
|
||||
pe: pefile.PE = pefile.PE(data=buf)
|
||||
yield from capa.features.extractors.pefile.extract_file_import_names(pe)
|
||||
@@ -52,7 +45,7 @@ def extract_file_import_names(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Fe
|
||||
logger.warning("unsupported format")
|
||||
|
||||
|
||||
def extract_file_section_names(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_section_names(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
|
||||
if buf.startswith(capa.features.extractors.common.MATCH_PE):
|
||||
pe: pefile.PE = pefile.PE(data=buf)
|
||||
yield from capa.features.extractors.pefile.extract_file_section_names(pe)
|
||||
@@ -63,15 +56,15 @@ def extract_file_section_names(_be2: BinExport2, buf: bytes) -> Iterator[tuple[F
|
||||
logger.warning("unsupported format")
|
||||
|
||||
|
||||
def extract_file_strings(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_strings(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.common.extract_file_strings(buf)
|
||||
|
||||
|
||||
def extract_file_format(_be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_format(_be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.common.extract_format(buf)
|
||||
|
||||
|
||||
def extract_features(be2: BinExport2, buf: bytes) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(be2: BinExport2, buf: bytes) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract file features"""
|
||||
for file_handler in FILE_HANDLERS:
|
||||
for feature, addr in file_handler(be2, buf):
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import List, Tuple, Iterator
|
||||
|
||||
from capa.features.file import FunctionName
|
||||
from capa.features.common import Feature, Characteristic
|
||||
@@ -23,7 +16,7 @@ from capa.features.extractors.base_extractor import FunctionHandle
|
||||
from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
|
||||
|
||||
|
||||
def extract_function_calls_to(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_calls_to(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
|
||||
be2: BinExport2 = fhi.ctx.be2
|
||||
@@ -39,7 +32,7 @@ def extract_function_calls_to(fh: FunctionHandle) -> Iterator[tuple[Feature, Add
|
||||
yield Characteristic("calls to"), AbsoluteVirtualAddress(caller_address)
|
||||
|
||||
|
||||
def extract_function_loop(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_loop(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
|
||||
be2: BinExport2 = fhi.ctx.be2
|
||||
@@ -47,7 +40,7 @@ def extract_function_loop(fh: FunctionHandle) -> Iterator[tuple[Feature, Address
|
||||
flow_graph_index: int = fhi.flow_graph_index
|
||||
flow_graph: BinExport2.FlowGraph = be2.flow_graph[flow_graph_index]
|
||||
|
||||
edges: list[tuple[int, int]] = []
|
||||
edges: List[Tuple[int, int]] = []
|
||||
for edge in flow_graph.edge:
|
||||
edges.append((edge.source_basic_block_index, edge.target_basic_block_index))
|
||||
|
||||
@@ -55,7 +48,7 @@ def extract_function_loop(fh: FunctionHandle) -> Iterator[tuple[Feature, Address
|
||||
yield Characteristic("loop"), fh.address
|
||||
|
||||
|
||||
def extract_function_name(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_name(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
|
||||
be2: BinExport2 = fhi.ctx.be2
|
||||
@@ -70,7 +63,7 @@ def extract_function_name(fh: FunctionHandle) -> Iterator[tuple[Feature, Address
|
||||
yield FunctionName(vertex.mangled_name), fh.address
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
for func_handler in FUNCTION_HANDLERS:
|
||||
for feature, addr in func_handler(fh):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import re
|
||||
from typing import Union, Iterator, Optional
|
||||
from typing import Set, Dict, List, Tuple, Union, Iterator, Optional
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
|
||||
@@ -29,7 +22,7 @@ HAS_ARCH_INTEL = {ARCH_I386, ARCH_AMD64}
|
||||
HAS_ARCH_ARM = {ARCH_AARCH64}
|
||||
|
||||
|
||||
def mask_immediate(arch: set[str], immediate: int) -> int:
|
||||
def mask_immediate(arch: Set[str], immediate: int) -> int:
|
||||
if arch & HAS_ARCH64:
|
||||
immediate &= 0xFFFFFFFFFFFFFFFF
|
||||
elif arch & HAS_ARCH32:
|
||||
@@ -37,7 +30,7 @@ def mask_immediate(arch: set[str], immediate: int) -> int:
|
||||
return immediate
|
||||
|
||||
|
||||
def twos_complement(arch: set[str], immediate: int, default: Optional[int] = None) -> int:
|
||||
def twos_complement(arch: Set[str], immediate: int, default: Optional[int] = None) -> int:
|
||||
if default is not None:
|
||||
return capa.features.extractors.helpers.twos_complement(immediate, default)
|
||||
elif arch & HAS_ARCH64:
|
||||
@@ -57,36 +50,17 @@ def is_vertex_type(vertex: BinExport2.CallGraph.Vertex, type_: BinExport2.CallGr
|
||||
return vertex.HasField("type") and vertex.type == type_
|
||||
|
||||
|
||||
# internal to `build_expression_tree`
|
||||
# this is unstable: it is subject to change, so don't rely on it!
|
||||
def _prune_expression_tree_references_to_tree_index(
|
||||
expression_tree: list[list[int]],
|
||||
tree_index: int,
|
||||
):
|
||||
# `i` is the index of the tree node that we'll search for `tree_index`
|
||||
# if we remove `tree_index` from it, and it is now empty,
|
||||
# then we'll need to prune references to `i`.
|
||||
for i, tree_node in enumerate(expression_tree):
|
||||
if tree_index in tree_node:
|
||||
tree_node.remove(tree_index)
|
||||
|
||||
if len(tree_node) == 0:
|
||||
# if the parent node is now empty,
|
||||
# remove references to that parent node.
|
||||
_prune_expression_tree_references_to_tree_index(expression_tree, i)
|
||||
|
||||
|
||||
# internal to `build_expression_tree`
|
||||
# this is unstable: it is subject to change, so don't rely on it!
|
||||
def _prune_expression_tree_empty_shifts(
|
||||
be2: BinExport2,
|
||||
operand: BinExport2.Operand,
|
||||
expression_tree: list[list[int]],
|
||||
expression_tree: List[List[int]],
|
||||
tree_index: int,
|
||||
):
|
||||
expression_index = operand.expression_index[tree_index]
|
||||
expression = be2.expression[expression_index]
|
||||
children_tree_indexes: list[int] = expression_tree[tree_index]
|
||||
children_tree_indexes: List[int] = expression_tree[tree_index]
|
||||
|
||||
if expression.type == BinExport2.Expression.OPERATOR:
|
||||
if len(children_tree_indexes) == 0 and expression.symbol in ("lsl", "lsr"):
|
||||
@@ -96,7 +70,9 @@ def _prune_expression_tree_empty_shifts(
|
||||
#
|
||||
# Which seems to be as if the shift wasn't there (shift of #0)
|
||||
# so we want to remove references to this node from any parent nodes.
|
||||
_prune_expression_tree_references_to_tree_index(expression_tree, tree_index)
|
||||
for tree_node in expression_tree:
|
||||
if tree_index in tree_node:
|
||||
tree_node.remove(tree_index)
|
||||
|
||||
return
|
||||
|
||||
@@ -106,37 +82,38 @@ def _prune_expression_tree_empty_shifts(
|
||||
|
||||
# internal to `build_expression_tree`
|
||||
# this is unstable: it is subject to change, so don't rely on it!
|
||||
def _fixup_expression_tree_references_to_tree_index(
|
||||
expression_tree: list[list[int]],
|
||||
existing_index: int,
|
||||
new_index: int,
|
||||
):
|
||||
for tree_node in expression_tree:
|
||||
for i, index in enumerate(tree_node):
|
||||
if index == existing_index:
|
||||
tree_node[i] = new_index
|
||||
|
||||
|
||||
# internal to `build_expression_tree`
|
||||
# this is unstable: it is subject to change, so don't rely on it!
|
||||
def _fixup_expression_tree_lonely_commas(
|
||||
def _prune_expression_tree_empty_commas(
|
||||
be2: BinExport2,
|
||||
operand: BinExport2.Operand,
|
||||
expression_tree: list[list[int]],
|
||||
expression_tree: List[List[int]],
|
||||
tree_index: int,
|
||||
):
|
||||
expression_index = operand.expression_index[tree_index]
|
||||
expression = be2.expression[expression_index]
|
||||
children_tree_indexes: list[int] = expression_tree[tree_index]
|
||||
children_tree_indexes: List[int] = expression_tree[tree_index]
|
||||
|
||||
if expression.type == BinExport2.Expression.OPERATOR:
|
||||
if len(children_tree_indexes) == 1 and expression.symbol == ",":
|
||||
existing_index = tree_index
|
||||
new_index = children_tree_indexes[0]
|
||||
_fixup_expression_tree_references_to_tree_index(expression_tree, existing_index, new_index)
|
||||
# Due to the above pruning of empty LSL or LSR expressions,
|
||||
# the parents might need to be fixed up.
|
||||
#
|
||||
# Specifically, if the pruned node was part of a comma list with two children,
|
||||
# now there's only a single child, which renders as an extra comma,
|
||||
# so we replace references to the comma node with the immediate child.
|
||||
#
|
||||
# A more correct way of doing this might be to walk up the parents and do fixups,
|
||||
# but I'm not quite sure how to do this yet. Just do two passes right now.
|
||||
child = children_tree_indexes[0]
|
||||
|
||||
for tree_node in expression_tree:
|
||||
tree_node.index
|
||||
if tree_index in tree_node:
|
||||
tree_node[tree_node.index(tree_index)] = child
|
||||
|
||||
return
|
||||
|
||||
for child_tree_index in children_tree_indexes:
|
||||
_fixup_expression_tree_lonely_commas(be2, operand, expression_tree, child_tree_index)
|
||||
_prune_expression_tree_empty_commas(be2, operand, expression_tree, child_tree_index)
|
||||
|
||||
|
||||
# internal to `build_expression_tree`
|
||||
@@ -144,17 +121,17 @@ def _fixup_expression_tree_lonely_commas(
|
||||
def _prune_expression_tree(
|
||||
be2: BinExport2,
|
||||
operand: BinExport2.Operand,
|
||||
expression_tree: list[list[int]],
|
||||
expression_tree: List[List[int]],
|
||||
):
|
||||
_prune_expression_tree_empty_shifts(be2, operand, expression_tree, 0)
|
||||
_fixup_expression_tree_lonely_commas(be2, operand, expression_tree, 0)
|
||||
_prune_expression_tree_empty_commas(be2, operand, expression_tree, 0)
|
||||
|
||||
|
||||
# this is unstable: it is subject to change, so don't rely on it!
|
||||
def _build_expression_tree(
|
||||
be2: BinExport2,
|
||||
operand: BinExport2.Operand,
|
||||
) -> list[list[int]]:
|
||||
) -> List[List[int]]:
|
||||
# The reconstructed expression tree layout, linking parent nodes to their children.
|
||||
#
|
||||
# There is one list of integers for each expression in the operand.
|
||||
@@ -182,7 +159,7 @@ def _build_expression_tree(
|
||||
# exist (see https://github.com/NationalSecurityAgency/ghidra/issues/6817)
|
||||
return []
|
||||
|
||||
tree: list[list[int]] = []
|
||||
tree: List[List[int]] = []
|
||||
for i, expression_index in enumerate(operand.expression_index):
|
||||
children = []
|
||||
|
||||
@@ -196,6 +173,7 @@ def _build_expression_tree(
|
||||
tree.append(children)
|
||||
|
||||
_prune_expression_tree(be2, operand, tree)
|
||||
_prune_expression_tree(be2, operand, tree)
|
||||
|
||||
return tree
|
||||
|
||||
@@ -203,34 +181,21 @@ def _build_expression_tree(
|
||||
def _fill_operand_expression_list(
|
||||
be2: BinExport2,
|
||||
operand: BinExport2.Operand,
|
||||
expression_tree: list[list[int]],
|
||||
expression_tree: List[List[int]],
|
||||
tree_index: int,
|
||||
expression_list: list[BinExport2.Expression],
|
||||
expression_list: List[BinExport2.Expression],
|
||||
):
|
||||
"""
|
||||
Walk the given expression tree and collect the expression nodes in-order.
|
||||
"""
|
||||
expression_index = operand.expression_index[tree_index]
|
||||
expression = be2.expression[expression_index]
|
||||
children_tree_indexes: list[int] = expression_tree[tree_index]
|
||||
children_tree_indexes: List[int] = expression_tree[tree_index]
|
||||
|
||||
if expression.type == BinExport2.Expression.REGISTER:
|
||||
assert len(children_tree_indexes) <= 1
|
||||
assert len(children_tree_indexes) == 0
|
||||
expression_list.append(expression)
|
||||
|
||||
if len(children_tree_indexes) == 0:
|
||||
return
|
||||
elif len(children_tree_indexes) == 1:
|
||||
# like for aarch64 with vector instructions, indicating vector data size:
|
||||
#
|
||||
# FADD V0.4S, V1.4S, V2.4S
|
||||
#
|
||||
# see: https://github.com/mandiant/capa/issues/2528
|
||||
child_index = children_tree_indexes[0]
|
||||
_fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
|
||||
return
|
||||
else:
|
||||
raise NotImplementedError(len(children_tree_indexes))
|
||||
return
|
||||
|
||||
elif expression.type == BinExport2.Expression.SYMBOL:
|
||||
assert len(children_tree_indexes) <= 1
|
||||
@@ -253,23 +218,9 @@ def _fill_operand_expression_list(
|
||||
raise NotImplementedError(len(children_tree_indexes))
|
||||
|
||||
elif expression.type == BinExport2.Expression.IMMEDIATE_INT:
|
||||
assert len(children_tree_indexes) <= 1
|
||||
assert len(children_tree_indexes) == 0
|
||||
expression_list.append(expression)
|
||||
|
||||
if len(children_tree_indexes) == 0:
|
||||
return
|
||||
elif len(children_tree_indexes) == 1:
|
||||
# the ghidra exporter can produce some weird expressions,
|
||||
# particularly for MSRs, like for:
|
||||
#
|
||||
# sreg(3, 0, c.0, c.4, 4)
|
||||
#
|
||||
# see: https://github.com/mandiant/capa/issues/2530
|
||||
child_index = children_tree_indexes[0]
|
||||
_fill_operand_expression_list(be2, operand, expression_tree, child_index, expression_list)
|
||||
return
|
||||
else:
|
||||
raise NotImplementedError(len(children_tree_indexes))
|
||||
return
|
||||
|
||||
elif expression.type == BinExport2.Expression.SIZE_PREFIX:
|
||||
# like: b4
|
||||
@@ -331,10 +282,10 @@ def _fill_operand_expression_list(
|
||||
raise NotImplementedError(expression.type)
|
||||
|
||||
|
||||
def get_operand_expressions(be2: BinExport2, op: BinExport2.Operand) -> list[BinExport2.Expression]:
|
||||
def get_operand_expressions(be2: BinExport2, op: BinExport2.Operand) -> List[BinExport2.Expression]:
|
||||
tree = _build_expression_tree(be2, op)
|
||||
|
||||
expressions: list[BinExport2.Expression] = []
|
||||
expressions: List[BinExport2.Expression] = []
|
||||
_fill_operand_expression_list(be2, op, tree, 0, expressions)
|
||||
|
||||
return expressions
|
||||
@@ -380,11 +331,11 @@ def get_instruction_mnemonic(be2: BinExport2, instruction: BinExport2.Instructio
|
||||
return be2.mnemonic[instruction.mnemonic_index].name.lower()
|
||||
|
||||
|
||||
def get_instruction_operands(be2: BinExport2, instruction: BinExport2.Instruction) -> list[BinExport2.Operand]:
|
||||
def get_instruction_operands(be2: BinExport2, instruction: BinExport2.Instruction) -> List[BinExport2.Operand]:
|
||||
return [be2.operand[operand_index] for operand_index in instruction.operand_index]
|
||||
|
||||
|
||||
def split_with_delimiters(s: str, delimiters: tuple[str, ...]) -> Iterator[str]:
|
||||
def split_with_delimiters(s: str, delimiters: Tuple[str, ...]) -> Iterator[str]:
|
||||
"""
|
||||
Splits a string by any of the provided delimiter characters,
|
||||
including the delimiters in the results.
|
||||
@@ -404,7 +355,7 @@ def split_with_delimiters(s: str, delimiters: tuple[str, ...]) -> Iterator[str]:
|
||||
yield s[start:]
|
||||
|
||||
|
||||
BinExport2OperandPattern = Union[str, tuple[str, ...]]
|
||||
BinExport2OperandPattern = Union[str, Tuple[str, ...]]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -431,8 +382,8 @@ class BinExport2InstructionPattern:
|
||||
This matcher uses the BinExport2 data layout under the hood.
|
||||
"""
|
||||
|
||||
mnemonics: tuple[str, ...]
|
||||
operands: tuple[Union[str, BinExport2OperandPattern], ...]
|
||||
mnemonics: Tuple[str, ...]
|
||||
operands: Tuple[Union[str, BinExport2OperandPattern], ...]
|
||||
capture: Optional[str]
|
||||
|
||||
@classmethod
|
||||
@@ -487,7 +438,7 @@ class BinExport2InstructionPattern:
|
||||
mnemonic, _, rest = pattern.partition(" ")
|
||||
mnemonics = mnemonic.split("|")
|
||||
|
||||
operands: list[Union[str, tuple[str, ...]]] = []
|
||||
operands: List[Union[str, Tuple[str, ...]]] = []
|
||||
while rest:
|
||||
rest = rest.strip()
|
||||
if not rest.startswith("["):
|
||||
@@ -558,7 +509,7 @@ class BinExport2InstructionPattern:
|
||||
expression: BinExport2.Expression
|
||||
|
||||
def match(
|
||||
self, mnemonic: str, operand_expressions: list[list[BinExport2.Expression]]
|
||||
self, mnemonic: str, operand_expressions: List[List[BinExport2.Expression]]
|
||||
) -> Optional["BinExport2InstructionPattern.MatchResult"]:
|
||||
"""
|
||||
Match the given BinExport2 data against this pattern.
|
||||
@@ -651,10 +602,10 @@ class BinExport2InstructionPattern:
|
||||
class BinExport2InstructionPatternMatcher:
|
||||
"""Index and match a collection of instruction patterns."""
|
||||
|
||||
def __init__(self, queries: list[BinExport2InstructionPattern]):
|
||||
def __init__(self, queries: List[BinExport2InstructionPattern]):
|
||||
self.queries = queries
|
||||
# shard the patterns by (mnemonic, #operands)
|
||||
self._index: dict[tuple[str, int], list[BinExport2InstructionPattern]] = defaultdict(list)
|
||||
self._index: Dict[Tuple[str, int], List[BinExport2InstructionPattern]] = defaultdict(list)
|
||||
|
||||
for query in queries:
|
||||
for mnemonic in query.mnemonics:
|
||||
@@ -672,7 +623,7 @@ class BinExport2InstructionPatternMatcher:
|
||||
)
|
||||
|
||||
def match(
|
||||
self, mnemonic: str, operand_expressions: list[list[BinExport2.Expression]]
|
||||
self, mnemonic: str, operand_expressions: List[List[BinExport2.Expression]]
|
||||
) -> Optional[BinExport2InstructionPattern.MatchResult]:
|
||||
queries = self._index.get((mnemonic.lower(), len(operand_expressions)), [])
|
||||
for query in queries:
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import List, Tuple, Iterator
|
||||
|
||||
import capa.features.extractors.helpers
|
||||
import capa.features.extractors.strings
|
||||
@@ -39,7 +32,7 @@ from capa.features.extractors.binexport2.binexport2_pb2 import BinExport2
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_insn_api_features(fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_api_features(fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
|
||||
@@ -75,7 +68,7 @@ def extract_insn_api_features(fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
|
||||
|
||||
def extract_insn_number_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
|
||||
if fhi.arch & HAS_ARCH_INTEL:
|
||||
@@ -84,7 +77,7 @@ def extract_insn_number_features(
|
||||
yield from capa.features.extractors.binexport2.arch.arm.insn.extract_insn_number_features(fh, bbh, ih)
|
||||
|
||||
|
||||
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
|
||||
@@ -99,7 +92,7 @@ def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
|
||||
# disassembler already identified string reference from instruction
|
||||
return
|
||||
|
||||
reference_addresses: list[int] = []
|
||||
reference_addresses: List[int] = []
|
||||
|
||||
if instruction_index in idx.data_reference_index_by_source_instruction_index:
|
||||
for data_reference_index in idx.data_reference_index_by_source_instruction_index[instruction_index]:
|
||||
@@ -149,7 +142,7 @@ def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
|
||||
|
||||
def extract_insn_string_features(
|
||||
fh: FunctionHandle, _bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
|
||||
@@ -168,7 +161,7 @@ def extract_insn_string_features(
|
||||
|
||||
def extract_insn_offset_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
|
||||
if fhi.arch & HAS_ARCH_INTEL:
|
||||
@@ -179,7 +172,7 @@ def extract_insn_offset_features(
|
||||
|
||||
def extract_insn_nzxor_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
|
||||
if fhi.arch & HAS_ARCH_INTEL:
|
||||
@@ -194,7 +187,7 @@ def extract_insn_nzxor_characteristic_features(
|
||||
|
||||
def extract_insn_mnemonic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
ii: InstructionContext = ih.inner
|
||||
|
||||
@@ -206,7 +199,7 @@ def extract_insn_mnemonic_features(
|
||||
yield Mnemonic(mnemonic_name), ih.address
|
||||
|
||||
|
||||
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract functions calls from features
|
||||
|
||||
most relevant at the function scope;
|
||||
@@ -228,7 +221,7 @@ def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
|
||||
|
||||
def extract_function_indirect_call_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
fhi: FunctionContext = fh.inner
|
||||
|
||||
if fhi.arch & HAS_ARCH_INTEL:
|
||||
@@ -241,7 +234,7 @@ def extract_function_indirect_call_characteristic_features(
|
||||
)
|
||||
|
||||
|
||||
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract instruction features"""
|
||||
for inst_handler in INSTRUCTION_HANDLERS:
|
||||
for feature, ea in inst_handler(f, bbh, insn):
|
||||
|
||||
@@ -1,36 +1,119 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
from typing import Iterator
|
||||
import string
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
from binaryninja import Function
|
||||
from binaryninja import BasicBlock as BinjaBasicBlock
|
||||
from binaryninja import (
|
||||
BinaryView,
|
||||
SymbolType,
|
||||
RegisterValueType,
|
||||
VariableSourceType,
|
||||
MediumLevelILOperation,
|
||||
MediumLevelILBasicBlock,
|
||||
MediumLevelILInstruction,
|
||||
)
|
||||
|
||||
from capa.features.common import Feature, Characteristic
|
||||
from capa.features.address import Address
|
||||
from capa.features.basicblock import BasicBlock
|
||||
from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
|
||||
from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
|
||||
|
||||
|
||||
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def get_printable_len_ascii(s: bytes) -> int:
|
||||
"""Return string length if all operand bytes are ascii or utf16-le printable"""
|
||||
count = 0
|
||||
for c in s:
|
||||
if c == 0:
|
||||
return count
|
||||
if c < 127 and chr(c) in string.printable:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def get_printable_len_wide(s: bytes) -> int:
|
||||
"""Return string length if all operand bytes are ascii or utf16-le printable"""
|
||||
if all(c == 0x00 for c in s[1::2]):
|
||||
return get_printable_len_ascii(s[::2])
|
||||
return 0
|
||||
|
||||
|
||||
def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
|
||||
bv: BinaryView = f.view
|
||||
|
||||
if il.operation != MediumLevelILOperation.MLIL_CALL:
|
||||
return 0
|
||||
|
||||
target = il.dest
|
||||
if target.operation not in [MediumLevelILOperation.MLIL_CONST, MediumLevelILOperation.MLIL_CONST_PTR]:
|
||||
return 0
|
||||
|
||||
addr = target.value.value
|
||||
sym = bv.get_symbol_at(addr)
|
||||
if not sym or sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.SymbolicFunctionSymbol]:
|
||||
return 0
|
||||
|
||||
if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
|
||||
return 0
|
||||
|
||||
if len(il.params) < 2:
|
||||
return 0
|
||||
|
||||
dest = il.params[0]
|
||||
if dest.operation in [MediumLevelILOperation.MLIL_ADDRESS_OF, MediumLevelILOperation.MLIL_VAR]:
|
||||
var = dest.src
|
||||
else:
|
||||
return 0
|
||||
|
||||
if var.source_type != VariableSourceType.StackVariableSourceType:
|
||||
return 0
|
||||
|
||||
src = il.params[1]
|
||||
if src.value.type != RegisterValueType.ConstantDataAggregateValue:
|
||||
return 0
|
||||
|
||||
s = f.get_constant_data(RegisterValueType.ConstantDataAggregateValue, src.value.value)
|
||||
return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
|
||||
|
||||
|
||||
def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
|
||||
"""check basic block for stackstring indicators
|
||||
|
||||
true if basic block contains enough moves of constant bytes to the stack
|
||||
"""
|
||||
count = 0
|
||||
for il in bb:
|
||||
count += get_stack_string_len(f, il)
|
||||
if count > MIN_STACKSTRING_LEN:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract stackstring indicators from basic block"""
|
||||
bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
|
||||
if bb[1] is not None and bb_contains_stackstring(fh.inner, bb[1]):
|
||||
yield Characteristic("stack string"), bbh.address
|
||||
|
||||
|
||||
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract tight loop indicators from a basic block"""
|
||||
bb: BinjaBasicBlock = bbh.inner
|
||||
for edge in bb.outgoing_edges:
|
||||
if edge.target.start == bb.start:
|
||||
bb: Tuple[BinjaBasicBlock, MediumLevelILBasicBlock] = bbh.inner
|
||||
for edge in bb[0].outgoing_edges:
|
||||
if edge.target.start == bb[0].start:
|
||||
yield Characteristic("tight loop"), bbh.address
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract basic block features"""
|
||||
for bb_handler in BASIC_BLOCK_HANDLERS:
|
||||
for feature, addr in bb_handler(fh, bbh):
|
||||
@@ -38,4 +121,7 @@ def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Featur
|
||||
yield BasicBlock(), bbh.address
|
||||
|
||||
|
||||
BASIC_BLOCK_HANDLERS = (extract_bb_tight_loop,)
|
||||
BASIC_BLOCK_HANDLERS = (
|
||||
extract_bb_tight_loop,
|
||||
extract_bb_stackstring,
|
||||
)
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import List, Tuple, Iterator
|
||||
|
||||
import binaryninja as binja
|
||||
|
||||
@@ -37,7 +30,7 @@ class BinjaFeatureExtractor(StaticFeatureExtractor):
|
||||
def __init__(self, bv: binja.BinaryView):
|
||||
super().__init__(hashes=SampleHashes.from_bytes(bv.file.raw.read(0, bv.file.raw.length)))
|
||||
self.bv = bv
|
||||
self.global_features: list[tuple[Feature, Address]] = []
|
||||
self.global_features: List[Tuple[Feature, Address]] = []
|
||||
self.global_features.extend(capa.features.extractors.binja.file.extract_file_format(self.bv))
|
||||
self.global_features.extend(capa.features.extractors.binja.global_.extract_os(self.bv))
|
||||
self.global_features.extend(capa.features.extractors.binja.global_.extract_arch(self.bv))
|
||||
@@ -55,24 +48,31 @@ class BinjaFeatureExtractor(StaticFeatureExtractor):
|
||||
for f in self.bv.functions:
|
||||
yield FunctionHandle(address=AbsoluteVirtualAddress(f.start), inner=f)
|
||||
|
||||
def extract_function_features(self, fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.binja.function.extract_features(fh)
|
||||
|
||||
def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
|
||||
f: binja.Function = fh.inner
|
||||
for bb in f.basic_blocks:
|
||||
yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=bb)
|
||||
# Set up a MLIL basic block dict look up to associate the disassembly basic block with its MLIL basic block
|
||||
mlil_lookup = {}
|
||||
for mlil_bb in f.mlil.basic_blocks:
|
||||
mlil_lookup[mlil_bb.source_block.start] = mlil_bb
|
||||
|
||||
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
for bb in f.basic_blocks:
|
||||
mlil_bb = mlil_lookup.get(bb.start)
|
||||
|
||||
yield BBHandle(address=AbsoluteVirtualAddress(bb.start), inner=(bb, mlil_bb))
|
||||
|
||||
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.binja.basicblock.extract_features(fh, bbh)
|
||||
|
||||
def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
|
||||
import capa.features.extractors.binja.helpers as binja_helpers
|
||||
|
||||
bb: binja.BasicBlock = bbh.inner
|
||||
addr = bb.start
|
||||
bb: Tuple[binja.BasicBlock, binja.MediumLevelILBasicBlock] = bbh.inner
|
||||
addr = bb[0].start
|
||||
|
||||
for text, length in bb:
|
||||
for text, length in bb[0]:
|
||||
insn = binja_helpers.DisassemblyInstruction(addr, length, text)
|
||||
yield InsnHandle(address=AbsoluteVirtualAddress(addr), inner=insn)
|
||||
addr += length
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
from binaryninja import Segment, BinaryView, SymbolType, SymbolBinding
|
||||
|
||||
@@ -20,22 +13,12 @@ import capa.features.extractors.common
|
||||
import capa.features.extractors.helpers
|
||||
import capa.features.extractors.strings
|
||||
from capa.features.file import Export, Import, Section, FunctionName
|
||||
from capa.features.common import (
|
||||
FORMAT_PE,
|
||||
FORMAT_ELF,
|
||||
FORMAT_SC32,
|
||||
FORMAT_SC64,
|
||||
FORMAT_BINJA_DB,
|
||||
Format,
|
||||
String,
|
||||
Feature,
|
||||
Characteristic,
|
||||
)
|
||||
from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
|
||||
from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
|
||||
from capa.features.extractors.binja.helpers import read_c_string, unmangle_c_name
|
||||
|
||||
|
||||
def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[tuple[Feature, Address]]:
|
||||
def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""check segment for embedded PE"""
|
||||
start = 0
|
||||
if bv.view_type == "PE" and seg.start == bv.start:
|
||||
@@ -49,13 +32,13 @@ def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[tuple[Feature
|
||||
yield Characteristic("embedded pe"), FileOffsetAddress(seg.start + offset)
|
||||
|
||||
|
||||
def extract_file_embedded_pe(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_embedded_pe(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract embedded PE features"""
|
||||
for seg in bv.segments:
|
||||
yield from check_segment_for_pe(bv, seg)
|
||||
|
||||
|
||||
def extract_file_export_names(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract function exports"""
|
||||
for sym in bv.get_symbols_of_type(SymbolType.FunctionSymbol) + bv.get_symbols_of_type(SymbolType.DataSymbol):
|
||||
if sym.binding in [SymbolBinding.GlobalBinding, SymbolBinding.WeakBinding]:
|
||||
@@ -89,7 +72,7 @@ def extract_file_export_names(bv: BinaryView) -> Iterator[tuple[Feature, Address
|
||||
yield Characteristic("forwarded export"), AbsoluteVirtualAddress(sym.address)
|
||||
|
||||
|
||||
def extract_file_import_names(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract function imports
|
||||
|
||||
1. imports by ordinal:
|
||||
@@ -113,19 +96,19 @@ def extract_file_import_names(bv: BinaryView) -> Iterator[tuple[Feature, Address
|
||||
yield Import(name), addr
|
||||
|
||||
|
||||
def extract_file_section_names(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_section_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract section names"""
|
||||
for name, section in bv.sections.items():
|
||||
yield Section(name), AbsoluteVirtualAddress(section.start)
|
||||
|
||||
|
||||
def extract_file_strings(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_strings(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract ASCII and UTF-16 LE strings"""
|
||||
for s in bv.strings:
|
||||
yield String(s.value), FileOffsetAddress(s.start)
|
||||
|
||||
|
||||
def extract_file_function_names(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_function_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract the names of statically-linked library functions.
|
||||
"""
|
||||
@@ -144,22 +127,12 @@ def extract_file_function_names(bv: BinaryView) -> Iterator[tuple[Feature, Addre
|
||||
yield FunctionName(name[1:]), sym.address
|
||||
|
||||
|
||||
def extract_file_format(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
if bv.file.database is not None:
|
||||
yield Format(FORMAT_BINJA_DB), NO_ADDRESS
|
||||
|
||||
def extract_file_format(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
view_type = bv.view_type
|
||||
if view_type in ["PE", "COFF"]:
|
||||
yield Format(FORMAT_PE), NO_ADDRESS
|
||||
elif view_type == "ELF":
|
||||
yield Format(FORMAT_ELF), NO_ADDRESS
|
||||
elif view_type == "Mapped":
|
||||
if bv.arch.name == "x86":
|
||||
yield Format(FORMAT_SC32), NO_ADDRESS
|
||||
elif bv.arch.name == "x86_64":
|
||||
yield Format(FORMAT_SC64), NO_ADDRESS
|
||||
else:
|
||||
raise NotImplementedError(f"unexpected raw file with arch: {bv.arch}")
|
||||
elif view_type == "Raw":
|
||||
# no file type to return when processing a binary file, but we want to continue processing
|
||||
return
|
||||
@@ -167,7 +140,7 @@ def extract_file_format(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
raise NotImplementedError(f"unexpected file format: {view_type}")
|
||||
|
||||
|
||||
def extract_features(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract file features"""
|
||||
for file_handler in FILE_HANDLERS:
|
||||
for feature, addr in file_handler(bv):
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
@@ -112,13 +105,13 @@ def find_binaryninja() -> Optional[Path]:
|
||||
logger.debug("detected OS: linux")
|
||||
elif sys.platform == "darwin":
|
||||
logger.warning("unsupported platform to find Binary Ninja: %s", sys.platform)
|
||||
return None
|
||||
return False
|
||||
elif sys.platform == "win32":
|
||||
logger.warning("unsupported platform to find Binary Ninja: %s", sys.platform)
|
||||
return None
|
||||
return False
|
||||
else:
|
||||
logger.warning("unsupported platform to find Binary Ninja: %s", sys.platform)
|
||||
return None
|
||||
return False
|
||||
|
||||
desktop_entry = get_desktop_entry("com.vector35.binaryninja.desktop")
|
||||
if not desktop_entry:
|
||||
|
||||
@@ -1,38 +1,18 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import string
|
||||
from typing import Iterator
|
||||
|
||||
from binaryninja import (
|
||||
Function,
|
||||
BinaryView,
|
||||
SymbolType,
|
||||
RegisterValueType,
|
||||
VariableSourceType,
|
||||
LowLevelILOperation,
|
||||
MediumLevelILOperation,
|
||||
MediumLevelILBasicBlock,
|
||||
MediumLevelILInstruction,
|
||||
)
|
||||
from binaryninja import Function, BinaryView, SymbolType, RegisterValueType, LowLevelILOperation
|
||||
|
||||
from capa.features.file import FunctionName
|
||||
from capa.features.common import Feature, Characteristic
|
||||
from capa.features.address import Address, AbsoluteVirtualAddress
|
||||
from capa.features.extractors import loops
|
||||
from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
|
||||
from capa.features.extractors.binja.helpers import get_llil_instr_at_addr
|
||||
from capa.features.extractors.base_extractor import FunctionHandle
|
||||
|
||||
|
||||
@@ -44,7 +24,7 @@ def extract_function_calls_to(fh: FunctionHandle):
|
||||
# Everything that is a code reference to the current function is considered a caller, which actually includes
|
||||
# many other references that are NOT a caller. For example, an instruction `push function_start` will also be
|
||||
# considered a caller to the function
|
||||
llil = get_llil_instr_at_addr(func.view, caller.address)
|
||||
llil = caller.llil
|
||||
if (llil is None) or llil.operation not in [
|
||||
LowLevelILOperation.LLIL_CALL,
|
||||
LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
|
||||
@@ -53,13 +33,14 @@ def extract_function_calls_to(fh: FunctionHandle):
|
||||
]:
|
||||
continue
|
||||
|
||||
if llil.dest.operation not in [
|
||||
LowLevelILOperation.LLIL_CONST,
|
||||
LowLevelILOperation.LLIL_CONST_PTR,
|
||||
if llil.dest.value.type not in [
|
||||
RegisterValueType.ImportedAddressValue,
|
||||
RegisterValueType.ConstantValue,
|
||||
RegisterValueType.ConstantPointerValue,
|
||||
]:
|
||||
continue
|
||||
|
||||
address = llil.dest.constant
|
||||
address = llil.dest.value.value
|
||||
if address != func.start:
|
||||
continue
|
||||
|
||||
@@ -114,102 +95,10 @@ def extract_function_name(fh: FunctionHandle):
|
||||
yield FunctionName(name[1:]), sym.address
|
||||
|
||||
|
||||
def get_printable_len_ascii(s: bytes) -> int:
|
||||
"""Return string length if all operand bytes are ascii or utf16-le printable"""
|
||||
count = 0
|
||||
for c in s:
|
||||
if c == 0:
|
||||
return count
|
||||
if c < 127 and chr(c) in string.printable:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def get_printable_len_wide(s: bytes) -> int:
|
||||
"""Return string length if all operand bytes are ascii or utf16-le printable"""
|
||||
if all(c == 0x00 for c in s[1::2]):
|
||||
return get_printable_len_ascii(s[::2])
|
||||
return 0
|
||||
|
||||
|
||||
def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
|
||||
bv: BinaryView = f.view
|
||||
|
||||
if il.operation != MediumLevelILOperation.MLIL_CALL:
|
||||
return 0
|
||||
|
||||
target = il.dest
|
||||
if target.operation not in [MediumLevelILOperation.MLIL_CONST, MediumLevelILOperation.MLIL_CONST_PTR]:
|
||||
return 0
|
||||
|
||||
addr = target.value.value
|
||||
sym = bv.get_symbol_at(addr)
|
||||
if not sym or sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.SymbolicFunctionSymbol]:
|
||||
return 0
|
||||
|
||||
if sym.name not in ["__builtin_strncpy", "__builtin_strcpy", "__builtin_wcscpy"]:
|
||||
return 0
|
||||
|
||||
if len(il.params) < 2:
|
||||
return 0
|
||||
|
||||
dest = il.params[0]
|
||||
if dest.operation in [MediumLevelILOperation.MLIL_ADDRESS_OF, MediumLevelILOperation.MLIL_VAR]:
|
||||
var = dest.src
|
||||
else:
|
||||
return 0
|
||||
|
||||
if var.source_type != VariableSourceType.StackVariableSourceType:
|
||||
return 0
|
||||
|
||||
src = il.params[1]
|
||||
if src.value.type != RegisterValueType.ConstantDataAggregateValue:
|
||||
return 0
|
||||
|
||||
s = f.get_constant_data(RegisterValueType.ConstantDataAggregateValue, src.value.value)
|
||||
return max(get_printable_len_ascii(bytes(s)), get_printable_len_wide(bytes(s)))
|
||||
|
||||
|
||||
def bb_contains_stackstring(f: Function, bb: MediumLevelILBasicBlock) -> bool:
|
||||
"""check basic block for stackstring indicators
|
||||
|
||||
true if basic block contains enough moves of constant bytes to the stack
|
||||
"""
|
||||
count = 0
|
||||
for il in bb:
|
||||
count += get_stack_string_len(f, il)
|
||||
if count > MIN_STACKSTRING_LEN:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def extract_stackstring(fh: FunctionHandle):
|
||||
"""extract stackstring indicators"""
|
||||
func: Function = fh.inner
|
||||
bv: BinaryView = func.view
|
||||
if bv is None:
|
||||
return
|
||||
|
||||
mlil = func.mlil
|
||||
if mlil is None:
|
||||
return
|
||||
|
||||
for block in mlil.basic_blocks:
|
||||
if bb_contains_stackstring(func, block):
|
||||
yield Characteristic("stack string"), block.source_block.start
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
for func_handler in FUNCTION_HANDLERS:
|
||||
for feature, addr in func_handler(fh):
|
||||
yield feature, addr
|
||||
|
||||
|
||||
FUNCTION_HANDLERS = (
|
||||
extract_function_calls_to,
|
||||
extract_function_loop,
|
||||
extract_recursive_call,
|
||||
extract_function_name,
|
||||
extract_stackstring,
|
||||
)
|
||||
FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call, extract_function_name)
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
from binaryninja import BinaryView
|
||||
|
||||
@@ -23,7 +16,7 @@ from capa.features.address import NO_ADDRESS, Address
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_os(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_os(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
name = bv.platform.name
|
||||
if "-" in name:
|
||||
name = name.split("-")[0]
|
||||
@@ -52,7 +45,7 @@ def extract_os(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
return
|
||||
|
||||
|
||||
def extract_arch(bv: BinaryView) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_arch(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
|
||||
arch = bv.arch.name
|
||||
if arch == "x86_64":
|
||||
yield Arch(ARCH_AMD64), NO_ADDRESS
|
||||
|
||||
@@ -1,22 +1,15 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import re
|
||||
from typing import Callable, Optional
|
||||
from typing import List, Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from binaryninja import BinaryView, LowLevelILFunction, LowLevelILInstruction
|
||||
from binaryninja import BinaryView, LowLevelILInstruction
|
||||
from binaryninja.architecture import InstructionTextToken
|
||||
|
||||
|
||||
@@ -24,7 +17,7 @@ from binaryninja.architecture import InstructionTextToken
|
||||
class DisassemblyInstruction:
|
||||
address: int
|
||||
length: int
|
||||
text: list[InstructionTextToken]
|
||||
text: List[InstructionTextToken]
|
||||
|
||||
|
||||
LLIL_VISITOR = Callable[[LowLevelILInstruction, LowLevelILInstruction, int], bool]
|
||||
@@ -61,7 +54,7 @@ def unmangle_c_name(name: str) -> str:
|
||||
|
||||
|
||||
def read_c_string(bv: BinaryView, offset: int, max_len: int) -> str:
|
||||
s: list[str] = []
|
||||
s: List[str] = []
|
||||
while len(s) < max_len:
|
||||
try:
|
||||
c = bv.read(offset + len(s), 1)[0]
|
||||
@@ -74,13 +67,3 @@ def read_c_string(bv: BinaryView, offset: int, max_len: int) -> str:
|
||||
s.append(chr(c))
|
||||
|
||||
return "".join(s)
|
||||
|
||||
|
||||
def get_llil_instr_at_addr(bv: BinaryView, addr: int) -> Optional[LowLevelILInstruction]:
|
||||
arch = bv.arch
|
||||
buffer = bv.read(addr, arch.max_instr_length)
|
||||
llil = LowLevelILFunction(arch=arch)
|
||||
llil.current_address = addr
|
||||
if arch.get_instruction_low_level_il(buffer, addr, llil) == 0:
|
||||
return None
|
||||
return llil[0]
|
||||
|
||||
@@ -1,22 +1,15 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from typing import Any, Optional
|
||||
from collections.abc import Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Any, List, Tuple, Iterator, Optional
|
||||
|
||||
import binaryninja as bn
|
||||
from binaryninja import Function
|
||||
from binaryninja import BasicBlock as BinjaBasicBlock
|
||||
from binaryninja import (
|
||||
Function,
|
||||
BinaryView,
|
||||
ILRegister,
|
||||
SymbolType,
|
||||
@@ -30,7 +23,7 @@ import capa.features.extractors.helpers
|
||||
from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
|
||||
from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
|
||||
from capa.features.address import Address, AbsoluteVirtualAddress
|
||||
from capa.features.extractors.binja.helpers import DisassemblyInstruction, visit_llil_exprs, get_llil_instr_at_addr
|
||||
from capa.features.extractors.binja.helpers import DisassemblyInstruction, visit_llil_exprs
|
||||
from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
|
||||
|
||||
# security cookie checks may perform non-zeroing XORs, these are expected within a certain
|
||||
@@ -43,27 +36,35 @@ SECURITY_COOKIE_BYTES_DELTA = 0x40
|
||||
# 2. The function must only make one call/jump to another address
|
||||
# If the function being checked is a stub function, returns the target address. Otherwise, return None.
|
||||
def is_stub_function(bv: BinaryView, addr: int) -> Optional[int]:
|
||||
llil = get_llil_instr_at_addr(bv, addr)
|
||||
if llil is None or llil.operation not in [
|
||||
LowLevelILOperation.LLIL_CALL,
|
||||
LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
|
||||
LowLevelILOperation.LLIL_JUMP,
|
||||
LowLevelILOperation.LLIL_TAILCALL,
|
||||
]:
|
||||
return None
|
||||
funcs = bv.get_functions_at(addr)
|
||||
for func in funcs:
|
||||
if len(func.basic_blocks) != 1:
|
||||
continue
|
||||
|
||||
# The LLIL instruction retrieved by `get_llil_instr_at_addr` did not go through a full analysis, so we cannot check
|
||||
# `llil.dest.value.type` here
|
||||
if llil.dest.operation not in [
|
||||
LowLevelILOperation.LLIL_CONST,
|
||||
LowLevelILOperation.LLIL_CONST_PTR,
|
||||
]:
|
||||
return None
|
||||
call_count = 0
|
||||
call_target = None
|
||||
for il in func.llil.instructions:
|
||||
if il.operation in [
|
||||
LowLevelILOperation.LLIL_CALL,
|
||||
LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
|
||||
LowLevelILOperation.LLIL_JUMP,
|
||||
LowLevelILOperation.LLIL_TAILCALL,
|
||||
]:
|
||||
call_count += 1
|
||||
if il.dest.value.type in [
|
||||
RegisterValueType.ImportedAddressValue,
|
||||
RegisterValueType.ConstantValue,
|
||||
RegisterValueType.ConstantPointerValue,
|
||||
]:
|
||||
call_target = il.dest.value.value
|
||||
|
||||
return llil.dest.constant
|
||||
if call_count == 1 and call_target is not None:
|
||||
return call_target
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction API features
|
||||
|
||||
@@ -122,7 +123,7 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
|
||||
|
||||
def extract_insn_number_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction number features
|
||||
example:
|
||||
@@ -130,7 +131,7 @@ def extract_insn_number_features(
|
||||
"""
|
||||
func: Function = fh.inner
|
||||
|
||||
results: list[tuple[Any[Number, OperandNumber], Address]] = []
|
||||
results: List[Tuple[Any[Number, OperandNumber], Address]] = []
|
||||
|
||||
def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
|
||||
if il.operation == LowLevelILOperation.LLIL_LOAD:
|
||||
@@ -161,7 +162,7 @@ def extract_insn_number_features(
|
||||
yield from results
|
||||
|
||||
|
||||
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse referenced byte sequences
|
||||
example:
|
||||
@@ -208,7 +209,7 @@ def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
|
||||
|
||||
def extract_insn_string_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction string features
|
||||
|
||||
@@ -265,7 +266,7 @@ def extract_insn_string_features(
|
||||
|
||||
def extract_insn_offset_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction structure offset features
|
||||
|
||||
@@ -274,7 +275,7 @@ def extract_insn_offset_features(
|
||||
"""
|
||||
func: Function = fh.inner
|
||||
|
||||
results: list[tuple[Any[Offset, OperandOffset], Address]] = []
|
||||
results: List[Tuple[Any[Offset, OperandOffset], Address]] = []
|
||||
address_size = func.view.arch.address_size * 8
|
||||
|
||||
def llil_checker(il: LowLevelILInstruction, parent: LowLevelILInstruction, index: int) -> bool:
|
||||
@@ -323,7 +324,7 @@ def extract_insn_offset_features(
|
||||
yield from results
|
||||
|
||||
|
||||
def is_nzxor_stack_cookie(f: Function, bb: bn.BasicBlock, llil: LowLevelILInstruction) -> bool:
|
||||
def is_nzxor_stack_cookie(f: Function, bb: BinjaBasicBlock, llil: LowLevelILInstruction) -> bool:
|
||||
"""check if nzxor exists within stack cookie delta"""
|
||||
# TODO(xusheng): use LLIL SSA to do more accurate analysis
|
||||
# https://github.com/mandiant/capa/issues/1609
|
||||
@@ -352,7 +353,7 @@ def is_nzxor_stack_cookie(f: Function, bb: bn.BasicBlock, llil: LowLevelILInstru
|
||||
|
||||
def extract_insn_nzxor_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction non-zeroing XOR instruction
|
||||
ignore expected non-zeroing XORs, e.g. security cookies
|
||||
@@ -366,7 +367,7 @@ def extract_insn_nzxor_characteristic_features(
|
||||
# e.g., <llil: eax = 0>, (LLIL_SET_REG). So we do not need to check whether the two operands are the same.
|
||||
if il.operation == LowLevelILOperation.LLIL_XOR:
|
||||
# Exclude cases related to the stack cookie
|
||||
if is_nzxor_stack_cookie(fh.inner, bbh.inner, il):
|
||||
if is_nzxor_stack_cookie(fh.inner, bbh.inner[0], il):
|
||||
return False
|
||||
results.append((Characteristic("nzxor"), ih.address))
|
||||
return False
|
||||
@@ -381,7 +382,7 @@ def extract_insn_nzxor_characteristic_features(
|
||||
|
||||
def extract_insn_mnemonic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction mnemonic features"""
|
||||
insn: DisassemblyInstruction = ih.inner
|
||||
yield Mnemonic(insn.text[0].text), ih.address
|
||||
@@ -389,7 +390,7 @@ def extract_insn_mnemonic_features(
|
||||
|
||||
def extract_insn_obfs_call_plus_5_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse call $+5 instruction from the given instruction.
|
||||
"""
|
||||
@@ -400,7 +401,7 @@ def extract_insn_obfs_call_plus_5_characteristic_features(
|
||||
|
||||
def extract_insn_peb_access_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction peb access
|
||||
|
||||
fs:[0x30] on x86, gs:[0x60] on x64
|
||||
@@ -443,7 +444,7 @@ def extract_insn_peb_access_characteristic_features(
|
||||
|
||||
def extract_insn_segment_access_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction fs or gs access"""
|
||||
func: Function = fh.inner
|
||||
|
||||
@@ -470,7 +471,7 @@ def extract_insn_segment_access_features(
|
||||
|
||||
def extract_insn_cross_section_cflow(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""inspect the instruction for a CALL or JMP that crosses section boundaries"""
|
||||
func: Function = fh.inner
|
||||
bv: BinaryView = func.view
|
||||
@@ -490,7 +491,7 @@ def extract_insn_cross_section_cflow(
|
||||
yield Characteristic("cross section flow"), ih.address
|
||||
|
||||
|
||||
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract functions calls from features
|
||||
|
||||
most relevant at the function scope, however, its most efficient to extract at the instruction scope
|
||||
@@ -533,7 +534,7 @@ def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
|
||||
|
||||
def extract_function_indirect_call_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
|
||||
does not include calls like => call ds:dword_ABD4974
|
||||
|
||||
@@ -561,7 +562,7 @@ def extract_function_indirect_call_characteristic_features(
|
||||
yield Characteristic("indirect call"), ih.address
|
||||
|
||||
|
||||
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract instruction features"""
|
||||
for inst_handler in INSTRUCTION_HANDLERS:
|
||||
for feature, ea in inst_handler(f, bbh, insn):
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import capa.features.extractors.helpers
|
||||
from capa.helpers import assert_never
|
||||
@@ -27,7 +20,7 @@ from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, Pr
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
this method extracts the given call's features (such as API name and arguments),
|
||||
and returns them as API, Number, and String features.
|
||||
@@ -62,7 +55,7 @@ def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -
|
||||
yield API(name), ch.address
|
||||
|
||||
|
||||
def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
for handler in CALL_HANDLERS:
|
||||
for feature, addr in handler(ph, th, ch):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Union, Iterator
|
||||
from typing import Dict, Tuple, Union, Iterator
|
||||
|
||||
import capa.features.extractors.cape.call
|
||||
import capa.features.extractors.cape.file
|
||||
@@ -22,8 +15,8 @@ import capa.features.extractors.cape.thread
|
||||
import capa.features.extractors.cape.global_
|
||||
import capa.features.extractors.cape.process
|
||||
from capa.exceptions import EmptyReportError, UnsupportedFormatError
|
||||
from capa.features.common import Feature
|
||||
from capa.features.address import Address, AbsoluteVirtualAddress, _NoAddress
|
||||
from capa.features.common import Feature, Characteristic
|
||||
from capa.features.address import NO_ADDRESS, Address, AbsoluteVirtualAddress, _NoAddress
|
||||
from capa.features.extractors.cape.models import Call, Static, Process, CapeReport
|
||||
from capa.features.extractors.base_extractor import (
|
||||
CallHandle,
|
||||
@@ -54,20 +47,19 @@ class CapeExtractor(DynamicFeatureExtractor):
|
||||
|
||||
def get_base_address(self) -> Union[AbsoluteVirtualAddress, _NoAddress, None]:
|
||||
# value according to the PE header, the actual trace may use a different imagebase
|
||||
assert self.report.static is not None
|
||||
assert self.report.static.pe is not None
|
||||
assert self.report.static is not None and self.report.static.pe is not None
|
||||
return AbsoluteVirtualAddress(self.report.static.pe.imagebase)
|
||||
|
||||
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from self.global_features
|
||||
|
||||
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.cape.file.extract_features(self.report)
|
||||
|
||||
def get_processes(self) -> Iterator[ProcessHandle]:
|
||||
yield from capa.features.extractors.cape.file.get_processes(self.report)
|
||||
|
||||
def extract_process_features(self, ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.cape.process.extract_features(ph)
|
||||
|
||||
def get_process_name(self, ph) -> str:
|
||||
@@ -77,15 +69,19 @@ class CapeExtractor(DynamicFeatureExtractor):
|
||||
def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
|
||||
yield from capa.features.extractors.cape.process.get_threads(ph)
|
||||
|
||||
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
yield from []
|
||||
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
if False:
|
||||
# force this routine to be a generator,
|
||||
# but we don't actually have any elements to generate.
|
||||
yield Characteristic("never"), NO_ADDRESS
|
||||
return
|
||||
|
||||
def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
|
||||
yield from capa.features.extractors.cape.thread.get_calls(ph, th)
|
||||
|
||||
def extract_call_features(
|
||||
self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.cape.call.extract_features(ph, th, ch)
|
||||
|
||||
def get_call_name(self, ph, th, ch) -> str:
|
||||
@@ -126,7 +122,7 @@ class CapeExtractor(DynamicFeatureExtractor):
|
||||
return "".join(parts)
|
||||
|
||||
@classmethod
|
||||
def from_report(cls, report: dict) -> "CapeExtractor":
|
||||
def from_report(cls, report: Dict) -> "CapeExtractor":
|
||||
cr = CapeReport.model_validate(report)
|
||||
|
||||
if cr.info.version not in TESTED_VERSIONS:
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
from capa.features.file import Export, Import, Section
|
||||
from capa.features.common import String, Feature
|
||||
@@ -48,7 +41,7 @@ def get_processes(report: CapeReport) -> Iterator[ProcessHandle]:
|
||||
seen_processes[addr].append(process)
|
||||
|
||||
|
||||
def extract_import_names(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_import_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract imported function names
|
||||
"""
|
||||
@@ -69,75 +62,57 @@ def extract_import_names(report: CapeReport) -> Iterator[tuple[Feature, Address]
|
||||
yield Import(name), AbsoluteVirtualAddress(function.address)
|
||||
|
||||
|
||||
def extract_export_names(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_export_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
assert report.static is not None and report.static.pe is not None
|
||||
for function in report.static.pe.exports:
|
||||
yield Export(function.name), AbsoluteVirtualAddress(function.address)
|
||||
|
||||
|
||||
def extract_section_names(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_section_names(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
assert report.static is not None and report.static.pe is not None
|
||||
for section in report.static.pe.sections:
|
||||
yield Section(section.name), AbsoluteVirtualAddress(section.virtual_address)
|
||||
|
||||
|
||||
def extract_file_strings(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_strings(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
if report.strings is not None:
|
||||
for string in report.strings:
|
||||
yield String(string), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_used_regkeys(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
if not report.behavior.summary:
|
||||
return
|
||||
|
||||
def extract_used_regkeys(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for regkey in report.behavior.summary.keys:
|
||||
yield String(regkey), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_used_files(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
if not report.behavior.summary:
|
||||
return
|
||||
|
||||
def extract_used_files(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for file in report.behavior.summary.files:
|
||||
yield String(file), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_used_mutexes(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
if not report.behavior.summary:
|
||||
return
|
||||
|
||||
def extract_used_mutexes(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for mutex in report.behavior.summary.mutexes:
|
||||
yield String(mutex), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_used_commands(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
if not report.behavior.summary:
|
||||
return
|
||||
|
||||
def extract_used_commands(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for cmd in report.behavior.summary.executed_commands:
|
||||
yield String(cmd), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_used_apis(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
if not report.behavior.summary:
|
||||
return
|
||||
|
||||
def extract_used_apis(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for symbol in report.behavior.summary.resolved_apis:
|
||||
yield String(symbol), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_used_services(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
if not report.behavior.summary:
|
||||
return
|
||||
|
||||
def extract_used_services(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for svc in report.behavior.summary.created_services:
|
||||
yield String(svc), NO_ADDRESS
|
||||
for svc in report.behavior.summary.started_services:
|
||||
yield String(svc), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_features(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for handler in FILE_HANDLERS:
|
||||
for feature, addr in handler(report):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
from capa.features.common import (
|
||||
OS,
|
||||
@@ -35,7 +28,7 @@ from capa.features.extractors.cape.models import CapeReport
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_arch(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_arch(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
if "Intel 80386" in report.target.file.type:
|
||||
yield Arch(ARCH_I386), NO_ADDRESS
|
||||
elif "x86-64" in report.target.file.type:
|
||||
@@ -47,7 +40,7 @@ def extract_arch(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
)
|
||||
|
||||
|
||||
def extract_format(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_format(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
if "PE" in report.target.file.type:
|
||||
yield Format(FORMAT_PE), NO_ADDRESS
|
||||
elif "ELF" in report.target.file.type:
|
||||
@@ -59,7 +52,7 @@ def extract_format(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
)
|
||||
|
||||
|
||||
def extract_os(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_os(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
# this variable contains the output of the file command
|
||||
file_output = report.target.file.type
|
||||
|
||||
@@ -87,7 +80,7 @@ def extract_os(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
yield OS(OS_ANY), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_features(report: CapeReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(report: CapeReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for global_handler in GLOBAL_HANDLER:
|
||||
for feature, addr in global_handler(report):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,24 +1,17 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
|
||||
from typing import Any
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from capa.features.extractors.base_extractor import ProcessHandle
|
||||
|
||||
|
||||
def find_process(processes: list[dict[str, Any]], ph: ProcessHandle) -> dict[str, Any]:
|
||||
def find_process(processes: List[Dict[str, Any]], ph: ProcessHandle) -> Dict[str, Any]:
|
||||
"""
|
||||
find a specific process identified by a process handler.
|
||||
|
||||
|
||||
@@ -1,20 +1,15 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Any, Union, Optional, Annotated, TypeAlias
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import binascii
|
||||
from typing import Any, Dict, List, Union, Literal, Optional
|
||||
|
||||
from pydantic import Field, BaseModel, ConfigDict
|
||||
from typing_extensions import Annotated, TypeAlias
|
||||
from pydantic.functional_validators import BeforeValidator
|
||||
|
||||
|
||||
@@ -26,7 +21,7 @@ def validate_hex_int(value):
|
||||
|
||||
|
||||
def validate_hex_bytes(value):
|
||||
return bytes.fromhex(value) if isinstance(value, str) else value
|
||||
return binascii.unhexlify(value) if isinstance(value, str) else value
|
||||
|
||||
|
||||
HexInt = Annotated[int, BeforeValidator(validate_hex_int)]
|
||||
@@ -64,48 +59,45 @@ Skip: TypeAlias = Optional[Any]
|
||||
# in a field with this type.
|
||||
# then we can update the model with the discovered shape.
|
||||
TODO: TypeAlias = None
|
||||
ListTODO: TypeAlias = list[None]
|
||||
ListTODO: TypeAlias = List[None]
|
||||
DictTODO: TypeAlias = ExactModel
|
||||
|
||||
Emptydict: TypeAlias = BaseModel
|
||||
EmptyList: TypeAlias = list[Any]
|
||||
EmptyDict: TypeAlias = BaseModel
|
||||
EmptyList: TypeAlias = List[Any]
|
||||
|
||||
|
||||
class Info(FlexibleModel):
|
||||
version: str
|
||||
|
||||
|
||||
class ImportedSymbol(FlexibleModel):
|
||||
class ImportedSymbol(ExactModel):
|
||||
address: HexInt
|
||||
name: Optional[str] = None
|
||||
|
||||
|
||||
class ImportedDll(FlexibleModel):
|
||||
class ImportedDll(ExactModel):
|
||||
dll: str
|
||||
imports: list[ImportedSymbol]
|
||||
imports: List[ImportedSymbol]
|
||||
|
||||
|
||||
"""
|
||||
class DirectoryEntry(FlexibleModel):
|
||||
class DirectoryEntry(ExactModel):
|
||||
name: str
|
||||
virtual_address: HexInt
|
||||
size: HexInt
|
||||
"""
|
||||
|
||||
|
||||
class Section(FlexibleModel):
|
||||
class Section(ExactModel):
|
||||
name: str
|
||||
# raw_address: HexInt
|
||||
raw_address: HexInt
|
||||
virtual_address: HexInt
|
||||
# virtual_size: HexInt
|
||||
# size_of_data: HexInt
|
||||
# characteristics: str
|
||||
# characteristics_raw: HexInt
|
||||
# entropy: float
|
||||
virtual_size: HexInt
|
||||
size_of_data: HexInt
|
||||
characteristics: str
|
||||
characteristics_raw: HexInt
|
||||
entropy: float
|
||||
|
||||
|
||||
"""
|
||||
class Resource(FlexibleModel):
|
||||
class Resource(ExactModel):
|
||||
name: str
|
||||
language: Optional[str] = None
|
||||
sublanguage: str
|
||||
@@ -143,7 +135,7 @@ class DigitalSigner(FlexibleModel):
|
||||
extensions_subjectKeyIdentifier: Optional[str] = None
|
||||
|
||||
|
||||
class AuxSigner(FlexibleModel):
|
||||
class AuxSigner(ExactModel):
|
||||
name: str
|
||||
issued_to: str = Field(alias="Issued to")
|
||||
issued_by: str = Field(alias="Issued by")
|
||||
@@ -151,70 +143,69 @@ class AuxSigner(FlexibleModel):
|
||||
sha1_hash: str = Field(alias="SHA1 hash")
|
||||
|
||||
|
||||
class Signer(FlexibleModel):
|
||||
class Signer(ExactModel):
|
||||
aux_sha1: Optional[str] = None
|
||||
aux_timestamp: Optional[str] = None
|
||||
aux_valid: Optional[bool] = None
|
||||
aux_error: Optional[bool] = None
|
||||
aux_error_desc: Optional[str] = None
|
||||
aux_signers: Optional[list[AuxSigner]] = None
|
||||
aux_signers: Optional[List[AuxSigner]] = None
|
||||
|
||||
|
||||
class Overlay(FlexibleModel):
|
||||
class Overlay(ExactModel):
|
||||
offset: HexInt
|
||||
size: HexInt
|
||||
|
||||
|
||||
class KV(FlexibleModel):
|
||||
class KV(ExactModel):
|
||||
name: str
|
||||
value: str
|
||||
"""
|
||||
|
||||
|
||||
class ExportedSymbol(FlexibleModel):
|
||||
class ExportedSymbol(ExactModel):
|
||||
address: HexInt
|
||||
name: str
|
||||
# ordinal: int
|
||||
ordinal: int
|
||||
|
||||
|
||||
class PE(FlexibleModel):
|
||||
# peid_signatures: TODO
|
||||
class PE(ExactModel):
|
||||
peid_signatures: TODO
|
||||
imagebase: HexInt
|
||||
# entrypoint: HexInt
|
||||
# reported_checksum: HexInt
|
||||
# actual_checksum: HexInt
|
||||
# osversion: str
|
||||
# pdbpath: Optional[str] = None
|
||||
# timestamp: str
|
||||
entrypoint: HexInt
|
||||
reported_checksum: HexInt
|
||||
actual_checksum: HexInt
|
||||
osversion: str
|
||||
pdbpath: Optional[str] = None
|
||||
timestamp: str
|
||||
|
||||
# list[ImportedDll], or dict[basename(dll), ImportedDll]
|
||||
imports: list[ImportedDll] | dict[str, ImportedDll] = Field(default_factory=list) # type: ignore
|
||||
# imported_dll_count: Optional[int] = None
|
||||
# imphash: str
|
||||
# List[ImportedDll], or Dict[basename(dll), ImportedDll]
|
||||
imports: Union[List[ImportedDll], Dict[str, ImportedDll]]
|
||||
imported_dll_count: Optional[int] = None
|
||||
imphash: str
|
||||
|
||||
# exported_dll_name: Optional[str] = None
|
||||
exports: list[ExportedSymbol] = Field(default_factory=list)
|
||||
exported_dll_name: Optional[str] = None
|
||||
exports: List[ExportedSymbol]
|
||||
|
||||
# dirents: list[DirectoryEntry]
|
||||
sections: list[Section] = Field(default_factory=list)
|
||||
dirents: List[DirectoryEntry]
|
||||
sections: List[Section]
|
||||
|
||||
# ep_bytes: Optional[HexBytes] = None
|
||||
ep_bytes: Optional[HexBytes] = None
|
||||
|
||||
# overlay: Optional[Overlay] = None
|
||||
# resources: list[Resource]
|
||||
# versioninfo: list[KV]
|
||||
overlay: Optional[Overlay] = None
|
||||
resources: List[Resource]
|
||||
versioninfo: List[KV]
|
||||
|
||||
# base64 encoded data
|
||||
# icon: Optional[str] = None
|
||||
icon: Optional[str] = None
|
||||
# MD5-like hash
|
||||
# icon_hash: Optional[str] = None
|
||||
icon_hash: Optional[str] = None
|
||||
# MD5-like hash
|
||||
# icon_fuzzy: Optional[str] = None
|
||||
icon_fuzzy: Optional[str] = None
|
||||
# short hex string
|
||||
# icon_dhash: Optional[str] = None
|
||||
icon_dhash: Optional[str] = None
|
||||
|
||||
# digital_signers: list[DigitalSigner]
|
||||
# guest_signers: Signer
|
||||
digital_signers: List[DigitalSigner]
|
||||
guest_signers: Signer
|
||||
|
||||
|
||||
# TODO(mr-tz): target.file.dotnet, target.file.extracted_files, target.file.extracted_files_tool,
|
||||
@@ -222,49 +213,48 @@ class PE(FlexibleModel):
|
||||
# https://github.com/mandiant/capa/issues/1814
|
||||
class File(FlexibleModel):
|
||||
type: str
|
||||
# cape_type_code: Optional[int] = None
|
||||
# cape_type: Optional[str] = None
|
||||
cape_type_code: Optional[int] = None
|
||||
cape_type: Optional[str] = None
|
||||
|
||||
# pid: Optional[Union[int, Literal[""]]] = None
|
||||
# name: Union[list[str], str]
|
||||
# path: str
|
||||
# guest_paths: Union[list[str], str, None]
|
||||
# timestamp: Optional[str] = None
|
||||
pid: Optional[Union[int, Literal[""]]] = None
|
||||
name: Union[List[str], str]
|
||||
path: str
|
||||
guest_paths: Union[List[str], str, None]
|
||||
timestamp: Optional[str] = None
|
||||
|
||||
#
|
||||
# hashes
|
||||
#
|
||||
# crc32: str
|
||||
crc32: str
|
||||
md5: str
|
||||
sha1: str
|
||||
sha256: str
|
||||
# sha512: str
|
||||
# sha3_384: Optional[str] = None
|
||||
# ssdeep: str
|
||||
sha512: str
|
||||
sha3_384: Optional[str] = None
|
||||
ssdeep: str
|
||||
# unsure why this would ever be "False"
|
||||
# tlsh: Optional[Union[str, bool]] = None
|
||||
# rh_hash: Optional[str] = None
|
||||
tlsh: Optional[Union[str, bool]] = None
|
||||
rh_hash: Optional[str] = None
|
||||
|
||||
#
|
||||
# other metadata, static analysis
|
||||
#
|
||||
# size: int
|
||||
size: int
|
||||
pe: Optional[PE] = None
|
||||
# ep_bytes: Optional[HexBytes] = None
|
||||
# entrypoint: Optional[int] = None
|
||||
# data: Optional[str] = None
|
||||
# strings: Optional[list[str]] = None
|
||||
ep_bytes: Optional[HexBytes] = None
|
||||
entrypoint: Optional[int] = None
|
||||
data: Optional[str] = None
|
||||
strings: Optional[List[str]] = None
|
||||
|
||||
#
|
||||
# detections (skip)
|
||||
#
|
||||
# yara: Skip = None
|
||||
# cape_yara: Skip = None
|
||||
# clamav: Skip = None
|
||||
# virustotal: Skip = None
|
||||
yara: Skip = None
|
||||
cape_yara: Skip = None
|
||||
clamav: Skip = None
|
||||
virustotal: Skip = None
|
||||
|
||||
|
||||
"""
|
||||
class ProcessFile(File):
|
||||
#
|
||||
# like a File, but also has dynamic analysis results
|
||||
@@ -277,82 +267,75 @@ class ProcessFile(File):
|
||||
target_pid: Optional[Union[int, str]] = None
|
||||
target_path: Optional[str] = None
|
||||
target_process: Optional[str] = None
|
||||
"""
|
||||
|
||||
|
||||
class Argument(FlexibleModel):
|
||||
class Argument(ExactModel):
|
||||
name: str
|
||||
# unsure why empty list is provided here
|
||||
value: Union[HexInt, int, str, EmptyList]
|
||||
pretty_value: Optional[str] = None
|
||||
|
||||
|
||||
class Call(FlexibleModel):
|
||||
# timestamp: str
|
||||
class Call(ExactModel):
|
||||
timestamp: str
|
||||
thread_id: int
|
||||
# category: str
|
||||
category: str
|
||||
|
||||
api: str
|
||||
|
||||
arguments: list[Argument]
|
||||
# status: bool
|
||||
arguments: List[Argument]
|
||||
status: bool
|
||||
return_: HexInt = Field(alias="return")
|
||||
pretty_return: Optional[str] = None
|
||||
|
||||
# repeated: int
|
||||
repeated: int
|
||||
|
||||
# virtual addresses
|
||||
# caller: HexInt
|
||||
# parentcaller: HexInt
|
||||
caller: HexInt
|
||||
parentcaller: HexInt
|
||||
|
||||
# index into calls array
|
||||
# id: int
|
||||
id: int
|
||||
|
||||
|
||||
# FlexibleModel to account for extended fields
|
||||
# refs: https://github.com/mandiant/capa/issues/2466
|
||||
# https://github.com/kevoreilly/CAPEv2/pull/2199
|
||||
class Process(FlexibleModel):
|
||||
class Process(ExactModel):
|
||||
process_id: int
|
||||
process_name: str
|
||||
parent_id: int
|
||||
# module_path: str
|
||||
# first_seen: str
|
||||
calls: list[Call]
|
||||
threads: list[int]
|
||||
environ: dict[str, str]
|
||||
module_path: str
|
||||
first_seen: str
|
||||
calls: List[Call]
|
||||
threads: List[int]
|
||||
environ: Dict[str, str]
|
||||
|
||||
|
||||
"""
|
||||
class ProcessTree(FlexibleModel):
|
||||
class ProcessTree(ExactModel):
|
||||
name: str
|
||||
pid: int
|
||||
parent_id: int
|
||||
module_path: str
|
||||
threads: list[int]
|
||||
environ: dict[str, str]
|
||||
children: list["ProcessTree"]
|
||||
"""
|
||||
threads: List[int]
|
||||
environ: Dict[str, str]
|
||||
children: List["ProcessTree"]
|
||||
|
||||
|
||||
class Summary(FlexibleModel):
|
||||
files: list[str]
|
||||
# read_files: list[str]
|
||||
# write_files: list[str]
|
||||
# delete_files: list[str]
|
||||
keys: list[str]
|
||||
# read_keys: list[str]
|
||||
# write_keys: list[str]
|
||||
# delete_keys: list[str]
|
||||
executed_commands: list[str]
|
||||
resolved_apis: list[str]
|
||||
mutexes: list[str]
|
||||
created_services: list[str]
|
||||
started_services: list[str]
|
||||
class Summary(ExactModel):
|
||||
files: List[str]
|
||||
read_files: List[str]
|
||||
write_files: List[str]
|
||||
delete_files: List[str]
|
||||
keys: List[str]
|
||||
read_keys: List[str]
|
||||
write_keys: List[str]
|
||||
delete_keys: List[str]
|
||||
executed_commands: List[str]
|
||||
resolved_apis: List[str]
|
||||
mutexes: List[str]
|
||||
created_services: List[str]
|
||||
started_services: List[str]
|
||||
|
||||
|
||||
"""
|
||||
class EncryptedBuffer(FlexibleModel):
|
||||
class EncryptedBuffer(ExactModel):
|
||||
process_name: str
|
||||
pid: int
|
||||
|
||||
@@ -360,41 +343,38 @@ class EncryptedBuffer(FlexibleModel):
|
||||
buffer: str
|
||||
buffer_size: Optional[int] = None
|
||||
crypt_key: Optional[Union[HexInt, str]] = None
|
||||
"""
|
||||
|
||||
|
||||
class Behavior(FlexibleModel):
|
||||
summary: Summary | None = None
|
||||
class Behavior(ExactModel):
|
||||
summary: Summary
|
||||
|
||||
# list of processes, of threads, of calls
|
||||
processes: list[Process]
|
||||
processes: List[Process]
|
||||
# tree of processes
|
||||
# processtree: list[ProcessTree]
|
||||
processtree: List[ProcessTree]
|
||||
|
||||
# anomaly: list[str]
|
||||
# encryptedbuffers: list[EncryptedBuffer]
|
||||
anomaly: List[str]
|
||||
encryptedbuffers: List[EncryptedBuffer]
|
||||
# these are small objects that describe atomic events,
|
||||
# like file move, registry access.
|
||||
# we'll detect the same with our API call analysis.
|
||||
# enhanced: Skip = None
|
||||
enhanced: Skip = None
|
||||
|
||||
|
||||
class Target(FlexibleModel):
|
||||
# category: str
|
||||
class Target(ExactModel):
|
||||
category: str
|
||||
file: File
|
||||
# pe: Optional[PE] = None
|
||||
|
||||
|
||||
class Static(FlexibleModel):
|
||||
pe: Optional[PE] = None
|
||||
# flare_capa: Skip = None
|
||||
|
||||
|
||||
"""
|
||||
class Cape(FlexibleModel):
|
||||
payloads: list[ProcessFile]
|
||||
class Static(ExactModel):
|
||||
pe: Optional[PE] = None
|
||||
flare_capa: Skip = None
|
||||
|
||||
|
||||
class Cape(ExactModel):
|
||||
payloads: List[ProcessFile]
|
||||
configs: Skip = None
|
||||
"""
|
||||
|
||||
|
||||
# flexible because there may be more sorts of analysis
|
||||
@@ -409,7 +389,7 @@ class CapeReport(FlexibleModel):
|
||||
# static analysis results
|
||||
#
|
||||
static: Optional[Static] = None
|
||||
strings: Optional[list[str]] = None
|
||||
strings: Optional[List[str]] = None
|
||||
|
||||
#
|
||||
# dynamic analysis results
|
||||
@@ -417,14 +397,15 @@ class CapeReport(FlexibleModel):
|
||||
# post-processed results: process tree, anomalies, etc
|
||||
behavior: Behavior
|
||||
|
||||
# post-processed results: payloads and extracted configs
|
||||
CAPE: Optional[Union[Cape, List]] = None
|
||||
dropped: Optional[List[File]] = None
|
||||
procdump: Optional[List[ProcessFile]] = None
|
||||
procmemory: ListTODO
|
||||
|
||||
# =========================================================================
|
||||
# information we won't use in capa
|
||||
#
|
||||
# post-processed results: payloads and extracted configs
|
||||
# CAPE: Optional[Union[Cape, list]] = None
|
||||
# dropped: Optional[list[File]] = None
|
||||
# procdump: Optional[list[ProcessFile]] = None
|
||||
# procmemory: Optional[ListTODO] = None
|
||||
|
||||
#
|
||||
# NBIs and HBIs
|
||||
@@ -433,32 +414,32 @@ class CapeReport(FlexibleModel):
|
||||
#
|
||||
# if we come up with a future use for this, go ahead and re-enable!
|
||||
#
|
||||
# network: Skip = None
|
||||
# suricata: Skip = None
|
||||
# curtain: Skip = None
|
||||
# sysmon: Skip = None
|
||||
# url_analysis: Skip = None
|
||||
network: Skip = None
|
||||
suricata: Skip = None
|
||||
curtain: Skip = None
|
||||
sysmon: Skip = None
|
||||
url_analysis: Skip = None
|
||||
|
||||
# screenshot hash values
|
||||
# deduplicated_shots: Skip = None
|
||||
deduplicated_shots: Skip = None
|
||||
# k-v pairs describing the time it took to run each stage.
|
||||
# statistics: Skip = None
|
||||
statistics: Skip = None
|
||||
# k-v pairs of ATT&CK ID to signature name or similar.
|
||||
# ttps: Skip = None
|
||||
ttps: Skip = None
|
||||
# debug log messages
|
||||
# debug: Skip = None
|
||||
debug: Skip = None
|
||||
|
||||
# various signature matches
|
||||
# we could potentially extend capa to use this info one day,
|
||||
# though it would be quite sandbox-specific,
|
||||
# and more detection-oriented than capability detection.
|
||||
# signatures: Skip = None
|
||||
# malfamily_tag: Optional[str] = None
|
||||
# malscore: float
|
||||
# detections: Skip = None
|
||||
# detections2pid: Optional[dict[int, list[str]]] = None
|
||||
signatures: Skip = None
|
||||
malfamily_tag: Optional[str] = None
|
||||
malscore: float
|
||||
detections: Skip = None
|
||||
detections2pid: Optional[Dict[int, List[str]]] = None
|
||||
# AV detections for the sample.
|
||||
# virustotal: Skip = None
|
||||
virustotal: Skip = None
|
||||
|
||||
@classmethod
|
||||
def from_buf(cls, buf: bytes) -> "CapeReport":
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import List, Tuple, Iterator
|
||||
|
||||
from capa.features.common import String, Feature
|
||||
from capa.features.address import Address, ThreadAddress
|
||||
@@ -29,14 +22,14 @@ def get_threads(ph: ProcessHandle) -> Iterator[ThreadHandle]:
|
||||
get the threads associated with a given process
|
||||
"""
|
||||
process: Process = ph.inner
|
||||
threads: list[int] = process.threads
|
||||
threads: List[int] = process.threads
|
||||
|
||||
for thread in threads:
|
||||
address: ThreadAddress = ThreadAddress(process=ph.address, tid=thread)
|
||||
yield ThreadHandle(address=address, inner={})
|
||||
|
||||
|
||||
def extract_environ_strings(ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_environ_strings(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract strings from a process' provided environment variables.
|
||||
"""
|
||||
@@ -46,7 +39,7 @@ def extract_environ_strings(ph: ProcessHandle) -> Iterator[tuple[Feature, Addres
|
||||
yield String(value), ph.address
|
||||
|
||||
|
||||
def extract_features(ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
for handler in PROCESS_HANDLERS:
|
||||
for feature, addr in handler(ph):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
|
||||
@@ -1,23 +1,16 @@
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import io
|
||||
import re
|
||||
import logging
|
||||
import binascii
|
||||
import contextlib
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import pefile
|
||||
|
||||
@@ -52,7 +45,7 @@ MATCH_RESULT = b'{"meta":'
|
||||
MATCH_JSON_OBJECT = b'{"'
|
||||
|
||||
|
||||
def extract_file_strings(buf: bytes, **kwargs) -> Iterator[tuple[String, Address]]:
|
||||
def extract_file_strings(buf: bytes, **kwargs) -> Iterator[Tuple[String, Address]]:
|
||||
"""
|
||||
extract ASCII and UTF-16 LE strings from file
|
||||
"""
|
||||
@@ -63,7 +56,7 @@ def extract_file_strings(buf: bytes, **kwargs) -> Iterator[tuple[String, Address
|
||||
yield String(s.s), FileOffsetAddress(s.offset)
|
||||
|
||||
|
||||
def extract_format(buf: bytes) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_format(buf: bytes) -> Iterator[Tuple[Feature, Address]]:
|
||||
if buf.startswith(MATCH_PE):
|
||||
yield Format(FORMAT_PE), NO_ADDRESS
|
||||
elif buf.startswith(MATCH_ELF):
|
||||
@@ -86,7 +79,7 @@ def extract_format(buf: bytes) -> Iterator[tuple[Feature, Address]]:
|
||||
return
|
||||
|
||||
|
||||
def extract_arch(buf) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_arch(buf) -> Iterator[Tuple[Feature, Address]]:
|
||||
if buf.startswith(MATCH_PE):
|
||||
yield from capa.features.extractors.pefile.extract_file_arch(pe=pefile.PE(data=buf))
|
||||
|
||||
@@ -118,7 +111,7 @@ def extract_arch(buf) -> Iterator[tuple[Feature, Address]]:
|
||||
return
|
||||
|
||||
|
||||
def extract_os(buf, os=OS_AUTO) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_os(buf, os=OS_AUTO) -> Iterator[Tuple[Feature, Address]]:
|
||||
if os != OS_AUTO:
|
||||
yield OS(os), NO_ADDRESS
|
||||
|
||||
|
||||
@@ -1,21 +1,14 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Union, Iterator, Optional
|
||||
from typing import Dict, List, Tuple, Union, Iterator, Optional
|
||||
from pathlib import Path
|
||||
|
||||
import dnfile
|
||||
@@ -48,11 +41,11 @@ from capa.features.extractors.dnfile.helpers import (
|
||||
|
||||
class DnFileFeatureExtractorCache:
|
||||
def __init__(self, pe: dnfile.dnPE):
|
||||
self.imports: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
self.native_imports: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
self.methods: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
self.fields: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
self.types: dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
self.imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
self.native_imports: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
self.methods: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
self.fields: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
self.types: Dict[int, Union[DnType, DnUnmanagedMethod]] = {}
|
||||
|
||||
for import_ in get_dotnet_managed_imports(pe):
|
||||
self.imports[import_.token] = import_
|
||||
@@ -91,7 +84,7 @@ class DnfileFeatureExtractor(StaticFeatureExtractor):
|
||||
self.token_cache: DnFileFeatureExtractorCache = DnFileFeatureExtractorCache(self.pe)
|
||||
|
||||
# pre-compute these because we'll yield them at *every* scope.
|
||||
self.global_features: list[tuple[Feature, Address]] = []
|
||||
self.global_features: List[Tuple[Feature, Address]] = []
|
||||
self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_format())
|
||||
self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_os(pe=self.pe))
|
||||
self.global_features.extend(capa.features.extractors.dotnetfile.extract_file_arch(pe=self.pe))
|
||||
@@ -107,7 +100,7 @@ class DnfileFeatureExtractor(StaticFeatureExtractor):
|
||||
|
||||
def get_functions(self) -> Iterator[FunctionHandle]:
|
||||
# create a method lookup table
|
||||
methods: dict[Address, FunctionHandle] = {}
|
||||
methods: Dict[Address, FunctionHandle] = {}
|
||||
for token, method in get_dotnet_managed_method_bodies(self.pe):
|
||||
fh: FunctionHandle = FunctionHandle(
|
||||
address=DNTokenAddress(token),
|
||||
@@ -143,7 +136,7 @@ class DnfileFeatureExtractor(StaticFeatureExtractor):
|
||||
|
||||
yield from methods.values()
|
||||
|
||||
def extract_function_features(self, fh) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_features(self, fh) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.dnfile.function.extract_features(fh)
|
||||
|
||||
def get_basic_blocks(self, f) -> Iterator[BBHandle]:
|
||||
@@ -164,5 +157,5 @@ class DnfileFeatureExtractor(StaticFeatureExtractor):
|
||||
inner=insn,
|
||||
)
|
||||
|
||||
def extract_insn_features(self, fh, bbh, ih) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_features(self, fh, bbh, ih) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.dnfile.insn.extract_features(fh, bbh, ih)
|
||||
|
||||
@@ -1,21 +1,14 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import dnfile
|
||||
|
||||
@@ -25,35 +18,35 @@ from capa.features.common import Class, Format, String, Feature, Namespace, Char
|
||||
from capa.features.address import Address
|
||||
|
||||
|
||||
def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[tuple[Import, Address]]:
|
||||
def extract_file_import_names(pe: dnfile.dnPE) -> Iterator[Tuple[Import, Address]]:
|
||||
yield from capa.features.extractors.dotnetfile.extract_file_import_names(pe=pe)
|
||||
|
||||
|
||||
def extract_file_format(pe: dnfile.dnPE) -> Iterator[tuple[Format, Address]]:
|
||||
def extract_file_format(pe: dnfile.dnPE) -> Iterator[Tuple[Format, Address]]:
|
||||
yield from capa.features.extractors.dotnetfile.extract_file_format(pe=pe)
|
||||
|
||||
|
||||
def extract_file_function_names(pe: dnfile.dnPE) -> Iterator[tuple[FunctionName, Address]]:
|
||||
def extract_file_function_names(pe: dnfile.dnPE) -> Iterator[Tuple[FunctionName, Address]]:
|
||||
yield from capa.features.extractors.dotnetfile.extract_file_function_names(pe=pe)
|
||||
|
||||
|
||||
def extract_file_strings(pe: dnfile.dnPE) -> Iterator[tuple[String, Address]]:
|
||||
def extract_file_strings(pe: dnfile.dnPE) -> Iterator[Tuple[String, Address]]:
|
||||
yield from capa.features.extractors.dotnetfile.extract_file_strings(pe=pe)
|
||||
|
||||
|
||||
def extract_file_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[tuple[Characteristic, Address]]:
|
||||
def extract_file_mixed_mode_characteristic_features(pe: dnfile.dnPE) -> Iterator[Tuple[Characteristic, Address]]:
|
||||
yield from capa.features.extractors.dotnetfile.extract_file_mixed_mode_characteristic_features(pe=pe)
|
||||
|
||||
|
||||
def extract_file_namespace_features(pe: dnfile.dnPE) -> Iterator[tuple[Namespace, Address]]:
|
||||
def extract_file_namespace_features(pe: dnfile.dnPE) -> Iterator[Tuple[Namespace, Address]]:
|
||||
yield from capa.features.extractors.dotnetfile.extract_file_namespace_features(pe=pe)
|
||||
|
||||
|
||||
def extract_file_class_features(pe: dnfile.dnPE) -> Iterator[tuple[Class, Address]]:
|
||||
def extract_file_class_features(pe: dnfile.dnPE) -> Iterator[Tuple[Class, Address]]:
|
||||
yield from capa.features.extractors.dotnetfile.extract_file_class_features(pe=pe)
|
||||
|
||||
|
||||
def extract_features(pe: dnfile.dnPE) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
|
||||
for file_handler in FILE_HANDLERS:
|
||||
for feature, address in file_handler(pe):
|
||||
yield feature, address
|
||||
|
||||
@@ -1,22 +1,15 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
from capa.features.common import Feature, Characteristic
|
||||
from capa.features.address import Address
|
||||
@@ -25,30 +18,30 @@ from capa.features.extractors.base_extractor import FunctionHandle
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_function_calls_to(fh: FunctionHandle) -> Iterator[tuple[Characteristic, Address]]:
|
||||
def extract_function_calls_to(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
|
||||
"""extract callers to a function"""
|
||||
for dest in fh.ctx["calls_to"]:
|
||||
yield Characteristic("calls to"), dest
|
||||
|
||||
|
||||
def extract_function_calls_from(fh: FunctionHandle) -> Iterator[tuple[Characteristic, Address]]:
|
||||
def extract_function_calls_from(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
|
||||
"""extract callers from a function"""
|
||||
for src in fh.ctx["calls_from"]:
|
||||
yield Characteristic("calls from"), src
|
||||
|
||||
|
||||
def extract_recursive_call(fh: FunctionHandle) -> Iterator[tuple[Characteristic, Address]]:
|
||||
def extract_recursive_call(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
|
||||
"""extract recursive function call"""
|
||||
if fh.address in fh.ctx["calls_to"]:
|
||||
yield Characteristic("recursive call"), fh.address
|
||||
|
||||
|
||||
def extract_function_loop(fh: FunctionHandle) -> Iterator[tuple[Characteristic, Address]]:
|
||||
def extract_function_loop(fh: FunctionHandle) -> Iterator[Tuple[Characteristic, Address]]:
|
||||
"""extract loop indicators from a function"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
for func_handler in FUNCTION_HANDLERS:
|
||||
for feature, addr in func_handler(fh):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,22 +1,15 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Union, Iterator, Optional
|
||||
from typing import Dict, Tuple, Union, Iterator, Optional
|
||||
|
||||
import dnfile
|
||||
from dncil.cil.body import CilMethodBody
|
||||
@@ -151,7 +144,7 @@ def get_dotnet_managed_imports(pe: dnfile.dnPE) -> Iterator[DnType]:
|
||||
)
|
||||
|
||||
|
||||
def get_dotnet_methoddef_property_accessors(pe: dnfile.dnPE) -> Iterator[tuple[int, str]]:
|
||||
def get_dotnet_methoddef_property_accessors(pe: dnfile.dnPE) -> Iterator[Tuple[int, str]]:
|
||||
"""get MethodDef methods used to access properties
|
||||
|
||||
see https://www.ntcore.com/files/dotnetformat.htm
|
||||
@@ -201,7 +194,7 @@ def get_dotnet_managed_methods(pe: dnfile.dnPE) -> Iterator[DnType]:
|
||||
"""
|
||||
nested_class_table = get_dotnet_nested_class_table_index(pe)
|
||||
|
||||
accessor_map: dict[int, str] = {}
|
||||
accessor_map: Dict[int, str] = {}
|
||||
for methoddef, methoddef_access in get_dotnet_methoddef_property_accessors(pe):
|
||||
accessor_map[methoddef] = methoddef_access
|
||||
|
||||
@@ -259,7 +252,7 @@ def get_dotnet_fields(pe: dnfile.dnPE) -> Iterator[DnType]:
|
||||
yield DnType(token, typedefname, namespace=typedefnamespace, member=field.row.Name)
|
||||
|
||||
|
||||
def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[tuple[int, CilMethodBody]]:
|
||||
def get_dotnet_managed_method_bodies(pe: dnfile.dnPE) -> Iterator[Tuple[int, CilMethodBody]]:
|
||||
"""get managed methods from MethodDef table"""
|
||||
for rid, method_def in iter_dotnet_table(pe, dnfile.mdtable.MethodDef.number):
|
||||
assert isinstance(method_def, dnfile.mdtable.MethodDefRow)
|
||||
@@ -339,7 +332,7 @@ def get_dotnet_table_row(pe: dnfile.dnPE, table_index: int, row_index: int) -> O
|
||||
|
||||
def resolve_nested_typedef_name(
|
||||
nested_class_table: dict, index: int, typedef: dnfile.mdtable.TypeDefRow, pe: dnfile.dnPE
|
||||
) -> tuple[str, tuple[str, ...]]:
|
||||
) -> Tuple[str, Tuple[str, ...]]:
|
||||
"""Resolves all nested TypeDef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
|
||||
|
||||
if index in nested_class_table:
|
||||
@@ -375,7 +368,7 @@ def resolve_nested_typedef_name(
|
||||
|
||||
def resolve_nested_typeref_name(
|
||||
index: int, typeref: dnfile.mdtable.TypeRefRow, pe: dnfile.dnPE
|
||||
) -> tuple[str, tuple[str, ...]]:
|
||||
) -> Tuple[str, Tuple[str, ...]]:
|
||||
"""Resolves all nested TypeRef class names. Returns the namespace as a str and the nested TypeRef name as a tuple"""
|
||||
# If the ResolutionScope decodes to a typeRef type then it is nested
|
||||
if isinstance(typeref.ResolutionScope.table, dnfile.mdtable.TypeRef):
|
||||
@@ -405,7 +398,7 @@ def resolve_nested_typeref_name(
|
||||
return str(typeref.TypeNamespace), (str(typeref.TypeName),)
|
||||
|
||||
|
||||
def get_dotnet_nested_class_table_index(pe: dnfile.dnPE) -> dict[int, int]:
|
||||
def get_dotnet_nested_class_table_index(pe: dnfile.dnPE) -> Dict[int, int]:
|
||||
"""Build index for EnclosingClass based off the NestedClass row index in the nestedclass table"""
|
||||
nested_class_table = {}
|
||||
|
||||
@@ -449,7 +442,7 @@ def is_dotnet_mixed_mode(pe: dnfile.dnPE) -> bool:
|
||||
return not bool(pe.net.Flags.CLR_ILONLY)
|
||||
|
||||
|
||||
def iter_dotnet_table(pe: dnfile.dnPE, table_index: int) -> Iterator[tuple[int, dnfile.base.MDTableRow]]:
|
||||
def iter_dotnet_table(pe: dnfile.dnPE, table_index: int) -> Iterator[Tuple[int, dnfile.base.MDTableRow]]:
|
||||
assert pe.net is not None
|
||||
assert pe.net.mdtables is not None
|
||||
|
||||
|
||||
@@ -1,22 +1,15 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Union, Iterator, Optional
|
||||
from typing import TYPE_CHECKING, Tuple, Union, Iterator, Optional
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from capa.features.extractors.dnfile.extractor import DnFileFeatureExtractorCache
|
||||
@@ -68,7 +61,7 @@ def get_callee(
|
||||
return callee
|
||||
|
||||
|
||||
def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction API features"""
|
||||
if ih.inner.opcode not in (
|
||||
OpCodes.Call,
|
||||
@@ -90,7 +83,7 @@ def extract_insn_api_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterato
|
||||
yield API(name), ih.address
|
||||
|
||||
|
||||
def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction property features"""
|
||||
name: Optional[str] = None
|
||||
access: Optional[str] = None
|
||||
@@ -125,7 +118,7 @@ def extract_insn_property_features(fh: FunctionHandle, bh, ih: InsnHandle) -> It
|
||||
|
||||
def extract_insn_namespace_class_features(
|
||||
fh: FunctionHandle, bh, ih: InsnHandle
|
||||
) -> Iterator[tuple[Union[Namespace, Class], Address]]:
|
||||
) -> Iterator[Tuple[Union[Namespace, Class], Address]]:
|
||||
"""parse instruction namespace and class features"""
|
||||
type_: Optional[Union[DnType, DnUnmanagedMethod]] = None
|
||||
|
||||
@@ -180,13 +173,13 @@ def extract_insn_namespace_class_features(
|
||||
yield Namespace(type_.namespace), ih.address
|
||||
|
||||
|
||||
def extract_insn_number_features(fh, bh, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_number_features(fh, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction number features"""
|
||||
if ih.inner.is_ldc():
|
||||
yield Number(ih.inner.get_ldc()), ih.address
|
||||
|
||||
|
||||
def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction string features"""
|
||||
if not ih.inner.is_ldstr():
|
||||
return
|
||||
@@ -204,7 +197,7 @@ def extract_insn_string_features(fh: FunctionHandle, bh, ih: InsnHandle) -> Iter
|
||||
|
||||
def extract_unmanaged_call_characteristic_features(
|
||||
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Characteristic, Address]]:
|
||||
) -> Iterator[Tuple[Characteristic, Address]]:
|
||||
if ih.inner.opcode not in (OpCodes.Call, OpCodes.Callvirt, OpCodes.Jmp):
|
||||
return
|
||||
|
||||
@@ -216,7 +209,7 @@ def extract_unmanaged_call_characteristic_features(
|
||||
yield Characteristic("unmanaged call"), ih.address
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract instruction features"""
|
||||
for inst_handler in INSTRUCTION_HANDLERS:
|
||||
for feature, addr in inst_handler(fh, bbh, ih):
|
||||
|
||||
@@ -1,29 +1,22 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
|
||||
from typing import Optional
|
||||
from typing import Tuple, Optional
|
||||
|
||||
|
||||
class DnType:
|
||||
def __init__(
|
||||
self, token: int, class_: tuple[str, ...], namespace: str = "", member: str = "", access: Optional[str] = None
|
||||
self, token: int, class_: Tuple[str, ...], namespace: str = "", member: str = "", access: Optional[str] = None
|
||||
):
|
||||
self.token: int = token
|
||||
self.access: Optional[str] = access
|
||||
self.namespace: str = namespace
|
||||
self.class_: tuple[str, ...] = class_
|
||||
self.class_: Tuple[str, ...] = class_
|
||||
|
||||
if member == ".ctor":
|
||||
member = "ctor"
|
||||
@@ -51,7 +44,7 @@ class DnType:
|
||||
return str(self)
|
||||
|
||||
@staticmethod
|
||||
def format_name(class_: tuple[str, ...], namespace: str = "", member: str = ""):
|
||||
def format_name(class_: Tuple[str, ...], namespace: str = "", member: str = ""):
|
||||
if len(class_) > 1:
|
||||
class_str = "/".join(class_) # Concat items in tuple, separated by a "/"
|
||||
else:
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
from pathlib import Path
|
||||
|
||||
import dnfile
|
||||
@@ -55,12 +48,12 @@ from capa.features.extractors.dnfile.helpers import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_file_format(**kwargs) -> Iterator[tuple[Format, Address]]:
|
||||
def extract_file_format(**kwargs) -> Iterator[Tuple[Format, Address]]:
|
||||
yield Format(FORMAT_DOTNET), NO_ADDRESS
|
||||
yield Format(FORMAT_PE), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Import, Address]]:
|
||||
def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Import, Address]]:
|
||||
for method in get_dotnet_managed_imports(pe):
|
||||
# like System.IO.File::OpenRead
|
||||
yield Import(str(method)), DNTokenAddress(method.token)
|
||||
@@ -71,12 +64,12 @@ def extract_file_import_names(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Impor
|
||||
yield Import(name), DNTokenAddress(imp.token)
|
||||
|
||||
|
||||
def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[FunctionName, Address]]:
|
||||
def extract_file_function_names(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[FunctionName, Address]]:
|
||||
for method in get_dotnet_managed_methods(pe):
|
||||
yield FunctionName(str(method)), DNTokenAddress(method.token)
|
||||
|
||||
|
||||
def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Namespace, Address]]:
|
||||
def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Namespace, Address]]:
|
||||
"""emit namespace features from TypeRef and TypeDef tables"""
|
||||
|
||||
# namespaces may be referenced multiple times, so we need to filter
|
||||
@@ -100,7 +93,7 @@ def extract_file_namespace_features(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple
|
||||
yield Namespace(namespace), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Class, Address]]:
|
||||
def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Class, Address]]:
|
||||
"""emit class features from TypeRef and TypeDef tables"""
|
||||
nested_class_table = get_dotnet_nested_class_table_index(pe)
|
||||
|
||||
@@ -123,11 +116,11 @@ def extract_file_class_features(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Cla
|
||||
yield Class(DnType.format_name(typerefname, namespace=typerefnamespace)), DNTokenAddress(token)
|
||||
|
||||
|
||||
def extract_file_os(**kwargs) -> Iterator[tuple[OS, Address]]:
|
||||
def extract_file_os(**kwargs) -> Iterator[Tuple[OS, Address]]:
|
||||
yield OS(OS_ANY), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Arch, Address]]:
|
||||
def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[Arch, Address]]:
|
||||
# to distinguish in more detail, see https://stackoverflow.com/a/23614024/10548020
|
||||
# .NET 4.5 added option: any CPU, 32-bit preferred
|
||||
assert pe.net is not None
|
||||
@@ -141,18 +134,18 @@ def extract_file_arch(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[Arch, Address
|
||||
yield Arch(ARCH_ANY), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[tuple[String, Address]]:
|
||||
def extract_file_strings(pe: dnfile.dnPE, **kwargs) -> Iterator[Tuple[String, Address]]:
|
||||
yield from capa.features.extractors.common.extract_file_strings(pe.__data__)
|
||||
|
||||
|
||||
def extract_file_mixed_mode_characteristic_features(
|
||||
pe: dnfile.dnPE, **kwargs
|
||||
) -> Iterator[tuple[Characteristic, Address]]:
|
||||
) -> Iterator[Tuple[Characteristic, Address]]:
|
||||
if is_dotnet_mixed_mode(pe):
|
||||
yield Characteristic("mixed mode"), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_file_features(pe: dnfile.dnPE) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
|
||||
for file_handler in FILE_HANDLERS:
|
||||
for feature, addr in file_handler(pe=pe): # type: ignore
|
||||
yield feature, addr
|
||||
@@ -169,7 +162,7 @@ FILE_HANDLERS = (
|
||||
)
|
||||
|
||||
|
||||
def extract_global_features(pe: dnfile.dnPE) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_global_features(pe: dnfile.dnPE) -> Iterator[Tuple[Feature, Address]]:
|
||||
for handler in GLOBAL_HANDLERS:
|
||||
for feature, va in handler(pe=pe): # type: ignore
|
||||
yield feature, va
|
||||
@@ -211,7 +204,7 @@ class DotnetFileFeatureExtractor(StaticFeatureExtractor):
|
||||
def is_mixed_mode(self) -> bool:
|
||||
return is_dotnet_mixed_mode(self.pe)
|
||||
|
||||
def get_runtime_version(self) -> tuple[int, int]:
|
||||
def get_runtime_version(self) -> Tuple[int, int]:
|
||||
assert self.pe.net is not None
|
||||
assert self.pe.net.struct is not None
|
||||
assert self.pe.net.struct.MajorRuntimeVersion is not None
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import capa.features.extractors.helpers
|
||||
from capa.features.insn import API, Number
|
||||
@@ -26,7 +19,7 @@ from capa.features.extractors.drakvuf.models import Call
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
This method extracts the given call's features (such as API name and arguments),
|
||||
and returns them as API, Number, and String features.
|
||||
@@ -56,7 +49,7 @@ def extract_call_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -
|
||||
yield API(name), ch.address
|
||||
|
||||
|
||||
def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(ph: ProcessHandle, th: ThreadHandle, ch: CallHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
for handler in CALL_HANDLERS:
|
||||
for feature, addr in handler(ph, th, ch):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,27 +1,20 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Union, Iterator
|
||||
from typing import Dict, List, Tuple, Union, Iterator
|
||||
|
||||
import capa.features.extractors.drakvuf.call
|
||||
import capa.features.extractors.drakvuf.file
|
||||
import capa.features.extractors.drakvuf.thread
|
||||
import capa.features.extractors.drakvuf.global_
|
||||
import capa.features.extractors.drakvuf.process
|
||||
from capa.features.common import Feature
|
||||
from capa.features.common import Feature, Characteristic
|
||||
from capa.features.address import NO_ADDRESS, Address, ThreadAddress, ProcessAddress, AbsoluteVirtualAddress, _NoAddress
|
||||
from capa.features.extractors.base_extractor import (
|
||||
CallHandle,
|
||||
@@ -46,7 +39,7 @@ class DrakvufExtractor(DynamicFeatureExtractor):
|
||||
self.report: DrakvufReport = report
|
||||
|
||||
# sort the api calls to prevent going through the entire list each time
|
||||
self.sorted_calls: dict[ProcessAddress, dict[ThreadAddress, list[Call]]] = index_calls(report)
|
||||
self.sorted_calls: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]] = index_calls(report)
|
||||
|
||||
# pre-compute these because we'll yield them at *every* scope.
|
||||
self.global_features = list(capa.features.extractors.drakvuf.global_.extract_features(self.report))
|
||||
@@ -55,16 +48,16 @@ class DrakvufExtractor(DynamicFeatureExtractor):
|
||||
# DRAKVUF currently does not yield information about the PE's address
|
||||
return NO_ADDRESS
|
||||
|
||||
def extract_global_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_global_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from self.global_features
|
||||
|
||||
def extract_file_features(self) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_features(self) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.drakvuf.file.extract_features(self.report)
|
||||
|
||||
def get_processes(self) -> Iterator[ProcessHandle]:
|
||||
yield from capa.features.extractors.drakvuf.file.get_processes(self.sorted_calls)
|
||||
|
||||
def extract_process_features(self, ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_process_features(self, ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.drakvuf.process.extract_features(ph)
|
||||
|
||||
def get_process_name(self, ph: ProcessHandle) -> str:
|
||||
@@ -73,8 +66,12 @@ class DrakvufExtractor(DynamicFeatureExtractor):
|
||||
def get_threads(self, ph: ProcessHandle) -> Iterator[ThreadHandle]:
|
||||
yield from capa.features.extractors.drakvuf.process.get_threads(self.sorted_calls, ph)
|
||||
|
||||
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
yield from []
|
||||
def extract_thread_features(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
if False:
|
||||
# force this routine to be a generator,
|
||||
# but we don't actually have any elements to generate.
|
||||
yield Characteristic("never"), NO_ADDRESS
|
||||
return
|
||||
|
||||
def get_calls(self, ph: ProcessHandle, th: ThreadHandle) -> Iterator[CallHandle]:
|
||||
yield from capa.features.extractors.drakvuf.thread.get_calls(self.sorted_calls, ph, th)
|
||||
@@ -90,10 +87,10 @@ class DrakvufExtractor(DynamicFeatureExtractor):
|
||||
|
||||
def extract_call_features(
|
||||
self, ph: ProcessHandle, th: ThreadHandle, ch: CallHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.drakvuf.call.extract_features(ph, th, ch)
|
||||
|
||||
@classmethod
|
||||
def from_report(cls, report: Iterator[dict]) -> "DrakvufExtractor":
|
||||
def from_report(cls, report: Iterator[Dict]) -> "DrakvufExtractor":
|
||||
dr = DrakvufReport.from_raw_report(report)
|
||||
return DrakvufExtractor(report=dr)
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Dict, List, Tuple, Iterator
|
||||
|
||||
from capa.features.file import Import
|
||||
from capa.features.common import Feature
|
||||
@@ -26,7 +19,7 @@ from capa.features.extractors.drakvuf.models import Call, DrakvufReport
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_processes(calls: dict[ProcessAddress, dict[ThreadAddress, list[Call]]]) -> Iterator[ProcessHandle]:
|
||||
def get_processes(calls: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]]) -> Iterator[ProcessHandle]:
|
||||
"""
|
||||
Get all the created processes for a sample.
|
||||
"""
|
||||
@@ -35,7 +28,7 @@ def get_processes(calls: dict[ProcessAddress, dict[ThreadAddress, list[Call]]])
|
||||
yield ProcessHandle(proc_addr, inner={"process_name": sample_call.process_name})
|
||||
|
||||
|
||||
def extract_import_names(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_import_names(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
Extract imported function names.
|
||||
"""
|
||||
@@ -50,7 +43,7 @@ def extract_import_names(report: DrakvufReport) -> Iterator[tuple[Feature, Addre
|
||||
yield Import(name), AbsoluteVirtualAddress(function_address)
|
||||
|
||||
|
||||
def extract_features(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for handler in FILE_HANDLERS:
|
||||
for feature, addr in handler(report):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
from capa.features.common import OS, FORMAT_PE, ARCH_AMD64, OS_WINDOWS, Arch, Format, Feature
|
||||
from capa.features.address import NO_ADDRESS, Address
|
||||
@@ -23,22 +16,22 @@ from capa.features.extractors.drakvuf.models import DrakvufReport
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_format(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_format(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
# DRAKVUF sandbox currently supports only Windows as the guest: https://drakvuf-sandbox.readthedocs.io/en/latest/usage/getting_started.html
|
||||
yield Format(FORMAT_PE), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_os(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_os(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
# DRAKVUF sandbox currently supports only PE files: https://drakvuf-sandbox.readthedocs.io/en/latest/usage/getting_started.html
|
||||
yield OS(OS_WINDOWS), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_arch(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_arch(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
# DRAKVUF sandbox currently supports only x64 Windows as the guest: https://drakvuf-sandbox.readthedocs.io/en/latest/usage/getting_started.html
|
||||
yield Arch(ARCH_AMD64), NO_ADDRESS
|
||||
|
||||
|
||||
def extract_features(report: DrakvufReport) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(report: DrakvufReport) -> Iterator[Tuple[Feature, Address]]:
|
||||
for global_handler in GLOBAL_HANDLER:
|
||||
for feature, addr in global_handler(report):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,28 +1,22 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import itertools
|
||||
from typing import Dict, List
|
||||
|
||||
from capa.features.address import ThreadAddress, ProcessAddress
|
||||
from capa.features.extractors.drakvuf.models import Call, DrakvufReport
|
||||
|
||||
|
||||
def index_calls(report: DrakvufReport) -> dict[ProcessAddress, dict[ThreadAddress, list[Call]]]:
|
||||
def index_calls(report: DrakvufReport) -> Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]]:
|
||||
# this method organizes calls into processes and threads, and then sorts them based on
|
||||
# timestamp so that we can address individual calls per index (CallAddress requires call index)
|
||||
result: dict[ProcessAddress, dict[ThreadAddress, list[Call]]] = {}
|
||||
result: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]] = {}
|
||||
for call in itertools.chain(report.syscalls, report.apicalls):
|
||||
if call.pid == 0:
|
||||
# DRAKVUF captures api/native calls from all processes running on the system.
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
from typing import Any, Iterator
|
||||
from typing import Any, Dict, List, Iterator
|
||||
|
||||
from pydantic import Field, BaseModel, ConfigDict, model_validator
|
||||
|
||||
@@ -54,7 +47,7 @@ class LoadedDLL(ConciseModel):
|
||||
plugin_name: str = Field(alias="Plugin")
|
||||
event: str = Field(alias="Event")
|
||||
name: str = Field(alias="DllName")
|
||||
imports: dict[str, int] = Field(alias="Rva")
|
||||
imports: Dict[str, int] = Field(alias="Rva")
|
||||
|
||||
|
||||
class Call(ConciseModel):
|
||||
@@ -65,18 +58,18 @@ class Call(ConciseModel):
|
||||
pid: int = Field(alias="PID")
|
||||
tid: int = Field(alias="TID")
|
||||
name: str = Field(alias="Method")
|
||||
arguments: dict[str, str]
|
||||
arguments: Dict[str, str]
|
||||
|
||||
|
||||
class WinApiCall(Call):
|
||||
# This class models Windows API calls captured by DRAKVUF (DLLs, etc.).
|
||||
arguments: dict[str, str] = Field(alias="Arguments")
|
||||
arguments: Dict[str, str] = Field(alias="Arguments")
|
||||
event: str = Field(alias="Event")
|
||||
return_value: str = Field(alias="ReturnValue")
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def build_arguments(cls, values: dict[str, Any]) -> dict[str, Any]:
|
||||
def build_arguments(cls, values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
args = values["Arguments"]
|
||||
values["Arguments"] = dict(arg.split("=", 1) for arg in args)
|
||||
return values
|
||||
@@ -107,7 +100,7 @@ class SystemCall(Call):
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def build_extra(cls, values: dict[str, Any]) -> dict[str, Any]:
|
||||
def build_extra(cls, values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
# DRAKVUF stores argument names and values as entries in the syscall's entry.
|
||||
# This model validator collects those arguments into a list in the model.
|
||||
values["arguments"] = {
|
||||
@@ -117,13 +110,13 @@ class SystemCall(Call):
|
||||
|
||||
|
||||
class DrakvufReport(ConciseModel):
|
||||
syscalls: list[SystemCall] = []
|
||||
apicalls: list[WinApiCall] = []
|
||||
discovered_dlls: list[DiscoveredDLL] = []
|
||||
loaded_dlls: list[LoadedDLL] = []
|
||||
syscalls: List[SystemCall] = []
|
||||
apicalls: List[WinApiCall] = []
|
||||
discovered_dlls: List[DiscoveredDLL] = []
|
||||
loaded_dlls: List[LoadedDLL] = []
|
||||
|
||||
@classmethod
|
||||
def from_raw_report(cls, entries: Iterator[dict]) -> "DrakvufReport":
|
||||
def from_raw_report(cls, entries: Iterator[Dict]) -> "DrakvufReport":
|
||||
report = cls()
|
||||
|
||||
for entry in entries:
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Dict, List, Tuple, Iterator
|
||||
|
||||
from capa.features.common import String, Feature
|
||||
from capa.features.address import Address, ThreadAddress, ProcessAddress
|
||||
@@ -25,7 +18,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_threads(
|
||||
calls: dict[ProcessAddress, dict[ThreadAddress, list[Call]]], ph: ProcessHandle
|
||||
calls: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]], ph: ProcessHandle
|
||||
) -> Iterator[ThreadHandle]:
|
||||
"""
|
||||
Get the threads associated with a given process.
|
||||
@@ -34,11 +27,11 @@ def get_threads(
|
||||
yield ThreadHandle(address=thread_addr, inner={})
|
||||
|
||||
|
||||
def extract_process_name(ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_process_name(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield String(ph.inner["process_name"]), ph.address
|
||||
|
||||
|
||||
def extract_features(ph: ProcessHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(ph: ProcessHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
for handler in PROCESS_HANDLERS:
|
||||
for feature, addr in handler(ph):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Dict, List, Iterator
|
||||
|
||||
from capa.features.address import ThreadAddress, ProcessAddress, DynamicCallAddress
|
||||
from capa.features.extractors.base_extractor import CallHandle, ThreadHandle, ProcessHandle
|
||||
@@ -24,7 +17,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_calls(
|
||||
sorted_calls: dict[ProcessAddress, dict[ThreadAddress, list[Call]]], ph: ProcessHandle, th: ThreadHandle
|
||||
sorted_calls: Dict[ProcessAddress, Dict[ThreadAddress, List[Call]]], ph: ProcessHandle, th: ThreadHandle
|
||||
) -> Iterator[CallHandle]:
|
||||
for i, call in enumerate(sorted_calls[ph.address][th.address]):
|
||||
call_addr = DynamicCallAddress(thread=th.address, id=i)
|
||||
|
||||
@@ -1,23 +1,16 @@
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import struct
|
||||
import logging
|
||||
import itertools
|
||||
import collections
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING, BinaryIO, Iterator, Optional
|
||||
from typing import TYPE_CHECKING, Set, Dict, List, Tuple, BinaryIO, Iterator, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -401,7 +394,7 @@ class ELF:
|
||||
return read_cstr(phdr.buf, 0)
|
||||
|
||||
@property
|
||||
def versions_needed(self) -> dict[str, set[str]]:
|
||||
def versions_needed(self) -> Dict[str, Set[str]]:
|
||||
# symbol version requirements are stored in the .gnu.version_r section,
|
||||
# which has type SHT_GNU_verneed (0x6ffffffe).
|
||||
#
|
||||
@@ -459,7 +452,7 @@ class ELF:
|
||||
return {}
|
||||
|
||||
@property
|
||||
def dynamic_entries(self) -> Iterator[tuple[int, int]]:
|
||||
def dynamic_entries(self) -> Iterator[Tuple[int, int]]:
|
||||
"""
|
||||
read the entries from the dynamic section,
|
||||
yielding the tag and value for each entry.
|
||||
@@ -554,7 +547,7 @@ class ELF:
|
||||
logger.warning("failed to read DT_NEEDED entry: %s", str(e))
|
||||
|
||||
@property
|
||||
def symtab(self) -> Optional[tuple[Shdr, Shdr]]:
|
||||
def symtab(self) -> Optional[Tuple[Shdr, Shdr]]:
|
||||
"""
|
||||
fetch the Shdr for the symtab and the associated strtab.
|
||||
"""
|
||||
@@ -689,7 +682,7 @@ class SymTab:
|
||||
symtab: Shdr,
|
||||
strtab: Shdr,
|
||||
) -> None:
|
||||
self.symbols: list[Symbol] = []
|
||||
self.symbols: List[Symbol] = []
|
||||
|
||||
self.symtab = symtab
|
||||
self.strtab = strtab
|
||||
@@ -1088,7 +1081,7 @@ def guess_os_from_go_buildinfo(elf: ELF) -> Optional[OS]:
|
||||
# and the 32-byte header is followed by varint-prefixed string data
|
||||
# for the two string values we care about.
|
||||
# https://github.com/mandiant/GoReSym/blob/0860a1b1b4f3495e9fb7e71eb4386bf3e0a7c500/buildinfo/buildinfo.go#L185-L193
|
||||
BUILDINFO_MAGIC = b"\xff Go buildinf:"
|
||||
BUILDINFO_MAGIC = b"\xFF Go buildinf:"
|
||||
|
||||
try:
|
||||
index = buf.index(BUILDINFO_MAGIC)
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import io
|
||||
import logging
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from elftools.elf.elffile import ELFFile, DynamicSegment, SymbolTableSection
|
||||
@@ -80,7 +73,8 @@ def extract_file_export_names(elf: ELFFile, **kwargs):
|
||||
|
||||
|
||||
def extract_file_import_names(elf: ELFFile, **kwargs):
|
||||
symbol_name_by_index: dict[int, str] = {}
|
||||
# Create a dictionary to store symbol names by their index
|
||||
symbol_names = {}
|
||||
|
||||
# Extract symbol names and store them in the dictionary
|
||||
for segment in elf.iter_segments():
|
||||
@@ -92,7 +86,7 @@ def extract_file_import_names(elf: ELFFile, **kwargs):
|
||||
logger.debug("Dynamic segment doesn't contain DT_SYMTAB")
|
||||
continue
|
||||
|
||||
for i, symbol in enumerate(segment.iter_symbols()):
|
||||
for _, symbol in enumerate(segment.iter_symbols()):
|
||||
# The following conditions are based on the following article
|
||||
# http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
|
||||
if not symbol.name:
|
||||
@@ -106,7 +100,7 @@ def extract_file_import_names(elf: ELFFile, **kwargs):
|
||||
if symbol.entry.st_name == 0:
|
||||
continue
|
||||
|
||||
symbol_name_by_index[i] = symbol.name
|
||||
symbol_names[_] = symbol.name
|
||||
|
||||
for segment in elf.iter_segments():
|
||||
if not isinstance(segment, DynamicSegment):
|
||||
@@ -126,17 +120,10 @@ def extract_file_import_names(elf: ELFFile, **kwargs):
|
||||
break
|
||||
|
||||
for relocation in relocations:
|
||||
if "r_info_sym" not in relocation.entry or "r_offset" not in relocation.entry:
|
||||
# Extract the symbol name from the symbol table using the symbol index in the relocation
|
||||
if relocation["r_info_sym"] not in symbol_names:
|
||||
continue
|
||||
|
||||
symbol_address: int = relocation["r_offset"]
|
||||
symbol_index: int = relocation["r_info_sym"]
|
||||
|
||||
if symbol_index not in symbol_name_by_index:
|
||||
continue
|
||||
symbol_name = symbol_name_by_index[symbol_index]
|
||||
|
||||
yield Import(symbol_name), FileOffsetAddress(symbol_address)
|
||||
yield Import(symbol_names[relocation["r_info_sym"]]), FileOffsetAddress(relocation["r_offset"])
|
||||
|
||||
|
||||
def extract_file_section_names(elf: ELFFile, **kwargs):
|
||||
@@ -179,7 +166,7 @@ def extract_file_arch(elf: ELFFile, **kwargs):
|
||||
logger.warning("unsupported architecture: %s", arch)
|
||||
|
||||
|
||||
def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[tuple[Feature, int]]:
|
||||
def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
|
||||
for file_handler in FILE_HANDLERS:
|
||||
for feature, addr in file_handler(elf=elf, buf=buf): # type: ignore
|
||||
yield feature, addr
|
||||
@@ -195,7 +182,7 @@ FILE_HANDLERS = (
|
||||
)
|
||||
|
||||
|
||||
def extract_global_features(elf: ELFFile, buf: bytes) -> Iterator[tuple[Feature, int]]:
|
||||
def extract_global_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, int]]:
|
||||
for global_handler in GLOBAL_HANDLERS:
|
||||
for feature, addr in global_handler(elf=elf, buf=buf): # type: ignore
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,21 +1,14 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import string
|
||||
import struct
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import ghidra
|
||||
from ghidra.program.model.lang import OperandType
|
||||
@@ -104,7 +97,7 @@ def _bb_has_tight_loop(bb: ghidra.program.model.block.CodeBlock):
|
||||
return False
|
||||
|
||||
|
||||
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract stackstring indicators from basic block"""
|
||||
bb: ghidra.program.model.block.CodeBlock = bbh.inner
|
||||
|
||||
@@ -112,7 +105,7 @@ def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[
|
||||
yield Characteristic("stack string"), bbh.address
|
||||
|
||||
|
||||
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""check basic block for tight loop indicators"""
|
||||
bb: ghidra.program.model.block.CodeBlock = bbh.inner
|
||||
|
||||
@@ -126,7 +119,7 @@ BASIC_BLOCK_HANDLERS = (
|
||||
)
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract features from the given basic block.
|
||||
|
||||
@@ -134,7 +127,7 @@ def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Featur
|
||||
bb: the basic block to process.
|
||||
|
||||
yields:
|
||||
tuple[Feature, int]: the features and their location found in this basic block.
|
||||
Tuple[Feature, int]: the features and their location found in this basic block.
|
||||
"""
|
||||
yield BasicBlock(), bbh.address
|
||||
for bb_handler in BASIC_BLOCK_HANDLERS:
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import List, Tuple, Iterator
|
||||
|
||||
import capa.features.extractors.ghidra.file
|
||||
import capa.features.extractors.ghidra.insn
|
||||
@@ -47,7 +40,7 @@ class GhidraFeatureExtractor(StaticFeatureExtractor):
|
||||
)
|
||||
)
|
||||
|
||||
self.global_features: list[tuple[Feature, Address]] = []
|
||||
self.global_features: List[Tuple[Feature, Address]] = []
|
||||
self.global_features.extend(capa.features.extractors.ghidra.file.extract_file_format())
|
||||
self.global_features.extend(capa.features.extractors.ghidra.global_.extract_os())
|
||||
self.global_features.extend(capa.features.extractors.ghidra.global_.extract_arch())
|
||||
@@ -80,7 +73,7 @@ class GhidraFeatureExtractor(StaticFeatureExtractor):
|
||||
func = getFunctionContaining(toAddr(addr)) # type: ignore [name-defined] # noqa: F821
|
||||
return FunctionHandle(address=AbsoluteVirtualAddress(func.getEntryPoint().getOffset()), inner=func)
|
||||
|
||||
def extract_function_features(self, fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.ghidra.function.extract_features(fh)
|
||||
|
||||
def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
|
||||
@@ -88,7 +81,7 @@ class GhidraFeatureExtractor(StaticFeatureExtractor):
|
||||
|
||||
yield from ghidra_helpers.get_function_blocks(fh)
|
||||
|
||||
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.ghidra.basicblock.extract_features(fh, bbh)
|
||||
|
||||
def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import re
|
||||
import struct
|
||||
from typing import Iterator
|
||||
from typing import List, Tuple, Iterator
|
||||
|
||||
from ghidra.program.model.symbol import SourceType, SymbolType
|
||||
|
||||
@@ -29,7 +22,7 @@ from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, Absolu
|
||||
MAX_OFFSET_PE_AFTER_MZ = 0x200
|
||||
|
||||
|
||||
def find_embedded_pe(block_bytez: bytes, mz_xor: list[tuple[bytes, bytes, int]]) -> Iterator[tuple[int, int]]:
|
||||
def find_embedded_pe(block_bytez: bytes, mz_xor: List[Tuple[bytes, bytes, int]]) -> Iterator[Tuple[int, int]]:
|
||||
"""check segment for embedded PE
|
||||
|
||||
adapted for Ghidra from:
|
||||
@@ -67,11 +60,11 @@ def find_embedded_pe(block_bytez: bytes, mz_xor: list[tuple[bytes, bytes, int]])
|
||||
yield off, i
|
||||
|
||||
|
||||
def extract_file_embedded_pe() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract embedded PE features"""
|
||||
|
||||
# pre-compute XOR pairs
|
||||
mz_xor: list[tuple[bytes, bytes, int]] = [
|
||||
mz_xor: List[Tuple[bytes, bytes, int]] = [
|
||||
(
|
||||
capa.features.extractors.helpers.xor_static(b"MZ", i),
|
||||
capa.features.extractors.helpers.xor_static(b"PE", i),
|
||||
@@ -91,14 +84,14 @@ def extract_file_embedded_pe() -> Iterator[tuple[Feature, Address]]:
|
||||
yield Characteristic("embedded pe"), FileOffsetAddress(ea)
|
||||
|
||||
|
||||
def extract_file_export_names() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract function exports"""
|
||||
st = currentProgram().getSymbolTable() # type: ignore [name-defined] # noqa: F821
|
||||
for addr in st.getExternalEntryPointIterator():
|
||||
yield Export(st.getPrimarySymbol(addr).getName()), AbsoluteVirtualAddress(addr.getOffset())
|
||||
|
||||
|
||||
def extract_file_import_names() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract function imports
|
||||
|
||||
1. imports by ordinal:
|
||||
@@ -123,14 +116,14 @@ def extract_file_import_names() -> Iterator[tuple[Feature, Address]]:
|
||||
yield Import(name), AbsoluteVirtualAddress(addr)
|
||||
|
||||
|
||||
def extract_file_section_names() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract section names"""
|
||||
|
||||
for block in currentProgram().getMemory().getBlocks(): # type: ignore [name-defined] # noqa: F821
|
||||
yield Section(block.getName()), AbsoluteVirtualAddress(block.getStart().getOffset())
|
||||
|
||||
|
||||
def extract_file_strings() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract ASCII and UTF-16 LE strings"""
|
||||
|
||||
for block in currentProgram().getMemory().getBlocks(): # type: ignore [name-defined] # noqa: F821
|
||||
@@ -148,7 +141,7 @@ def extract_file_strings() -> Iterator[tuple[Feature, Address]]:
|
||||
yield String(s.s), FileOffsetAddress(offset)
|
||||
|
||||
|
||||
def extract_file_function_names() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract the names of statically-linked library functions.
|
||||
"""
|
||||
@@ -169,7 +162,7 @@ def extract_file_function_names() -> Iterator[tuple[Feature, Address]]:
|
||||
yield FunctionName(name[1:]), addr
|
||||
|
||||
|
||||
def extract_file_format() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
|
||||
ef = currentProgram().getExecutableFormat() # type: ignore [name-defined] # noqa: F821
|
||||
if "PE" in ef:
|
||||
yield Format(FORMAT_PE), NO_ADDRESS
|
||||
@@ -182,7 +175,7 @@ def extract_file_format() -> Iterator[tuple[Feature, Address]]:
|
||||
raise NotImplementedError(f"unexpected file format: {ef}")
|
||||
|
||||
|
||||
def extract_features() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract file features"""
|
||||
for file_handler in FILE_HANDLERS:
|
||||
for feature, addr in file_handler():
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import ghidra
|
||||
from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
|
||||
@@ -56,7 +49,7 @@ def extract_recursive_call(fh: FunctionHandle):
|
||||
yield Characteristic("recursive call"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
for func_handler in FUNCTION_HANDLERS:
|
||||
for feature, addr in func_handler(fh):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
import contextlib
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import capa.ghidra.helpers
|
||||
import capa.features.extractors.elf
|
||||
@@ -25,7 +18,7 @@ from capa.features.address import NO_ADDRESS, Address
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_os() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_os() -> Iterator[Tuple[Feature, Address]]:
|
||||
format_name: str = currentProgram().getExecutableFormat() # type: ignore [name-defined] # noqa: F821
|
||||
|
||||
if "PE" in format_name:
|
||||
@@ -52,7 +45,7 @@ def extract_os() -> Iterator[tuple[Feature, Address]]:
|
||||
return
|
||||
|
||||
|
||||
def extract_arch() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_arch() -> Iterator[Tuple[Feature, Address]]:
|
||||
lang_id = currentProgram().getMetadata().get("Language ID") # type: ignore [name-defined] # noqa: F821
|
||||
|
||||
if "x86" in lang_id and "64" in lang_id:
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Dict, List, Iterator
|
||||
|
||||
import ghidra
|
||||
import java.lang
|
||||
@@ -27,7 +20,7 @@ from capa.features.address import AbsoluteVirtualAddress
|
||||
from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
|
||||
|
||||
|
||||
def ints_to_bytes(bytez: list[int]) -> bytes:
|
||||
def ints_to_bytes(bytez: List[int]) -> bytes:
|
||||
"""convert Java signed ints to Python bytes
|
||||
|
||||
args:
|
||||
@@ -90,10 +83,10 @@ def get_insn_in_range(bbh: BBHandle) -> Iterator[InsnHandle]:
|
||||
yield InsnHandle(address=AbsoluteVirtualAddress(insn.getAddress().getOffset()), inner=insn)
|
||||
|
||||
|
||||
def get_file_imports() -> dict[int, list[str]]:
|
||||
def get_file_imports() -> Dict[int, List[str]]:
|
||||
"""get all import names & addrs"""
|
||||
|
||||
import_dict: dict[int, list[str]] = {}
|
||||
import_dict: Dict[int, List[str]] = {}
|
||||
|
||||
for f in currentProgram().getFunctionManager().getExternalFunctions(): # type: ignore [name-defined] # noqa: F821
|
||||
for r in f.getSymbol().getReferences():
|
||||
@@ -117,7 +110,7 @@ def get_file_imports() -> dict[int, list[str]]:
|
||||
return import_dict
|
||||
|
||||
|
||||
def get_file_externs() -> dict[int, list[str]]:
|
||||
def get_file_externs() -> Dict[int, List[str]]:
|
||||
"""
|
||||
Gets function names & addresses of statically-linked library functions
|
||||
|
||||
@@ -131,7 +124,7 @@ def get_file_externs() -> dict[int, list[str]]:
|
||||
- Note: See Symbol Table labels
|
||||
"""
|
||||
|
||||
extern_dict: dict[int, list[str]] = {}
|
||||
extern_dict: Dict[int, List[str]] = {}
|
||||
|
||||
for sym in currentProgram().getSymbolTable().getAllSymbols(True): # type: ignore [name-defined] # noqa: F821
|
||||
# .isExternal() misses more than this config for the function symbols
|
||||
@@ -150,7 +143,7 @@ def get_file_externs() -> dict[int, list[str]]:
|
||||
return extern_dict
|
||||
|
||||
|
||||
def map_fake_import_addrs() -> dict[int, list[int]]:
|
||||
def map_fake_import_addrs() -> Dict[int, List[int]]:
|
||||
"""
|
||||
Map ghidra's fake import entrypoints to their
|
||||
real addresses
|
||||
@@ -169,7 +162,7 @@ def map_fake_import_addrs() -> dict[int, list[int]]:
|
||||
- 0x473090 -> PTR_CreateServiceW_00473090
|
||||
- 'EXTERNAL:00000025' -> External Address (ghidra.program.model.address.SpecialAddress)
|
||||
"""
|
||||
fake_dict: dict[int, list[int]] = {}
|
||||
fake_dict: Dict[int, List[int]] = {}
|
||||
|
||||
for f in currentProgram().getFunctionManager().getExternalFunctions(): # type: ignore [name-defined] # noqa: F821
|
||||
for r in f.getSymbol().getReferences():
|
||||
@@ -181,9 +174,9 @@ def map_fake_import_addrs() -> dict[int, list[int]]:
|
||||
|
||||
def check_addr_for_api(
|
||||
addr: ghidra.program.model.address.Address,
|
||||
fakes: dict[int, list[int]],
|
||||
imports: dict[int, list[str]],
|
||||
externs: dict[int, list[str]],
|
||||
fakes: Dict[int, List[int]],
|
||||
imports: Dict[int, List[str]],
|
||||
externs: Dict[int, List[str]],
|
||||
) -> bool:
|
||||
offset = addr.getOffset()
|
||||
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2023 Google LLC
|
||||
#
|
||||
# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Any, Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Any, Dict, Tuple, Iterator
|
||||
|
||||
import ghidra
|
||||
from ghidra.program.model.lang import OperandType
|
||||
@@ -33,21 +26,21 @@ SECURITY_COOKIE_BYTES_DELTA = 0x40
|
||||
OPERAND_TYPE_DYNAMIC_ADDRESS = OperandType.DYNAMIC | OperandType.ADDRESS
|
||||
|
||||
|
||||
def get_imports(ctx: dict[str, Any]) -> dict[int, Any]:
|
||||
def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
|
||||
"""Populate the import cache for this context"""
|
||||
if "imports_cache" not in ctx:
|
||||
ctx["imports_cache"] = capa.features.extractors.ghidra.helpers.get_file_imports()
|
||||
return ctx["imports_cache"]
|
||||
|
||||
|
||||
def get_externs(ctx: dict[str, Any]) -> dict[int, Any]:
|
||||
def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
|
||||
"""Populate the externs cache for this context"""
|
||||
if "externs_cache" not in ctx:
|
||||
ctx["externs_cache"] = capa.features.extractors.ghidra.helpers.get_file_externs()
|
||||
return ctx["externs_cache"]
|
||||
|
||||
|
||||
def get_fakes(ctx: dict[str, Any]) -> dict[int, Any]:
|
||||
def get_fakes(ctx: Dict[str, Any]) -> Dict[int, Any]:
|
||||
"""Populate the fake import addrs cache for this context"""
|
||||
if "fakes_cache" not in ctx:
|
||||
ctx["fakes_cache"] = capa.features.extractors.ghidra.helpers.map_fake_import_addrs()
|
||||
@@ -55,7 +48,7 @@ def get_fakes(ctx: dict[str, Any]) -> dict[int, Any]:
|
||||
|
||||
|
||||
def check_for_api_call(
|
||||
insn, externs: dict[int, Any], fakes: dict[int, Any], imports: dict[int, Any], imp_or_ex: bool
|
||||
insn, externs: Dict[int, Any], fakes: Dict[int, Any], imports: Dict[int, Any], imp_or_ex: bool
|
||||
) -> Iterator[Any]:
|
||||
"""check instruction for API call
|
||||
|
||||
@@ -117,7 +110,7 @@ def check_for_api_call(
|
||||
yield info
|
||||
|
||||
|
||||
def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
insn: ghidra.program.database.code.InstructionDB = ih.inner
|
||||
|
||||
if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
|
||||
@@ -138,7 +131,7 @@ def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle)
|
||||
yield API(ext), ih.address
|
||||
|
||||
|
||||
def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction number features
|
||||
example:
|
||||
@@ -193,7 +186,7 @@ def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandl
|
||||
yield OperandOffset(i, const), addr
|
||||
|
||||
|
||||
def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction structure offset features
|
||||
|
||||
@@ -226,7 +219,7 @@ def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandl
|
||||
yield OperandOffset(i, op_off), ih.address
|
||||
|
||||
|
||||
def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse referenced byte sequences
|
||||
|
||||
@@ -241,7 +234,7 @@ def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
|
||||
yield Bytes(extracted_bytes), ih.address
|
||||
|
||||
|
||||
def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction string features
|
||||
|
||||
@@ -256,7 +249,7 @@ def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandl
|
||||
|
||||
def extract_insn_mnemonic_features(
|
||||
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction mnemonic features"""
|
||||
insn: ghidra.program.database.code.InstructionDB = ih.inner
|
||||
|
||||
@@ -265,7 +258,7 @@ def extract_insn_mnemonic_features(
|
||||
|
||||
def extract_insn_obfs_call_plus_5_characteristic_features(
|
||||
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse call $+5 instruction from the given instruction.
|
||||
"""
|
||||
@@ -286,7 +279,7 @@ def extract_insn_obfs_call_plus_5_characteristic_features(
|
||||
|
||||
def extract_insn_segment_access_features(
|
||||
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction fs or gs access"""
|
||||
insn: ghidra.program.database.code.InstructionDB = ih.inner
|
||||
|
||||
@@ -301,7 +294,7 @@ def extract_insn_segment_access_features(
|
||||
|
||||
def extract_insn_peb_access_characteristic_features(
|
||||
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction peb access
|
||||
|
||||
fs:[0x30] on x86, gs:[0x60] on x64
|
||||
@@ -317,7 +310,7 @@ def extract_insn_peb_access_characteristic_features(
|
||||
|
||||
def extract_insn_cross_section_cflow(
|
||||
fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""inspect the instruction for a CALL or JMP that crosses section boundaries"""
|
||||
insn: ghidra.program.database.code.InstructionDB = ih.inner
|
||||
|
||||
@@ -371,7 +364,7 @@ def extract_function_calls_from(
|
||||
fh: FunctionHandle,
|
||||
bb: BBHandle,
|
||||
ih: InsnHandle,
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract functions calls from features
|
||||
|
||||
most relevant at the function scope, however, its most efficient to extract at the instruction scope
|
||||
@@ -400,7 +393,7 @@ def extract_function_indirect_call_characteristic_features(
|
||||
fh: FunctionHandle,
|
||||
bb: BBHandle,
|
||||
ih: InsnHandle,
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
|
||||
does not include calls like => call ds:dword_ABD4974
|
||||
|
||||
@@ -419,36 +412,37 @@ def extract_function_indirect_call_characteristic_features(
|
||||
def check_nzxor_security_cookie_delta(
|
||||
fh: ghidra.program.database.function.FunctionDB, insn: ghidra.program.database.code.InstructionDB
|
||||
):
|
||||
"""
|
||||
Get the first and last blocks of the function
|
||||
Check if insn within first addr of first bb + delta
|
||||
Check if insn within last addr of last bb - delta
|
||||
"""Get the function containing the insn
|
||||
Get the last block of the function that contains the insn
|
||||
|
||||
Check the bb containing the insn
|
||||
Check the last bb of the function containing the insn
|
||||
"""
|
||||
|
||||
model = SimpleBlockModel(currentProgram()) # type: ignore [name-defined] # noqa: F821
|
||||
insn_addr = insn.getAddress()
|
||||
func_asv = fh.getBody()
|
||||
|
||||
first_addr = func_asv.getMinAddress()
|
||||
if insn_addr < first_addr.add(SECURITY_COOKIE_BYTES_DELTA):
|
||||
first_bb = model.getFirstCodeBlockContaining(first_addr, monitor()) # type: ignore [name-defined] # noqa: F821
|
||||
if first_bb.contains(insn_addr):
|
||||
return True
|
||||
|
||||
last_addr = func_asv.getMaxAddress()
|
||||
if insn_addr > last_addr.add(SECURITY_COOKIE_BYTES_DELTA * -1):
|
||||
last_bb = model.getFirstCodeBlockContaining(last_addr, monitor()) # type: ignore [name-defined] # noqa: F821
|
||||
if last_bb.contains(insn_addr):
|
||||
return True
|
||||
|
||||
return False
|
||||
if model.getFirstCodeBlockContaining(
|
||||
first_addr, monitor() # type: ignore [name-defined] # noqa: F821
|
||||
) == model.getFirstCodeBlockContaining(
|
||||
last_addr, monitor() # type: ignore [name-defined] # noqa: F821
|
||||
):
|
||||
if insn_addr < first_addr.add(SECURITY_COOKIE_BYTES_DELTA):
|
||||
return True
|
||||
else:
|
||||
return insn_addr > last_addr.add(SECURITY_COOKIE_BYTES_DELTA * -1)
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def extract_insn_nzxor_characteristic_features(
|
||||
fh: FunctionHandle,
|
||||
bb: BBHandle,
|
||||
ih: InsnHandle,
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
f: ghidra.program.database.function.FunctionDB = fh.inner
|
||||
insn: ghidra.program.database.code.InstructionDB = ih.inner
|
||||
|
||||
@@ -467,7 +461,7 @@ def extract_features(
|
||||
fh: FunctionHandle,
|
||||
bb: BBHandle,
|
||||
insn: InsnHandle,
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
for insn_handler in INSTRUCTION_HANDLERS:
|
||||
for feature, addr in insn_handler(fh, bb, insn):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,21 +1,14 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import struct
|
||||
import builtins
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
MIN_STACKSTRING_LEN = 8
|
||||
|
||||
@@ -126,7 +119,7 @@ def twos_complement(val: int, bits: int) -> int:
|
||||
return val
|
||||
|
||||
|
||||
def carve_pe(pbytes: bytes, offset: int = 0) -> Iterator[tuple[int, int]]:
|
||||
def carve_pe(pbytes: bytes, offset: int = 0) -> Iterator[Tuple[int, int]]:
|
||||
"""
|
||||
Generate (offset, key) tuples of embedded PEs
|
||||
|
||||
|
||||
@@ -1,21 +1,14 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import string
|
||||
import struct
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import idaapi
|
||||
|
||||
@@ -87,19 +80,19 @@ def bb_contains_stackstring(f: idaapi.func_t, bb: idaapi.BasicBlock) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract stackstring indicators from basic block"""
|
||||
if bb_contains_stackstring(fh.inner, bbh.inner):
|
||||
yield Characteristic("stack string"), bbh.address
|
||||
|
||||
|
||||
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract tight loop indicators from a basic block"""
|
||||
if capa.features.extractors.ida.helpers.is_basic_block_tight_loop(bbh.inner):
|
||||
yield Characteristic("tight loop"), bbh.address
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract basic block features"""
|
||||
for bb_handler in BASIC_BLOCK_HANDLERS:
|
||||
for feature, addr in bb_handler(fh, bbh):
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import List, Tuple, Iterator
|
||||
|
||||
import idaapi
|
||||
|
||||
@@ -43,7 +36,7 @@ class IdaFeatureExtractor(StaticFeatureExtractor):
|
||||
sha256=capa.ida.helpers.retrieve_input_file_sha256(),
|
||||
)
|
||||
)
|
||||
self.global_features: list[tuple[Feature, Address]] = []
|
||||
self.global_features: List[Tuple[Feature, Address]] = []
|
||||
self.global_features.extend(capa.features.extractors.ida.file.extract_file_format())
|
||||
self.global_features.extend(capa.features.extractors.ida.global_.extract_os())
|
||||
self.global_features.extend(capa.features.extractors.ida.global_.extract_arch())
|
||||
@@ -68,7 +61,7 @@ class IdaFeatureExtractor(StaticFeatureExtractor):
|
||||
f = idaapi.get_func(ea)
|
||||
return FunctionHandle(address=AbsoluteVirtualAddress(f.start_ea), inner=f)
|
||||
|
||||
def extract_function_features(self, fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.ida.function.extract_features(fh)
|
||||
|
||||
def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
|
||||
@@ -77,7 +70,7 @@ class IdaFeatureExtractor(StaticFeatureExtractor):
|
||||
for bb in ida_helpers.get_function_blocks(fh.inner):
|
||||
yield BBHandle(address=AbsoluteVirtualAddress(bb.start_ea), inner=bb)
|
||||
|
||||
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
yield from capa.features.extractors.ida.basicblock.extract_features(fh, bbh)
|
||||
|
||||
def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import struct
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import idc
|
||||
import idaapi
|
||||
@@ -33,7 +26,7 @@ from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, Absolu
|
||||
MAX_OFFSET_PE_AFTER_MZ = 0x200
|
||||
|
||||
|
||||
def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[tuple[int, int]]:
|
||||
def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[Tuple[int, int]]:
|
||||
"""check segment for embedded PE
|
||||
|
||||
adapted for IDA from:
|
||||
@@ -78,7 +71,7 @@ def check_segment_for_pe(seg: idaapi.segment_t) -> Iterator[tuple[int, int]]:
|
||||
yield off, i
|
||||
|
||||
|
||||
def extract_file_embedded_pe() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract embedded PE features
|
||||
|
||||
IDA must load resource sections for this to be complete
|
||||
@@ -90,7 +83,7 @@ def extract_file_embedded_pe() -> Iterator[tuple[Feature, Address]]:
|
||||
yield Characteristic("embedded pe"), FileOffsetAddress(ea)
|
||||
|
||||
|
||||
def extract_file_export_names() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract function exports"""
|
||||
for _, ordinal, ea, name in idautils.Entries():
|
||||
forwarded_name = ida_entry.get_entry_forwarder(ordinal)
|
||||
@@ -102,7 +95,7 @@ def extract_file_export_names() -> Iterator[tuple[Feature, Address]]:
|
||||
yield Characteristic("forwarded export"), AbsoluteVirtualAddress(ea)
|
||||
|
||||
|
||||
def extract_file_import_names() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract function imports
|
||||
|
||||
1. imports by ordinal:
|
||||
@@ -138,7 +131,7 @@ def extract_file_import_names() -> Iterator[tuple[Feature, Address]]:
|
||||
yield Import(info[1]), AbsoluteVirtualAddress(ea)
|
||||
|
||||
|
||||
def extract_file_section_names() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract section names
|
||||
|
||||
IDA must load resource sections for this to be complete
|
||||
@@ -149,7 +142,7 @@ def extract_file_section_names() -> Iterator[tuple[Feature, Address]]:
|
||||
yield Section(idaapi.get_segm_name(seg)), AbsoluteVirtualAddress(seg.start_ea)
|
||||
|
||||
|
||||
def extract_file_strings() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract ASCII and UTF-16 LE strings
|
||||
|
||||
IDA must load resource sections for this to be complete
|
||||
@@ -167,7 +160,7 @@ def extract_file_strings() -> Iterator[tuple[Feature, Address]]:
|
||||
yield String(s.s), FileOffsetAddress(seg.start_ea + s.offset)
|
||||
|
||||
|
||||
def extract_file_function_names() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
extract the names of statically-linked library functions.
|
||||
"""
|
||||
@@ -184,7 +177,7 @@ def extract_file_function_names() -> Iterator[tuple[Feature, Address]]:
|
||||
yield FunctionName(name[1:]), addr
|
||||
|
||||
|
||||
def extract_file_format() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
|
||||
filetype = capa.ida.helpers.get_filetype()
|
||||
|
||||
if filetype in (idaapi.f_PE, idaapi.f_COFF):
|
||||
@@ -198,7 +191,7 @@ def extract_file_format() -> Iterator[tuple[Feature, Address]]:
|
||||
raise NotImplementedError(f"unexpected file format: {filetype}")
|
||||
|
||||
|
||||
def extract_features() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features() -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract file features"""
|
||||
for file_handler in FILE_HANDLERS:
|
||||
for feature, addr in file_handler():
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Iterator
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import idaapi
|
||||
import idautils
|
||||
@@ -50,7 +43,7 @@ def extract_recursive_call(fh: FunctionHandle):
|
||||
yield Characteristic("recursive call"), fh.address
|
||||
|
||||
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
for func_handler in FUNCTION_HANDLERS:
|
||||
for feature, addr in func_handler(fh):
|
||||
yield feature, addr
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import logging
|
||||
import contextlib
|
||||
from typing import Iterator
|
||||
from typing import Tuple, Iterator
|
||||
|
||||
import ida_loader
|
||||
|
||||
@@ -26,7 +19,7 @@ from capa.features.address import NO_ADDRESS, Address
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_os() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_os() -> Iterator[Tuple[Feature, Address]]:
|
||||
format_name: str = ida_loader.get_file_type_name()
|
||||
|
||||
if "PE" in format_name:
|
||||
@@ -53,7 +46,7 @@ def extract_os() -> Iterator[tuple[Feature, Address]]:
|
||||
return
|
||||
|
||||
|
||||
def extract_arch() -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_arch() -> Iterator[Tuple[Feature, Address]]:
|
||||
procname = capa.ida.helpers.get_processor_name()
|
||||
if procname == "metapc" and capa.ida.helpers.is_64bit():
|
||||
yield Arch(ARCH_AMD64), NO_ADDRESS
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import functools
|
||||
from typing import Any, Iterator, Optional
|
||||
from typing import Any, Dict, Tuple, Iterator, Optional
|
||||
|
||||
import idc
|
||||
import idaapi
|
||||
@@ -48,15 +41,7 @@ if hasattr(ida_bytes, "parse_binpat_str"):
|
||||
return
|
||||
|
||||
while True:
|
||||
ea = ida_bytes.bin_search(start, end, patterns, ida_bytes.BIN_SEARCH_FORWARD)
|
||||
if isinstance(ea, int):
|
||||
# "ea_t" in IDA 8.4, 8.3
|
||||
pass
|
||||
elif isinstance(ea, tuple):
|
||||
# "drc_t" in IDA 9
|
||||
ea = ea[0]
|
||||
else:
|
||||
raise NotImplementedError(f"bin_search returned unhandled type: {type(ea)}")
|
||||
ea, _ = ida_bytes.bin_search(start, end, patterns, ida_bytes.BIN_SEARCH_FORWARD)
|
||||
if ea == idaapi.BADADDR:
|
||||
break
|
||||
start = ea + 1
|
||||
@@ -139,9 +124,9 @@ def inspect_import(imports, library, ea, function, ordinal):
|
||||
return True
|
||||
|
||||
|
||||
def get_file_imports() -> dict[int, tuple[str, str, int]]:
|
||||
def get_file_imports() -> Dict[int, Tuple[str, str, int]]:
|
||||
"""get file imports"""
|
||||
imports: dict[int, tuple[str, str, int]] = {}
|
||||
imports: Dict[int, Tuple[str, str, int]] = {}
|
||||
|
||||
for idx in range(idaapi.get_import_module_qty()):
|
||||
library = idaapi.get_import_module_name(idx)
|
||||
@@ -162,7 +147,7 @@ def get_file_imports() -> dict[int, tuple[str, str, int]]:
|
||||
return imports
|
||||
|
||||
|
||||
def get_file_externs() -> dict[int, tuple[str, str, int]]:
|
||||
def get_file_externs() -> Dict[int, Tuple[str, str, int]]:
|
||||
externs = {}
|
||||
|
||||
for seg in get_segments(skip_header_segments=True):
|
||||
@@ -263,7 +248,7 @@ def find_string_at(ea: int, min_: int = 4) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def get_op_phrase_info(op: idaapi.op_t) -> dict:
|
||||
def get_op_phrase_info(op: idaapi.op_t) -> Dict:
|
||||
"""parse phrase features from operand
|
||||
|
||||
Pretty much dup of sark's implementation:
|
||||
@@ -338,7 +323,7 @@ def is_frame_register(reg: int) -> bool:
|
||||
return reg in (idautils.procregs.sp.reg, idautils.procregs.bp.reg)
|
||||
|
||||
|
||||
def get_insn_ops(insn: idaapi.insn_t, target_ops: Optional[tuple[Any]] = None) -> idaapi.op_t:
|
||||
def get_insn_ops(insn: idaapi.insn_t, target_ops: Optional[Tuple[Any]] = None) -> idaapi.op_t:
|
||||
"""yield op_t for instruction, filter on type if specified"""
|
||||
for op in insn.ops:
|
||||
if op.type == idaapi.o_void:
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# Copyright 2024 Google LLC
|
||||
#
|
||||
# Copyright (C) 2024 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
import re
|
||||
from typing import Any, Iterator, Optional
|
||||
from typing import Any, Dict, Tuple, Iterator, Optional
|
||||
|
||||
import idc
|
||||
import ida_ua
|
||||
@@ -32,19 +25,19 @@ from capa.features.extractors.base_extractor import BBHandle, InsnHandle, Functi
|
||||
SECURITY_COOKIE_BYTES_DELTA = 0x40
|
||||
|
||||
|
||||
def get_imports(ctx: dict[str, Any]) -> dict[int, Any]:
|
||||
def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
|
||||
if "imports_cache" not in ctx:
|
||||
ctx["imports_cache"] = capa.features.extractors.ida.helpers.get_file_imports()
|
||||
return ctx["imports_cache"]
|
||||
|
||||
|
||||
def get_externs(ctx: dict[str, Any]) -> dict[int, Any]:
|
||||
def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
|
||||
if "externs_cache" not in ctx:
|
||||
ctx["externs_cache"] = capa.features.extractors.ida.helpers.get_file_externs()
|
||||
return ctx["externs_cache"]
|
||||
|
||||
|
||||
def check_for_api_call(insn: idaapi.insn_t, funcs: dict[int, Any]) -> Optional[tuple[str, str]]:
|
||||
def check_for_api_call(insn: idaapi.insn_t, funcs: Dict[int, Any]) -> Optional[Tuple[str, str]]:
|
||||
"""check instruction for API call"""
|
||||
info = None
|
||||
ref = insn.ea
|
||||
@@ -72,7 +65,7 @@ def check_for_api_call(insn: idaapi.insn_t, funcs: dict[int, Any]) -> Optional[t
|
||||
return info
|
||||
|
||||
|
||||
def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction API features
|
||||
|
||||
@@ -142,7 +135,7 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
|
||||
|
||||
def extract_insn_number_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction number features
|
||||
example:
|
||||
@@ -188,7 +181,7 @@ def extract_insn_number_features(
|
||||
yield OperandOffset(i, const), ih.address
|
||||
|
||||
|
||||
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse referenced byte sequences
|
||||
example:
|
||||
@@ -210,7 +203,7 @@ def extract_insn_bytes_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
|
||||
|
||||
def extract_insn_string_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction string features
|
||||
|
||||
@@ -228,7 +221,7 @@ def extract_insn_string_features(
|
||||
|
||||
def extract_insn_offset_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction structure offset features
|
||||
|
||||
@@ -376,7 +369,7 @@ def is_nzxor_stack_cookie(f: idaapi.func_t, bb: idaapi.BasicBlock, insn: idaapi.
|
||||
|
||||
def extract_insn_nzxor_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse instruction non-zeroing XOR instruction
|
||||
ignore expected non-zeroing XORs, e.g. security cookies
|
||||
@@ -394,14 +387,14 @@ def extract_insn_nzxor_characteristic_features(
|
||||
|
||||
def extract_insn_mnemonic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction mnemonic features"""
|
||||
yield Mnemonic(idc.print_insn_mnem(ih.inner.ea)), ih.address
|
||||
|
||||
|
||||
def extract_insn_obfs_call_plus_5_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""
|
||||
parse call $+5 instruction from the given instruction.
|
||||
"""
|
||||
@@ -416,7 +409,7 @@ def extract_insn_obfs_call_plus_5_characteristic_features(
|
||||
|
||||
def extract_insn_peb_access_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction peb access
|
||||
|
||||
fs:[0x30] on x86, gs:[0x60] on x64
|
||||
@@ -444,7 +437,7 @@ def extract_insn_peb_access_characteristic_features(
|
||||
|
||||
def extract_insn_segment_access_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""parse instruction fs or gs access
|
||||
|
||||
TODO:
|
||||
@@ -473,7 +466,7 @@ def extract_insn_segment_access_features(
|
||||
|
||||
def extract_insn_cross_section_cflow(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""inspect the instruction for a CALL or JMP that crosses section boundaries"""
|
||||
insn: idaapi.insn_t = ih.inner
|
||||
|
||||
@@ -489,7 +482,7 @@ def extract_insn_cross_section_cflow(
|
||||
yield Characteristic("cross section flow"), ih.address
|
||||
|
||||
|
||||
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract functions calls from features
|
||||
|
||||
most relevant at the function scope, however, its most efficient to extract at the instruction scope
|
||||
@@ -503,7 +496,7 @@ def extract_function_calls_from(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandl
|
||||
|
||||
def extract_function_indirect_call_characteristic_features(
|
||||
fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle
|
||||
) -> Iterator[tuple[Feature, Address]]:
|
||||
) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
|
||||
does not include calls like => call ds:dword_ABD4974
|
||||
|
||||
@@ -516,7 +509,7 @@ def extract_function_indirect_call_characteristic_features(
|
||||
yield Characteristic("indirect call"), ih.address
|
||||
|
||||
|
||||
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[tuple[Feature, Address]]:
|
||||
def extract_features(f: FunctionHandle, bbh: BBHandle, insn: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
|
||||
"""extract instruction features"""
|
||||
for inst_handler in INSTRUCTION_HANDLERS:
|
||||
for feature, ea in inst_handler(f, bbh, insn):
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import networkx
|
||||
from networkx.algorithms.components import strongly_connected_components
|
||||
|
||||
@@ -1,20 +1,15 @@
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Copyright (C) 2022 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Union, TypeAlias
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
from typing import Dict, List, Tuple, Union
|
||||
from dataclasses import dataclass
|
||||
|
||||
from typing_extensions import TypeAlias
|
||||
|
||||
from capa.features.common import Feature
|
||||
from capa.features.address import NO_ADDRESS, Address, ThreadAddress, ProcessAddress, DynamicCallAddress
|
||||
from capa.features.extractors.base_extractor import (
|
||||
@@ -32,19 +27,19 @@ from capa.features.extractors.base_extractor import (
|
||||
|
||||
@dataclass
|
||||
class InstructionFeatures:
|
||||
features: list[tuple[Address, Feature]]
|
||||
features: List[Tuple[Address, Feature]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class BasicBlockFeatures:
|
||||
features: list[tuple[Address, Feature]]
|
||||
instructions: dict[Address, InstructionFeatures]
|
||||
features: List[Tuple[Address, Feature]]
|
||||
instructions: Dict[Address, InstructionFeatures]
|
||||
|
||||
|
||||
@dataclass
|
||||
class FunctionFeatures:
|
||||
features: list[tuple[Address, Feature]]
|
||||
basic_blocks: dict[Address, BasicBlockFeatures]
|
||||
features: List[Tuple[Address, Feature]]
|
||||
basic_blocks: Dict[Address, BasicBlockFeatures]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -57,9 +52,9 @@ class NullStaticFeatureExtractor(StaticFeatureExtractor):
|
||||
|
||||
base_address: Address
|
||||
sample_hashes: SampleHashes
|
||||
global_features: list[Feature]
|
||||
file_features: list[tuple[Address, Feature]]
|
||||
functions: dict[Address, FunctionFeatures]
|
||||
global_features: List[Feature]
|
||||
file_features: List[Tuple[Address, Feature]]
|
||||
functions: Dict[Address, FunctionFeatures]
|
||||
|
||||
def get_base_address(self):
|
||||
return self.base_address
|
||||
@@ -103,19 +98,19 @@ class NullStaticFeatureExtractor(StaticFeatureExtractor):
|
||||
@dataclass
|
||||
class CallFeatures:
|
||||
name: str
|
||||
features: list[tuple[Address, Feature]]
|
||||
features: List[Tuple[Address, Feature]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ThreadFeatures:
|
||||
features: list[tuple[Address, Feature]]
|
||||
calls: dict[Address, CallFeatures]
|
||||
features: List[Tuple[Address, Feature]]
|
||||
calls: Dict[Address, CallFeatures]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessFeatures:
|
||||
features: list[tuple[Address, Feature]]
|
||||
threads: dict[Address, ThreadFeatures]
|
||||
features: List[Tuple[Address, Feature]]
|
||||
threads: Dict[Address, ThreadFeatures]
|
||||
name: str
|
||||
|
||||
|
||||
@@ -123,9 +118,9 @@ class ProcessFeatures:
|
||||
class NullDynamicFeatureExtractor(DynamicFeatureExtractor):
|
||||
base_address: Address
|
||||
sample_hashes: SampleHashes
|
||||
global_features: list[Feature]
|
||||
file_features: list[tuple[Address, Feature]]
|
||||
processes: dict[Address, ProcessFeatures]
|
||||
global_features: List[Feature]
|
||||
file_features: List[Tuple[Address, Feature]]
|
||||
processes: Dict[Address, ProcessFeatures]
|
||||
|
||||
def extract_global_features(self):
|
||||
for feature in self.global_features:
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# Copyright 2021 Google LLC
|
||||
#
|
||||
# Copyright (C) 2021 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
@@ -115,7 +108,10 @@ def extract_file_function_names(**kwargs):
|
||||
"""
|
||||
extract the names of statically-linked library functions.
|
||||
"""
|
||||
yield from []
|
||||
if False:
|
||||
# using a `yield` here to force this to be a generator, not function.
|
||||
yield NotImplementedError("pefile doesn't have library matching")
|
||||
return
|
||||
|
||||
|
||||
def extract_file_os(**kwargs):
|
||||
@@ -152,11 +148,11 @@ def extract_file_features(pe, buf):
|
||||
buf: the raw sample bytes
|
||||
|
||||
yields:
|
||||
tuple[Feature, VA]: a feature and its location.
|
||||
Tuple[Feature, VA]: a feature and its location.
|
||||
"""
|
||||
|
||||
for file_handler in FILE_HANDLERS:
|
||||
# file_handler: type: (pe, bytes) -> Iterable[tuple[Feature, Address]]
|
||||
# file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
|
||||
for feature, va in file_handler(pe=pe, buf=buf): # type: ignore
|
||||
yield feature, va
|
||||
|
||||
@@ -181,10 +177,10 @@ def extract_global_features(pe, buf):
|
||||
buf: the raw sample bytes
|
||||
|
||||
yields:
|
||||
tuple[Feature, VA]: a feature and its location.
|
||||
Tuple[Feature, VA]: a feature and its location.
|
||||
"""
|
||||
for handler in GLOBAL_HANDLERS:
|
||||
# file_handler: type: (pe, bytes) -> Iterable[tuple[Feature, Address]]
|
||||
# file_handler: type: (pe, bytes) -> Iterable[Tuple[Feature, Address]]
|
||||
for feature, va in handler(pe=pe, buf=buf): # type: ignore
|
||||
yield feature, va
|
||||
|
||||
|
||||
@@ -1,98 +1,51 @@
|
||||
# strings code from FLOSS, https://github.com/mandiant/flare-floss
|
||||
#
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Copyright (C) 2020 Mandiant, Inc. All Rights Reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at: [package root]/LICENSE.txt
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
import re
|
||||
import string
|
||||
import contextlib
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Iterator
|
||||
from collections import namedtuple
|
||||
|
||||
ASCII_BYTE = r" !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t".encode(
|
||||
"ascii"
|
||||
)
|
||||
ASCII_RE_4 = re.compile(b"([%s]{%d,})" % (ASCII_BYTE, 4))
|
||||
UNICODE_RE_4 = re.compile(b"((?:[%s]\x00){%d,})" % (ASCII_BYTE, 4))
|
||||
REPEATS = {ord("A"), 0x00, 0xFE, 0xFF}
|
||||
REPEATS = [b"A", b"\x00", b"\xfe", b"\xff"]
|
||||
SLICE_SIZE = 4096
|
||||
PRINTABLE_CHAR_SET = set(string.printable)
|
||||
|
||||
String = namedtuple("String", ["s", "offset"])
|
||||
|
||||
|
||||
@dataclass
|
||||
class String:
|
||||
s: str
|
||||
offset: int
|
||||
|
||||
|
||||
def buf_filled_with(buf: bytes, character: int) -> bool:
|
||||
"""Check if the given buffer is filled with the given character, repeatedly.
|
||||
|
||||
Args:
|
||||
buf: The bytes buffer to check
|
||||
character: The byte value (0-255) to check for
|
||||
|
||||
Returns:
|
||||
True if all bytes in the buffer match the character, False otherwise.
|
||||
The empty buffer contains no bytes, therefore always returns False.
|
||||
"""
|
||||
if not buf:
|
||||
return False
|
||||
|
||||
if not (0 <= character <= 255):
|
||||
raise ValueError(f"Character value {character} outside valid byte range (0-255)")
|
||||
|
||||
if len(buf) < SLICE_SIZE:
|
||||
return all(b == character for b in buf)
|
||||
|
||||
# single big allocation, re-used each loop
|
||||
dupe_chunk = bytes(character) * SLICE_SIZE
|
||||
|
||||
def buf_filled_with(buf, character):
|
||||
dupe_chunk = character * SLICE_SIZE
|
||||
for offset in range(0, len(buf), SLICE_SIZE):
|
||||
# bytes objects are immutable, so the slices share the underlying array,
|
||||
# and therefore this is cheap.
|
||||
current_chunk = buf[offset : offset + SLICE_SIZE]
|
||||
|
||||
if len(current_chunk) == SLICE_SIZE:
|
||||
# chunk-aligned comparison
|
||||
|
||||
if dupe_chunk != current_chunk:
|
||||
return False
|
||||
|
||||
else:
|
||||
# last loop, final chunk size is not aligned
|
||||
if not all(b == character for b in current_chunk):
|
||||
return False
|
||||
|
||||
new_chunk = buf[offset : offset + SLICE_SIZE]
|
||||
if dupe_chunk[: len(new_chunk)] != new_chunk:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def extract_ascii_strings(buf: bytes, n: int = 4) -> Iterator[String]:
|
||||
def extract_ascii_strings(buf, n=4):
|
||||
"""
|
||||
Extract ASCII strings from the given binary data.
|
||||
|
||||
Params:
|
||||
buf: the bytes from which to extract strings
|
||||
n: minimum string length
|
||||
:param buf: A bytestring.
|
||||
:type buf: str
|
||||
:param n: The minimum length of strings to extract.
|
||||
:type n: int
|
||||
:rtype: Sequence[String]
|
||||
"""
|
||||
|
||||
if not buf:
|
||||
return
|
||||
|
||||
if n < 1:
|
||||
raise ValueError("minimum string length must be positive")
|
||||
|
||||
if (buf[0] in REPEATS) and buf_filled_with(buf, buf[0]):
|
||||
return
|
||||
|
||||
@@ -106,21 +59,20 @@ def extract_ascii_strings(buf: bytes, n: int = 4) -> Iterator[String]:
|
||||
yield String(match.group().decode("ascii"), match.start())
|
||||
|
||||
|
||||
def extract_unicode_strings(buf: bytes, n: int = 4) -> Iterator[String]:
|
||||
def extract_unicode_strings(buf, n=4):
|
||||
"""
|
||||
Extract naive UTF-16 strings from the given binary data.
|
||||
|
||||
Params:
|
||||
buf: the bytes from which to extract strings
|
||||
n: minimum string length
|
||||
:param buf: A bytestring.
|
||||
:type buf: str
|
||||
:param n: The minimum length of strings to extract.
|
||||
:type n: int
|
||||
:rtype: Sequence[String]
|
||||
"""
|
||||
|
||||
if not buf:
|
||||
return
|
||||
|
||||
if n < 1:
|
||||
raise ValueError("minimum string length must be positive")
|
||||
|
||||
if (buf[0] in REPEATS) and buf_filled_with(buf, buf[0]):
|
||||
return
|
||||
|
||||
@@ -132,7 +84,3 @@ def extract_unicode_strings(buf: bytes, n: int = 4) -> Iterator[String]:
|
||||
for match in r.finditer(buf):
|
||||
with contextlib.suppress(UnicodeDecodeError):
|
||||
yield String(match.group().decode("utf-16"), match.start())
|
||||
|
||||
|
||||
def is_printable_str(s: str) -> bool:
|
||||
return set(s).issubset(PRINTABLE_CHAR_SET)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user