adding capa quickstart reference (#1802)

binja: add support for forwarded exports

.devcontainer/devcontainer.json

@@ -41,7 +41,7 @@
 	// "forwardPorts": [],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "git submodule update --init && pip3 install --user -e .[dev]",
+	"postCreateCommand": "git submodule update --init && pip3 install --user -e .[dev] && pre-commit install",
 
 	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",

.github/mypy/mypy.ini

@@ -86,3 +86,6 @@ ignore_missing_imports = True
 
 [mypy-netnode.*]
 ignore_missing_imports = True
+
+[mypy-ghidra.*]
+ignore_missing_imports = True

.github/workflows/publish.yml

@@ -33,12 +33,6 @@ jobs:
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           path: dist/*
-      - name: upload package to GH Release
-        uses: svenstaro/upload-release-action@2728235f7dc9ff598bd86ce3c274b74f802d2208 # v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN}}
-          file: dist/*
-          tag: ${{ github.ref }}
       - name: publish package
         uses: pypa/gh-action-pypi-publish@f5622bde02b04381239da3573277701ceca8f6a0  # release/v1
         with:

.github/workflows/tests.yml

@@ -139,3 +139,62 @@ jobs:
       env:
         BN_LICENSE: ${{ secrets.BN_LICENSE }}
       run: pytest -v tests/test_binja_features.py  # explicitly refer to the binja tests for performance. other tests run above.
+
+  ghidra-tests:
+    name: Ghidra tests for ${{ matrix.python-version }}
+    runs-on: ubuntu-20.04
+    needs: [code_style, rule_linter]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.11"]
+        java-version: ["17"]
+        gradle-version: ["7.3"]
+        ghidra-version: ["10.3"]
+        public-version: ["PUBLIC_20230510"] # for ghidra releases
+        jep-version: ["4.1.1"]
+        ghidrathon-version: ["3.0.0"]
+    steps:
+    - name: Checkout capa with submodules
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      with:
+        submodules: true
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Set up Java ${{ matrix.java-version }}
+      uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2 # v3
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java-version }}
+    - name: Set up Gradle ${{ matrix.gradle-version }} 
+      uses: gradle/gradle-build-action@40b6781dcdec2762ad36556682ac74e31030cfe2 # v2.5.1
+      with:
+        gradle-version: ${{ matrix.gradle-version }}
+    - name: Install Jep ${{ matrix.jep-version }} 
+      run : pip install jep==${{ matrix.jep-version }}
+    - name: Install Ghidra ${{ matrix.ghidra-version }} 
+      run: |
+        mkdir ./.github/ghidra
+        wget "https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_${{ matrix.ghidra-version }}_build/ghidra_${{ matrix.ghidra-version }}_${{ matrix.public-version }}.zip" -O ./.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip
+        unzip .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC.zip -d .github/ghidra/
+    - name: Install Ghidrathon
+      run : |
+        mkdir ./.github/ghidrathon
+        curl -o ./.github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip "https://codeload.github.com/mandiant/Ghidrathon/zip/refs/tags/v${{ matrix.ghidrathon-version }}"
+        unzip .github/ghidrathon/ghidrathon-${{ matrix.ghidrathon-version }}.zip -d .github/ghidrathon/
+        gradle -p ./.github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/ -PGHIDRA_INSTALL_DIR=$(pwd)/.github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC
+        unzip .github/ghidrathon/Ghidrathon-${{ matrix.ghidrathon-version }}/dist/*.zip -d .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/Ghidra/Extensions
+    - name: Install pyyaml
+      run: sudo apt-get install -y libyaml-dev
+    - name: Install capa
+      run: pip install -e .[dev] 
+    - name: Run tests
+      run: | 
+        mkdir ./.github/ghidra/project
+        .github/ghidra/ghidra_${{ matrix.ghidra-version }}_PUBLIC/support/analyzeHeadless .github/ghidra/project ghidra_test -Import ./tests/data/mimikatz.exe_ -ScriptPath ./tests/ -PostScript test_ghidra_features.py > ../output.log
+        cat ../output.log
+        exit_code=$(cat ../output.log | grep exit | awk '{print $NF}')
+        exit $exit_code
+ 

CHANGELOG.md

@@ -3,22 +3,72 @@
 ## master (unreleased)
 
 ### New Features
+- ghidra: add Ghidra feature extractor and supporting code #1770 @colton-gabertan
+- ghidra: add entry script helping users run capa against a loaded Ghidra database #1767 @mike-hunhoff
+- binja: add support for forwarded exports #1646 @xusheng6
+- binja: add support for symtab names #1504 @xusheng6
 
 ### Breaking Changes
 
-### New Rules (0)
+### New Rules (1)
 
+- nursery/get-ntoskrnl-base-address @mr-tz
 -
 
 ### Bug Fixes
+- ghidra: fix ints_to_bytes performance #1761 @mike-hunhoff
+- binja: improve function call site detection @xusheng6
+- binja: use binaryninja.load to open files @xusheng6
+- binja: bump binja version to 3.5 #1789 @xusheng6
 
 ### capa explorer IDA Pro plugin
 
 ### Development
 
 ### Raw diffs
-- [capa v6.0.0...master](https://github.com/mandiant/capa/compare/v6.0.0...master)
-- [capa-rules v6.0.0...master](https://github.com/mandiant/capa-rules/compare/v6.0.0...master)
+- [capa v6.1.0...master](https://github.com/mandiant/capa/compare/v6.1.0...master)
+- [capa-rules v6.1.0...master](https://github.com/mandiant/capa-rules/compare/v6.1.0...master)
+
+## v6.1.0
+
+capa v6.1.0 is a bug fix release, most notably fixing unhandled exceptions in the capa explorer IDA Pro plugin.
+@Aayush-Goel-04 put a lot of effort into improving code quality and adding a script for rule authors.
+The script shows which features are present in a sample but not referenced by any existing rule.
+You could use this script to find opportunities for new rules.
+
+Speaking of new rules, we have eight additions, coming from Ronnie, Jakub, Moritz, Ervin, and still@teamt5.org!
+
+### New Features
+- ELF: implement import and export name extractor #1607 #1608 @Aayush-Goel-04
+- bump pydantic from 1.10.9 to 2.1.1 #1582 @Aayush-Goel-04
+- develop script to highlight features not used during matching #331 @Aayush-Goel-04
+
+### New Rules (8)
+
+- executable/pe/export/forwarded-export ronnie.salomonsen@mandiant.com
+- host-interaction/bootloader/get-uefi-variable jakub.jozwiak@mandiant.com
+- host-interaction/bootloader/set-uefi-variable jakub.jozwiak@mandiant.com
+- nursery/enumerate-device-drivers-on-linux @mr-tz
+- anti-analysis/anti-vm/vm-detection/check-for-foreground-window-switch ervin.ocampo@mandiant.com
+- linking/static/sqlite3/linked-against-cppsqlite3 still@teamt5.org
+- linking/static/sqlite3/linked-against-sqlite3 still@teamt5.org
+
+### Bug Fixes
+
+- rules: fix forwarded export characteristic #1656 @RonnieSalomonsen
+- Binary Ninja: Fix stack string detection #1473 @xusheng6
+- linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin 
+- OS: detect Android ELF files #1705 @williballenthin
+- ELF: fix parsing of symtab #1704 @williballenthin
+- result document: don't use deprecated pydantic functions #1718 @williballenthin
+- pytest: don't mark IDA tests as pytest tests #1719 @williballenthin
+
+### capa explorer IDA Pro plugin
+- fix unhandled exception when resolving rule path #1693 @mike-hunhoff
+
+### Raw diffs
+- [capa v6.0.0...v6.1.0](https://github.com/mandiant/capa/compare/v6.0.0...v6.1.0)
+- [capa-rules v6.0.0...v6.1.0](https://github.com/mandiant/capa-rules/compare/v6.0.0...v6.1.0)
 
 ## v6.0.0
 
@@ -82,6 +132,7 @@ For those that use capa as a library, we've introduced some limited breaking cha
 - output: don't leave behind traces of progress bar @williballenthin
 - import-to-ida: fix bug introduced with JSON report changes in v5 #1584 @williballenthin
 - main: don't show spinner when emitting debug messages #1636 @williballenthin
+- rules: add forwarded export characteristics to rule syntax file scope #1653 @RonnieSalomonsen
 
 ### capa explorer IDA Pro plugin
 
@@ -95,8 +146,8 @@ For those that use capa as a library, we've introduced some limited breaking cha
 
 
 ### Raw diffs
-- [capa v5.1.0...v6.0.0](https://github.com/mandiant/capa/compare/v5.1.0...v6.0.0a1)
-- [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0a1)
+- [capa v5.1.0...v6.0.0](https://github.com/mandiant/capa/compare/v5.1.0...v6.0.0)
+- [capa-rules v5.1.0...v6.0.0](https://github.com/mandiant/capa-rules/compare/v5.1.0...v6.0.0)
 
 ## v5.1.0
 capa version 5.1.0 adds a Protocol Buffers (protobuf) format for result documents. Additionally, the [Vector35](https://vector35.com/) team contributed a new feature extractor using Binary Ninja. Other new features are a new CLI flag to override the detected operating system, functionality to read and render existing result documents, and a output color format that's easier to read.

README.md

@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-823-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-832-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
@@ -170,6 +170,8 @@ capa explorer helps you identify interesting areas of a program and build new ca
 
 ![capa + IDA Pro integration](https://github.com/mandiant/capa/blob/master/doc/img/explorer_expanded.png)
 
+If you use Ghidra, you can use the Python 3 [Ghidra feature extractor](/capa/ghidra/). This integration enables capa to extract features directly from your Ghidra database, which can help you identify capabilities in programs that you analyze using Ghidra.
+
 # further information
 ## capa
 - [Installation](https://github.com/mandiant/capa/blob/master/doc/installation.md)

capa/features/common.py

@@ -136,8 +136,8 @@ class Feature(abc.ABC):  # noqa: B024
         import capa.features.freeze.features
 
         return (
-            capa.features.freeze.features.feature_from_capa(self).json()
-            < capa.features.freeze.features.feature_from_capa(other).json()
+            capa.features.freeze.features.feature_from_capa(self).model_dump_json()
+            < capa.features.freeze.features.feature_from_capa(other).model_dump_json()
         )
 
     def get_name_str(self) -> str:

capa/features/extractors/binja/basicblock.py

@@ -75,10 +75,11 @@ def get_stack_string_len(f: Function, il: MediumLevelILInstruction) -> int:
         return 0
 
     dest = il.params[0]
-    if dest.operation != MediumLevelILOperation.MLIL_ADDRESS_OF:
+    if dest.operation in [MediumLevelILOperation.MLIL_ADDRESS_OF, MediumLevelILOperation.MLIL_VAR]:
+        var = dest.src
+    else:
         return 0
 
-    var = dest.src
     if var.source_type != VariableSourceType.StackVariableSourceType:
         return 0
 

capa/features/extractors/binja/file.py

@@ -17,7 +17,7 @@ import capa.features.extractors.strings
 from capa.features.file import Export, Import, Section, FunctionName
 from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
 from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
-from capa.features.extractors.binja.helpers import unmangle_c_name
+from capa.features.extractors.binja.helpers import read_c_string, unmangle_c_name
 
 
 def check_segment_for_pe(bv: BinaryView, seg: Segment) -> Iterator[Tuple[int, int]]:
@@ -82,6 +82,24 @@ def extract_file_export_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address
             if name != unmangled_name:
                 yield Export(unmangled_name), AbsoluteVirtualAddress(sym.address)
 
+    for sym in bv.get_symbols_of_type(SymbolType.DataSymbol):
+        if sym.binding not in [SymbolBinding.GlobalBinding]:
+            continue
+
+        name = sym.short_name
+        if not name.startswith("__forwarder_name"):
+            continue
+
+        # Due to https://github.com/Vector35/binaryninja-api/issues/4641, in binja version 3.5, the symbol's name
+        # does not contain the DLL name. As a workaround, we read the C string at the symbol's address, which contains
+        # both the DLL name and the function name.
+        # Once the above issue is closed in the next binjs stable release, we can update the code here to use the
+        # symbol name directly.
+        name = read_c_string(bv, sym.address, 1024)
+        forwarded_name = capa.features.extractors.helpers.reformat_forwarded_export_name(name)
+        yield Export(forwarded_name), AbsoluteVirtualAddress(sym.address)
+        yield Characteristic("forwarded export"), AbsoluteVirtualAddress(sym.address)
+
 
 def extract_file_import_names(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:
     """extract function imports
@@ -125,15 +143,17 @@ def extract_file_function_names(bv: BinaryView) -> Iterator[Tuple[Feature, Addre
     """
     for sym_name in bv.symbols:
         for sym in bv.symbols[sym_name]:
-            if sym.type == SymbolType.LibraryFunctionSymbol:
-                name = sym.short_name
-                yield FunctionName(name), sym.address
-                if name.startswith("_"):
-                    # some linkers may prefix linked routines with a `_` to avoid name collisions.
-                    # extract features for both the mangled and un-mangled representations.
-                    # e.g. `_fwrite` -> `fwrite`
-                    # see: https://stackoverflow.com/a/2628384/87207
-                    yield FunctionName(name[1:]), sym.address
+            if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
+                continue
+
+            name = sym.short_name
+            yield FunctionName(name), sym.address
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                yield FunctionName(name[1:]), sym.address
 
 
 def extract_file_format(bv: BinaryView) -> Iterator[Tuple[Feature, Address]]:

capa/features/extractors/binja/function.py

@@ -7,8 +7,9 @@
 # See the License for the specific language governing permissions and limitations under the License.
 from typing import Tuple, Iterator
 
-from binaryninja import Function, BinaryView, LowLevelILOperation
+from binaryninja import Function, BinaryView, SymbolType, RegisterValueType, LowLevelILOperation
 
+from capa.features.file import FunctionName
 from capa.features.common import Feature, Characteristic
 from capa.features.address import Address, AbsoluteVirtualAddress
 from capa.features.extractors import loops
@@ -23,13 +24,27 @@ def extract_function_calls_to(fh: FunctionHandle):
         # Everything that is a code reference to the current function is considered a caller, which actually includes
         # many other references that are NOT a caller. For example, an instruction `push function_start` will also be
         # considered a caller to the function
-        if caller.llil is not None and caller.llil.operation in [
+        llil = caller.llil
+        if (llil is None) or llil.operation not in [
             LowLevelILOperation.LLIL_CALL,
             LowLevelILOperation.LLIL_CALL_STACK_ADJUST,
             LowLevelILOperation.LLIL_JUMP,
             LowLevelILOperation.LLIL_TAILCALL,
         ]:
-            yield Characteristic("calls to"), AbsoluteVirtualAddress(caller.address)
+            continue
+
+        if llil.dest.value.type not in [
+            RegisterValueType.ImportedAddressValue,
+            RegisterValueType.ConstantValue,
+            RegisterValueType.ConstantPointerValue,
+        ]:
+            continue
+
+        address = llil.dest.value.value
+        if address != func.start:
+            continue
+
+        yield Characteristic("calls to"), AbsoluteVirtualAddress(caller.address)
 
 
 def extract_function_loop(fh: FunctionHandle):
@@ -59,10 +74,31 @@ def extract_recursive_call(fh: FunctionHandle):
             yield Characteristic("recursive call"), fh.address
 
 
+def extract_function_name(fh: FunctionHandle):
+    """extract function names (e.g., symtab names)"""
+    func: Function = fh.inner
+    bv: BinaryView = func.view
+    if bv is None:
+        return
+
+    for sym in bv.get_symbols(func.start):
+        if sym.type not in [SymbolType.LibraryFunctionSymbol, SymbolType.FunctionSymbol]:
+            continue
+
+        name = sym.short_name
+        yield FunctionName(name), sym.address
+        if name.startswith("_"):
+            # some linkers may prefix linked routines with a `_` to avoid name collisions.
+            # extract features for both the mangled and un-mangled representations.
+            # e.g. `_fwrite` -> `fwrite`
+            # see: https://stackoverflow.com/a/2628384/87207
+            yield FunctionName(name[1:]), sym.address
+
+
 def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
     for func_handler in FUNCTION_HANDLERS:
         for feature, addr in func_handler(fh):
             yield feature, addr
 
 
-FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call, extract_function_name)

capa/features/extractors/binja/helpers.py

@@ -9,7 +9,7 @@ import re
 from typing import List, Callable
 from dataclasses import dataclass
 
-from binaryninja import LowLevelILInstruction
+from binaryninja import BinaryView, LowLevelILInstruction
 from binaryninja.architecture import InstructionTextToken
 
 
@@ -51,3 +51,19 @@ def unmangle_c_name(name: str) -> str:
             return match.group(1)
 
     return name
+
+
+def read_c_string(bv: BinaryView, offset: int, max_len: int) -> str:
+    s: List[str] = []
+    while len(s) < max_len:
+        try:
+            c = bv.read(offset + len(s), 1)[0]
+        except Exception:
+            break
+
+        if c == 0:
+            break
+
+        s.append(chr(c))
+
+    return "".join(s)

capa/features/extractors/binja/insn.py

@@ -94,28 +94,32 @@ def extract_insn_api_features(fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle)
                 candidate_addrs.append(stub_addr)
 
             for address in candidate_addrs:
-                sym = func.view.get_symbol_at(address)
-                if sym is None or sym.type not in [SymbolType.ImportAddressSymbol, SymbolType.ImportedFunctionSymbol]:
-                    continue
+                for sym in func.view.get_symbols(address):
+                    if sym is None or sym.type not in [
+                        SymbolType.ImportAddressSymbol,
+                        SymbolType.ImportedFunctionSymbol,
+                        SymbolType.FunctionSymbol,
+                    ]:
+                        continue
 
-                sym_name = sym.short_name
+                    sym_name = sym.short_name
 
-                lib_name = ""
-                import_lib = bv.lookup_imported_object_library(sym.address)
-                if import_lib is not None:
-                    lib_name = import_lib[0].name
-                    if lib_name.endswith(".dll"):
-                        lib_name = lib_name[:-4]
-                    elif lib_name.endswith(".so"):
-                        lib_name = lib_name[:-3]
+                    lib_name = ""
+                    import_lib = bv.lookup_imported_object_library(sym.address)
+                    if import_lib is not None:
+                        lib_name = import_lib[0].name
+                        if lib_name.endswith(".dll"):
+                            lib_name = lib_name[:-4]
+                        elif lib_name.endswith(".so"):
+                            lib_name = lib_name[:-3]
 
-                for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name):
-                    yield API(name), ih.address
-
-                if sym_name.startswith("_"):
-                    for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name[1:]):
+                    for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name):
                         yield API(name), ih.address
 
+                    if sym_name.startswith("_"):
+                        for name in capa.features.extractors.helpers.generate_symbols(lib_name, sym_name[1:]):
+                            yield API(name), ih.address
+
 
 def extract_insn_number_features(
     fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle

capa/features/extractors/elf.py

@@ -13,6 +13,8 @@ from enum import Enum
 from typing import Set, Dict, List, Tuple, BinaryIO, Iterator, Optional
 from dataclasses import dataclass
 
+import Elf  # from vivisect
+
 logger = logging.getLogger(__name__)
 
 
@@ -54,6 +56,7 @@ class OS(str, Enum):
     CLOUD = "cloud"
     SYLLABLE = "syllable"
     NACL = "nacl"
+    ANDROID = "android"
 
 
 # via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658
@@ -709,17 +712,17 @@ class SymTab:
         yield from self.symbols
 
     @classmethod
-    def from_Elf(cls, ElfBinary) -> Optional["SymTab"]:
-        endian = "<" if ElfBinary.getEndian() == 0 else ">"
-        bitness = ElfBinary.bits
+    def from_viv(cls, elf: Elf.Elf) -> Optional["SymTab"]:
+        endian = "<" if elf.getEndian() == 0 else ">"
+        bitness = elf.bits
 
         SHT_SYMTAB = 0x2
-        for section in ElfBinary.sections:
-            if section.sh_info & SHT_SYMTAB:
-                strtab_section = ElfBinary.sections[section.sh_link]
-                sh_symtab = Shdr.from_viv(section, ElfBinary.readAtOffset(section.sh_offset, section.sh_size))
+        for section in elf.sections:
+            if section.sh_type == SHT_SYMTAB:
+                strtab_section = elf.sections[section.sh_link]
+                sh_symtab = Shdr.from_viv(section, elf.readAtOffset(section.sh_offset, section.sh_size))
                 sh_strtab = Shdr.from_viv(
-                    strtab_section, ElfBinary.readAtOffset(strtab_section.sh_offset, strtab_section.sh_size)
+                    strtab_section, elf.readAtOffset(strtab_section.sh_offset, strtab_section.sh_size)
                 )
 
         try:
@@ -764,6 +767,11 @@ def guess_os_from_ph_notes(elf: ELF) -> Optional[OS]:
         elif note.name == "FreeBSD":
             logger.debug("note owner: %s", "FREEBSD")
             return OS.FREEBSD
+        elif note.name == "Android":
+            logger.debug("note owner: %s", "Android")
+            # see the following for parsing the structure:
+            # https://android.googlesource.com/platform/ndk/+/master/parse_elfnote.py
+            return OS.ANDROID
         elif note.name == "GNU":
             abi_tag = note.abi_tag
             if abi_tag:
@@ -855,6 +863,8 @@ def guess_os_from_needed_dependencies(elf: ELF) -> Optional[OS]:
             return OS.HURD
         if needed.startswith("libhurduser.so"):
             return OS.HURD
+        if needed.startswith("libandroid.so"):
+            return OS.ANDROID
 
     return None
 
@@ -888,7 +898,7 @@ def guess_os_from_symtab(elf: ELF) -> Optional[OS]:
 
 def detect_elf_os(f) -> str:
     """
-    f: type Union[BinaryIO, IDAIO]
+    f: type Union[BinaryIO, IDAIO, GHIDRAIO]
     """
     try:
         elf = ELF(f)

capa/features/extractors/elffile.py

@@ -11,9 +11,10 @@ from typing import Tuple, Iterator
 from pathlib import Path
 
 from elftools.elf.elffile import ELFFile, SymbolTableSection
+from elftools.elf.relocation import RelocationSection
 
 import capa.features.extractors.common
-from capa.features.file import Import, Section
+from capa.features.file import Export, Import, Section
 from capa.features.common import OS, FORMAT_ELF, Arch, Format, Feature
 from capa.features.address import NO_ADDRESS, FileOffsetAddress, AbsoluteVirtualAddress
 from capa.features.extractors.base_extractor import FeatureExtractor
@@ -21,11 +22,8 @@ from capa.features.extractors.base_extractor import FeatureExtractor
 logger = logging.getLogger(__name__)
 
 
-def extract_file_import_names(elf, **kwargs):
-    # see https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/scripts/readelf.py#L372
-    symbol_tables = [(idx, s) for idx, s in enumerate(elf.iter_sections()) if isinstance(s, SymbolTableSection)]
-
-    for _, section in symbol_tables:
+def extract_file_export_names(elf: ELFFile, **kwargs):
+    for section in elf.iter_sections():
         if not isinstance(section, SymbolTableSection):
             continue
 
@@ -35,14 +33,64 @@ def extract_file_import_names(elf, **kwargs):
 
         logger.debug("Symbol table '%s' contains %s entries:", section.name, section.num_symbols())
 
+        for symbol in section.iter_symbols():
+            # The following conditions are based on the following article
+            # http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
+            if not symbol.name:
+                continue
+            if symbol.entry.st_info.type not in ["STT_FUNC", "STT_OBJECT", "STT_IFUNC"]:
+                continue
+            if symbol.entry.st_value == 0:
+                continue
+            if symbol.entry.st_shndx == "SHN_UNDEF":
+                continue
+
+            yield Export(symbol.name), AbsoluteVirtualAddress(symbol.entry.st_value)
+
+
+def extract_file_import_names(elf: ELFFile, **kwargs):
+    # Create a dictionary to store symbol names by their index
+    symbol_names = {}
+
+    # Extract symbol names and store them in the dictionary
+    for section in elf.iter_sections():
+        if not isinstance(section, SymbolTableSection):
+            continue
+
         for _, symbol in enumerate(section.iter_symbols()):
-            if symbol.name and symbol.entry.st_info.type == "STT_FUNC":
-                # TODO(williballenthin): extract symbol address
-                # https://github.com/mandiant/capa/issues/1608
-                yield Import(symbol.name), FileOffsetAddress(0x0)
+            # The following conditions are based on the following article
+            # http://www.m4b.io/elf/export/binary/analysis/2015/05/25/what-is-an-elf-export.html
+            if not symbol.name:
+                continue
+            if symbol.entry.st_info.type not in ["STT_FUNC", "STT_OBJECT", "STT_IFUNC"]:
+                continue
+            if symbol.entry.st_value != 0:
+                continue
+            if symbol.entry.st_shndx != "SHN_UNDEF":
+                continue
+            if symbol.entry.st_name == 0:
+                continue
+
+            symbol_names[_] = symbol.name
+
+    for section in elf.iter_sections():
+        if not isinstance(section, RelocationSection):
+            continue
+
+        if section["sh_entsize"] == 0:
+            logger.debug("Symbol table '%s' has a sh_entsize of zero!", section.name)
+            continue
+
+        logger.debug("Symbol table '%s' contains %s entries:", section.name, section.num_relocations())
+
+        for relocation in section.iter_relocations():
+            # Extract the symbol name from the symbol table using the symbol index in the relocation
+            if relocation["r_info_sym"] not in symbol_names:
+                continue
+            yield Import(symbol_names[relocation["r_info_sym"]]), FileOffsetAddress(relocation["r_offset"])
 
 
-def extract_file_section_names(elf, **kwargs):
+def extract_file_section_names(elf: ELFFile, **kwargs):
     for section in elf.iter_sections():
         if section.name:
             yield Section(section.name), AbsoluteVirtualAddress(section.header.sh_addr)
@@ -54,7 +102,7 @@ def extract_file_strings(buf, **kwargs):
     yield from capa.features.extractors.common.extract_file_strings(buf)
 
 
-def extract_file_os(elf, buf, **kwargs):
+def extract_file_os(elf: ELFFile, buf, **kwargs):
     # our current approach does not always get an OS value, e.g. for packed samples
     # for file limitation purposes, we're more lax here
     try:
@@ -68,7 +116,7 @@ def extract_file_format(**kwargs):
     yield Format(FORMAT_ELF), NO_ADDRESS
 
 
-def extract_file_arch(elf, **kwargs):
+def extract_file_arch(elf: ELFFile, **kwargs):
     arch = elf.get_machine_arch()
     if arch == "x86":
         yield Arch("i386"), NO_ADDRESS
@@ -85,8 +133,7 @@ def extract_file_features(elf: ELFFile, buf: bytes) -> Iterator[Tuple[Feature, i
 
 
 FILE_HANDLERS = (
-    # TODO(williballenthin): implement extract_file_export_names
-    # https://github.com/mandiant/capa/issues/1607
+    extract_file_export_names,
     extract_file_import_names,
     extract_file_section_names,
     extract_file_strings,

capa/features/extractors/ghidra/basicblock.py

@@ -0,0 +1,152 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import string
+import struct
+from typing import Tuple, Iterator
+
+import ghidra
+from ghidra.program.model.lang import OperandType
+
+import capa.features.extractors.ghidra.helpers
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address
+from capa.features.basicblock import BasicBlock
+from capa.features.extractors.helpers import MIN_STACKSTRING_LEN
+from capa.features.extractors.base_extractor import BBHandle, FunctionHandle
+
+
+def get_printable_len(op: ghidra.program.model.scalar.Scalar) -> int:
+    """Return string length if all operand bytes are ascii or utf16-le printable"""
+    op_bit_len = op.bitLength()
+    op_byte_len = op_bit_len // 8
+    op_val = op.getValue()
+
+    if op_bit_len == 8:
+        chars = struct.pack("<B", op_val & 0xFF)
+    elif op_bit_len == 16:
+        chars = struct.pack("<H", op_val & 0xFFFF)
+    elif op_bit_len == 32:
+        chars = struct.pack("<I", op_val & 0xFFFFFFFF)
+    elif op_bit_len == 64:
+        chars = struct.pack("<Q", op_val & 0xFFFFFFFFFFFFFFFF)
+    else:
+        raise ValueError(f"Unhandled operand data type 0x{op_bit_len:x}.")
+
+    def is_printable_ascii(chars_: bytes):
+        return all(c < 127 and chr(c) in string.printable for c in chars_)
+
+    def is_printable_utf16le(chars_: bytes):
+        if all(c == 0x00 for c in chars_[1::2]):
+            return is_printable_ascii(chars_[::2])
+
+    if is_printable_ascii(chars):
+        return op_byte_len
+
+    if is_printable_utf16le(chars):
+        return op_byte_len
+
+    return 0
+
+
+def is_mov_imm_to_stack(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    """verify instruction moves immediate onto stack"""
+
+    # Ghidra will Bitwise OR the OperandTypes to assign multiple
+    # i.e., the first operand is a stackvar (dynamically allocated),
+    # and the second is a scalar value (single int/char/float/etc.)
+    mov_its_ops = [(OperandType.ADDRESS | OperandType.DYNAMIC), OperandType.SCALAR]
+    found = False
+
+    # MOV dword ptr [EBP + local_*], 0x65
+    if insn.getMnemonicString().startswith("MOV"):
+        found = all(insn.getOperandType(i) == mov_its_ops[i] for i in range(2))
+
+    return found
+
+
+def bb_contains_stackstring(bb: ghidra.program.model.block.CodeBlock) -> bool:
+    """check basic block for stackstring indicators
+
+    true if basic block contains enough moves of constant bytes to the stack
+    """
+    count = 0
+    for insn in currentProgram().getListing().getInstructions(bb, True):  # type: ignore [name-defined] # noqa: F821
+        if is_mov_imm_to_stack(insn):
+            count += get_printable_len(insn.getScalar(1))
+        if count > MIN_STACKSTRING_LEN:
+            return True
+    return False
+
+
+def _bb_has_tight_loop(bb: ghidra.program.model.block.CodeBlock):
+    """
+    parse tight loops, true if last instruction in basic block branches to bb start
+    """
+    # Reverse Ordered, first InstructionDB
+    last_insn = currentProgram().getListing().getInstructions(bb, False).next()  # type: ignore [name-defined] # noqa: F821
+
+    if last_insn.getFlowType().isJump():
+        return last_insn.getAddress(0) == bb.getMinAddress()
+
+    return False
+
+
+def extract_bb_stackstring(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """extract stackstring indicators from basic block"""
+    bb: ghidra.program.model.block.CodeBlock = bbh.inner
+
+    if bb_contains_stackstring(bb):
+        yield Characteristic("stack string"), bbh.address
+
+
+def extract_bb_tight_loop(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """check basic block for tight loop indicators"""
+    bb: ghidra.program.model.block.CodeBlock = bbh.inner
+
+    if _bb_has_tight_loop(bb):
+        yield Characteristic("tight loop"), bbh.address
+
+
+BASIC_BLOCK_HANDLERS = (
+    extract_bb_tight_loop,
+    extract_bb_stackstring,
+)
+
+
+def extract_features(fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract features from the given basic block.
+
+    args:
+        bb: the basic block to process.
+
+    yields:
+      Tuple[Feature, int]: the features and their location found in this basic block.
+    """
+    yield BasicBlock(), bbh.address
+    for bb_handler in BASIC_BLOCK_HANDLERS:
+        for feature, addr in bb_handler(fh, bbh):
+            yield feature, addr
+
+
+def main():
+    features = []
+    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
+
+    for fh in GhidraFeatureExtractor().get_functions():
+        for bbh in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
+            features.extend(list(extract_features(fh, bbh)))
+
+    import pprint
+
+    pprint.pprint(features)  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ghidra/extractor.py

@@ -0,0 +1,75 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import List, Tuple, Iterator
+
+import capa.features.extractors.ghidra.file
+import capa.features.extractors.ghidra.insn
+import capa.features.extractors.ghidra.global_
+import capa.features.extractors.ghidra.function
+import capa.features.extractors.ghidra.basicblock
+from capa.features.common import Feature
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle, FeatureExtractor
+
+
+class GhidraFeatureExtractor(FeatureExtractor):
+    def __init__(self):
+        super().__init__()
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        self.global_features: List[Tuple[Feature, Address]] = []
+        self.global_features.extend(capa.features.extractors.ghidra.file.extract_file_format())
+        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_os())
+        self.global_features.extend(capa.features.extractors.ghidra.global_.extract_arch())
+        self.imports = ghidra_helpers.get_file_imports()
+        self.externs = ghidra_helpers.get_file_externs()
+        self.fakes = ghidra_helpers.map_fake_import_addrs()
+
+    def get_base_address(self):
+        return AbsoluteVirtualAddress(currentProgram().getImageBase().getOffset())  # type: ignore [name-defined] # noqa: F821
+
+    def extract_global_features(self):
+        yield from self.global_features
+
+    def extract_file_features(self):
+        yield from capa.features.extractors.ghidra.file.extract_features()
+
+    def get_functions(self) -> Iterator[FunctionHandle]:
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        for fhandle in ghidra_helpers.get_function_symbols():
+            fh: FunctionHandle = FunctionHandle(
+                address=AbsoluteVirtualAddress(fhandle.getEntryPoint().getOffset()),
+                inner=fhandle,
+                ctx={"imports_cache": self.imports, "externs_cache": self.externs, "fakes_cache": self.fakes},
+            )
+            yield fh
+
+    @staticmethod
+    def get_function(addr: int) -> FunctionHandle:
+        func = getFunctionContaining(toAddr(addr))  # type: ignore [name-defined] # noqa: F821
+        return FunctionHandle(address=AbsoluteVirtualAddress(func.getEntryPoint().getOffset()), inner=func)
+
+    def extract_function_features(self, fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.ghidra.function.extract_features(fh)
+
+    def get_basic_blocks(self, fh: FunctionHandle) -> Iterator[BBHandle]:
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        yield from ghidra_helpers.get_function_blocks(fh)
+
+    def extract_basic_block_features(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[Tuple[Feature, Address]]:
+        yield from capa.features.extractors.ghidra.basicblock.extract_features(fh, bbh)
+
+    def get_instructions(self, fh: FunctionHandle, bbh: BBHandle) -> Iterator[InsnHandle]:
+        import capa.features.extractors.ghidra.helpers as ghidra_helpers
+
+        yield from ghidra_helpers.get_insn_in_range(bbh)
+
+    def extract_insn_features(self, fh: FunctionHandle, bbh: BBHandle, ih: InsnHandle):
+        yield from capa.features.extractors.ghidra.insn.extract_features(fh, bbh, ih)

capa/features/extractors/ghidra/file.py

@@ -0,0 +1,202 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import re
+import struct
+from typing import List, Tuple, Iterator
+
+from ghidra.program.model.symbol import SourceType, SymbolType
+
+import capa.features.extractors.common
+import capa.features.extractors.helpers
+import capa.features.extractors.strings
+import capa.features.extractors.ghidra.helpers
+from capa.features.file import Export, Import, Section, FunctionName
+from capa.features.common import FORMAT_PE, FORMAT_ELF, Format, String, Feature, Characteristic
+from capa.features.address import NO_ADDRESS, Address, FileOffsetAddress, AbsoluteVirtualAddress
+
+MAX_OFFSET_PE_AFTER_MZ = 0x200
+
+
+def find_embedded_pe(block_bytez: bytes, mz_xor: List[Tuple[bytes, bytes, int]]) -> Iterator[Tuple[int, int]]:
+    """check segment for embedded PE
+
+    adapted for Ghidra from:
+    https://github.com/vivisect/vivisect/blob/91e8419a861f4977https://github.com/vivisect/vivisect/blob/91e8419a861f49779f18316f155311967e696836/PE/carve.py#L259f18316f155311967e696836/PE/carve.py#L25
+    """
+    todo = []
+
+    for mzx, pex, i in mz_xor:
+        for match in re.finditer(re.escape(mzx), block_bytez):
+            todo.append((match.start(), mzx, pex, i))
+
+    seg_max = len(block_bytez)  # type: ignore [name-defined] # noqa: F821
+    while len(todo):
+        off, mzx, pex, i = todo.pop()
+
+        # MZ header has one field we will check e_lfanew is at 0x3c
+        e_lfanew = off + 0x3C
+
+        if seg_max < e_lfanew + 4:
+            continue
+
+        e_lfanew_bytes = block_bytez[e_lfanew : e_lfanew + 4]
+        newoff = struct.unpack("<I", capa.features.extractors.helpers.xor_static(e_lfanew_bytes, i))[0]
+
+        # assume XOR'd "PE" bytes exist within threshold
+        if newoff > MAX_OFFSET_PE_AFTER_MZ:
+            continue
+
+        peoff = off + newoff
+        if seg_max < peoff + 2:
+            continue
+
+        pe_bytes = block_bytez[peoff : peoff + 2]
+        if pe_bytes == pex:
+            yield off, i
+
+
+def extract_file_embedded_pe() -> Iterator[Tuple[Feature, Address]]:
+    """extract embedded PE features"""
+
+    # pre-compute XOR pairs
+    mz_xor: List[Tuple[bytes, bytes, int]] = [
+        (
+            capa.features.extractors.helpers.xor_static(b"MZ", i),
+            capa.features.extractors.helpers.xor_static(b"PE", i),
+            i,
+        )
+        for i in range(256)
+    ]
+
+    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
+        if not all((block.isLoaded(), block.isInitialized(), "Headers" not in block.getName())):
+            continue
+
+        for off, _ in find_embedded_pe(capa.features.extractors.ghidra.helpers.get_block_bytes(block), mz_xor):
+            # add offset back to block start
+            ea: int = block.getStart().add(off).getOffset()
+
+            yield Characteristic("embedded pe"), FileOffsetAddress(ea)
+
+
+def extract_file_export_names() -> Iterator[Tuple[Feature, Address]]:
+    """extract function exports"""
+    st = currentProgram().getSymbolTable()  # type: ignore [name-defined] # noqa: F821
+    for addr in st.getExternalEntryPointIterator():
+        yield Export(st.getPrimarySymbol(addr).getName()), AbsoluteVirtualAddress(addr.getOffset())
+
+
+def extract_file_import_names() -> Iterator[Tuple[Feature, Address]]:
+    """extract function imports
+
+    1. imports by ordinal:
+     - modulename.#ordinal
+
+    2. imports by name, results in two features to support importname-only
+       matching:
+     - modulename.importname
+     - importname
+    """
+
+    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
+        for r in f.getSymbol().getReferences():
+            if r.getReferenceType().isData():
+                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
+
+        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_*
+        if "Ordinal_" in fstr[1]:
+            fstr[1] = f"#{fstr[1].split('_')[1]}"
+
+        for name in capa.features.extractors.helpers.generate_symbols(fstr[0][:-4], fstr[1]):
+            yield Import(name), AbsoluteVirtualAddress(addr)
+
+
+def extract_file_section_names() -> Iterator[Tuple[Feature, Address]]:
+    """extract section names"""
+
+    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
+        yield Section(block.getName()), AbsoluteVirtualAddress(block.getStart().getOffset())
+
+
+def extract_file_strings() -> Iterator[Tuple[Feature, Address]]:
+    """extract ASCII and UTF-16 LE strings"""
+
+    for block in currentProgram().getMemory().getBlocks():  # type: ignore [name-defined] # noqa: F821
+        if block.isInitialized():
+            p_bytes = capa.features.extractors.ghidra.helpers.get_block_bytes(block)
+
+        for s in capa.features.extractors.strings.extract_ascii_strings(p_bytes):
+            offset = block.getStart().getOffset() + s.offset
+            yield String(s.s), FileOffsetAddress(offset)
+
+        for s in capa.features.extractors.strings.extract_unicode_strings(p_bytes):
+            offset = block.getStart().getOffset() + s.offset
+            yield String(s.s), FileOffsetAddress(offset)
+
+
+def extract_file_function_names() -> Iterator[Tuple[Feature, Address]]:
+    """
+    extract the names of statically-linked library functions.
+    """
+
+    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
+        # .isExternal() misses more than this config for the function symbols
+        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
+            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
+            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
+                name = name[13:]
+            addr = AbsoluteVirtualAddress(sym.getAddress().getOffset())
+            yield FunctionName(name), addr
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                yield FunctionName(name[1:]), addr
+
+
+def extract_file_format() -> Iterator[Tuple[Feature, Address]]:
+    ef = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+    if "PE" in ef:
+        yield Format(FORMAT_PE), NO_ADDRESS
+    elif "ELF" in ef:
+        yield Format(FORMAT_ELF), NO_ADDRESS
+    elif "Raw" in ef:
+        # no file type to return when processing a binary file, but we want to continue processing
+        return
+    else:
+        raise NotImplementedError(f"unexpected file format: {ef}")
+
+
+def extract_features() -> Iterator[Tuple[Feature, Address]]:
+    """extract file features"""
+    for file_handler in FILE_HANDLERS:
+        for feature, addr in file_handler():
+            yield feature, addr
+
+
+FILE_HANDLERS = (
+    extract_file_embedded_pe,
+    extract_file_export_names,
+    extract_file_import_names,
+    extract_file_section_names,
+    extract_file_strings,
+    extract_file_function_names,
+    extract_file_format,
+)
+
+
+def main():
+    """ """
+    import pprint
+
+    pprint.pprint(list(extract_features()))  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ghidra/function.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Tuple, Iterator
+
+import ghidra
+from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
+
+import capa.features.extractors.ghidra.helpers
+from capa.features.common import Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors import loops
+from capa.features.extractors.base_extractor import FunctionHandle
+
+
+def extract_function_calls_to(fh: FunctionHandle):
+    """extract callers to a function"""
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+    for ref in f.getSymbol().getReferences():
+        if ref.getReferenceType().isCall():
+            yield Characteristic("calls to"), AbsoluteVirtualAddress(ref.getFromAddress().getOffset())
+
+
+def extract_function_loop(fh: FunctionHandle):
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+
+    edges = []
+    for block in SimpleBlockIterator(BasicBlockModel(currentProgram()), f.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
+        dests = block.getDestinations(monitor())  # type: ignore [name-defined] # noqa: F821
+        s_addrs = block.getStartAddresses()
+
+        while dests.hasNext():  # For loop throws Python TypeError
+            for addr in s_addrs:
+                edges.append((addr.getOffset(), dests.next().getDestinationAddress().getOffset()))
+
+    if loops.has_loop(edges):
+        yield Characteristic("loop"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
+
+
+def extract_recursive_call(fh: FunctionHandle):
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+
+    for func in f.getCalledFunctions(monitor()):  # type: ignore [name-defined] # noqa: F821
+        if func.getEntryPoint().getOffset() == f.getEntryPoint().getOffset():
+            yield Characteristic("recursive call"), AbsoluteVirtualAddress(f.getEntryPoint().getOffset())
+
+
+def extract_features(fh: FunctionHandle) -> Iterator[Tuple[Feature, Address]]:
+    for func_handler in FUNCTION_HANDLERS:
+        for feature, addr in func_handler(fh):
+            yield feature, addr
+
+
+FUNCTION_HANDLERS = (extract_function_calls_to, extract_function_loop, extract_recursive_call)
+
+
+def main():
+    """ """
+    features = []
+    for fhandle in capa.features.extractors.ghidra.helpers.get_function_symbols():
+        features.extend(list(extract_features(fhandle)))
+
+    import pprint
+
+    pprint.pprint(features)  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/ghidra/global_.py

@@ -0,0 +1,67 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+import contextlib
+from typing import Tuple, Iterator
+
+import capa.ghidra.helpers
+import capa.features.extractors.elf
+import capa.features.extractors.ghidra.helpers
+from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch, Feature
+from capa.features.address import NO_ADDRESS, Address
+
+logger = logging.getLogger(__name__)
+
+
+def extract_os() -> Iterator[Tuple[Feature, Address]]:
+    format_name: str = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+
+    if "PE" in format_name:
+        yield OS(OS_WINDOWS), NO_ADDRESS
+
+    elif "ELF" in format_name:
+        with contextlib.closing(capa.ghidra.helpers.GHIDRAIO()) as f:
+            os = capa.features.extractors.elf.detect_elf_os(f)
+
+        yield OS(os), NO_ADDRESS
+
+    else:
+        # we likely end up here:
+        #  1. handling shellcode, or
+        #  2. handling a new file format (e.g. macho)
+        #
+        # for (1) we can't do much - its shellcode and all bets are off.
+        # we could maybe accept a further CLI argument to specify the OS,
+        # but i think this would be rarely used.
+        # rules that rely on OS conditions will fail to match on shellcode.
+        #
+        # for (2), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported file format: %s, will not guess OS", format_name)
+        return
+
+
+def extract_arch() -> Iterator[Tuple[Feature, Address]]:
+    lang_id = currentProgram().getMetadata().get("Language ID")  # type: ignore [name-defined] # noqa: F821
+
+    if "x86" in lang_id and "64" in lang_id:
+        yield Arch(ARCH_AMD64), NO_ADDRESS
+
+    elif "x86" in lang_id and "32" in lang_id:
+        yield Arch(ARCH_I386), NO_ADDRESS
+
+    elif "x86" not in lang_id:
+        logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel")
+        return
+
+    else:
+        # we likely end up here:
+        #  1. handling a new architecture (e.g. aarch64)
+        #
+        # for (1), this logic will need to be updated as the format is implemented.
+        logger.debug("unsupported architecture: %s", lang_id)
+        return

capa/features/extractors/ghidra/helpers.py

@@ -0,0 +1,277 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Dict, List, Iterator
+
+import ghidra
+import java.lang
+from ghidra.program.model.lang import OperandType
+from ghidra.program.model.block import BasicBlockModel, SimpleBlockIterator
+from ghidra.program.model.symbol import SourceType, SymbolType
+from ghidra.program.model.address import AddressSpace
+
+import capa.features.extractors.helpers
+from capa.features.common import THUNK_CHAIN_DEPTH_DELTA
+from capa.features.address import AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+
+def ints_to_bytes(bytez: List[int]) -> bytes:
+    """convert Java signed ints to Python bytes
+
+    args:
+        bytez: list of Java signed ints
+    """
+    return bytes([b & 0xFF for b in bytez])
+
+
+def find_byte_sequence(addr: ghidra.program.model.address.Address, seq: bytes) -> Iterator[int]:
+    """yield all ea of a given byte sequence
+
+    args:
+        addr: start address
+        seq: bytes to search e.g. b"\x01\x03"
+    """
+    seqstr = "".join([f"\\x{b:02x}" for b in seq])
+    eas = findBytes(addr, seqstr, java.lang.Integer.MAX_VALUE, 1)  # type: ignore [name-defined] # noqa: F821
+
+    yield from eas
+
+
+def get_bytes(addr: ghidra.program.model.address.Address, length: int) -> bytes:
+    """yield length bytes at addr
+
+    args:
+        addr: Address to begin pull from
+        length: length of bytes to pull
+    """
+    try:
+        return ints_to_bytes(getBytes(addr, length))  # type: ignore [name-defined] # noqa: F821
+    except RuntimeError:
+        return b""
+
+
+def get_block_bytes(block: ghidra.program.model.mem.MemoryBlock) -> bytes:
+    """yield all bytes in a given block
+
+    args:
+        block: MemoryBlock to pull from
+    """
+    return get_bytes(block.getStart(), block.getSize())
+
+
+def get_function_symbols():
+    """yield all non-external function symbols"""
+    yield from currentProgram().getFunctionManager().getFunctionsNoStubs(True)  # type: ignore [name-defined] # noqa: F821
+
+
+def get_function_blocks(fh: FunctionHandle) -> Iterator[BBHandle]:
+    """yield BBHandle for each bb in a given function"""
+
+    func: ghidra.program.database.function.FunctionDB = fh.inner
+    for bb in SimpleBlockIterator(BasicBlockModel(currentProgram()), func.getBody(), monitor()):  # type: ignore [name-defined] # noqa: F821
+        yield BBHandle(address=AbsoluteVirtualAddress(bb.getMinAddress().getOffset()), inner=bb)
+
+
+def get_insn_in_range(bbh: BBHandle) -> Iterator[InsnHandle]:
+    """yield InshHandle for each insn in a given basicblock"""
+    for insn in currentProgram().getListing().getInstructions(bbh.inner, True):  # type: ignore [name-defined] # noqa: F821
+        yield InsnHandle(address=AbsoluteVirtualAddress(insn.getAddress().getOffset()), inner=insn)
+
+
+def get_file_imports() -> Dict[int, List[str]]:
+    """get all import names & addrs"""
+
+    import_dict: Dict[int, List[str]] = {}
+
+    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
+        for r in f.getSymbol().getReferences():
+            if r.getReferenceType().isData():
+                addr = r.getFromAddress().getOffset()  # gets pointer to fake external addr
+
+        ex_loc = f.getExternalLocation().getAddress()  # map external locations as well (offset into module files)
+
+        fstr = f.toString().split("::")  # format: MODULE.dll::import / MODULE::Ordinal_* / <EXTERNAL>::import
+        if "Ordinal_" in fstr[1]:
+            fstr[1] = f"#{fstr[1].split('_')[1]}"
+
+        # <EXTERNAL> mostly shows up in ELF files, otherwise, strip '.dll' w/ [:-4]
+        fstr[0] = "*" if "<EXTERNAL>" in fstr[0] else fstr[0][:-4]
+
+        for name in capa.features.extractors.helpers.generate_symbols(fstr[0], fstr[1]):
+            import_dict.setdefault(addr, []).append(name)
+            if ex_loc:
+                import_dict.setdefault(ex_loc.getOffset(), []).append(name)
+
+    return import_dict
+
+
+def get_file_externs() -> Dict[int, List[str]]:
+    """
+    Gets function names & addresses of statically-linked library functions
+
+    Ghidra's external namespace is mostly reserved for dynamically-linked
+    imports. Statically-linked functions are part of the global namespace.
+    Filtering on the type, source, and namespace of the symbols yield more
+    statically-linked library functions.
+
+    Example: (PMA Lab 16-01.exe_) 7faafc7e4a5c736ebfee6abbbc812d80:0x407490
+    - __aulldiv
+    - Note: See Symbol Table labels
+    """
+
+    extern_dict: Dict[int, List[str]] = {}
+
+    for sym in currentProgram().getSymbolTable().getAllSymbols(True):  # type: ignore [name-defined] # noqa: F821
+        # .isExternal() misses more than this config for the function symbols
+        if sym.getSymbolType() == SymbolType.FUNCTION and sym.getSource() == SourceType.ANALYSIS and sym.isGlobal():
+            name = sym.getName()  # starts to resolve names based on Ghidra's FidDB
+            if name.startswith("FID_conflict:"):  # format: FID_conflict:<function-name>
+                name = name[13:]
+            extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name)
+            if name.startswith("_"):
+                # some linkers may prefix linked routines with a `_` to avoid name collisions.
+                # extract features for both the mangled and un-mangled representations.
+                # e.g. `_fwrite` -> `fwrite`
+                # see: https://stackoverflow.com/a/2628384/87207
+                extern_dict.setdefault(sym.getAddress().getOffset(), []).append(name[1:])
+
+    return extern_dict
+
+
+def map_fake_import_addrs() -> Dict[int, List[int]]:
+    """
+    Map ghidra's fake import entrypoints to their
+    real addresses
+
+    Helps as many Ghidra Scripting API calls end up returning
+    these external (fake) addresses.
+
+    Undocumented but intended Ghidra behavior:
+     - Import entryPoint fields are stored in the 'EXTERNAL:' AddressSpace.
+       'getEntryPoint()' returns the entryPoint field, which is an offset
+       from the beginning of the assigned AddressSpace. In the case of externals,
+       they start from 1 and increment.
+    https://github.com/NationalSecurityAgency/ghidra/blob/26d4bd9104809747c21f2528cab8aba9aef9acd5/Ghidra/Features/Base/src/test.slow/java/ghidra/program/database/function/ExternalFunctionDBTest.java#L90
+
+    Example: (mimikatz.exe_) 5f66b82558ca92e54e77f216ef4c066c:0x473090
+    - 0x473090 -> PTR_CreateServiceW_00473090
+    - 'EXTERNAL:00000025' -> External Address (ghidra.program.model.address.SpecialAddress)
+    """
+    fake_dict: Dict[int, List[int]] = {}
+
+    for f in currentProgram().getFunctionManager().getExternalFunctions():  # type: ignore [name-defined] # noqa: F821
+        for r in f.getSymbol().getReferences():
+            if r.getReferenceType().isData():
+                fake_dict.setdefault(f.getEntryPoint().getOffset(), []).append(r.getFromAddress().getOffset())
+
+    return fake_dict
+
+
+def check_addr_for_api(
+    addr: ghidra.program.model.address.Address,
+    fakes: Dict[int, List[int]],
+    imports: Dict[int, List[str]],
+    externs: Dict[int, List[str]],
+) -> bool:
+    offset = addr.getOffset()
+
+    fake = fakes.get(offset)
+    if fake:
+        return True
+
+    imp = imports.get(offset)
+    if imp:
+        return True
+
+    extern = externs.get(offset)
+    if extern:
+        return True
+
+    return False
+
+
+def is_call_or_jmp(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    return any(mnem in insn.getMnemonicString() for mnem in ["CALL", "J"])  # JMP, JNE, JNZ, etc
+
+
+def is_sp_modified(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == OperandType.REGISTER:
+            return "SP" in insn.getRegister(i).getName() and insn.getOperandRefType(i).isWrite()
+    return False
+
+
+def is_stack_referenced(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    """generic catch-all for stack references"""
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == OperandType.REGISTER:
+            if "BP" in insn.getRegister(i).getName():
+                return True
+            else:
+                continue
+
+    return any(ref.isStackReference() for ref in insn.getReferencesFrom())
+
+
+def is_zxor(insn: ghidra.program.database.code.InstructionDB) -> bool:
+    # assume XOR insn
+    # XOR's against the same operand zero out
+    ops = []
+    operands = []
+    for i in range(insn.getNumOperands()):
+        ops.append(insn.getOpObjects(i))
+
+    # Operands stored in a 2D array
+    for j in range(len(ops)):
+        for k in range(len(ops[j])):
+            operands.append(ops[j][k])
+
+    return all(n == operands[0] for n in operands)
+
+
+def handle_thunk(addr: ghidra.program.model.address.Address):
+    """Follow thunk chains down to a reasonable depth"""
+    ref = addr
+    for _ in range(THUNK_CHAIN_DEPTH_DELTA):
+        thunk_jmp = getInstructionAt(ref)  # type: ignore [name-defined] # noqa: F821
+        if thunk_jmp and is_call_or_jmp(thunk_jmp):
+            if OperandType.isAddress(thunk_jmp.getOperandType(0)):
+                ref = thunk_jmp.getAddress(0)
+        else:
+            thunk_dat = getDataContaining(ref)  # type: ignore [name-defined] # noqa: F821
+            if thunk_dat and thunk_dat.isDefined() and thunk_dat.isPointer():
+                ref = thunk_dat.getValue()
+                break  # end of thunk chain reached
+    return ref
+
+
+def dereference_ptr(insn: ghidra.program.database.code.InstructionDB):
+    addr_code = OperandType.ADDRESS | OperandType.CODE
+    to_deref = insn.getAddress(0)
+    dat = getDataContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
+
+    if insn.getOperandType(0) == addr_code:
+        thfunc = getFunctionContaining(to_deref)  # type: ignore [name-defined] # noqa: F821
+        if thfunc and thfunc.isThunk():
+            return handle_thunk(to_deref)
+        else:
+            # if it doesn't poin to a thunk, it's usually a jmp to a label
+            return to_deref
+    if not dat:
+        return to_deref
+    if dat.isDefined() and dat.isPointer():
+        addr = dat.getValue()
+        # now we need to check the addr space to see if it is truly resolvable
+        # ghidra sometimes likes to hand us direct RAM addrs, which typically point
+        # to api calls that we can't actually resolve as such
+        if addr.getAddressSpace().getType() == AddressSpace.TYPE_RAM:
+            return to_deref
+        else:
+            return addr
+    else:
+        return to_deref

capa/features/extractors/ghidra/insn.py

@@ -0,0 +1,521 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+from typing import Any, Dict, Tuple, Iterator
+
+import ghidra
+from ghidra.program.model.lang import OperandType
+from ghidra.program.model.block import SimpleBlockModel
+
+import capa.features.extractors.helpers
+import capa.features.extractors.ghidra.helpers
+from capa.features.insn import API, MAX_STRUCTURE_SIZE, Number, Offset, Mnemonic, OperandNumber, OperandOffset
+from capa.features.common import MAX_BYTES_FEATURE_SIZE, Bytes, String, Feature, Characteristic
+from capa.features.address import Address, AbsoluteVirtualAddress
+from capa.features.extractors.base_extractor import BBHandle, InsnHandle, FunctionHandle
+
+# security cookie checks may perform non-zeroing XORs, these are expected within a certain
+# byte range within the first and returning basic blocks, this helps to reduce FP features
+SECURITY_COOKIE_BYTES_DELTA = 0x40
+
+
+def get_imports(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    """Populate the import cache for this context"""
+    if "imports_cache" not in ctx:
+        ctx["imports_cache"] = capa.features.extractors.ghidra.helpers.get_file_imports()
+    return ctx["imports_cache"]
+
+
+def get_externs(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    """Populate the externs cache for this context"""
+    if "externs_cache" not in ctx:
+        ctx["externs_cache"] = capa.features.extractors.ghidra.helpers.get_file_externs()
+    return ctx["externs_cache"]
+
+
+def get_fakes(ctx: Dict[str, Any]) -> Dict[int, Any]:
+    """Populate the fake import addrs cache for this context"""
+    if "fakes_cache" not in ctx:
+        ctx["fakes_cache"] = capa.features.extractors.ghidra.helpers.map_fake_import_addrs()
+    return ctx["fakes_cache"]
+
+
+def check_for_api_call(
+    insn, externs: Dict[int, Any], fakes: Dict[int, Any], imports: Dict[int, Any], imp_or_ex: bool
+) -> Iterator[Any]:
+    """check instruction for API call
+
+    params:
+        externs - external library functions cache
+        fakes - mapped fake import addresses cache
+        imports - imported functions cache
+        imp_or_ex - flag to check imports or externs
+
+    yields:
+        matched api calls
+    """
+    info = ()
+    funcs = imports if imp_or_ex else externs
+
+    # assume only CALLs or JMPs are passed
+    ref_type = insn.getOperandType(0)
+    addr_data = OperandType.ADDRESS | OperandType.DATA  # needs dereferencing
+    addr_code = OperandType.ADDRESS | OperandType.CODE  # needs dereferencing
+
+    if OperandType.isRegister(ref_type):
+        if OperandType.isAddress(ref_type):
+            # If it's an address in a register, check the mapped fake addrs
+            # since they're dereferenced to their fake addrs
+            op_ref = insn.getAddress(0).getOffset()
+            ref = fakes.get(op_ref)  # obtain the real addr
+            if not ref:
+                return
+        else:
+            return
+    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
+        # we must dereference and check if the addr is a pointer to an api function
+        addr_ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
+        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
+            return
+        ref = addr_ref.getOffset()
+    elif ref_type == OperandType.DYNAMIC | OperandType.ADDRESS or ref_type == OperandType.DYNAMIC:
+        return  # cannot resolve dynamics statically
+    else:
+        # pure address does not need to get dereferenced/ handled
+        addr_ref = insn.getAddress(0)
+        if not addr_ref:
+            # If it returned null, it was an indirect
+            # that had no address reference.
+            # This check is faster than checking for (indirect and not address)
+            return
+        if not capa.features.extractors.ghidra.helpers.check_addr_for_api(addr_ref, fakes, imports, externs):
+            return
+        ref = addr_ref.getOffset()
+
+    if isinstance(ref, list):  # ref from REG | ADDR
+        for r in ref:
+            info = funcs.get(r)  # type: ignore
+            if info:
+                yield info
+    else:
+        info = funcs.get(ref)  # type: ignore
+        if info:
+            yield info
+
+
+def extract_insn_api_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    externs = get_externs(fh.ctx)
+    fakes = get_fakes(fh.ctx)
+    imports = get_imports(fh.ctx)
+
+    # check calls to imported functions
+    for api in check_for_api_call(insn, externs, fakes, imports, True):
+        for imp in api:
+            yield API(imp), ih.address
+
+    # check calls to extern functions
+    for api in check_for_api_call(insn, externs, fakes, imports, False):
+        for ext in api:
+            yield API(ext), ih.address
+
+
+def extract_insn_number_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction number features
+    example:
+        push    3136B0h         ; dwControlCode
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("RET"):
+        # skip things like:
+        #   .text:0042250E retn 8
+        return
+
+    if capa.features.extractors.ghidra.helpers.is_sp_modified(insn):
+        # skip things like:
+        #   .text:00401145 add esp, 0Ch
+        return
+
+    for i in range(insn.getNumOperands()):
+        # Exceptions for LEA insn:
+        # invalid operand encoding, considered numbers instead of offsets
+        # see: mimikatz.exe_:0x4018C0
+        if insn.getOperandType(i) == OperandType.DYNAMIC and insn.getMnemonicString().startswith("LEA"):
+            # Additional check, avoid yielding "wide" values (ex. mimikatz.exe:0x471EE6 LEA EBX, [ECX + EAX*0x4])
+            op_objs = insn.getOpObjects(i)
+            if len(op_objs) == 3:  # ECX, EAX, 0x4
+                continue
+
+            if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
+                const = op_objs[-1].getUnsignedValue()
+                addr = ih.address
+
+                yield Number(const), addr
+                yield OperandNumber(i, const), addr
+        elif not OperandType.isScalar(insn.getOperandType(i)):
+            # skip things like:
+            #   references, void types
+            continue
+        else:
+            const = insn.getScalar(i).getUnsignedValue()
+            addr = ih.address
+
+            yield Number(const), addr
+            yield OperandNumber(i, const), addr
+
+            if insn.getMnemonicString().startswith("ADD") and 0 < const < MAX_STRUCTURE_SIZE:
+                # for pattern like:
+                #
+                #     add eax, 0x10
+                #
+                # assume 0x10 is also an offset (imagine eax is a pointer).
+                yield Offset(const), addr
+                yield OperandOffset(i, const), addr
+
+
+def extract_insn_offset_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction structure offset features
+
+    example:
+        .text:0040112F cmp [esi+4], ebx
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("LEA"):
+        return
+
+    # ignore any stack references
+    if not capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
+        # Ghidra stores operands in 2D arrays if they contain offsets
+        for i in range(insn.getNumOperands()):
+            if insn.getOperandType(i) == OperandType.DYNAMIC:  # e.g. [esi + 4]
+                # manual extraction, since the default api calls only work on the 1st dimension of the array
+                op_objs = insn.getOpObjects(i)
+                if isinstance(op_objs[-1], ghidra.program.model.scalar.Scalar):
+                    op_off = op_objs[-1].getValue()
+                    yield Offset(op_off), ih.address
+                    yield OperandOffset(i, op_off), ih.address
+                else:
+                    yield Offset(0), ih.address
+                    yield OperandOffset(i, 0), ih.address
+
+
+def extract_insn_bytes_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse referenced byte sequences
+    example:
+        push    offset iid_004118d4_IShellLinkA ; riid
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    ref = insn.getAddress()  # init to insn addr
+    for i in range(insn.getNumOperands()):
+        if OperandType.isAddress(insn.getOperandType(i)):
+            ref = insn.getAddress(i)  # pulls pointer if there is one
+
+    if ref != insn.getAddress():  # bail out if there's no pointer
+        ghidra_dat = getDataAt(ref)  # type: ignore [name-defined] # noqa: F821
+        if (
+            ghidra_dat and not ghidra_dat.hasStringValue() and not ghidra_dat.isPointer()
+        ):  # avoid if the data itself is a pointer
+            extracted_bytes = capa.features.extractors.ghidra.helpers.get_bytes(ref, MAX_BYTES_FEATURE_SIZE)
+            if extracted_bytes and not capa.features.extractors.helpers.all_zeros(extracted_bytes):
+                # don't extract byte features for obvious strings
+                yield Bytes(extracted_bytes), ih.address
+
+
+def extract_insn_string_features(fh: FunctionHandle, bb: BBHandle, ih: InsnHandle) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse instruction string features
+
+    example:
+        push offset aAcr     ; "ACR  > "
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+    dyn_addr = OperandType.DYNAMIC | OperandType.ADDRESS
+
+    ref = insn.getAddress()
+    for i in range(insn.getNumOperands()):
+        if OperandType.isScalarAsAddress(insn.getOperandType(i)):
+            ref = insn.getAddress(i)
+        # strings are also referenced dynamically via pointers & arrays, so we need to deref them
+        if insn.getOperandType(i) == dyn_addr:
+            ref = insn.getAddress(i)
+            dat = getDataAt(ref)  # type: ignore [name-defined] # noqa: F821
+            if dat and dat.isPointer():
+                ref = dat.getValue()
+
+    if ref != insn.getAddress():
+        ghidra_dat = getDataAt(ref)  # type: ignore [name-defined] # noqa: F821
+        if ghidra_dat and ghidra_dat.hasStringValue():
+            yield String(ghidra_dat.getValue()), ih.address
+
+
+def extract_insn_mnemonic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction mnemonic features"""
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    yield Mnemonic(insn.getMnemonicString().lower()), ih.address
+
+
+def extract_insn_obfs_call_plus_5_characteristic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """
+    parse call $+5 instruction from the given instruction.
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    code_ref = OperandType.ADDRESS | OperandType.CODE
+    ref = insn.getAddress()
+    for i in range(insn.getNumOperands()):
+        if insn.getOperandType(i) == code_ref:
+            ref = insn.getAddress(i)
+
+    if insn.getAddress().add(5) == ref:
+        yield Characteristic("call $+5"), ih.address
+
+
+def extract_insn_segment_access_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction fs or gs access"""
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    insn_str = insn.toString()
+
+    if "FS:" in insn_str:
+        yield Characteristic("fs access"), ih.address
+
+    if "GS:" in insn_str:
+        yield Characteristic("gs access"), ih.address
+
+
+def extract_insn_peb_access_characteristic_features(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """parse instruction peb access
+
+    fs:[0x30] on x86, gs:[0x60] on x64
+
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    insn_str = insn.toString()
+    if insn_str.startswith(("PUSH", "MOV")):
+        if "FS:[0x30]" in insn_str or "GS:[0x60]" in insn_str:
+            yield Characteristic("peb access"), ih.address
+
+
+def extract_insn_cross_section_cflow(
+    fh: FunctionHandle, bb: BBHandle, ih: InsnHandle
+) -> Iterator[Tuple[Feature, Address]]:
+    """inspect the instruction for a CALL or JMP that crosses section boundaries"""
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if not capa.features.extractors.ghidra.helpers.is_call_or_jmp(insn):
+        return
+
+    externs = get_externs(fh.ctx)
+    fakes = get_fakes(fh.ctx)
+    imports = get_imports(fh.ctx)
+
+    # OperandType to dereference
+    addr_data = OperandType.ADDRESS | OperandType.DATA
+    addr_code = OperandType.ADDRESS | OperandType.CODE
+
+    ref_type = insn.getOperandType(0)
+
+    # both OperandType flags must be present
+    # bail on REGISTER alone
+    if OperandType.isRegister(ref_type):
+        if OperandType.isAddress(ref_type):
+            ref = insn.getAddress(0)  # Ghidra dereferences REG | ADDR
+            if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
+                return
+        else:
+            return
+    elif ref_type in (addr_data, addr_code) or (OperandType.isIndirect(ref_type) and OperandType.isAddress(ref_type)):
+        # we must dereference and check if the addr is a pointer to an api function
+        ref = capa.features.extractors.ghidra.helpers.dereference_ptr(insn)
+        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
+            return
+    elif ref_type == OperandType.DYNAMIC | OperandType.ADDRESS or ref_type == OperandType.DYNAMIC:
+        return  # cannot resolve dynamics statically
+    else:
+        # pure address does not need to get dereferenced/ handled
+        ref = insn.getAddress(0)
+        if not ref:
+            # If it returned null, it was an indirect
+            # that had no address reference.
+            # This check is faster than checking for (indirect and not address)
+            return
+        if capa.features.extractors.ghidra.helpers.check_addr_for_api(ref, fakes, imports, externs):
+            return
+
+    this_mem_block = getMemoryBlock(insn.getAddress())  # type: ignore [name-defined] # noqa: F821
+    ref_block = getMemoryBlock(ref)  # type: ignore [name-defined] # noqa: F821
+    if ref_block != this_mem_block:
+        yield Characteristic("cross section flow"), ih.address
+
+
+def extract_function_calls_from(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    ih: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    """extract functions calls from features
+
+    most relevant at the function scope, however, its most efficient to extract at the instruction scope
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("CALL"):
+        # This method of "dereferencing" addresses/ pointers
+        # is not as robust as methods in other functions,
+        # but works just fine for this one
+        reference = 0
+        for ref in insn.getReferencesFrom():
+            addr = ref.getToAddress()
+
+            # avoid returning fake addrs
+            if not addr.isExternalAddress():
+                reference = addr.getOffset()
+
+            # if a reference is < 0, then ghidra pulled an offset from a DYNAMIC | ADDR (usually a stackvar)
+            # these cannot be resolved to actual addrs
+            if reference > 0:
+                yield Characteristic("calls from"), AbsoluteVirtualAddress(reference)
+
+
+def extract_function_indirect_call_characteristic_features(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    ih: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    """extract indirect function calls (e.g., call eax or call dword ptr [edx+4])
+    does not include calls like => call ds:dword_ABD4974
+
+    most relevant at the function or basic block scope;
+    however, its most efficient to extract at the instruction scope
+    """
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if insn.getMnemonicString().startswith("CALL"):
+        if OperandType.isRegister(insn.getOperandType(0)):
+            yield Characteristic("indirect call"), ih.address
+        if OperandType.isIndirect(insn.getOperandType(0)):
+            yield Characteristic("indirect call"), ih.address
+
+
+def check_nzxor_security_cookie_delta(
+    fh: ghidra.program.database.function.FunctionDB, insn: ghidra.program.database.code.InstructionDB
+):
+    """Get the function containing the insn
+    Get the last block of the function that contains the insn
+
+    Check the bb containing the insn
+    Check the last bb of the function containing the insn
+    """
+
+    model = SimpleBlockModel(currentProgram())  # type: ignore [name-defined] # noqa: F821
+    insn_addr = insn.getAddress()
+    func_asv = fh.getBody()
+    first_addr = func_asv.getMinAddress()
+    last_addr = func_asv.getMaxAddress()
+
+    if model.getFirstCodeBlockContaining(
+        first_addr, monitor()  # type: ignore [name-defined] # noqa: F821
+    ) == model.getFirstCodeBlockContaining(
+        last_addr, monitor()  # type: ignore [name-defined] # noqa: F821
+    ):
+        if insn_addr < first_addr.add(SECURITY_COOKIE_BYTES_DELTA):
+            return True
+        else:
+            return insn_addr > last_addr.add(SECURITY_COOKIE_BYTES_DELTA * -1)
+    else:
+        return False
+
+
+def extract_insn_nzxor_characteristic_features(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    ih: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    f: ghidra.program.database.function.FunctionDB = fh.inner
+    insn: ghidra.program.database.code.InstructionDB = ih.inner
+
+    if "XOR" not in insn.getMnemonicString():
+        return
+    if capa.features.extractors.ghidra.helpers.is_stack_referenced(insn):
+        return
+    if capa.features.extractors.ghidra.helpers.is_zxor(insn):
+        return
+    if check_nzxor_security_cookie_delta(f, insn):
+        return
+    yield Characteristic("nzxor"), ih.address
+
+
+def extract_features(
+    fh: FunctionHandle,
+    bb: BBHandle,
+    insn: InsnHandle,
+) -> Iterator[Tuple[Feature, Address]]:
+    for insn_handler in INSTRUCTION_HANDLERS:
+        for feature, addr in insn_handler(fh, bb, insn):
+            yield feature, addr
+
+
+INSTRUCTION_HANDLERS = (
+    extract_insn_api_features,
+    extract_insn_number_features,
+    extract_insn_bytes_features,
+    extract_insn_string_features,
+    extract_insn_offset_features,
+    extract_insn_nzxor_characteristic_features,
+    extract_insn_mnemonic_features,
+    extract_insn_obfs_call_plus_5_characteristic_features,
+    extract_insn_peb_access_characteristic_features,
+    extract_insn_cross_section_cflow,
+    extract_insn_segment_access_features,
+    extract_function_calls_from,
+    extract_function_indirect_call_characteristic_features,
+)
+
+
+def main():
+    """ """
+    features = []
+    from capa.features.extractors.ghidra.extractor import GhidraFeatureExtractor
+
+    for fh in GhidraFeatureExtractor().get_functions():
+        for bb in capa.features.extractors.ghidra.helpers.get_function_blocks(fh):
+            for insn in capa.features.extractors.ghidra.helpers.get_insn_in_range(bb):
+                features.extend(list(extract_features(fh, bb, insn)))
+
+    import pprint
+
+    pprint.pprint(features)  # noqa: T203
+
+
+if __name__ == "__main__":
+    main()

capa/features/extractors/viv/function.py

@@ -38,7 +38,7 @@ def extract_function_symtab_names(fh: FunctionHandle) -> Iterator[Tuple[Feature,
         # this is in order to eliminate the computational overhead of refetching symtab each time.
         if "symtab" not in fh.ctx["cache"]:
             try:
-                fh.ctx["cache"]["symtab"] = SymTab.from_Elf(fh.inner.vw.parsedbin)
+                fh.ctx["cache"]["symtab"] = SymTab.from_viv(fh.inner.vw.parsedbin)
             except Exception:
                 fh.ctx["cache"]["symtab"] = None
 

capa/features/extractors/viv/insn.py

@@ -115,7 +115,7 @@ def extract_insn_api_features(fh: FunctionHandle, bb, ih: InsnHandle) -> Iterato
                 # the symbol table gets stored as a function's attribute in order to avoid running
                 # this code everytime the call is made, thus preventing the computational overhead.
                 try:
-                    fh.ctx["cache"]["symtab"] = SymTab.from_Elf(f.vw.parsedbin)
+                    fh.ctx["cache"]["symtab"] = SymTab.from_viv(f.vw.parsedbin)
                 except Exception:
                     fh.ctx["cache"]["symtab"] = None
 

capa/features/freeze/__init__.py

@@ -14,7 +14,7 @@ import logging
 from enum import Enum
 from typing import List, Tuple, Union
 
-from pydantic import Field, BaseModel
+from pydantic import Field, BaseModel, ConfigDict
 
 import capa.helpers
 import capa.version
@@ -31,8 +31,7 @@ logger = logging.getLogger(__name__)
 
 
 class HashableModel(BaseModel):
-    class Config:
-        frozen = True
+    model_config = ConfigDict(frozen=True)
 
 
 class AddressType(str, Enum):
@@ -46,7 +45,7 @@ class AddressType(str, Enum):
 
 class Address(HashableModel):
     type: AddressType
-    value: Union[int, Tuple[int, int], None]
+    value: Union[int, Tuple[int, int], None] = None  # None default value to support deserialization of NO_ADDRESS
 
     @classmethod
     def from_capa(cls, a: capa.features.address.Address) -> "Address":
@@ -159,9 +158,7 @@ class BasicBlockFeature(HashableModel):
     basic_block: Address = Field(alias="basic block")
     address: Address
     feature: Feature
-
-    class Config:
-        allow_population_by_field_name = True
+    model_config = ConfigDict(populate_by_name=True)
 
 
 class InstructionFeature(HashableModel):
@@ -194,26 +191,20 @@ class FunctionFeatures(BaseModel):
     address: Address
     features: Tuple[FunctionFeature, ...]
     basic_blocks: Tuple[BasicBlockFeatures, ...] = Field(alias="basic blocks")
-
-    class Config:
-        allow_population_by_field_name = True
+    model_config = ConfigDict(populate_by_name=True)
 
 
 class Features(BaseModel):
     global_: Tuple[GlobalFeature, ...] = Field(alias="global")
     file: Tuple[FileFeature, ...]
     functions: Tuple[FunctionFeatures, ...]
-
-    class Config:
-        allow_population_by_field_name = True
+    model_config = ConfigDict(populate_by_name=True)
 
 
 class Extractor(BaseModel):
     name: str
     version: str = capa.version.__version__
-
-    class Config:
-        allow_population_by_field_name = True
+    model_config = ConfigDict(populate_by_name=True)
 
 
 class Freeze(BaseModel):
@@ -221,9 +212,7 @@ class Freeze(BaseModel):
     base_address: Address = Field(alias="base address")
     extractor: Extractor
     features: Features
-
-    class Config:
-        allow_population_by_field_name = True
+    model_config = ConfigDict(populate_by_name=True)
 
 
 def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -> str:
@@ -324,14 +313,14 @@ def dumps(extractor: capa.features.extractors.base_extractor.FeatureExtractor) -
     )  # type: ignore
     # Mypy is unable to recognise `base_address` as a argument due to alias
 
-    return freeze.json()
+    return freeze.model_dump_json()
 
 
 def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor:
     """deserialize a set of features (as a NullFeatureExtractor) from a string."""
     import capa.features.extractors.null as null
 
-    freeze = Freeze.parse_raw(s)
+    freeze = Freeze.model_validate_json(s)
     if freeze.version != 2:
         raise ValueError(f"unsupported freeze format version: {freeze.version}")
 

capa/features/freeze/features.py

@@ -8,7 +8,7 @@
 import binascii
 from typing import Union, Optional
 
-from pydantic import Field, BaseModel
+from pydantic import Field, BaseModel, ConfigDict
 
 import capa.features.file
 import capa.features.insn
@@ -17,9 +17,7 @@ import capa.features.basicblock
 
 
 class FeatureModel(BaseModel):
-    class Config:
-        frozen = True
-        allow_population_by_field_name = True
+    model_config = ConfigDict(frozen=True, populate_by_name=True)
 
     def to_capa(self) -> capa.features.common.Feature:
         if isinstance(self, OSFeature):
@@ -213,141 +211,141 @@ def feature_from_capa(f: capa.features.common.Feature) -> "Feature":
 class OSFeature(FeatureModel):
     type: str = "os"
     os: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class ArchFeature(FeatureModel):
     type: str = "arch"
     arch: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class FormatFeature(FeatureModel):
     type: str = "format"
     format: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class MatchFeature(FeatureModel):
     type: str = "match"
     match: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class CharacteristicFeature(FeatureModel):
     type: str = "characteristic"
     characteristic: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class ExportFeature(FeatureModel):
     type: str = "export"
     export: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class ImportFeature(FeatureModel):
     type: str = "import"
     import_: str = Field(alias="import")
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class SectionFeature(FeatureModel):
     type: str = "section"
     section: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class FunctionNameFeature(FeatureModel):
     type: str = "function name"
     function_name: str = Field(alias="function name")
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class SubstringFeature(FeatureModel):
     type: str = "substring"
     substring: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class RegexFeature(FeatureModel):
     type: str = "regex"
     regex: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class StringFeature(FeatureModel):
     type: str = "string"
     string: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class ClassFeature(FeatureModel):
     type: str = "class"
     class_: str = Field(alias="class")
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class NamespaceFeature(FeatureModel):
     type: str = "namespace"
     namespace: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class BasicBlockFeature(FeatureModel):
     type: str = "basic block"
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class APIFeature(FeatureModel):
     type: str = "api"
     api: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class PropertyFeature(FeatureModel):
     type: str = "property"
-    access: Optional[str]
+    access: Optional[str] = None
     property: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class NumberFeature(FeatureModel):
     type: str = "number"
     number: Union[int, float]
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class BytesFeature(FeatureModel):
     type: str = "bytes"
     bytes: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class OffsetFeature(FeatureModel):
     type: str = "offset"
     offset: int
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class MnemonicFeature(FeatureModel):
     type: str = "mnemonic"
     mnemonic: str
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class OperandNumberFeature(FeatureModel):
     type: str = "operand number"
     index: int
     operand_number: int = Field(alias="operand number")
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 class OperandOffsetFeature(FeatureModel):
     type: str = "operand offset"
     index: int
     operand_offset: int = Field(alias="operand offset")
-    description: Optional[str]
+    description: Optional[str] = None
 
 
 Feature = Union[

capa/ghidra/README.md

@@ -0,0 +1,172 @@
+<div align="center">
+    <img src="/doc/img/ghidra_backend_logo.png" width=300 height=175>
+</div>
+
+The Ghidra feature extractor is an application of the FLARE team's open-source project, Ghidrathon, to integrate capa with Ghidra using Python 3. capa is a framework that uses a well-defined collection of rules to identify capabilities in a program. You can run capa against a PE file, ELF file, or shellcode and it tells you what it thinks the program can do. For example, it might suggest that the program is a backdoor, can install services, or relies on HTTP to communicate. The Ghidra feature extractor can be used to run capa analysis on your Ghidra databases without needing access to the original binary file.
+
+<img src="/doc/img/ghidra_script_mngr_output.png">
+
+## Getting Started
+
+### Installation
+
+Please ensure that you have the following dependencies installed before continuing:
+
+| Dependency | Version | Source |
+|------------|---------|--------|
+| Ghidrathon | `>= 3.0.0` | https://github.com/mandiant/Ghidrathon |
+| Python | `>= 3.8` | https://www.python.org/downloads |
+| Ghidra | `>= 10.2` | https://ghidra-sre.org |
+
+In order to run capa using using Ghidra, you must install capa as a library, obtain the official capa rules that match the capa version you have installed, and configure the Python 3 script [capa_ghidra.py](/capa/ghidra/capa_ghidra.py). You can do this by completing the following steps using the Python 3 interpreter that you have configured for your Ghidrathon installation:
+
+1. Install capa and its dependencies from PyPI using the following command:
+```bash
+$ pip install flare-capa
+```
+
+2. Download and extract the [official capa rules](https://github.com/mandiant/capa-rules/releases) that match the capa version you have installed. Use the following command to view the version of capa you have installed:
+```bash
+$ pip show flare-capa
+OR
+$ capa --version
+```
+
+3. Copy [capa_ghidra.py](/capa/ghidra/capa_ghidra.py) to your `$USER_HOME/ghidra_scripts` directory or manually add `</path/to/ghidra_capa.py/>` to the Ghidra Script Manager.
+
+## Usage
+
+After completing the installation steps you can execute `capa_ghidra.py` using the Ghidra Script Manager or Headless Analyzer.
+
+### Ghidra Script Manager
+
+To execute `capa_ghidra.py` using the Ghidra Script Manager, first open the Ghidra Script Manager by navigating to `Window > Script Manager` in the Ghidra Code Browser. Next, locate `capa_ghidra.py` by selecting the `Python 3 > capa` category or using the Ghidra Script Manager search funtionality. Finally, double-click `capa_ghidra.py` to execute the script. If you don't see `capa_ghidra.py`, make sure you have copied the script to your `$USER_HOME/ghidra_scripts` directory or manually added `</path/to/ghidra_capa.py/>` to the Ghidra Script Manager
+
+When executed, `capa_ghidra.py` asks you to provide your capa rules directory and preferred output format. `capa_ghidra.py` supports `default`, `verbose`, and `vverbose` output formats when executed from the Ghidra Script Manager. `capa_ghidra.py` writes output to the Ghidra Console Window.
+
+#### Example
+
+The following is an example of running `capa_ghidra.py` using the Ghidra Script Manager:
+
+Selecting capa rules:
+<img src="/doc/img/ghidra_script_mngr_rules.png">
+
+Choosing output format:
+<img src="/doc/img/ghidra_script_mngr_verbosity.png">
+
+Viewing results in Ghidra Console Window:
+<img src="/doc/img/ghidra_script_mngr_output.png">
+
+### Ghidra Headless Analyzer
+
+To execute `capa_ghidra.py` using the Ghidra Headless Analyzer, you can use the Ghidra `analyzeHeadless` script located in your `$GHIDRA_HOME/support` directory. You will need to provide the following arguments to the Ghidra `analyzeHeadless` script:
+
+1. `</path/to/ghidra/project/>`: path to Ghidra project
+2. `<ghidra_project_name>`: name of Ghidra Project
+3. `-process <sample_name>`: name of sample `<sample_name>`
+4. `-ScriptPath </path/to/capa_ghidra/>`: OPTIONAL argument specifying path `</path/to/capa_ghidra/>` to `capa_ghidra.py`
+5. `-PostScript capa_ghidra.py`: executes `capa_ghidra.py` as post-analysis script
+6. `"<capa_args>"`: single, quoted string containing capa arguments that must specify capa rules directory and output format, e.g. `"<path/to/capa/rules> --verbose"`. `capa_ghidra.py` supports `default`, `verbose`, `vverbose` and `json` formats when executed using the Ghidra Headless Analyzer. `capa_ghidra.py` writes output to the console window used to execute the Ghidra `analyzeHeadless` script.
+7. `-processor <languageID>`: required ONLY if sample `<sample_name>` is shellcode. More information on specifying the `<languageID>` can be found in the `$GHIDRA_HOME/support/analyzeHeadlessREADME.html` documentation.
+
+The following is an example of combining these arguments into a single `analyzeHeadless` script command:
+
+```
+$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -process <sample_name> -PostScript capa_ghidra.py "/path/to/capa/rules/ --verbose"
+```
+
+You may also want to run capa against a sample that you have not yet imported into your Ghidra project. The following is an example of importing a sample and running `capa_ghidra.py` using a single `analyzeHeadless` script command:
+
+```
+$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -Import </path/to/sample> -PostScript capa_ghidra.py "/path/to/capa/rules/ --verbose"
+```
+
+You can also provide `capa_ghidra.py` the single argument `"help"` to view supported arguments when running the script using the Ghidra Headless Analyzer:
+```
+$GHIDRA_HOME/support/analyzeHeadless </path/to/ghidra/project/> <ghidra_project_name> -process <sample_name> -PostScript capa_ghidra.py "help"
+```
+
+#### Example
+
+The following is an example of running `capa_ghidra.py` against a shellcode sample using the Ghidra `analyzeHeadless` script:
+```
+$ analyzeHeadless /home/wumbo/Desktop/ghidra_projects/ capa_test -process 499c2a85f6e8142c3f48d4251c9c7cd6.raw32 -processor x86:LE:32:default -PostScript capa_ghidra.py "/home/wumbo/capa/rules -vv"
+[...]
+
+INFO  REPORT: Analysis succeeded for file: /499c2a85f6e8142c3f48d4251c9c7cd6.raw32 (HeadlessAnalyzer)  
+INFO  SCRIPT: /home/wumbo/ghidra_scripts/capa_ghidra.py (HeadlessAnalyzer)  
+md5                     499c2a85f6e8142c3f48d4251c9c7cd6                                                                                                                                                                                                    
+sha1
+sha256                  e8e02191c1b38c808d27a899ac164b3675eb5cadd3a8907b0ffa863714000e72
+path                    /home/wumbo/capa/tests/data/499c2a85f6e8142c3f48d4251c9c7cd6.raw32
+timestamp               2023-08-29 17:57:00.946588
+capa version            6.1.0
+os                      unknown os
+format                  Raw Binary
+arch                    x86
+extractor               ghidra
+base address            global
+rules                   /home/wumbo/capa/rules
+function count          42
+library function count  0
+total feature count     1970
+
+contain loop (24 matches, only showing first match of library rule)
+author  moritz.raabe@mandiant.com
+scope   function
+function @ 0x0
+  or:
+    characteristic: loop @ 0x0
+    characteristic: tight loop @ 0x278
+
+contain obfuscated stackstrings
+namespace  anti-analysis/obfuscation/string/stackstring
+author     moritz.raabe@mandiant.com
+scope      basic block
+att&ck     Defense Evasion::Obfuscated Files or Information::Indicator Removal from Tools [T1027.005]
+mbc        Anti-Static Analysis::Executable Code Obfuscation::Argument Obfuscation [B0032.020], Anti-Static Analysis::Executable Code Obfuscation::Stack Strings [B0032.017]
+basic block @ 0x0 in function 0x0
+  characteristic: stack string @ 0x0
+
+encode data using XOR
+namespace  data-manipulation/encoding/xor
+author     moritz.raabe@mandiant.com
+scope      basic block
+att&ck     Defense Evasion::Obfuscated Files or Information [T1027]
+mbc        Defense Evasion::Obfuscated Files or Information::Encoding-Standard Algorithm [E1027.m02], Data::Encode Data::XOR [C0026.002]
+basic block @ 0x8AF in function 0x8A1
+  and:
+    characteristic: tight loop @ 0x8AF
+    characteristic: nzxor @ 0x8C0
+    not: = filter for potential false positives
+      or:
+        or: = unsigned bitwise negation operation (~i)
+          number: 0xFFFFFFFF = bitwise negation for unsigned 32 bits
+          number: 0xFFFFFFFFFFFFFFFF = bitwise negation for unsigned 64 bits
+        or: = signed bitwise negation operation (~i)
+          number: 0xFFFFFFF = bitwise negation for signed 32 bits
+          number: 0xFFFFFFFFFFFFFFF = bitwise negation for signed 64 bits
+        or: = Magic constants used in the implementation of strings functions.
+          number: 0x7EFEFEFF = optimized string constant for 32 bits
+          number: 0x81010101 = -0x81010101 = 0x7EFEFEFF
+          number: 0x81010100 = 0x81010100 = ~0x7EFEFEFF
+          number: 0x7EFEFEFEFEFEFEFF = optimized string constant for 64 bits
+          number: 0x8101010101010101 = -0x8101010101010101 = 0x7EFEFEFEFEFEFEFF
+          number: 0x8101010101010100 = 0x8101010101010100 = ~0x7EFEFEFEFEFEFEFF
+
+get OS information via KUSER_SHARED_DATA
+namespace   host-interaction/os/version
+author      @mr-tz
+scope       function
+att&ck      Discovery::System Information Discovery [T1082]
+references  https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/ntexapi_x/kuser_shared_data/index.htm
+function @ 0x1CA6
+  or:
+    number: 0x7FFE026C = NtMajorVersion @ 0x1D18
+
+
+
+Script /home/wumbo/ghidra_scripts/capa_ghidra.py called exit with code 0
+
+[...]
+```

capa/ghidra/capa_ghidra.py

@@ -0,0 +1,166 @@
+# Run capa against loaded Ghidra database
+# @author Mike Hunhoff (mehunhoff@google.com)
+# @category Python 3.capa
+
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import sys
+import logging
+import pathlib
+import argparse
+
+import capa
+import capa.main
+import capa.rules
+import capa.ghidra.helpers
+import capa.render.default
+import capa.features.extractors.ghidra.extractor
+
+logger = logging.getLogger("capa_ghidra")
+
+
+def run_headless():
+    parser = argparse.ArgumentParser(description="The FLARE team's open-source tool to integrate capa with Ghidra.")
+
+    parser.add_argument(
+        "rules",
+        type=str,
+        help="path to rule file or directory",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", help="enable verbose result document (no effect with --json)"
+    )
+    parser.add_argument(
+        "-vv", "--vverbose", action="store_true", help="enable very verbose result document (no effect with --json)"
+    )
+    parser.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR")
+    parser.add_argument("-q", "--quiet", action="store_true", help="disable all output but errors")
+    parser.add_argument("-j", "--json", action="store_true", help="emit JSON instead of text")
+
+    script_args = list(getScriptArgs())  # type: ignore [name-defined] # noqa: F821
+    if not script_args or len(script_args) > 1:
+        script_args = []
+    else:
+        script_args = script_args[0].split()
+        for idx, arg in enumerate(script_args):
+            if arg.lower() == "help":
+                script_args[idx] = "--help"
+
+    args = parser.parse_args(args=script_args)
+
+    if args.quiet:
+        logging.basicConfig(level=logging.WARNING)
+        logging.getLogger().setLevel(logging.WARNING)
+    elif args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+        logging.getLogger().setLevel(logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+        logging.getLogger().setLevel(logging.INFO)
+
+    logger.debug("running in Ghidra headless mode")
+
+    rules_path = pathlib.Path(args.rules)
+
+    logger.debug("rule path: %s", rules_path)
+    rules = capa.main.get_rules([rules_path])
+
+    meta = capa.ghidra.helpers.collect_metadata([rules_path])
+    extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
+
+    capabilities, counts = capa.main.find_capabilities(rules, extractor, False)
+
+    meta.analysis.feature_counts = counts["feature_counts"]
+    meta.analysis.library_functions = counts["library_functions"]
+    meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
+
+    if capa.main.has_file_limitation(rules, capabilities, is_standalone=True):
+        logger.info("capa encountered warnings during analysis")
+
+    if args.json:
+        print(capa.render.json.render(meta, rules, capabilities))  # noqa: T201
+    elif args.vverbose:
+        print(capa.render.vverbose.render(meta, rules, capabilities))  # noqa: T201
+    elif args.verbose:
+        print(capa.render.verbose.render(meta, rules, capabilities))  # noqa: T201
+    else:
+        print(capa.render.default.render(meta, rules, capabilities))  # noqa: T201
+
+    return 0
+
+
+def run_ui():
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger().setLevel(logging.INFO)
+
+    rules_dir: str = ""
+    try:
+        selected_dir = askDirectory("Choose capa rules directory", "Ok")  # type: ignore [name-defined] # noqa: F821
+        if selected_dir:
+            rules_dir = selected_dir.getPath()
+    except RuntimeError:
+        # RuntimeError thrown when user selects "Cancel"
+        pass
+
+    if not rules_dir:
+        logger.info("You must choose a capa rules directory before running capa.")
+        return capa.main.E_MISSING_RULES
+
+    verbose = askChoice(  # type: ignore [name-defined] # noqa: F821
+        "capa output verbosity", "Choose capa output verbosity", ["default", "verbose", "vverbose"], "default"
+    )
+
+    rules_path: pathlib.Path = pathlib.Path(rules_dir)
+    logger.info("running capa using rules from %s", str(rules_path))
+
+    rules = capa.main.get_rules([rules_path])
+
+    meta = capa.ghidra.helpers.collect_metadata([rules_path])
+    extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
+
+    capabilities, counts = capa.main.find_capabilities(rules, extractor, True)
+
+    meta.analysis.feature_counts = counts["feature_counts"]
+    meta.analysis.library_functions = counts["library_functions"]
+    meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
+
+    if capa.main.has_file_limitation(rules, capabilities, is_standalone=False):
+        logger.info("capa encountered warnings during analysis")
+
+    if verbose == "vverbose":
+        print(capa.render.vverbose.render(meta, rules, capabilities))  # noqa: T201
+    elif verbose == "verbose":
+        print(capa.render.verbose.render(meta, rules, capabilities))  # noqa: T201
+    else:
+        print(capa.render.default.render(meta, rules, capabilities))  # noqa: T201
+
+    return 0
+
+
+def main():
+    if not capa.ghidra.helpers.is_supported_ghidra_version():
+        return capa.main.E_UNSUPPORTED_GHIDRA_VERSION
+
+    if not capa.ghidra.helpers.is_supported_file_type():
+        return capa.main.E_INVALID_FILE_TYPE
+
+    if not capa.ghidra.helpers.is_supported_arch_type():
+        return capa.main.E_INVALID_FILE_ARCH
+
+    if isRunningHeadless():  # type: ignore [name-defined] # noqa: F821
+        return run_headless()
+    else:
+        return run_ui()
+
+
+if __name__ == "__main__":
+    if sys.version_info < (3, 8):
+        from capa.exceptions import UnsupportedRuntimeError
+
+        raise UnsupportedRuntimeError("This version of capa can only be used with Python 3.8+")
+    sys.exit(main())

capa/ghidra/helpers.py

@@ -0,0 +1,159 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import logging
+import datetime
+import contextlib
+from typing import List
+from pathlib import Path
+
+import capa
+import capa.version
+import capa.features.common
+import capa.features.freeze
+import capa.render.result_document as rdoc
+import capa.features.extractors.ghidra.helpers
+
+logger = logging.getLogger("capa")
+
+# file type as returned by Ghidra
+SUPPORTED_FILE_TYPES = ("Executable and Linking Format (ELF)", "Portable Executable (PE)", "Raw Binary")
+
+
+class GHIDRAIO:
+    """
+    An object that acts as a file-like object,
+    using bytes from the current Ghidra listing.
+    """
+
+    def __init__(self):
+        super().__init__()
+
+        self.offset = 0
+        self.bytes_ = self.get_bytes()
+
+    def seek(self, offset, whence=0):
+        assert whence == 0
+        self.offset = offset
+
+    def read(self, size):
+        logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, currentProgram().getImageBase().add(self.offset).getOffset())  # type: ignore [name-defined] # noqa: F821
+
+        if size > len(self.bytes_) - self.offset:
+            logger.debug("cannot read 0x%x bytes at 0x%x (ea: BADADDR)", size, self.offset)
+            return b""
+        else:
+            return self.bytes_[self.offset : self.offset + size]
+
+    def close(self):
+        return
+
+    def get_bytes(self):
+        file_bytes = currentProgram().getMemory().getAllFileBytes()[0]  # type: ignore [name-defined] # noqa: F821
+
+        # getOriginalByte() allows for raw file parsing on the Ghidra side
+        # other functions will fail as Ghidra will think that it's reading uninitialized memory
+        bytes_ = [file_bytes.getOriginalByte(i) for i in range(file_bytes.getSize())]
+
+        return capa.features.extractors.ghidra.helpers.ints_to_bytes(bytes_)
+
+
+def is_supported_ghidra_version():
+    version = float(getGhidraVersion()[:4])  # type: ignore [name-defined] # noqa: F821
+    if version < 10.2:
+        warning_msg = "capa does not support this Ghidra version"
+        logger.warning(warning_msg)
+        logger.warning("Your Ghidra version is: %s. Supported versions are: Ghidra >= 10.2", version)
+        return False
+    return True
+
+
+def is_running_headless():
+    return isRunningHeadless()  # type: ignore [name-defined] # noqa: F821
+
+
+def is_supported_file_type():
+    file_info = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+    if file_info not in SUPPORTED_FILE_TYPES:
+        logger.error("-" * 80)
+        logger.error(" Input file does not appear to be a supported file type.")
+        logger.error(" ")
+        logger.error(
+            " capa currently only supports analyzing PE, ELF, or binary files containing x86 (32- and 64-bit) shellcode."
+        )
+        logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.")
+        logger.error("-" * 80)
+        return False
+    return True
+
+
+def is_supported_arch_type():
+    lang_id = str(currentProgram().getLanguageID()).lower()  # type: ignore [name-defined] # noqa: F821
+
+    if not all((lang_id.startswith("x86"), any(arch in lang_id for arch in ("32", "64")))):
+        logger.error("-" * 80)
+        logger.error(" Input file does not appear to target a supported architecture.")
+        logger.error(" ")
+        logger.error(" capa currently only supports analyzing x86 (32- and 64-bit).")
+        logger.error("-" * 80)
+        return False
+    return True
+
+
+def get_file_md5():
+    return currentProgram().getExecutableMD5()  # type: ignore [name-defined] # noqa: F821
+
+
+def get_file_sha256():
+    return currentProgram().getExecutableSHA256()  # type: ignore [name-defined] # noqa: F821
+
+
+def collect_metadata(rules: List[Path]):
+    md5 = get_file_md5()
+    sha256 = get_file_sha256()
+
+    info = currentProgram().getLanguageID().toString()  # type: ignore [name-defined] # noqa: F821
+    if "x86" in info and "64" in info:
+        arch = "x86_64"
+    elif "x86" in info and "32" in info:
+        arch = "x86"
+    else:
+        arch = "unknown arch"
+
+    format_name: str = currentProgram().getExecutableFormat()  # type: ignore [name-defined] # noqa: F821
+    if "PE" in format_name:
+        os = "windows"
+    elif "ELF" in format_name:
+        with contextlib.closing(capa.ghidra.helpers.GHIDRAIO()) as f:
+            os = capa.features.extractors.elf.detect_elf_os(f)
+    else:
+        os = "unknown os"
+
+    return rdoc.Metadata(
+        timestamp=datetime.datetime.now(),
+        version=capa.version.__version__,
+        argv=(),
+        sample=rdoc.Sample(
+            md5=md5,
+            sha1="",
+            sha256=sha256,
+            path=currentProgram().getExecutablePath(),  # type: ignore [name-defined] # noqa: F821
+        ),
+        analysis=rdoc.Analysis(
+            format=currentProgram().getExecutableFormat(),  # type: ignore [name-defined] # noqa: F821
+            arch=arch,
+            os=os,
+            extractor="ghidra",
+            rules=tuple(r.resolve().absolute().as_posix() for r in rules),
+            base_address=capa.features.freeze.Address.from_capa(currentProgram().getImageBase().getOffset()),  # type: ignore [name-defined] # noqa: F821
+            layout=rdoc.Layout(
+                functions=(),
+            ),
+            feature_counts=rdoc.FeatureCounts(file=0, functions=()),
+            library_functions=(),
+        ),
+    )

capa/helpers.py

@@ -43,6 +43,14 @@ def is_runtime_ida():
     return importlib.util.find_spec("idc") is not None
 
 
+def is_runtime_ghidra():
+    try:
+        currentProgram  # type: ignore [name-defined] # noqa: F821
+    except NameError:
+        return False
+    return True
+
+
 def assert_never(value) -> NoReturn:
     # careful: python -O will remove this assertion.
     # but this is only used for type checking, so it's ok.

capa/ida/helpers.py

@@ -5,7 +5,6 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-import json
 import logging
 import datetime
 import contextlib
@@ -223,7 +222,7 @@ def load_and_verify_cached_results() -> Optional[rdoc.ResultDocument]:
     logger.debug("loading cached capa results from netnode '%s'", CAPA_NETNODE)
 
     n = netnode.Netnode(CAPA_NETNODE)
-    doc = rdoc.ResultDocument.parse_obj(json.loads(n[NETNODE_RESULTS]))
+    doc = rdoc.ResultDocument.model_validate_json(n[NETNODE_RESULTS])
 
     for rule in rutils.capability_rules(doc):
         for location_, _ in rule.matches:

capa/ida/plugin/form.py

@@ -573,10 +573,11 @@ class CapaExplorerForm(idaapi.PluginForm):
 
     def ensure_capa_settings_rule_path(self):
         try:
-            path: Path = Path(settings.user.get(CAPA_SETTINGS_RULE_PATH, ""))
+            path: str = settings.user.get(CAPA_SETTINGS_RULE_PATH, "")
 
             # resolve rules directory - check self and settings first, then ask user
-            if not path.exists():
+            # pathlib.Path considers "" equivalent to "." so we first check if rule path is an empty string
+            if not path or not Path(path).exists():
                 # configure rules selection messagebox
                 rules_message = QtWidgets.QMessageBox()
                 rules_message.setIcon(QtWidgets.QMessageBox.Information)
@@ -594,15 +595,15 @@ class CapaExplorerForm(idaapi.PluginForm):
                 if pressed == QtWidgets.QMessageBox.Cancel:
                     raise UserCancelledError()
 
-                path = Path(self.ask_user_directory())
+                path = self.ask_user_directory()
                 if not path:
                     raise UserCancelledError()
 
-                if not path.exists():
+                if not Path(path).exists():
                     logger.error("rule path %s does not exist or cannot be accessed", path)
                     return False
 
-                settings.user[CAPA_SETTINGS_RULE_PATH] = str(path)
+                settings.user[CAPA_SETTINGS_RULE_PATH] = path
         except UserCancelledError:
             capa.ida.helpers.inform_user_ida_ui("Analysis requires capa rules")
             logger.warning(
@@ -1304,7 +1305,7 @@ class CapaExplorerForm(idaapi.PluginForm):
             idaapi.info("No program analysis to save.")
             return
 
-        s = self.resdoc_cache.json().encode("utf-8")
+        s = self.resdoc_cache.model_dump_json().encode("utf-8")
 
         path = Path(self.ask_user_capa_json_file())
         if not path.exists():

capa/main.py

@@ -97,6 +97,7 @@ E_INVALID_FILE_TYPE = 16
 E_INVALID_FILE_ARCH = 17
 E_INVALID_FILE_OS = 18
 E_UNSUPPORTED_IDA_VERSION = 19
+E_UNSUPPORTED_GHIDRA_VERSION = 20
 
 logger = logging.getLogger("capa")
 
@@ -256,6 +257,11 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
     with redirecting_print_to_tqdm(disable_progress):
         with tqdm.contrib.logging.logging_redirect_tqdm():
             pbar = tqdm.tqdm
+            if capa.helpers.is_runtime_ghidra():
+                # Ghidrathon interpreter cannot properly handle
+                # the TMonitor thread that is created via a monitor_interval
+                # > 0
+                pbar.monitor_interval = 0
             if disable_progress:
                 # do not use tqdm to avoid unnecessary side effects when caller intends
                 # to disable progress completely
@@ -552,7 +558,8 @@ def get_extractor(
                 sys.path.append(str(bn_api))
 
         try:
-            from binaryninja import BinaryView, BinaryViewType
+            import binaryninja
+            from binaryninja import BinaryView
         except ImportError:
             raise RuntimeError(
                 "Cannot import binaryninja module. Please install the Binary Ninja Python API first: "
@@ -562,7 +569,7 @@ def get_extractor(
         import capa.features.extractors.binja.extractor
 
         with halo.Halo(text="analyzing program", spinner="simpleDots", stream=sys.stderr, enabled=not disable_progress):
-            bv: BinaryView = BinaryViewType.get_view_of_file(str(path))
+            bv: BinaryView = binaryninja.load(str(path))
             if bv is None:
                 raise RuntimeError(f"Binary Ninja cannot open file {path}")
 
@@ -1223,7 +1230,7 @@ def main(argv: Optional[List[str]] = None):
 
     if format_ == FORMAT_RESULT:
         # result document directly parses into meta, capabilities
-        result_doc = capa.render.result_document.ResultDocument.parse_file(args.sample)
+        result_doc = capa.render.result_document.ResultDocument.from_file(Path(args.sample))
         meta, capabilities = result_doc.to_capa()
 
     else:
@@ -1338,8 +1345,47 @@ def ida_main():
     print(capa.render.default.render(meta, rules, capabilities))
 
 
+def ghidra_main():
+    import capa.rules
+    import capa.ghidra.helpers
+    import capa.render.default
+    import capa.features.extractors.ghidra.extractor
+
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger().setLevel(logging.INFO)
+
+    logger.debug("-" * 80)
+    logger.debug(" Using default embedded rules.")
+    logger.debug(" ")
+    logger.debug(" You can see the current default rule set here:")
+    logger.debug("     https://github.com/mandiant/capa-rules")
+    logger.debug("-" * 80)
+
+    rules_path = get_default_root() / "rules"
+    logger.debug("rule path: %s", rules_path)
+    rules = get_rules([rules_path])
+
+    meta = capa.ghidra.helpers.collect_metadata([rules_path])
+
+    capabilities, counts = find_capabilities(
+        rules,
+        capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor(),
+        not capa.ghidra.helpers.is_running_headless(),
+    )
+
+    meta.analysis.feature_counts = counts["feature_counts"]
+    meta.analysis.library_functions = counts["library_functions"]
+
+    if has_file_limitation(rules, capabilities, is_standalone=False):
+        logger.info("capa encountered warnings during analysis")
+
+    print(capa.render.default.render(meta, rules, capabilities))
+
+
 if __name__ == "__main__":
     if capa.helpers.is_runtime_ida():
         ida_main()
+    elif capa.helpers.is_runtime_ghidra():
+        ghidra_main()
     else:
         sys.exit(main())

capa/render/json.py

@@ -11,4 +11,4 @@ from capa.engine import MatchResults
 
 
 def render(meta, rules: RuleSet, capabilities: MatchResults) -> str:
-    return rd.ResultDocument.from_capa(meta, rules, capabilities).json(exclude_none=True)
+    return rd.ResultDocument.from_capa(meta, rules, capabilities).model_dump_json(exclude_none=True)

capa/render/proto/__init__.py

@@ -126,7 +126,7 @@ def metadata_to_pb2(meta: rd.Metadata) -> capa_pb2.Metadata:
         timestamp=str(meta.timestamp),
         version=meta.version,
         argv=meta.argv,
-        sample=google.protobuf.json_format.ParseDict(meta.sample.dict(), capa_pb2.Sample()),
+        sample=google.protobuf.json_format.ParseDict(meta.sample.model_dump(), capa_pb2.Sample()),
         analysis=capa_pb2.Analysis(
             format=meta.analysis.format,
             arch=meta.analysis.arch,
@@ -393,7 +393,7 @@ def match_to_pb2(match: rd.Match) -> capa_pb2.Match:
 def rule_metadata_to_pb2(rule_metadata: rd.RuleMetadata) -> capa_pb2.RuleMetadata:
     # after manual type conversions to the RuleMetadata, we can rely on the protobuf json parser
     # conversions include tuple -> list and rd.Enum -> proto.enum
-    meta = dict_tuple_to_list_values(rule_metadata.dict())
+    meta = dict_tuple_to_list_values(rule_metadata.model_dump())
     meta["scope"] = scope_to_pb2(meta["scope"])
     meta["attack"] = list(map(dict_tuple_to_list_values, meta.get("attack", [])))
     meta["mbc"] = list(map(dict_tuple_to_list_values, meta.get("mbc", [])))

capa/render/result_document.py

@@ -7,9 +7,10 @@
 # See the License for the specific language governing permissions and limitations under the License.
 import datetime
 import collections
-from typing import Dict, List, Tuple, Union, Optional
+from typing import Dict, List, Tuple, Union, Literal, Optional
+from pathlib import Path
 
-from pydantic import Field, BaseModel
+from pydantic import Field, BaseModel, ConfigDict
 
 import capa.rules
 import capa.engine
@@ -23,14 +24,11 @@ from capa.helpers import assert_never
 
 
 class FrozenModel(BaseModel):
-    class Config:
-        frozen = True
-        extra = "forbid"
+    model_config = ConfigDict(frozen=True, extra="forbid")
 
 
 class Model(BaseModel):
-    class Config:
-        extra = "forbid"
+    model_config = ConfigDict(extra="forbid")
 
 
 class Sample(Model):
@@ -105,13 +103,13 @@ class CompoundStatement(StatementModel):
 
 
 class SomeStatement(StatementModel):
-    type = "some"
+    type: Literal["some"] = "some"
     description: Optional[str] = None
     count: int
 
 
 class RangeStatement(StatementModel):
-    type = "range"
+    type: Literal["range"] = "range"
     description: Optional[str] = None
     min: int
     max: int
@@ -119,7 +117,7 @@ class RangeStatement(StatementModel):
 
 
 class SubscopeStatement(StatementModel):
-    type = "subscope"
+    type: Literal["subscope"] = "subscope"
     description: Optional[str] = None
     scope: capa.rules.Scope
 
@@ -134,7 +132,7 @@ Statement = Union[
 
 
 class StatementNode(FrozenModel):
-    type = "statement"
+    type: Literal["statement"] = "statement"
     statement: Statement
 
 
@@ -171,7 +169,7 @@ def statement_from_capa(node: capa.engine.Statement) -> Statement:
 
 
 class FeatureNode(FrozenModel):
-    type = "feature"
+    type: Literal["feature"] = "feature"
     feature: frz.Feature
 
 
@@ -500,15 +498,12 @@ class MaecMetadata(FrozenModel):
     malware_family: Optional[str] = Field(None, alias="malware-family")
     malware_category: Optional[str] = Field(None, alias="malware-category")
     malware_category_ov: Optional[str] = Field(None, alias="malware-category-ov")
-
-    class Config:
-        frozen = True
-        allow_population_by_field_name = True
+    model_config = ConfigDict(frozen=True, populate_by_name=True)
 
 
 class RuleMetadata(FrozenModel):
     name: str
-    namespace: Optional[str]
+    namespace: Optional[str] = None
     authors: Tuple[str, ...]
     scope: capa.rules.Scope
     attack: Tuple[AttackSpec, ...] = Field(alias="att&ck")
@@ -546,9 +541,7 @@ class RuleMetadata(FrozenModel):
         )  # type: ignore
         # Mypy is unable to recognise arguments due to alias
 
-    class Config:
-        frozen = True
-        allow_population_by_field_name = True
+    model_config = ConfigDict(frozen=True, populate_by_name=True)
 
 
 class RuleMatches(FrozenModel):
@@ -604,3 +597,7 @@ class ResultDocument(FrozenModel):
                 capabilities[rule_name].append((addr.to_capa(), result))
 
         return self.meta, capabilities
+
+    @classmethod
+    def from_file(cls, path: Path) -> "ResultDocument":
+        return cls.model_validate_json(path.read_text(encoding="utf-8"))

capa/render/vverbose.py

@@ -88,7 +88,7 @@ def render_statement(ostream, match: rd.Match, statement: rd.Statement, indent=0
         # so, we have to inline some of the feature rendering here.
 
         child = statement.child
-        value = child.dict(by_alias=True).get(child.type)
+        value = child.model_dump(by_alias=True).get(child.type)
 
         if value:
             if isinstance(child, frzf.StringFeature):
@@ -141,7 +141,7 @@ def render_feature(ostream, match: rd.Match, feature: frzf.Feature, indent=0):
         value = feature.class_
     else:
         # convert attributes to dictionary using aliased names, if applicable
-        value = feature.dict(by_alias=True).get(key)
+        value = feature.model_dump(by_alias=True).get(key)
 
     if value is None:
         raise ValueError(f"{key} contains None")

capa/rules/__init__.py

@@ -106,6 +106,7 @@ SUPPORTED_FEATURES: Dict[str, Set] = {
         capa.features.common.Class,
         capa.features.common.Namespace,
         capa.features.common.Characteristic("mixed mode"),
+        capa.features.common.Characteristic("forwarded export"),
     },
     FUNCTION_SCOPE: {
         capa.features.common.MatchedRule,
@@ -737,6 +738,33 @@ class Rule:
 
         yield from self._extract_subscope_rules_rec(self.statement)
 
+    def _extract_all_features_rec(self, statement) -> Set[Feature]:
+        feature_set: Set[Feature] = set()
+
+        for child in statement.get_children():
+            if isinstance(child, Statement):
+                feature_set.update(self._extract_all_features_rec(child))
+            else:
+                feature_set.add(child)
+        return feature_set
+
+    def extract_all_features(self) -> Set[Feature]:
+        """
+        recursively extracts all feature statements in this rule.
+
+        returns:
+            set: A set of all feature statements contained within this rule.
+        """
+        if not isinstance(self.statement, ceng.Statement):
+            # For rules with single feature like
+            # anti-analysis\obfuscation\obfuscated-with-advobfuscator.yml
+            # contains a single feature - substring , which is of type String
+            return {
+                self.statement,
+            }
+
+        return self._extract_all_features_rec(self.statement)
+
     def evaluate(self, features: FeatureSet, short_circuit=True):
         capa.perf.counters["evaluate.feature"] += 1
         capa.perf.counters["evaluate.feature.rule"] += 1

capa/version.py

@@ -5,7 +5,7 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-__version__ = "6.0.0a2"
+__version__ = "6.1.0"
 
 
 def get_major_version():

pyproject.toml

@@ -18,6 +18,7 @@ authors = [
     {name = "Mike Hunhoff", email = "michael.hunhoff@mandiant.com"},
 ]
 description = "The FLARE team's open-source tool to identify capabilities in executable files."
+readme = {file = "README.md", content-type = "text/markdown"}
 license = {file = "LICENSE.txt"}
 requires-python = ">=3.8"
 keywords = ["malware analysis", "reverse engineering", "capability detection", "software behaviors", "capa", "FLARE"]
@@ -31,8 +32,8 @@ classifiers = [
     "Topic :: Security",
 ]
 dependencies = [
-    "tqdm==4.65.0",
-    "pyyaml==6.0",
+    "tqdm==4.66.1",
+    "pyyaml==6.0.1",
     "tabulate==0.9.0",
     "colorama==0.4.6",
     "termcolor==2.3.0",
@@ -44,29 +45,28 @@ dependencies = [
     "ruamel.yaml==0.17.32",
     "vivisect==1.1.1",
     "pefile==2023.2.7",
-    "pyelftools==0.29",
+    "pyelftools==0.30",
     "dnfile==0.13.0",
     "dncil==1.0.2",
-    "pydantic==1.10.9",
+    "pydantic==2.1.1",
     "protobuf==4.23.4",
 ]
-dynamic = ["version", "readme"]
+dynamic = ["version"]
 
 [tool.setuptools.dynamic]
 version = {attr = "capa.version.__version__"}
-readme = {file = "README.md"}
 
 [tool.setuptools]
 packages = ["capa"]
 
 [project.optional-dependencies]
 dev = [
-    "pre-commit==3.3.3",
-    "pytest==7.4.0",
+    "pre-commit==3.4.0",
+    "pytest==7.4.2",
     "pytest-sugar==0.9.7",
     "pytest-instafail==0.5.0",
     "pytest-cov==4.1.0",
-    "flake8==6.0.0",
+    "flake8==6.1.0",
     "flake8-bugbear==23.7.10",
     "flake8-encodings==0.5.0.post1",
     "flake8-comprehensions==3.14.0",
@@ -77,28 +77,28 @@ dev = [
     "flake8-simplify==0.20.0",
     "flake8-use-pathlib==0.3.0",
     "flake8-copyright==0.2.4",
-    "ruff==0.0.278",
+    "ruff==0.0.291",
     "black==23.7.0",
     "isort==5.11.4",
-    "mypy==1.4.1",
+    "mypy==1.5.1",
     "psutil==5.9.2",
     "stix2==3.0.1",
     "requests==2.31.0",
-    "mypy-protobuf==3.4.0",
+    "mypy-protobuf==3.5.0",
     # type stubs for mypy
     "types-backports==0.1.3",
     "types-colorama==0.4.15.11",
     "types-PyYAML==6.0.8",
-    "types-tabulate==0.9.0.1",
+    "types-tabulate==0.9.0.3",
     "types-termcolor==1.1.4",
     "types-psutil==5.8.23",
-    "types_requests==2.31.0.1",
-    "types-protobuf==4.23.0.1",
+    "types_requests==2.31.0.2",
+    "types-protobuf==4.23.0.3",
 ]
 build = [
     "pyinstaller==5.10.1",
     "setuptools==68.0.0",
-    "build==0.10.0"
+    "build==1.0.3"
 ]
 
 [project.urls]

scripts/bulk-process.py

@@ -144,8 +144,7 @@ def get_capa_results(args):
     meta.analysis.layout = capa.main.compute_layout(rules, extractor, capabilities)
 
     doc = rd.ResultDocument.from_capa(meta, rules, capabilities)
-
-    return {"path": path, "status": "ok", "ok": doc.dict(exclude_none=True)}
+    return {"path": path, "status": "ok", "ok": doc.model_dump()}
 
 
 def main(argv=None):
@@ -214,7 +213,9 @@ def main(argv=None):
             if result["status"] == "error":
                 logger.warning(result["error"])
             elif result["status"] == "ok":
-                results[result["path"].as_posix()] = rd.ResultDocument.parse_obj(result["ok"]).json(exclude_none=True)
+                results[result["path"].as_posix()] = rd.ResultDocument.model_validate(result["ok"]).model_dump_json(
+                    exclude_none=True
+                )
             else:
                 raise ValueError(f"unexpected status: {result['status']}")
 

scripts/detect_duplicate_features.py

@@ -8,38 +8,17 @@
 import sys
 import logging
 import argparse
+from typing import Set
 from pathlib import Path
 
 import capa.main
 import capa.rules
-import capa.engine as ceng
+from capa.features.common import Feature
 
 logger = logging.getLogger("detect_duplicate_features")
 
 
-def get_child_features(feature: ceng.Statement) -> list:
-    """
-    Recursively extracts all feature statements from a given rule statement.
-
-    Args:
-        feature (capa.engine.Statement): The feature statement to extract features from.
-
-    Returns:
-        list: A list of all feature statements contained within the given feature statement.
-    """
-    children = []
-
-    if isinstance(feature, (ceng.And, ceng.Or, ceng.Some)):
-        for child in feature.children:
-            children.extend(get_child_features(child))
-    elif isinstance(feature, (ceng.Subscope, ceng.Range, ceng.Not)):
-        children.extend(get_child_features(feature.child))
-    else:
-        children.append(feature)
-    return children
-
-
-def get_features(rule_path: str) -> list:
+def get_features(rule_path: str) -> Set[Feature]:
     """
     Extracts all features from a given rule file.
 
@@ -47,17 +26,15 @@ def get_features(rule_path: str) -> list:
         rule_path (str): The path to the rule file to extract features from.
 
     Returns:
-        list: A list of all feature statements contained within the rule file.
+        set: A set of all feature statements contained within the rule file.
     """
-    feature_list = []
     with Path(rule_path).open("r", encoding="utf-8") as f:
         try:
             new_rule = capa.rules.Rule.from_yaml(f.read())
-            feature_list = get_child_features(new_rule.statement)
+            return new_rule.extract_all_features()
         except Exception as e:
             logger.error("Error: New rule %s %s %s", rule_path, str(type(e)), str(e))
             sys.exit(-1)
-    return feature_list
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
@@ -67,7 +44,6 @@ def find_overlapping_rules(new_rule_path, rules_path):
 
     # Loads features of new rule in a list.
     new_rule_features = get_features(new_rule_path)
-
     count = 0
     overlapping_rules = []
 
@@ -75,7 +51,7 @@ def find_overlapping_rules(new_rule_path, rules_path):
     ruleset = capa.main.get_rules(rules_path)
 
     for rule_name, rule in ruleset.rules.items():
-        rule_features = get_child_features(rule.statement)
+        rule_features = rule.extract_all_features()
 
         if not len(rule_features):
             continue

scripts/import-to-ida.py

@@ -30,6 +30,7 @@ See the License for the specific language governing permissions and limitations
 """
 import logging
 import binascii
+from pathlib import Path
 
 import ida_nalt
 import ida_funcs
@@ -68,7 +69,7 @@ def main():
     if not path:
         return 0
 
-    result_doc = capa.render.result_document.ResultDocument.parse_file(path)
+    result_doc = capa.render.result_document.ResultDocument.from_file(Path(path))
     meta, capabilities = result_doc.to_capa()
 
     # in IDA 7.4, the MD5 hash may be truncated, for example:

scripts/lint.py

@@ -309,7 +309,7 @@ def get_sample_capabilities(ctx: Context, path: Path) -> Set[str]:
 
     logger.debug("analyzing sample: %s", nice_path)
     extractor = capa.main.get_extractor(
-        nice_path, format_, OS_AUTO, "", DEFAULT_SIGNATURES, False, disable_progress=True
+        nice_path, format_, OS_AUTO, capa.main.BACKEND_VIV, DEFAULT_SIGNATURES, False, disable_progress=True
     )
 
     capabilities, _ = capa.main.find_capabilities(ctx.rules, extractor, disable_progress=True)
@@ -569,6 +569,10 @@ class FeatureNtdllNtoskrnlApi(Lint):
                     "ZwCreateProcess",
                     "ZwCreateUserProcess",
                     "RtlCreateUserProcess",
+                    "NtProtectVirtualMemory",
+                    "NtEnumerateSystemEnvironmentValuesEx",
+                    "NtQuerySystemEnvironmentValueEx",
+                    "NtQuerySystemEnvironmentValue",
                 ):
                     # ntoskrnl.exe does not export these routines
                     continue
@@ -579,6 +583,7 @@ class FeatureNtdllNtoskrnlApi(Lint):
                     "KeStackAttachProcess",
                     "ObfDereferenceObject",
                     "KeUnstackDetachProcess",
+                    "ExGetFirmwareEnvironmentVariable",
                 ):
                     # ntdll.dll does not export these routines
                     continue

scripts/proto-from-results.py

@@ -31,6 +31,7 @@ Example:
 import sys
 import logging
 import argparse
+from pathlib import Path
 
 import capa.render.proto
 import capa.render.result_document
@@ -64,7 +65,7 @@ def main(argv=None):
         logging.basicConfig(level=logging.INFO)
         logging.getLogger().setLevel(logging.INFO)
 
-    rd = capa.render.result_document.ResultDocument.parse_file(args.json)
+    rd = capa.render.result_document.ResultDocument.from_file(Path(args.json))
     pb = capa.render.proto.doc_to_pb2(rd)
 
     sys.stdout.buffer.write(pb.SerializeToString(deterministic=True))

scripts/proto-to-results.py

@@ -78,7 +78,7 @@ def main(argv=None):
     rdpb.ParseFromString(pb)
 
     rd = capa.render.proto.doc_from_pb2(rdpb)
-    print(rd.json(exclude_none=True, indent=2, sort_keys=True))
+    print(rd.model_dump_json(exclude_none=True, indent=2))
 
 
 if __name__ == "__main__":

scripts/show-features.py

@@ -199,6 +199,21 @@ def ida_main():
     return 0
 
 
+def ghidra_main():
+    import capa.features.extractors.ghidra.extractor
+
+    extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
+
+    for feature, addr in extractor.extract_file_features():
+        print(f"file: {format_address(addr)}: {feature}")
+
+    function_handles = tuple(extractor.get_functions())
+
+    print_features(function_handles, extractor)
+
+    return 0
+
+
 def print_features(functions, extractor: capa.features.extractors.base_extractor.FeatureExtractor):
     for f in functions:
         if extractor.is_library_function(f.address):
@@ -248,5 +263,7 @@ def print_features(functions, extractor: capa.features.extractors.base_extractor
 if __name__ == "__main__":
     if capa.helpers.is_runtime_ida():
         ida_main()
+    elif capa.helpers.is_runtime_ghidra():
+        ghidra_main()
     else:
         sys.exit(main())

scripts/show-unused-features.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""
+Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+You may obtain a copy of the License at: [package root]/LICENSE.txt
+Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+"""
+import os
+import sys
+import typing
+import logging
+import argparse
+from typing import Set, Tuple
+from pathlib import Path
+from collections import Counter
+
+import tabulate
+from termcolor import colored
+
+import capa.main
+import capa.rules
+import capa.helpers
+import capa.features
+import capa.exceptions
+import capa.render.verbose as v
+import capa.features.common
+import capa.features.freeze
+import capa.features.address
+import capa.features.extractors.pefile
+import capa.features.extractors.base_extractor
+from capa.helpers import log_unsupported_runtime_error
+from capa.features.common import Feature
+from capa.features.extractors.base_extractor import FunctionHandle
+
+logger = logging.getLogger("show-unused-features")
+
+
+def format_address(addr: capa.features.address.Address) -> str:
+    return v.format_address(capa.features.freeze.Address.from_capa((addr)))
+
+
+def get_rules_feature_set(rules_path) -> Set[Feature]:
+    ruleset = capa.main.get_rules(rules_path)
+    rules_feature_set: Set[Feature] = set()
+    for _, rule in ruleset.rules.items():
+        rules_feature_set.update(rule.extract_all_features())
+
+    return rules_feature_set
+
+
+def get_file_features(
+    functions: Tuple[FunctionHandle, ...], extractor: capa.features.extractors.base_extractor.FeatureExtractor
+) -> typing.Counter[Feature]:
+    feature_map: typing.Counter[Feature] = Counter()
+
+    for f in functions:
+        if extractor.is_library_function(f.address):
+            function_name = extractor.get_function_name(f.address)
+            logger.debug("skipping library function %s (%s)", format_address(f.address), function_name)
+            continue
+
+        for feature, _ in extractor.extract_function_features(f):
+            if capa.features.common.is_global_feature(feature):
+                continue
+            feature_map.update([feature])
+
+        for bb in extractor.get_basic_blocks(f):
+            for feature, _ in extractor.extract_basic_block_features(f, bb):
+                if capa.features.common.is_global_feature(feature):
+                    continue
+                feature_map.update([feature])
+
+            for insn in extractor.get_instructions(f, bb):
+                for feature, _ in extractor.extract_insn_features(f, bb, insn):
+                    if capa.features.common.is_global_feature(feature):
+                        continue
+                    feature_map.update([feature])
+    return feature_map
+
+
+def get_colored(s: str):
+    if "(" in s and ")" in s:
+        s_split = s.split("(", 1)
+        s_color = colored(s_split[1][:-1], "cyan")
+        return f"{s_split[0]}({s_color})"
+    else:
+        return colored(s, "cyan")
+
+
+def print_unused_features(feature_map: typing.Counter[Feature], rules_feature_set: Set[Feature]):
+    unused_features = []
+    for feature, count in reversed(feature_map.most_common()):
+        if feature in rules_feature_set:
+            continue
+        unused_features.append((str(count), get_colored(str(feature))))
+    print("\n")
+    print(tabulate.tabulate(unused_features, headers=["Count", "Feature"], tablefmt="plain"))
+    print("\n")
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(description="Show the features that capa doesn't have rules for yet")
+    capa.main.install_common_args(parser, wanted={"format", "os", "sample", "signatures", "backend", "rules"})
+
+    parser.add_argument("-F", "--function", type=str, help="Show features for specific function")
+    args = parser.parse_args(args=argv)
+    capa.main.handle_common_args(args)
+
+    if args.function and args.backend == "pefile":
+        print("pefile backend does not support extracting function features")
+        return -1
+
+    try:
+        taste = capa.helpers.get_file_taste(Path(args.sample))
+    except IOError as e:
+        logger.error("%s", str(e))
+        return -1
+
+    try:
+        sig_paths = capa.main.get_signatures(args.signatures)
+    except IOError as e:
+        logger.error("%s", str(e))
+        return -1
+
+    if (args.format == "freeze") or (
+        args.format == capa.features.common.FORMAT_AUTO and capa.features.freeze.is_freeze(taste)
+    ):
+        extractor = capa.features.freeze.load(Path(args.sample).read_bytes())
+    else:
+        should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None)
+        try:
+            extractor = capa.main.get_extractor(
+                args.sample, args.format, args.os, args.backend, sig_paths, should_save_workspace
+            )
+        except capa.exceptions.UnsupportedFormatError:
+            capa.helpers.log_unsupported_format_error()
+            return -1
+        except capa.exceptions.UnsupportedRuntimeError:
+            log_unsupported_runtime_error()
+            return -1
+
+    feature_map: typing.Counter[Feature] = Counter()
+
+    feature_map.update([feature for feature, _ in extractor.extract_global_features()])
+
+    function_handles: Tuple[FunctionHandle, ...]
+    if isinstance(extractor, capa.features.extractors.pefile.PefileFeatureExtractor):
+        # pefile extractor doesn't extract function features
+        function_handles = ()
+    else:
+        function_handles = tuple(extractor.get_functions())
+
+    if args.function:
+        if args.format == "freeze":
+            function_handles = tuple(filter(lambda fh: fh.address == args.function, function_handles))
+        else:
+            function_handles = tuple(filter(lambda fh: format_address(fh.address) == args.function, function_handles))
+
+            if args.function not in [format_address(fh.address) for fh in function_handles]:
+                print(f"{args.function} not a function")
+                return -1
+
+        if len(function_handles) == 0:
+            print(f"{args.function} not a function")
+            return -1
+
+    feature_map.update(get_file_features(function_handles, extractor))
+
+    rules_feature_set = get_rules_feature_set(args.rules)
+
+    print_unused_features(feature_map, rules_feature_set)
+    return 0
+
+
+def ida_main():
+    import idc
+
+    import capa.main
+    import capa.features.extractors.ida.extractor
+
+    function = idc.get_func_attr(idc.here(), idc.FUNCATTR_START)
+    print(f"getting features for current function {hex(function)}")
+
+    extractor = capa.features.extractors.ida.extractor.IdaFeatureExtractor()
+    feature_map: typing.Counter[Feature] = Counter()
+
+    feature_map.update([feature for feature, _ in extractor.extract_file_features()])
+
+    function_handles = tuple(extractor.get_functions())
+
+    if function:
+        function_handles = tuple(filter(lambda fh: fh.inner.start_ea == function, function_handles))
+
+        if len(function_handles) == 0:
+            print(f"{hex(function)} not a function")
+            return -1
+
+    feature_map.update(get_file_features(function_handles, extractor))
+
+    rules_path = capa.main.get_default_root() / "rules"
+    rules_feature_set = get_rules_feature_set([rules_path])
+
+    print_unused_features(feature_map, rules_feature_set)
+
+    return 0
+
+
+if __name__ == "__main__":
+    if capa.helpers.is_runtime_ida():
+        ida_main()
+    else:
+        sys.exit(main())

tests/fixtures.py

@@ -159,7 +159,8 @@ def get_dnfile_extractor(path: Path):
 
 @lru_cache(maxsize=1)
 def get_binja_extractor(path: Path):
-    from binaryninja import Settings, BinaryViewType
+    import binaryninja
+    from binaryninja import Settings
 
     import capa.features.extractors.binja.extractor
 
@@ -168,7 +169,7 @@ def get_binja_extractor(path: Path):
     if path.name.endswith("kernel32-64.dll_"):
         old_pdb = settings.get_bool("pdb.loadGlobalSymbols")
         settings.set_bool("pdb.loadGlobalSymbols", False)
-    bv = BinaryViewType.get_view_of_file(str(path))
+    bv = binaryninja.load(str(path))
     if path.name.endswith("kernel32-64.dll_"):
         settings.set_bool("pdb.loadGlobalSymbols", old_pdb)
 
@@ -180,6 +181,16 @@ def get_binja_extractor(path: Path):
     return extractor
 
 
+@lru_cache(maxsize=1)
+def get_ghidra_extractor(path: Path):
+    import capa.features.extractors.ghidra.extractor
+
+    extractor = capa.features.extractors.ghidra.extractor.GhidraFeatureExtractor()
+    setattr(extractor, "path", path.as_posix())
+
+    return extractor
+
+
 def extract_global_features(extractor):
     features = collections.defaultdict(set)
     for feature, va in extractor.extract_global_features():
@@ -308,6 +319,8 @@ def get_data_path_by_name(name) -> Path:
         return CD / "data" / "2bf18d0403677378adad9001b1243211.elf_"
     elif name.startswith("ea2876"):
         return CD / "data" / "ea2876e9175410b6f6719f80ee44b9553960758c7d0f7bed73c0fe9a78d8e669.dll_"
+    elif name.startswith("1038a2"):
+        return CD / "data" / "1038a23daad86042c66bfe6c9d052d27048de9653bde5750dc0f240c792d9ac8.elf_"
     else:
         raise ValueError(f"unexpected sample fixture: {name}")
 
@@ -355,7 +368,7 @@ def get_sample_md5_by_name(name):
     elif name.startswith("3b13b"):
         # file name is SHA256 hash
         return "56a6ffe6a02941028cc8235204eef31d"
-    elif name == "7351f.elf":
+    elif name.startswith("7351f"):
         return "7351f8a40c5450557b24622417fc478d"
     elif name.startswith("79abd"):
         return "79abd17391adc6251ecdc58d13d76baf"
@@ -1051,6 +1064,14 @@ FEATURE_COUNT_TESTS_DOTNET = [
 ]
 
 
+FEATURE_COUNT_TESTS_GHIDRA = [
+    # Ghidra may render functions as labels, as well as provide differing amounts of call references
+    # (Colton) TODO: Add more test cases
+    ("mimikatz", "function=0x4702FD", capa.features.common.Characteristic("calls from"), 0),
+    ("mimikatz", "function=0x4556E5", capa.features.common.Characteristic("calls to"), 0),
+]
+
+
 def do_test_feature_presence(get_extractor, sample, scope, feature, expected):
     extractor = get_extractor(sample)
     features = scope(extractor)
@@ -1180,8 +1201,8 @@ def _039a6_dotnetfile_extractor():
     return get_dnfile_extractor(get_data_path_by_name("_039a6"))
 
 
-def get_result_doc(path):
-    return capa.render.result_document.ResultDocument.parse_file(path)
+def get_result_doc(path: Path):
+    return capa.render.result_document.ResultDocument.from_file(path)
 
 
 @pytest.fixture

tests/test_binja_features.py

@@ -36,19 +36,10 @@ except ImportError:
 @pytest.mark.skipif(binja_present is False, reason="Skip binja tests if the binaryninja Python API is not installed")
 @fixtures.parametrize(
     "sample,scope,feature,expected",
-    fixtures.FEATURE_PRESENCE_TESTS,
+    fixtures.FEATURE_PRESENCE_TESTS + fixtures.FEATURE_SYMTAB_FUNC_TESTS,
     indirect=["sample", "scope"],
 )
 def test_binja_features(sample, scope, feature, expected):
-    if feature == capa.features.common.Characteristic("stack string"):
-        pytest.xfail("skip failing Binja stack string detection temporarily, see #1473")
-
-    if isinstance(feature, capa.features.file.Export) and "." in str(feature.value):
-        pytest.xfail("skip Binja unsupported forwarded export feature, see #1646")
-
-    if feature == capa.features.common.Characteristic("forwarded export"):
-        pytest.xfail("skip Binja unsupported forwarded export feature, see #1646")
-
     fixtures.do_test_feature_presence(fixtures.get_binja_extractor, sample, scope, feature, expected)
 
 
@@ -72,4 +63,4 @@ def test_standalone_binja_backend():
 @pytest.mark.skipif(binja_present is False, reason="Skip binja tests if the binaryninja Python API is not installed")
 def test_binja_version():
     version = binaryninja.core_version_info()
-    assert version.major == 3 and version.minor == 4
+    assert version.major == 3 and version.minor == 5

tests/test_elffile_features.py

@@ -0,0 +1,71 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+import io
+from pathlib import Path
+
+from elftools.elf.elffile import ELFFile
+
+from capa.features.extractors.elffile import extract_file_export_names, extract_file_import_names
+
+CD = Path(__file__).resolve().parent
+SAMPLE_PATH = CD / "data" / "055da8e6ccfe5a9380231ea04b850e18.elf_"
+
+
+def test_elffile_import_features():
+    expected_imports = [
+        "memfrob",
+        "puts",
+        "__libc_start_main",
+        "malloc",
+        "__cxa_finalize",
+    ]
+    path = Path(SAMPLE_PATH)
+    elf = ELFFile(io.BytesIO(path.read_bytes()))
+    # Extract imports
+    imports = list(extract_file_import_names(elf))
+
+    # Verify that at least one import was found
+    assert len(imports) > 0, "No imports were found."
+
+    # Extract the symbol names from the extracted imports
+    extracted_symbol_names = [imported[0].value for imported in imports]
+
+    # Check if all expected symbol names are found
+    for symbol_name in expected_imports:
+        assert symbol_name in extracted_symbol_names, f"Symbol '{symbol_name}' not found in imports."
+
+
+def test_elffile_export_features():
+    expected_exports = [
+        "deregister_tm_clones",
+        "register_tm_clones",
+        "__do_global_dtors_aux",
+        "completed.8060",
+        "__do_global_dtors_aux_fini_array_entry",
+        "frame_dummy",
+        "_init",
+        "__libc_csu_fini",
+        "_fini",
+        "__dso_handle",
+        "_IO_stdin_used",
+        "__libc_csu_init",
+    ]
+    path = Path(SAMPLE_PATH)
+    elf = ELFFile(io.BytesIO(path.read_bytes()))
+    # Extract imports
+    exports = list(extract_file_export_names(elf))
+
+    # Verify that at least one export was found
+    assert len(exports) > 0, "No exports were found."
+
+    # Extract the symbol names from the extracted imports
+    extracted_symbol_names = [exported[0].value for exported in exports]
+
+    # Check if all expected symbol names are found
+    for symbol_name in expected_exports:
+        assert symbol_name in extracted_symbol_names, f"Symbol '{symbol_name}' not found in exports."

tests/test_ghidra_features.py

@@ -0,0 +1,98 @@
+# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at: [package root]/LICENSE.txt
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+"""
+Must invoke this script from within the Ghidra Runtime Enviornment
+"""
+import sys
+import logging
+from pathlib import Path
+
+import pytest
+
+try:
+    sys.path.append(str(Path(__file__).parent))
+    import fixtures
+finally:
+    sys.path.pop()
+
+
+logger = logging.getLogger("test_ghidra_features")
+
+ghidra_present: bool = False
+try:
+    import ghidra  # noqa: F401
+
+    ghidra_present = True
+except ImportError:
+    pass
+
+
+def standardize_posix_str(psx_str):
+    """fixture test passes the PosixPath to the test data
+
+    params: psx_str - PosixPath() to the test data
+    return: string that matches test-id sample name
+    """
+
+    if "Practical Malware Analysis Lab" in str(psx_str):
+        # <PosixPath>/'Practical Malware Analysis Lab 16-01.exe_' -> 'pma16-01'
+        wanted_str = "pma" + str(psx_str).split("/")[-1][len("Practical Malware Analysis Lab ") : -5]
+    else:
+        # <PosixPath>/mimikatz.exe_ -> mimikatz
+        wanted_str = str(psx_str).split("/")[-1][:-5]
+
+    if "_" in wanted_str:
+        # al-khaser_x86 -> al-khaser x86
+        wanted_str = wanted_str.replace("_", " ")
+
+    return wanted_str
+
+
+def check_input_file(wanted):
+    """check that test is running on the loaded sample
+
+    params: wanted - PosixPath() passed from test arg
+    """
+
+    import capa.ghidra.helpers as ghidra_helpers
+
+    found = ghidra_helpers.get_file_md5()
+    sample_name = standardize_posix_str(wanted)
+
+    if not found.startswith(fixtures.get_sample_md5_by_name(sample_name)):
+        raise RuntimeError(f"please run the tests against sample with MD5: `{found}`")
+
+
+@pytest.mark.skipif(ghidra_present is False, reason="Ghidra tests must be ran within Ghidra")
+@fixtures.parametrize("sample,scope,feature,expected", fixtures.FEATURE_PRESENCE_TESTS, indirect=["sample", "scope"])
+def test_ghidra_features(sample, scope, feature, expected):
+    try:
+        check_input_file(sample)
+    except RuntimeError:
+        pytest.skip(reason="Test must be ran against sample loaded in Ghidra")
+
+    fixtures.do_test_feature_presence(fixtures.get_ghidra_extractor, sample, scope, feature, expected)
+
+
+@pytest.mark.skipif(ghidra_present is False, reason="Ghidra tests must be ran within Ghidra")
+@fixtures.parametrize(
+    "sample,scope,feature,expected", fixtures.FEATURE_COUNT_TESTS_GHIDRA, indirect=["sample", "scope"]
+)
+def test_ghidra_feature_counts(sample, scope, feature, expected):
+    try:
+        check_input_file(sample)
+    except RuntimeError:
+        pytest.skip(reason="Test must be ran against sample loaded in Ghidra")
+
+    fixtures.do_test_feature_count(fixtures.get_ghidra_extractor, sample, scope, feature, expected)
+
+
+if __name__ == "__main__":
+    # No support for faulthandler module in Ghidrathon, see:
+    # https://github.com/mandiant/Ghidrathon/issues/70
+    sys.exit(pytest.main(["--pyargs", "-p no:faulthandler", "test_ghidra_features"]))

tests/test_ida_features.py

@@ -92,6 +92,15 @@ def get_ida_extractor(_path):
     return capa.features.extractors.ida.extractor.IdaFeatureExtractor()
 
 
+def nocollect(f):
+    "don't collect the decorated function as a pytest test"
+    f.__test__ = False
+    return f
+
+
+# although these look like pytest tests, they're not, because they don't run within pytest
+# (the runner is below) and they use `yield`, which is deprecated.
+@nocollect
 @pytest.mark.skip(reason="IDA Pro tests must be run within IDA")
 def test_ida_features():
     # we're guaranteed to be in a function here, so there's a stack frame
@@ -118,6 +127,7 @@ def test_ida_features():
             yield this_name, id, "pass", None
 
 
+@nocollect
 @pytest.mark.skip(reason="IDA Pro tests must be run within IDA")
 def test_ida_feature_counts():
     # we're guaranteed to be in a function here, so there's a stack frame

tests/test_os_detection.py

@@ -80,6 +80,18 @@ def test_elf_symbol_table():
         assert capa.features.extractors.elf.detect_elf_os(f) == "linux"
 
 
+def test_elf_android_notes():
+    # DEBUG:capa.features.extractors.elf:guess: osabi: None
+    # DEBUG:capa.features.extractors.elf:guess: ph notes: OS.ANDROID
+    # DEBUG:capa.features.extractors.elf:guess: sh notes: None
+    # DEBUG:capa.features.extractors.elf:guess: linker: None
+    # DEBUG:capa.features.extractors.elf:guess: ABI versions needed: None
+    # DEBUG:capa.features.extractors.elf:guess: needed dependencies: OS.ANDROID
+    path = get_data_path_by_name("1038a2")
+    with Path(path).open("rb") as f:
+        assert capa.features.extractors.elf.detect_elf_os(f) == "android"
+
+
 def test_elf_parse_capa_pyinstaller_header():
     # error after misidentified large pydata section with address 0; fixed in #1454
     # compressed ELF header of capa-v5.1.0-linux

tests/test_result_document.py

@@ -236,22 +236,22 @@ def test_basic_block_node_from_capa():
 def assert_round_trip(rd: rdoc.ResultDocument):
     one = rd
 
-    doc = one.json(exclude_none=True)
-    two = rdoc.ResultDocument.parse_raw(doc)
+    doc = one.model_dump_json(exclude_none=True)
+    two = rdoc.ResultDocument.model_validate_json(doc)
 
     # show the round trip works
     # first by comparing the objects directly,
     # which works thanks to pydantic model equality.
     assert one == two
     # second by showing their json representations are the same.
-    assert one.json(exclude_none=True) == two.json(exclude_none=True)
+    assert one.model_dump_json(exclude_none=True) == two.model_dump_json(exclude_none=True)
 
     # now show that two different versions are not equal.
     three = copy.deepcopy(two)
     three.meta.__dict__.update({"version": "0.0.0"})
     assert one.meta.version != three.meta.version
     assert one != three
-    assert one.json(exclude_none=True) != three.json(exclude_none=True)
+    assert one.model_dump_json(exclude_none=True) != three.model_dump_json(exclude_none=True)
 
 
 @pytest.mark.parametrize(
@@ -272,13 +272,13 @@ def test_round_trip(request, rd_file):
 
 def test_json_to_rdoc():
     path = fixtures.get_data_path_by_name("pma01-01-rd")
-    assert isinstance(rdoc.ResultDocument.parse_file(path), rdoc.ResultDocument)
+    assert isinstance(rdoc.ResultDocument.from_file(path), rdoc.ResultDocument)
 
 
 def test_rdoc_to_capa():
     path = fixtures.get_data_path_by_name("pma01-01-rd")
 
-    rd = rdoc.ResultDocument.parse_file(path)
+    rd = rdoc.ResultDocument.from_file(path)
 
     meta, capabilites = rd.to_capa()
     assert isinstance(meta, rdoc.Metadata)

tests/test_scripts.py

@@ -45,6 +45,7 @@ def get_rule_path():
         pytest.param("show-capabilities-by-function.py", [get_file_path()]),
         pytest.param("show-features.py", [get_file_path()]),
         pytest.param("show-features.py", ["-F", "0x407970", get_file_path()]),
+        pytest.param("show-unused-features.py", [get_file_path()]),
         pytest.param("capa_as_library.py", [get_file_path()]),
     ],
 )