From aefe97e09ef5e997a483c96f394002cc5bb572c2 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 08:39:56 -0600 Subject: [PATCH 01/52] rules: fix typos --- capa/rules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/capa/rules.py b/capa/rules.py index 6fa9a4c4..479a0773 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -22,7 +22,7 @@ except ImportError: # https://github.com/python/mypy/issues/1153 from backports.functools_lru_cache import lru_cache # type: ignore -from typing import Any, Set, Dict, List, Union, Iterator +from typing import Any, Dict, List, Union, Iterator import yaml import ruamel.yaml @@ -153,14 +153,14 @@ def ensure_feature_valid_for_scope(scope: str, feature: Union[Feature, Statement and isinstance(feature.value, str) and capa.features.common.Characteristic(feature.value) not in SUPPORTED_FEATURES[scope] ): - raise InvalidRule("feature %s not support for scope %s" % (feature, scope)) + raise InvalidRule("feature %s not supported for scope %s" % (feature, scope)) if not isinstance(feature, capa.features.common.Characteristic): # features of this scope that are not Characteristics will be Type instances. # check that the given feature is one of these types. types_for_scope = filter(lambda t: isinstance(t, type), SUPPORTED_FEATURES[scope]) if not isinstance(feature, tuple(types_for_scope)): # type: ignore - raise InvalidRule("feature %s not support for scope %s" % (feature, scope)) + raise InvalidRule("feature %s not supported for scope %s" % (feature, scope)) def parse_int(s: str) -> int: From a1eca58d7aa9d40b95d9d99af9c5b5dfeed6a6d8 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 08:40:40 -0600 Subject: [PATCH 02/52] features: support characteristic(os/*) features --- capa/features/common.py | 6 ++++++ capa/rules.py | 11 ++++++++++- tests/test_rules.py | 19 ++++++++++++++++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/capa/features/common.py b/capa/features/common.py index 221f15c1..b3a0735a 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -27,6 +27,11 @@ ARCH_X32 = "x32" ARCH_X64 = "x64" VALID_ARCH = (ARCH_X32, ARCH_X64) +OS_WINDOWS = "os/windows" +OS_LINUX = "os/linux" +OS_MACOS = "os/macos" +VALID_OS = (OS_WINDOWS, OS_LINUX, OS_MACOS) + def bytes_to_str(b: bytes) -> str: return str(codecs.encode(b, "hex").decode("utf-8")) @@ -131,6 +136,7 @@ class MatchedRule(Feature): class Characteristic(Feature): def __init__(self, value: str, description=None): + super(Characteristic, self).__init__(value, description=description) diff --git a/capa/rules.py b/capa/rules.py index 479a0773..3d62b7f5 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -34,7 +34,7 @@ import capa.features.insn import capa.features.common import capa.features.basicblock from capa.engine import Statement, FeatureSet -from capa.features.common import MAX_BYTES_FEATURE_SIZE, Feature +from capa.features.common import OS_LINUX, OS_MACOS, OS_WINDOWS, MAX_BYTES_FEATURE_SIZE, Feature logger = logging.getLogger(__name__) @@ -78,6 +78,9 @@ SUPPORTED_FEATURES = { capa.features.file.FunctionName, capa.features.common.Characteristic("embedded pe"), capa.features.common.String, + capa.features.common.Characteristic(OS_WINDOWS), + capa.features.common.Characteristic(OS_LINUX), + capa.features.common.Characteristic(OS_MACOS), }, FUNCTION_SCOPE: { # plus basic block scope features, see below @@ -86,6 +89,9 @@ SUPPORTED_FEATURES = { capa.features.common.Characteristic("calls to"), capa.features.common.Characteristic("loop"), capa.features.common.Characteristic("recursive call"), + capa.features.common.Characteristic(OS_WINDOWS), + capa.features.common.Characteristic(OS_LINUX), + capa.features.common.Characteristic(OS_MACOS), }, BASIC_BLOCK_SCOPE: { capa.features.common.MatchedRule, @@ -103,6 +109,9 @@ SUPPORTED_FEATURES = { capa.features.common.Characteristic("tight loop"), capa.features.common.Characteristic("stack string"), capa.features.common.Characteristic("indirect call"), + capa.features.common.Characteristic(OS_WINDOWS), + capa.features.common.Characteristic(OS_LINUX), + capa.features.common.Characteristic(OS_MACOS), }, } diff --git a/tests/test_rules.py b/tests/test_rules.py index 12791d2e..6b57b89a 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -15,7 +15,7 @@ import capa.engine import capa.features.common from capa.features.file import FunctionName from capa.features.insn import Number, Offset -from capa.features.common import ARCH_X32, ARCH_X64, String +from capa.features.common import ARCH_X32, ARCH_X64, OS_WINDOWS, String, Characteristic def test_rule_ctor(): @@ -944,3 +944,20 @@ def test_function_name_features(): assert (FunctionName("strcpy") in children) == True assert (FunctionName("strcmp", description="copy from here to there") in children) == True assert (FunctionName("strdup", description="duplicate a string") in children) == True + + +def test_os_features(): + rule = textwrap.dedent( + """ + rule: + meta: + name: test rule + scope: file + features: + - and: + - characteristic: os/windows + """ + ) + r = capa.rules.Rule.from_yaml(rule) + children = list(r.statement.get_children()) + assert (Characteristic(OS_WINDOWS) in children) == True From e797a67e9766d5373b551906e1e326564aa497ab Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 09:08:37 -0600 Subject: [PATCH 03/52] features: define CHARACTERISTIC_OS constants for ease of use --- capa/features/common.py | 5 +++++ capa/rules.py | 21 +++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/capa/features/common.py b/capa/features/common.py index b3a0735a..7a995f81 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -140,6 +140,11 @@ class Characteristic(Feature): super(Characteristic, self).__init__(value, description=description) +CHARACTERISTIC_WINDOWS = Characteristic(OS_WINDOWS) +CHARACTERISTIC_LINUX = Characteristic(OS_LINUX) +CHARACTERISTIC_MACOS = Characteristic(OS_MACOS) + + class String(Feature): def __init__(self, value: str, description=None): super(String, self).__init__(value, description=description) diff --git a/capa/rules.py b/capa/rules.py index 3d62b7f5..6d4f0872 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -34,7 +34,8 @@ import capa.features.insn import capa.features.common import capa.features.basicblock from capa.engine import Statement, FeatureSet -from capa.features.common import OS_LINUX, OS_MACOS, OS_WINDOWS, MAX_BYTES_FEATURE_SIZE, Feature +from capa.features.common import MAX_BYTES_FEATURE_SIZE, Feature +from capa.features.common import CHARACTERISTIC_WINDOWS, CHARACTERISTIC_LINUX, CHARACTERISTIC_MACOS logger = logging.getLogger(__name__) @@ -78,9 +79,9 @@ SUPPORTED_FEATURES = { capa.features.file.FunctionName, capa.features.common.Characteristic("embedded pe"), capa.features.common.String, - capa.features.common.Characteristic(OS_WINDOWS), - capa.features.common.Characteristic(OS_LINUX), - capa.features.common.Characteristic(OS_MACOS), + CHARACTERISTIC_WINDOWS, + CHARACTERISTIC_LINUX, + CHARACTERISTIC_MACOS, }, FUNCTION_SCOPE: { # plus basic block scope features, see below @@ -89,9 +90,9 @@ SUPPORTED_FEATURES = { capa.features.common.Characteristic("calls to"), capa.features.common.Characteristic("loop"), capa.features.common.Characteristic("recursive call"), - capa.features.common.Characteristic(OS_WINDOWS), - capa.features.common.Characteristic(OS_LINUX), - capa.features.common.Characteristic(OS_MACOS), + CHARACTERISTIC_WINDOWS, + CHARACTERISTIC_LINUX, + CHARACTERISTIC_MACOS, }, BASIC_BLOCK_SCOPE: { capa.features.common.MatchedRule, @@ -109,9 +110,9 @@ SUPPORTED_FEATURES = { capa.features.common.Characteristic("tight loop"), capa.features.common.Characteristic("stack string"), capa.features.common.Characteristic("indirect call"), - capa.features.common.Characteristic(OS_WINDOWS), - capa.features.common.Characteristic(OS_LINUX), - capa.features.common.Characteristic(OS_MACOS), + CHARACTERISTIC_WINDOWS, + CHARACTERISTIC_LINUX, + CHARACTERISTIC_MACOS, }, } From 06f8943bc41952fa4f874ced33830ea33894e672 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 09:10:04 -0600 Subject: [PATCH 04/52] features: add format/pe and format/elf characteristics --- capa/features/common.py | 7 +++++++ capa/rules.py | 8 ++++++++ tests/test_rules.py | 19 ++++++++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/capa/features/common.py b/capa/features/common.py index 7a995f81..5f6e19cb 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -32,6 +32,10 @@ OS_LINUX = "os/linux" OS_MACOS = "os/macos" VALID_OS = (OS_WINDOWS, OS_LINUX, OS_MACOS) +FORMAT_PE = "format/pe" +FORMAT_ELF = "format/elf" +VALID_FORMAT = (FORMAT_PE, FORMAT_ELF) + def bytes_to_str(b: bytes) -> str: return str(codecs.encode(b, "hex").decode("utf-8")) @@ -144,6 +148,9 @@ CHARACTERISTIC_WINDOWS = Characteristic(OS_WINDOWS) CHARACTERISTIC_LINUX = Characteristic(OS_LINUX) CHARACTERISTIC_MACOS = Characteristic(OS_MACOS) +CHARACTERISTIC_PE = Characteristic(FORMAT_PE) +CHARACTERISTIC_ELF = Characteristic(FORMAT_ELF) + class String(Feature): def __init__(self, value: str, description=None): diff --git a/capa/rules.py b/capa/rules.py index 6d4f0872..e3264998 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -36,6 +36,8 @@ import capa.features.basicblock from capa.engine import Statement, FeatureSet from capa.features.common import MAX_BYTES_FEATURE_SIZE, Feature from capa.features.common import CHARACTERISTIC_WINDOWS, CHARACTERISTIC_LINUX, CHARACTERISTIC_MACOS +from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_ELF + logger = logging.getLogger(__name__) @@ -82,6 +84,8 @@ SUPPORTED_FEATURES = { CHARACTERISTIC_WINDOWS, CHARACTERISTIC_LINUX, CHARACTERISTIC_MACOS, + CHARACTERISTIC_PE, + CHARACTERISTIC_ELF, }, FUNCTION_SCOPE: { # plus basic block scope features, see below @@ -93,6 +97,8 @@ SUPPORTED_FEATURES = { CHARACTERISTIC_WINDOWS, CHARACTERISTIC_LINUX, CHARACTERISTIC_MACOS, + CHARACTERISTIC_PE, + CHARACTERISTIC_ELF, }, BASIC_BLOCK_SCOPE: { capa.features.common.MatchedRule, @@ -113,6 +119,8 @@ SUPPORTED_FEATURES = { CHARACTERISTIC_WINDOWS, CHARACTERISTIC_LINUX, CHARACTERISTIC_MACOS, + CHARACTERISTIC_PE, + CHARACTERISTIC_ELF, }, } diff --git a/tests/test_rules.py b/tests/test_rules.py index 6b57b89a..abc37968 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -15,7 +15,7 @@ import capa.engine import capa.features.common from capa.features.file import FunctionName from capa.features.insn import Number, Offset -from capa.features.common import ARCH_X32, ARCH_X64, OS_WINDOWS, String, Characteristic +from capa.features.common import ARCH_X32, ARCH_X64, OS_WINDOWS, FORMAT_PE, String, Characteristic def test_rule_ctor(): @@ -961,3 +961,20 @@ def test_os_features(): r = capa.rules.Rule.from_yaml(rule) children = list(r.statement.get_children()) assert (Characteristic(OS_WINDOWS) in children) == True + + +def test_format_features(): + rule = textwrap.dedent( + """ + rule: + meta: + name: test rule + scope: file + features: + - and: + - characteristic: format/pe + """ + ) + r = capa.rules.Rule.from_yaml(rule) + children = list(r.statement.get_children()) + assert (Characteristic(FORMAT_PE) in children) == True \ No newline at end of file From 20859d2796f3a20d6a96ff6485382aee892fd11d Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 09:11:29 -0600 Subject: [PATCH 05/52] extractors: pefile: extract OS and format --- capa/features/extractors/pefile.py | 14 ++++++++++++++ tests/fixtures.py | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/capa/features/extractors/pefile.py b/capa/features/extractors/pefile.py index e27d6c20..0d4a5243 100644 --- a/capa/features/extractors/pefile.py +++ b/capa/features/extractors/pefile.py @@ -9,12 +9,14 @@ import logging import pefile +import capa.features.common import capa.features.extractors import capa.features.extractors.helpers import capa.features.extractors.strings from capa.features.file import Export, Import, Section from capa.features.common import String, Characteristic from capa.features.extractors.base_extractor import FeatureExtractor +from capa.features.common import CHARACTERISTIC_WINDOWS, CHARACTERISTIC_PE logger = logging.getLogger(__name__) @@ -110,6 +112,16 @@ def extract_file_function_names(pe, file_path): return +def extract_os(pe, file_path): + # assuming PE -> Windows + # though i suppose they're also used by UEFI + yield CHARACTERISTIC_WINDOWS, 0x0 + + +def extract_format(pe, file_path): + yield CHARACTERISTIC_PE, 0x0 + + def extract_file_features(pe, file_path): """ extract file features from given workspace @@ -134,6 +146,8 @@ FILE_HANDLERS = ( extract_file_section_names, extract_file_strings, extract_file_function_names, + extract_os, + extract_format, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index f15e7f06..1d9ed580 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -22,6 +22,7 @@ import capa.features.insn import capa.features.common import capa.features.basicblock from capa.features.common import ARCH_X32, ARCH_X64 +from capa.features.common import CHARACTERISTIC_WINDOWS, CHARACTERISTIC_PE CD = os.path.dirname(__file__) @@ -499,6 +500,13 @@ FEATURE_PRESENCE_TESTS = sorted( ("mimikatz", "function=0x456BB9", capa.features.common.Characteristic("calls to"), False), # file/function-name ("pma16-01", "file", capa.features.file.FunctionName("__aulldiv"), True), + # os & format + ("pma16-01", "file", CHARACTERISTIC_WINDOWS, True), + ("pma16-01", "function=0x404356", CHARACTERISTIC_WINDOWS, True), + ("pma16-01", "function=0x404356,bb=0x4043B9", CHARACTERISTIC_WINDOWS, True), + ("pma16-01", "file", CHARACTERISTIC_PE, True), + ("pma16-01", "function=0x404356", CHARACTERISTIC_PE, True), + ("pma16-01", "function=0x404356,bb=0x4043B9", CHARACTERISTIC_PE, True), ], # order tests by (file, item) # so that our LRU cache is most effective. From 97092c91db959cd74ee5c8e6bf761e58fc272ce4 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 09:13:56 -0600 Subject: [PATCH 06/52] tests: assert absence of the wrong os/format --- tests/fixtures.py | 4 +++- tests/test_rules.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 1d9ed580..3e8f1b08 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -22,7 +22,7 @@ import capa.features.insn import capa.features.common import capa.features.basicblock from capa.features.common import ARCH_X32, ARCH_X64 -from capa.features.common import CHARACTERISTIC_WINDOWS, CHARACTERISTIC_PE +from capa.features.common import CHARACTERISTIC_WINDOWS, CHARACTERISTIC_LINUX, CHARACTERISTIC_PE, CHARACTERISTIC_ELF CD = os.path.dirname(__file__) @@ -502,9 +502,11 @@ FEATURE_PRESENCE_TESTS = sorted( ("pma16-01", "file", capa.features.file.FunctionName("__aulldiv"), True), # os & format ("pma16-01", "file", CHARACTERISTIC_WINDOWS, True), + ("pma16-01", "file", CHARACTERISTIC_LINUX, False), ("pma16-01", "function=0x404356", CHARACTERISTIC_WINDOWS, True), ("pma16-01", "function=0x404356,bb=0x4043B9", CHARACTERISTIC_WINDOWS, True), ("pma16-01", "file", CHARACTERISTIC_PE, True), + ("pma16-01", "file", CHARACTERISTIC_ELF, False), ("pma16-01", "function=0x404356", CHARACTERISTIC_PE, True), ("pma16-01", "function=0x404356,bb=0x4043B9", CHARACTERISTIC_PE, True), ], diff --git a/tests/test_rules.py b/tests/test_rules.py index abc37968..022292ca 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -15,7 +15,7 @@ import capa.engine import capa.features.common from capa.features.file import FunctionName from capa.features.insn import Number, Offset -from capa.features.common import ARCH_X32, ARCH_X64, OS_WINDOWS, FORMAT_PE, String, Characteristic +from capa.features.common import ARCH_X32, ARCH_X64, CHARACTERISTIC_PE, CHARACTERISTIC_WINDOWS, OS_WINDOWS, FORMAT_PE, String, Characteristic def test_rule_ctor(): @@ -960,7 +960,8 @@ def test_os_features(): ) r = capa.rules.Rule.from_yaml(rule) children = list(r.statement.get_children()) - assert (Characteristic(OS_WINDOWS) in children) == True + assert (CHARACTERISTIC_WINDOWS in children) == True + assert (CHARACTERISTIC_LINUX not in children) == True def test_format_features(): @@ -977,4 +978,5 @@ def test_format_features(): ) r = capa.rules.Rule.from_yaml(rule) children = list(r.statement.get_children()) - assert (Characteristic(FORMAT_PE) in children) == True \ No newline at end of file + assert (CHARACTERISTIC_PE in children) == True + assert (CHARACTERISTIC_ELF not in children) == True \ No newline at end of file From 753b003107e51e8a6eb6b3aba6d68b3d801b7461 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 09:23:41 -0600 Subject: [PATCH 07/52] pep8 --- capa/features/extractors/pefile.py | 3 +-- capa/rules.py | 13 +++++++++---- tests/fixtures.py | 10 ++++++++-- tests/test_rules.py | 13 +++++++++++-- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/capa/features/extractors/pefile.py b/capa/features/extractors/pefile.py index 0d4a5243..b1727574 100644 --- a/capa/features/extractors/pefile.py +++ b/capa/features/extractors/pefile.py @@ -14,9 +14,8 @@ import capa.features.extractors import capa.features.extractors.helpers import capa.features.extractors.strings from capa.features.file import Export, Import, Section -from capa.features.common import String, Characteristic +from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_WINDOWS, String, Characteristic from capa.features.extractors.base_extractor import FeatureExtractor -from capa.features.common import CHARACTERISTIC_WINDOWS, CHARACTERISTIC_PE logger = logging.getLogger(__name__) diff --git a/capa/rules.py b/capa/rules.py index e3264998..8e6267bf 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -34,10 +34,15 @@ import capa.features.insn import capa.features.common import capa.features.basicblock from capa.engine import Statement, FeatureSet -from capa.features.common import MAX_BYTES_FEATURE_SIZE, Feature -from capa.features.common import CHARACTERISTIC_WINDOWS, CHARACTERISTIC_LINUX, CHARACTERISTIC_MACOS -from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_ELF - +from capa.features.common import ( + CHARACTERISTIC_PE, + CHARACTERISTIC_ELF, + CHARACTERISTIC_LINUX, + CHARACTERISTIC_MACOS, + CHARACTERISTIC_WINDOWS, + MAX_BYTES_FEATURE_SIZE, + Feature, +) logger = logging.getLogger(__name__) diff --git a/tests/fixtures.py b/tests/fixtures.py index 3e8f1b08..3bf065f0 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -21,8 +21,14 @@ import capa.features.file import capa.features.insn import capa.features.common import capa.features.basicblock -from capa.features.common import ARCH_X32, ARCH_X64 -from capa.features.common import CHARACTERISTIC_WINDOWS, CHARACTERISTIC_LINUX, CHARACTERISTIC_PE, CHARACTERISTIC_ELF +from capa.features.common import ( + ARCH_X32, + ARCH_X64, + CHARACTERISTIC_PE, + CHARACTERISTIC_ELF, + CHARACTERISTIC_LINUX, + CHARACTERISTIC_WINDOWS, +) CD = os.path.dirname(__file__) diff --git a/tests/test_rules.py b/tests/test_rules.py index 022292ca..95a39fb1 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -15,7 +15,16 @@ import capa.engine import capa.features.common from capa.features.file import FunctionName from capa.features.insn import Number, Offset -from capa.features.common import ARCH_X32, ARCH_X64, CHARACTERISTIC_PE, CHARACTERISTIC_WINDOWS, OS_WINDOWS, FORMAT_PE, String, Characteristic +from capa.features.common import ( + ARCH_X32, + ARCH_X64, + FORMAT_PE, + OS_WINDOWS, + CHARACTERISTIC_PE, + CHARACTERISTIC_WINDOWS, + String, + Characteristic, +) def test_rule_ctor(): @@ -979,4 +988,4 @@ def test_format_features(): r = capa.rules.Rule.from_yaml(rule) children = list(r.statement.get_children()) assert (CHARACTERISTIC_PE in children) == True - assert (CHARACTERISTIC_ELF not in children) == True \ No newline at end of file + assert (CHARACTERISTIC_ELF not in children) == True From 05f8e2445aa716e575834fe5e8577b0b41e09103 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 09:29:05 -0600 Subject: [PATCH 08/52] fixtures: add tests demonstrating extraction of features from ELF files --- tests/fixtures.py | 9 +++++++++ tests/test_pefile_features.py | 7 +++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 3bf065f0..f6e59e25 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -203,6 +203,8 @@ def get_data_path_by_name(name): return os.path.join(CD, "data", "773290480d5445f11d3dc1b800728966.exe_") elif name.startswith("3b13b"): return os.path.join(CD, "data", "3b13b6f1d7cd14dc4a097a12e2e505c0a4cff495262261e2bfc991df238b9b04.dll_") + elif name == "7351f.elf": + return os.path.join(CD, "data", "7351f8a40c5450557b24622417fc478d.elf_") else: raise ValueError("unexpected sample fixture: %s" % name) @@ -248,6 +250,8 @@ def get_sample_md5_by_name(name): elif name.startswith("3b13b"): # file name is SHA256 hash return "56a6ffe6a02941028cc8235204eef31d" + elif name == "7351f.elf": + return "7351f8a40c5450557b24622417fc478d" else: raise ValueError("unexpected sample fixture: %s" % name) @@ -515,6 +519,11 @@ FEATURE_PRESENCE_TESTS = sorted( ("pma16-01", "file", CHARACTERISTIC_ELF, False), ("pma16-01", "function=0x404356", CHARACTERISTIC_PE, True), ("pma16-01", "function=0x404356,bb=0x4043B9", CHARACTERISTIC_PE, True), + # elf support + ("7351f.elf", "file", CHARACTERISTIC_LINUX, True), + ("7351f.elf", "file", CHARACTERISTIC_ELF, True), + ("7351f.elf", "function=0x408753", capa.features.common.String("/dev/null"), True), + ("7351f.elf", "function=0x408753,bb=0x408781", capa.features.insn.API("open"), True), ], # order tests by (file, item) # so that our LRU cache is most effective. diff --git a/tests/test_pefile_features.py b/tests/test_pefile_features.py index 8bb46d43..2e1afc7b 100644 --- a/tests/test_pefile_features.py +++ b/tests/test_pefile_features.py @@ -20,9 +20,12 @@ import capa.features.file ) def test_pefile_features(sample, scope, feature, expected): if scope.__name__ != "file": - pytest.xfail("pefile only extract file scope features") + pytest.xfail("pefile only extracts file scope features") if isinstance(feature, capa.features.file.FunctionName): - pytest.xfail("pefile only doesn't extract function names") + pytest.xfail("pefile doesn't extract function names") + + if ".elf" in sample: + pytest.xfail("pefile doesn't handle ELF files") fixtures.do_test_feature_presence(fixtures.get_pefile_extractor, sample, scope, feature, expected) From baaa8ba2c10b2dd27554223e5469ce712c5c1902 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 13:52:50 -0600 Subject: [PATCH 09/52] scripts: add script to detect ELF OS closes #724 --- scripts/detect-elf-os.py | 332 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100644 scripts/detect-elf-os.py diff --git a/scripts/detect-elf-os.py b/scripts/detect-elf-os.py new file mode 100644 index 00000000..f63dda4b --- /dev/null +++ b/scripts/detect-elf-os.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python2 +""" +Copyright (C) 2021 FireEye, Inc. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. +You may obtain a copy of the License at: [package root]/LICENSE.txt +Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. + +detect-elf-os + +Attempt to detect the underlying OS that the given ELF file targets. +""" +import sys +import struct +import logging +import argparse +import contextlib +from enum import Enum +from typing import BinaryIO + +logger = logging.getLogger("capa.detect-elf-os") + + +def align(v, alignment): + remainder = v % alignment + if remainder == 0: + return v + else: + return v + remainder + + +class IDAIO: + """ + An object that acts as a file-like object, + using bytes from the current IDB workspace. + """ + + def __init__(self): + assert IDAIO.is_runtime_ida() == True + + super(IDAIO, self).__init__() + import idc + import ida_bytes + import ida_loader + + self.offset = 0 + + def seek(self, offset, whence=0): + assert whence == 0 + self.offset = offset + + def read(self, size): + ea = ida_loader.get_fileregion_ea(self.offset) + if ea == idc.BADADDR: + # best guess, such as if file is mapped at address 0x0. + ea = self.offset + + logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, ea) + return ida_bytes.get_bytes(ea, size) + + def close(self): + return + + @staticmethod + def is_runtime_ida(): + try: + import idc + except ImportError: + return False + else: + return True + + +class CorruptElfFile(ValueError): + pass + + +class OS(str, Enum): + HPUX = "HPUX" + NETBSD = "NETBSD" + LINUX = "LINUX" + HURD = "HURD" + _86OPEN = "86OPEN" + SOLARIS = "SOLARIS" + AIX = "AIX" + IRIX = "IRIX" + FREEBSD = "FREEBSD" + TRU64 = "TRU64" + MODESTO = "MODESTO" + OPENBSD = "OPENBSD" + OPENVMS = "OPENVMS" + NSK = "NSK" + AROS = "AROS" + FENIXOS = "FENIXOS" + CLOUD = "CLOUD" + SORTFIX = "SORTFIX" + ARM_AEABI = "ARM_AEABI" + SYLLABLE = "SYLLABLE" + NACL = "NACL" + + +def detect_elf_os(f: BinaryIO) -> str: + f.seek(0x0) + file_header = f.read(0x40) + + # we'll set this to the detected OS + # prefer the first heuristics, + # but rather than short circuiting, + # we'll still parse out the remainder, for debugging. + ret = None + + if not file_header.startswith(b"\x7fELF"): + raise CorruptElfFile("missing magic header") + + ei_class, ei_data = struct.unpack_from("BB", file_header, 4) + logger.debug("ei_class: 0x%02x ei_data: 0x%02x", ei_class, ei_data) + if ei_class == 1: + bitness = 32 + elif ei_class == 2: + bitness = 64 + else: + raise CorruptElfFile("invalid ei_class: 0x%02x" % ei_class) + + if ei_data == 1: + endian = "<" + elif ei_data == 2: + endian = ">" + else: + raise CorruptElfFile("not an ELF file: invalid ei_data: 0x%02x" % ei_data) + + if bitness == 32: + (e_phoff,) = struct.unpack_from(endian + "I", file_header, 0x1C) + e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x2A) + elif bitness == 64: + (e_phoff,) = struct.unpack_from(endian + "Q", file_header, 0x20) + e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x36) + else: + raise NotImplemented + + logger.debug("e_phoff: 0x%02x e_phentsize: 0x%02x e_phnum: %d", e_phoff, e_phentsize, e_phnum) + + (ei_osabi,) = struct.unpack_from(endian + "B", file_header, 7) + OSABI = { + # via pyelftools: https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/elftools/elf/enums.py#L35-L58 + # 0: "SYSV", + 1: OS.HPUX, + 2: OS.NETBSD, + 3: OS.LINUX, + 4: OS.HURD, + 5: OS._86OPEN, + 6: OS.SOLARIS, + 7: OS.AIX, + 8: OS.IRIX, + 9: OS.FREEBSD, + 10: OS.TRU64, + 11: OS.MODESTO, + 12: OS.OPENBSD, + 13: OS.OPENVMS, + 14: OS.NSK, + 15: OS.AROS, + 16: OS.FENIXOS, + 17: OS.CLOUD, + # 53: "SORTFIX", + # 64: "ARM_AEABI", + # 97: "ARM", + # 255: "STANDALONE", + } + logger.debug("ei_osabi: 0x%02x (%s)", ei_osabi, OSABI.get(ei_osabi, "unknown")) + + if ei_osabi in OSABI and ei_osabi != 0x0: + # update only if not set + # so we can get the debugging output of subsequent strategies + ret = OSABI[ei_osabi] if not ret else ret + + f.seek(e_phoff) + program_header_size = e_phnum * e_phentsize + program_headers = f.read(program_header_size) + if len(program_headers) != program_header_size: + logger.warning("failed to read program headers") + e_phnum = 0 + + for i in range(e_phnum): + offset = i * e_phentsize + phent = program_headers[offset : offset + e_phentsize] + + PT_NOTE = 0x4 + + (p_type,) = struct.unpack_from(endian + "I", phent, 0x0) + logger.debug("p_type: 0x%04x", p_type) + if p_type != PT_NOTE: + continue + + if bitness == 32: + p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4) + elif bitness == 64: + p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8) + else: + raise NotImplemented + + logger.debug("p_offset: 0x%02x p_filesz: 0x%04x", p_offset, p_filesz) + + f.seek(p_offset) + note = f.read(p_filesz) + if len(note) != p_filesz: + logger.warning("failed to read note content") + continue + + namesz, descsz, type_ = struct.unpack_from(endian + "III", note, 0x0) + name_offset = 0xC + desc_offset = name_offset + align(namesz, 0x4) + + logger.debug("namesz: 0x%02x descsz: 0x%02x type: 0x%04x", namesz, descsz, type_) + + name = note[name_offset : name_offset + namesz].partition(b"\x00")[0].decode("ascii") + logger.debug("name: %s", name) + + if type_ != 1: + continue + + if name == "GNU": + if descsz < 16: + continue + + desc = note[desc_offset : desc_offset + descsz] + abi_tag, kmajor, kminor, kpatch = struct.unpack_from(endian + "IIII", desc, 0x0) + # via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658 + # and here: https://github.com/bminor/binutils-gdb/blob/34c54daa337da9fadf87d2706d6a590ae1f88f4d/include/elf/common.h#L933-L939 + GNU_ABI_TAG = { + 0: OS.LINUX, + 1: OS.HURD, + 2: OS.SOLARIS, + 3: OS.FREEBSD, + 4: OS.NETBSD, + 5: OS.SYLLABLE, + 6: OS.NACL, + } + logger.debug("GNU_ABI_TAG: 0x%02x", abi_tag) + + if abi_tag in GNU_ABI_TAG: + # update only if not set + # so we can get the debugging output of subsequent strategies + ret = GNU_ABI_TAG[abi_tag] if not ret else ret + logger.debug("abi tag: %s earliest compatible kernel: %d.%d.%d", ret, kmajor, kminor, kpatch) + elif name == "OpenBSD": + logger.debug("note owner: %s", "OPENBSD") + ret = OS.OPENBSD if not ret else ret + elif name == "NetBSD": + logger.debug("note owner: %s", "NETBSD") + ret = OS.NETBSD if not ret else ret + + for i in range(e_phnum): + offset = i * e_phentsize + phent = program_headers[offset : offset + e_phentsize] + + PT_INTERP = 0x3 + + (p_type,) = struct.unpack_from(endian + "I", phent, 0x0) + if p_type != PT_INTERP: + continue + + if bitness == 32: + p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4) + elif bitness == 64: + p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8) + else: + raise NotImplemented + + f.seek(p_offset) + interp = f.read(p_filesz) + if len(interp) != p_filesz: + logger.warning("failed to read interp content") + continue + + linker = interp.partition(b"\x00")[0].decode("ascii") + logger.debug("linker: %s", linker) + if "ld-linux" in linker: + # update only if not set + # so we can get the debugging output of subsequent strategies + ret = OS.LINUX if ret is None else ret + + return ret.value if ret is not None else "unknown" + + +def main(argv=None): + if IDAIO.is_runtime_ida(): + f: BinaryIO = IDAIO() + + else: + print("not ida") + if argv is None: + argv = sys.argv[1:] + + parser = argparse.ArgumentParser(description="Detect the underlying OS for the given ELF file") + parser.add_argument("sample", type=str, help="path to ELF file") + + logging_group = parser.add_argument_group("logging arguments") + + logging_group.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR") + logging_group.add_argument( + "-q", "--quiet", action="store_true", help="disable all status output except fatal errors" + ) + + args = parser.parse_args(args=argv) + + if args.quiet: + logging.basicConfig(level=logging.WARNING) + logging.getLogger().setLevel(logging.WARNING) + elif args.debug: + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + logging.getLogger().setLevel(logging.INFO) + + f = open(args.sample, "rb") + + with contextlib.closing(f): + try: + print(detect_elf_os(f)) + return 0 + except CorruptElfFile as e: + logger.error("corrupt ELF file: %s", str(e.args[0])) + return -1 + + +if __name__ == "__main__": + if IDAIO.is_runtime_ida(): + main() + else: + sys.exit(main()) From 37bc47c77259abac600a8eb36736ae464c2bf412 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 14:41:11 -0600 Subject: [PATCH 10/52] extractors: viv: extract from bytes not file path --- capa/features/extractors/viv/extractor.py | 4 ++- capa/features/extractors/viv/file.py | 30 +++++++++-------------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/capa/features/extractors/viv/extractor.py b/capa/features/extractors/viv/extractor.py index 15f3973a..05fec5a3 100644 --- a/capa/features/extractors/viv/extractor.py +++ b/capa/features/extractors/viv/extractor.py @@ -37,13 +37,15 @@ class VivisectFeatureExtractor(FeatureExtractor): super(VivisectFeatureExtractor, self).__init__() self.vw = vw self.path = path + with open(self.path, "rb") as f: + self.buf = f.read() def get_base_address(self): # assume there is only one file loaded into the vw return list(self.vw.filemeta.values())[0]["imagebase"] def extract_file_features(self): - for feature, va in capa.features.extractors.viv.file.extract_features(self.vw, self.path): + for feature, va in capa.features.extractors.viv.file.extract_features(self.vw, self.buf): yield feature, va def get_functions(self): diff --git a/capa/features/extractors/viv/file.py b/capa/features/extractors/viv/file.py index 00eb57ed..9ec9d672 100644 --- a/capa/features/extractors/viv/file.py +++ b/capa/features/extractors/viv/file.py @@ -17,20 +17,17 @@ from capa.features.file import Export, Import, Section, FunctionName from capa.features.common import String, Characteristic -def extract_file_embedded_pe(vw, file_path): - with open(file_path, "rb") as f: - fbytes = f.read() - - for offset, i in pe_carve.carve(fbytes, 1): +def extract_file_embedded_pe(vw, buf): + for offset, i in pe_carve.carve(buf, 1): yield Characteristic("embedded pe"), offset -def extract_file_export_names(vw, file_path): +def extract_file_export_names(vw, buf): for va, etype, name, _ in vw.getExports(): yield Export(name), va -def extract_file_import_names(vw, file_path): +def extract_file_import_names(vw, buf): """ extract imported function names 1. imports by ordinal: @@ -64,26 +61,23 @@ def is_viv_ord_impname(impname: str) -> bool: return True -def extract_file_section_names(vw, file_path): +def extract_file_section_names(vw, buf): for va, _, segname, _ in vw.getSegments(): yield Section(segname), va -def extract_file_strings(vw, file_path): +def extract_file_strings(vw, buf): """ extract ASCII and UTF-16 LE strings from file """ - with open(file_path, "rb") as f: - b = f.read() - - for s in capa.features.extractors.strings.extract_ascii_strings(b): + for s in capa.features.extractors.strings.extract_ascii_strings(buf): yield String(s.s), s.offset - for s in capa.features.extractors.strings.extract_unicode_strings(b): + for s in capa.features.extractors.strings.extract_unicode_strings(buf): yield String(s.s), s.offset -def extract_file_function_names(vw, file_path): +def extract_file_function_names(vw, buf): """ extract the names of statically-linked library functions. """ @@ -93,20 +87,20 @@ def extract_file_function_names(vw, file_path): yield FunctionName(name), va -def extract_features(vw, file_path): +def extract_features(vw, buf: bytes): """ extract file features from given workspace args: vw (vivisect.VivWorkspace): the vivisect workspace - file_path: path to the input file + buf: the raw input file bytes yields: Tuple[Feature, VA]: a feature and its location. """ for file_handler in FILE_HANDLERS: - for feature, va in file_handler(vw, file_path): + for feature, va in file_handler(vw, buf): yield feature, va From 7205862dbfa2133025c09ab6fce34dd3fc3f6575 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 14:42:29 -0600 Subject: [PATCH 11/52] helpers: move ELF and IDA helpers out of script and into common module --- capa/helpers.py | 9 ++ capa/ida/helpers.py | 29 +++++ scripts/detect-elf-os.py | 274 ++------------------------------------- 3 files changed, 46 insertions(+), 266 deletions(-) diff --git a/capa/helpers.py b/capa/helpers.py index dd0ff79e..eaee04bd 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -21,3 +21,12 @@ def get_file_taste(sample_path: str) -> bytes: with open(sample_path, "rb") as f: taste = f.read(8) return taste + + +def is_runtime_ida(): + try: + import idc + except ImportError: + return False + else: + return True \ No newline at end of file diff --git a/capa/ida/helpers.py b/capa/ida/helpers.py index b2e8f0fc..800d1383 100644 --- a/capa/ida/helpers.py +++ b/capa/ida/helpers.py @@ -12,6 +12,8 @@ import datetime import idc import idaapi import idautils +import ida_bytes +import ida_loader import capa import capa.version @@ -121,3 +123,30 @@ def collect_metadata(): }, "version": capa.version.__version__, } + + +class IDAIO: + """ + An object that acts as a file-like object, + using bytes from the current IDB workspace. + """ + + def __init__(self): + super(IDAIO, self).__init__() + self.offset = 0 + + def seek(self, offset, whence=0): + assert whence == 0 + self.offset = offset + + def read(self, size): + ea = ida_loader.get_fileregion_ea(self.offset) + if ea == idc.BADADDR: + # best guess, such as if file is mapped at address 0x0. + ea = self.offset + + logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, ea) + return ida_bytes.get_bytes(ea, size) + + def close(self): + return diff --git a/scripts/detect-elf-os.py b/scripts/detect-elf-os.py index f63dda4b..950d64df 100644 --- a/scripts/detect-elf-os.py +++ b/scripts/detect-elf-os.py @@ -13,278 +13,20 @@ detect-elf-os Attempt to detect the underlying OS that the given ELF file targets. """ import sys -import struct import logging import argparse import contextlib -from enum import Enum from typing import BinaryIO +import capa.helpers +import capa.features.extractors.elf + logger = logging.getLogger("capa.detect-elf-os") -def align(v, alignment): - remainder = v % alignment - if remainder == 0: - return v - else: - return v + remainder - - -class IDAIO: - """ - An object that acts as a file-like object, - using bytes from the current IDB workspace. - """ - - def __init__(self): - assert IDAIO.is_runtime_ida() == True - - super(IDAIO, self).__init__() - import idc - import ida_bytes - import ida_loader - - self.offset = 0 - - def seek(self, offset, whence=0): - assert whence == 0 - self.offset = offset - - def read(self, size): - ea = ida_loader.get_fileregion_ea(self.offset) - if ea == idc.BADADDR: - # best guess, such as if file is mapped at address 0x0. - ea = self.offset - - logger.debug("reading 0x%x bytes at 0x%x (ea: 0x%x)", size, self.offset, ea) - return ida_bytes.get_bytes(ea, size) - - def close(self): - return - - @staticmethod - def is_runtime_ida(): - try: - import idc - except ImportError: - return False - else: - return True - - -class CorruptElfFile(ValueError): - pass - - -class OS(str, Enum): - HPUX = "HPUX" - NETBSD = "NETBSD" - LINUX = "LINUX" - HURD = "HURD" - _86OPEN = "86OPEN" - SOLARIS = "SOLARIS" - AIX = "AIX" - IRIX = "IRIX" - FREEBSD = "FREEBSD" - TRU64 = "TRU64" - MODESTO = "MODESTO" - OPENBSD = "OPENBSD" - OPENVMS = "OPENVMS" - NSK = "NSK" - AROS = "AROS" - FENIXOS = "FENIXOS" - CLOUD = "CLOUD" - SORTFIX = "SORTFIX" - ARM_AEABI = "ARM_AEABI" - SYLLABLE = "SYLLABLE" - NACL = "NACL" - - -def detect_elf_os(f: BinaryIO) -> str: - f.seek(0x0) - file_header = f.read(0x40) - - # we'll set this to the detected OS - # prefer the first heuristics, - # but rather than short circuiting, - # we'll still parse out the remainder, for debugging. - ret = None - - if not file_header.startswith(b"\x7fELF"): - raise CorruptElfFile("missing magic header") - - ei_class, ei_data = struct.unpack_from("BB", file_header, 4) - logger.debug("ei_class: 0x%02x ei_data: 0x%02x", ei_class, ei_data) - if ei_class == 1: - bitness = 32 - elif ei_class == 2: - bitness = 64 - else: - raise CorruptElfFile("invalid ei_class: 0x%02x" % ei_class) - - if ei_data == 1: - endian = "<" - elif ei_data == 2: - endian = ">" - else: - raise CorruptElfFile("not an ELF file: invalid ei_data: 0x%02x" % ei_data) - - if bitness == 32: - (e_phoff,) = struct.unpack_from(endian + "I", file_header, 0x1C) - e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x2A) - elif bitness == 64: - (e_phoff,) = struct.unpack_from(endian + "Q", file_header, 0x20) - e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x36) - else: - raise NotImplemented - - logger.debug("e_phoff: 0x%02x e_phentsize: 0x%02x e_phnum: %d", e_phoff, e_phentsize, e_phnum) - - (ei_osabi,) = struct.unpack_from(endian + "B", file_header, 7) - OSABI = { - # via pyelftools: https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/elftools/elf/enums.py#L35-L58 - # 0: "SYSV", - 1: OS.HPUX, - 2: OS.NETBSD, - 3: OS.LINUX, - 4: OS.HURD, - 5: OS._86OPEN, - 6: OS.SOLARIS, - 7: OS.AIX, - 8: OS.IRIX, - 9: OS.FREEBSD, - 10: OS.TRU64, - 11: OS.MODESTO, - 12: OS.OPENBSD, - 13: OS.OPENVMS, - 14: OS.NSK, - 15: OS.AROS, - 16: OS.FENIXOS, - 17: OS.CLOUD, - # 53: "SORTFIX", - # 64: "ARM_AEABI", - # 97: "ARM", - # 255: "STANDALONE", - } - logger.debug("ei_osabi: 0x%02x (%s)", ei_osabi, OSABI.get(ei_osabi, "unknown")) - - if ei_osabi in OSABI and ei_osabi != 0x0: - # update only if not set - # so we can get the debugging output of subsequent strategies - ret = OSABI[ei_osabi] if not ret else ret - - f.seek(e_phoff) - program_header_size = e_phnum * e_phentsize - program_headers = f.read(program_header_size) - if len(program_headers) != program_header_size: - logger.warning("failed to read program headers") - e_phnum = 0 - - for i in range(e_phnum): - offset = i * e_phentsize - phent = program_headers[offset : offset + e_phentsize] - - PT_NOTE = 0x4 - - (p_type,) = struct.unpack_from(endian + "I", phent, 0x0) - logger.debug("p_type: 0x%04x", p_type) - if p_type != PT_NOTE: - continue - - if bitness == 32: - p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4) - elif bitness == 64: - p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8) - else: - raise NotImplemented - - logger.debug("p_offset: 0x%02x p_filesz: 0x%04x", p_offset, p_filesz) - - f.seek(p_offset) - note = f.read(p_filesz) - if len(note) != p_filesz: - logger.warning("failed to read note content") - continue - - namesz, descsz, type_ = struct.unpack_from(endian + "III", note, 0x0) - name_offset = 0xC - desc_offset = name_offset + align(namesz, 0x4) - - logger.debug("namesz: 0x%02x descsz: 0x%02x type: 0x%04x", namesz, descsz, type_) - - name = note[name_offset : name_offset + namesz].partition(b"\x00")[0].decode("ascii") - logger.debug("name: %s", name) - - if type_ != 1: - continue - - if name == "GNU": - if descsz < 16: - continue - - desc = note[desc_offset : desc_offset + descsz] - abi_tag, kmajor, kminor, kpatch = struct.unpack_from(endian + "IIII", desc, 0x0) - # via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658 - # and here: https://github.com/bminor/binutils-gdb/blob/34c54daa337da9fadf87d2706d6a590ae1f88f4d/include/elf/common.h#L933-L939 - GNU_ABI_TAG = { - 0: OS.LINUX, - 1: OS.HURD, - 2: OS.SOLARIS, - 3: OS.FREEBSD, - 4: OS.NETBSD, - 5: OS.SYLLABLE, - 6: OS.NACL, - } - logger.debug("GNU_ABI_TAG: 0x%02x", abi_tag) - - if abi_tag in GNU_ABI_TAG: - # update only if not set - # so we can get the debugging output of subsequent strategies - ret = GNU_ABI_TAG[abi_tag] if not ret else ret - logger.debug("abi tag: %s earliest compatible kernel: %d.%d.%d", ret, kmajor, kminor, kpatch) - elif name == "OpenBSD": - logger.debug("note owner: %s", "OPENBSD") - ret = OS.OPENBSD if not ret else ret - elif name == "NetBSD": - logger.debug("note owner: %s", "NETBSD") - ret = OS.NETBSD if not ret else ret - - for i in range(e_phnum): - offset = i * e_phentsize - phent = program_headers[offset : offset + e_phentsize] - - PT_INTERP = 0x3 - - (p_type,) = struct.unpack_from(endian + "I", phent, 0x0) - if p_type != PT_INTERP: - continue - - if bitness == 32: - p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4) - elif bitness == 64: - p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8) - else: - raise NotImplemented - - f.seek(p_offset) - interp = f.read(p_filesz) - if len(interp) != p_filesz: - logger.warning("failed to read interp content") - continue - - linker = interp.partition(b"\x00")[0].decode("ascii") - logger.debug("linker: %s", linker) - if "ld-linux" in linker: - # update only if not set - # so we can get the debugging output of subsequent strategies - ret = OS.LINUX if ret is None else ret - - return ret.value if ret is not None else "unknown" - - def main(argv=None): - if IDAIO.is_runtime_ida(): + if capa.helpers.is_runtime_ida(): + from capa.ida.helpers import IDAIO f: BinaryIO = IDAIO() else: @@ -318,15 +60,15 @@ def main(argv=None): with contextlib.closing(f): try: - print(detect_elf_os(f)) + print(capa.features.extractors.elf.detect_elf_os(f)) return 0 - except CorruptElfFile as e: + except capa.features.extractors.elf.CorruptElfFile as e: logger.error("corrupt ELF file: %s", str(e.args[0])) return -1 if __name__ == "__main__": - if IDAIO.is_runtime_ida(): + if capa.helpers.is_runtime_ida(): main() else: sys.exit(main()) From fa8b4a4203bb76f47e7bd54161cf11afc720dcda Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 14:43:13 -0600 Subject: [PATCH 12/52] extractors: add common routine to extract OS from ELF --- capa/features/extractors/elf.py | 224 ++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 capa/features/extractors/elf.py diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py new file mode 100644 index 00000000..45388003 --- /dev/null +++ b/capa/features/extractors/elf.py @@ -0,0 +1,224 @@ +import struct +import logging +from enum import Enum +from typing import BinaryIO + +logger = logging.getLogger(__name__) + + +def align(v, alignment): + remainder = v % alignment + if remainder == 0: + return v + else: + return v + remainder + + +class CorruptElfFile(ValueError): + pass + + +class OS(str, Enum): + HPUX = "HPUX" + NETBSD = "NETBSD" + LINUX = "LINUX" + HURD = "HURD" + _86OPEN = "86OPEN" + SOLARIS = "SOLARIS" + AIX = "AIX" + IRIX = "IRIX" + FREEBSD = "FREEBSD" + TRU64 = "TRU64" + MODESTO = "MODESTO" + OPENBSD = "OPENBSD" + OPENVMS = "OPENVMS" + NSK = "NSK" + AROS = "AROS" + FENIXOS = "FENIXOS" + CLOUD = "CLOUD" + SORTFIX = "SORTFIX" + ARM_AEABI = "ARM_AEABI" + SYLLABLE = "SYLLABLE" + NACL = "NACL" + + +def detect_elf_os(f: BinaryIO) -> str: + f.seek(0x0) + file_header = f.read(0x40) + + # we'll set this to the detected OS + # prefer the first heuristics, + # but rather than short circuiting, + # we'll still parse out the remainder, for debugging. + ret = None + + if not file_header.startswith(b"\x7fELF"): + raise CorruptElfFile("missing magic header") + + ei_class, ei_data = struct.unpack_from("BB", file_header, 4) + logger.debug("ei_class: 0x%02x ei_data: 0x%02x", ei_class, ei_data) + if ei_class == 1: + bitness = 32 + elif ei_class == 2: + bitness = 64 + else: + raise CorruptElfFile("invalid ei_class: 0x%02x" % ei_class) + + if ei_data == 1: + endian = "<" + elif ei_data == 2: + endian = ">" + else: + raise CorruptElfFile("not an ELF file: invalid ei_data: 0x%02x" % ei_data) + + if bitness == 32: + (e_phoff,) = struct.unpack_from(endian + "I", file_header, 0x1C) + e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x2A) + elif bitness == 64: + (e_phoff,) = struct.unpack_from(endian + "Q", file_header, 0x20) + e_phentsize, e_phnum = struct.unpack_from(endian + "HH", file_header, 0x36) + else: + raise NotImplemented + + logger.debug("e_phoff: 0x%02x e_phentsize: 0x%02x e_phnum: %d", e_phoff, e_phentsize, e_phnum) + + (ei_osabi,) = struct.unpack_from(endian + "B", file_header, 7) + OSABI = { + # via pyelftools: https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/elftools/elf/enums.py#L35-L58 + # 0: "SYSV", + 1: OS.HPUX, + 2: OS.NETBSD, + 3: OS.LINUX, + 4: OS.HURD, + 5: OS._86OPEN, + 6: OS.SOLARIS, + 7: OS.AIX, + 8: OS.IRIX, + 9: OS.FREEBSD, + 10: OS.TRU64, + 11: OS.MODESTO, + 12: OS.OPENBSD, + 13: OS.OPENVMS, + 14: OS.NSK, + 15: OS.AROS, + 16: OS.FENIXOS, + 17: OS.CLOUD, + # 53: "SORTFIX", + # 64: "ARM_AEABI", + # 97: "ARM", + # 255: "STANDALONE", + } + logger.debug("ei_osabi: 0x%02x (%s)", ei_osabi, OSABI.get(ei_osabi, "unknown")) + + if ei_osabi in OSABI and ei_osabi != 0x0: + # update only if not set + # so we can get the debugging output of subsequent strategies + ret = OSABI[ei_osabi] if not ret else ret + + f.seek(e_phoff) + program_header_size = e_phnum * e_phentsize + program_headers = f.read(program_header_size) + if len(program_headers) != program_header_size: + logger.warning("failed to read program headers") + e_phnum = 0 + + for i in range(e_phnum): + offset = i * e_phentsize + phent = program_headers[offset : offset + e_phentsize] + + PT_NOTE = 0x4 + + (p_type,) = struct.unpack_from(endian + "I", phent, 0x0) + logger.debug("p_type: 0x%04x", p_type) + if p_type != PT_NOTE: + continue + + if bitness == 32: + p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4) + elif bitness == 64: + p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8) + else: + raise NotImplemented + + logger.debug("p_offset: 0x%02x p_filesz: 0x%04x", p_offset, p_filesz) + + f.seek(p_offset) + note = f.read(p_filesz) + if len(note) != p_filesz: + logger.warning("failed to read note content") + continue + + namesz, descsz, type_ = struct.unpack_from(endian + "III", note, 0x0) + name_offset = 0xC + desc_offset = name_offset + align(namesz, 0x4) + + logger.debug("namesz: 0x%02x descsz: 0x%02x type: 0x%04x", namesz, descsz, type_) + + name = note[name_offset : name_offset + namesz].partition(b"\x00")[0].decode("ascii") + logger.debug("name: %s", name) + + if type_ != 1: + continue + + if name == "GNU": + if descsz < 16: + continue + + desc = note[desc_offset : desc_offset + descsz] + abi_tag, kmajor, kminor, kpatch = struct.unpack_from(endian + "IIII", desc, 0x0) + # via readelf: https://github.com/bminor/binutils-gdb/blob/c0e94211e1ac05049a4ce7c192c9d14d1764eb3e/binutils/readelf.c#L19635-L19658 + # and here: https://github.com/bminor/binutils-gdb/blob/34c54daa337da9fadf87d2706d6a590ae1f88f4d/include/elf/common.h#L933-L939 + GNU_ABI_TAG = { + 0: OS.LINUX, + 1: OS.HURD, + 2: OS.SOLARIS, + 3: OS.FREEBSD, + 4: OS.NETBSD, + 5: OS.SYLLABLE, + 6: OS.NACL, + } + logger.debug("GNU_ABI_TAG: 0x%02x", abi_tag) + + if abi_tag in GNU_ABI_TAG: + # update only if not set + # so we can get the debugging output of subsequent strategies + ret = GNU_ABI_TAG[abi_tag] if not ret else ret + logger.debug("abi tag: %s earliest compatible kernel: %d.%d.%d", ret, kmajor, kminor, kpatch) + elif name == "OpenBSD": + logger.debug("note owner: %s", "OPENBSD") + ret = OS.OPENBSD if not ret else ret + elif name == "NetBSD": + logger.debug("note owner: %s", "NETBSD") + ret = OS.NETBSD if not ret else ret + + for i in range(e_phnum): + offset = i * e_phentsize + phent = program_headers[offset : offset + e_phentsize] + + PT_INTERP = 0x3 + + (p_type,) = struct.unpack_from(endian + "I", phent, 0x0) + if p_type != PT_INTERP: + continue + + if bitness == 32: + p_offset, _, _, p_filesz = struct.unpack_from(endian + "IIII", phent, 0x4) + elif bitness == 64: + p_offset, _, _, p_filesz = struct.unpack_from(endian + "QQQQ", phent, 0x8) + else: + raise NotImplemented + + f.seek(p_offset) + interp = f.read(p_filesz) + if len(interp) != p_filesz: + logger.warning("failed to read interp content") + continue + + linker = interp.partition(b"\x00")[0].decode("ascii") + logger.debug("linker: %s", linker) + if "ld-linux" in linker: + # update only if not set + # so we can get the debugging output of subsequent strategies + ret = OS.LINUX if ret is None else ret + + return ret.value if ret is not None else "unknown" \ No newline at end of file From 294f74b209bb83b5b9eac4fcc8003ddbfde87ece Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 14:44:41 -0600 Subject: [PATCH 13/52] extractors: viv: extract format and OS at all scopes --- capa/features/extractors/viv/common.py | 31 +++++++++++++++++++++++ capa/features/extractors/viv/extractor.py | 9 +++++++ scripts/show-features.py | 19 ++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 capa/features/extractors/viv/common.py diff --git a/capa/features/extractors/viv/common.py b/capa/features/extractors/viv/common.py new file mode 100644 index 00000000..fb47f303 --- /dev/null +++ b/capa/features/extractors/viv/common.py @@ -0,0 +1,31 @@ +import io +import logging +import binascii +import contextlib + +import capa.features.extractors.elf +from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_ELF, CHARACTERISTIC_WINDOWS, Characteristic + + +logger = logging.getLogger(__name__) + + +def extract_format(buf): + if buf.startswith(b"MZ"): + yield CHARACTERISTIC_PE, 0x0 + elif buf.startswith(b"\x7fELF"): + yield CHARACTERISTIC_ELF, 0x0 + else: + raise NotImplementedError("file format: %s", binascii.hexlify(buf[:4]).decode("ascii")) + + +def extract_os(buf): + if buf.startswith(b"MZ"): + yield CHARACTERISTIC_WINDOWS, 0x0 + elif buf.startswith(b"\x7fELF"): + with contextlib.closing(io.BytesIO(buf)) as f: + os = capa.features.extractors.elf.detect_elf_os(f) + + yield Characteristic("os/%s" % (os.lower())), 0x0 + else: + raise NotImplementedError("file format: %s", binascii.hexlify(buf[:4]).decode("ascii")) diff --git a/capa/features/extractors/viv/extractor.py b/capa/features/extractors/viv/extractor.py index 05fec5a3..ae8aa4fb 100644 --- a/capa/features/extractors/viv/extractor.py +++ b/capa/features/extractors/viv/extractor.py @@ -12,6 +12,7 @@ import viv_utils.flirt import capa.features.extractors.viv.file import capa.features.extractors.viv.insn +import capa.features.extractors.viv.common import capa.features.extractors.viv.function import capa.features.extractors.viv.basicblock from capa.features.extractors.base_extractor import FeatureExtractor @@ -40,6 +41,10 @@ class VivisectFeatureExtractor(FeatureExtractor): with open(self.path, "rb") as f: self.buf = f.read() + self.global_features = [] + self.global_features.extend(capa.features.extractors.viv.common.extract_os(self.buf)) + self.global_features.extend(capa.features.extractors.viv.common.extract_format(self.buf)) + def get_base_address(self): # assume there is only one file loaded into the vw return list(self.vw.filemeta.values())[0]["imagebase"] @@ -47,6 +52,7 @@ class VivisectFeatureExtractor(FeatureExtractor): def extract_file_features(self): for feature, va in capa.features.extractors.viv.file.extract_features(self.vw, self.buf): yield feature, va + yield from self.global_features def get_functions(self): for va in sorted(self.vw.getFunctions()): @@ -55,6 +61,7 @@ class VivisectFeatureExtractor(FeatureExtractor): def extract_function_features(self, f): for feature, va in capa.features.extractors.viv.function.extract_features(f): yield feature, va + yield from self.global_features def get_basic_blocks(self, f): return f.basic_blocks @@ -62,6 +69,7 @@ class VivisectFeatureExtractor(FeatureExtractor): def extract_basic_block_features(self, f, bb): for feature, va in capa.features.extractors.viv.basicblock.extract_features(f, bb): yield feature, va + yield from self.global_features def get_instructions(self, f, bb): for insn in bb.instructions: @@ -70,6 +78,7 @@ class VivisectFeatureExtractor(FeatureExtractor): def extract_insn_features(self, f, bb, insn): for feature, va in capa.features.extractors.viv.insn.extract_features(f, bb, insn): yield feature, va + yield from self.global_features def is_library_function(self, va): return viv_utils.flirt.is_library_function(self.vw, va) diff --git a/scripts/show-features.py b/scripts/show-features.py index 3090a471..1d37588c 100644 --- a/scripts/show-features.py +++ b/scripts/show-features.py @@ -76,6 +76,7 @@ import capa.engine import capa.helpers import capa.features import capa.features.freeze +import capa.features.common logger = logging.getLogger("capa.show-features") @@ -193,6 +194,15 @@ def ida_main(): return 0 +def is_global_feature(feature): + if (isinstance(feature, capa.features.common.Characteristic) + and isinstance(feature.value, str) + and (feature.value.startswith("os/") + or feature.value.startswith("format/"))): + return True + return False + + def print_features(functions, extractor): for f in functions: function_address = int(f) @@ -203,14 +213,23 @@ def print_features(functions, extractor): continue for feature, va in extractor.extract_function_features(f): + if is_global_feature(feature): + continue + print("func: 0x%08x: %s" % (va, feature)) for bb in extractor.get_basic_blocks(f): for feature, va in extractor.extract_basic_block_features(f, bb): + if is_global_feature(feature): + continue + print("bb : 0x%08x: %s" % (va, feature)) for insn in extractor.get_instructions(f, bb): for feature, va in extractor.extract_insn_features(f, bb, insn): + if is_global_feature(feature): + continue + try: print("insn: 0x%08x: %s" % (va, feature)) except UnicodeEncodeError: From a7678e779eb79799cc8fa1754b6af90dc07b3973 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 14:52:36 -0600 Subject: [PATCH 14/52] extractors: smda: extract format and OS characteristics at all scopes --- capa/features/extractors/{viv => }/common.py | 0 capa/features/extractors/smda/extractor.py | 11 +++++++++++ capa/features/extractors/viv/extractor.py | 6 +++--- 3 files changed, 14 insertions(+), 3 deletions(-) rename capa/features/extractors/{viv => }/common.py (100%) diff --git a/capa/features/extractors/viv/common.py b/capa/features/extractors/common.py similarity index 100% rename from capa/features/extractors/viv/common.py rename to capa/features/extractors/common.py diff --git a/capa/features/extractors/smda/extractor.py b/capa/features/extractors/smda/extractor.py index b4355d8f..2f7443aa 100644 --- a/capa/features/extractors/smda/extractor.py +++ b/capa/features/extractors/smda/extractor.py @@ -1,5 +1,6 @@ from smda.common.SmdaReport import SmdaReport +import capa.features.extractors.common import capa.features.extractors.smda.file import capa.features.extractors.smda.insn import capa.features.extractors.smda.function @@ -12,6 +13,12 @@ class SmdaFeatureExtractor(FeatureExtractor): super(SmdaFeatureExtractor, self).__init__() self.smda_report = smda_report self.path = path + with open(self.path, "rb") as f: + self.buf = f.read() + + self.global_features = [] + self.global_features.extend(capa.features.extractors.common.extract_os(self.buf)) + self.global_features.extend(capa.features.extractors.common.extract_format(self.buf)) def get_base_address(self): return self.smda_report.base_addr @@ -19,6 +26,7 @@ class SmdaFeatureExtractor(FeatureExtractor): def extract_file_features(self): for feature, va in capa.features.extractors.smda.file.extract_features(self.smda_report, self.path): yield feature, va + yield from self.global_features def get_functions(self): for function in self.smda_report.getFunctions(): @@ -27,6 +35,7 @@ class SmdaFeatureExtractor(FeatureExtractor): def extract_function_features(self, f): for feature, va in capa.features.extractors.smda.function.extract_features(f): yield feature, va + yield from self.global_features def get_basic_blocks(self, f): for bb in f.getBlocks(): @@ -35,6 +44,7 @@ class SmdaFeatureExtractor(FeatureExtractor): def extract_basic_block_features(self, f, bb): for feature, va in capa.features.extractors.smda.basicblock.extract_features(f, bb): yield feature, va + yield from self.global_features def get_instructions(self, f, bb): for smda_ins in bb.getInstructions(): @@ -43,3 +53,4 @@ class SmdaFeatureExtractor(FeatureExtractor): def extract_insn_features(self, f, bb, insn): for feature, va in capa.features.extractors.smda.insn.extract_features(f, bb, insn): yield feature, va + yield from self.global_features diff --git a/capa/features/extractors/viv/extractor.py b/capa/features/extractors/viv/extractor.py index ae8aa4fb..f6960390 100644 --- a/capa/features/extractors/viv/extractor.py +++ b/capa/features/extractors/viv/extractor.py @@ -10,9 +10,9 @@ import logging import viv_utils import viv_utils.flirt +import capa.features.extractors.common import capa.features.extractors.viv.file import capa.features.extractors.viv.insn -import capa.features.extractors.viv.common import capa.features.extractors.viv.function import capa.features.extractors.viv.basicblock from capa.features.extractors.base_extractor import FeatureExtractor @@ -42,8 +42,8 @@ class VivisectFeatureExtractor(FeatureExtractor): self.buf = f.read() self.global_features = [] - self.global_features.extend(capa.features.extractors.viv.common.extract_os(self.buf)) - self.global_features.extend(capa.features.extractors.viv.common.extract_format(self.buf)) + self.global_features.extend(capa.features.extractors.common.extract_os(self.buf)) + self.global_features.extend(capa.features.extractors.common.extract_format(self.buf)) def get_base_address(self): # assume there is only one file loaded into the vw From 769d3547926f4038175cac73dc67627bd90579a7 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 14:56:01 -0600 Subject: [PATCH 15/52] detect-elf-os: remove extra print statement --- scripts/detect-elf-os.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/detect-elf-os.py b/scripts/detect-elf-os.py index 950d64df..dd8c4d74 100644 --- a/scripts/detect-elf-os.py +++ b/scripts/detect-elf-os.py @@ -30,7 +30,6 @@ def main(argv=None): f: BinaryIO = IDAIO() else: - print("not ida") if argv is None: argv = sys.argv[1:] From c1910d47f0c6da53e13b4589f14ffd6aac455cad Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 15:02:10 -0600 Subject: [PATCH 16/52] move is_global_feature into capa.features.common --- capa/features/common.py | 13 +++++++++++++ scripts/show-features.py | 15 +++------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/capa/features/common.py b/capa/features/common.py index 5f6e19cb..59bf5009 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -278,3 +278,16 @@ class Bytes(Feature): @classmethod def freeze_deserialize(cls, args): return cls(*[codecs.decode(x, "hex") for x in args]) + + +def is_global_feature(feature): + """ + is this a feature that is extracted at every scope? + today, this are OS and file format features. + """ + if (isinstance(feature, Characteristic) + and isinstance(feature.value, str) + and (feature.value.startswith("os/") + or feature.value.startswith("format/"))): + return True + return False \ No newline at end of file diff --git a/scripts/show-features.py b/scripts/show-features.py index 1d37588c..44808aad 100644 --- a/scripts/show-features.py +++ b/scripts/show-features.py @@ -194,15 +194,6 @@ def ida_main(): return 0 -def is_global_feature(feature): - if (isinstance(feature, capa.features.common.Characteristic) - and isinstance(feature.value, str) - and (feature.value.startswith("os/") - or feature.value.startswith("format/"))): - return True - return False - - def print_features(functions, extractor): for f in functions: function_address = int(f) @@ -213,21 +204,21 @@ def print_features(functions, extractor): continue for feature, va in extractor.extract_function_features(f): - if is_global_feature(feature): + if capa.features.common.is_global_feature(feature): continue print("func: 0x%08x: %s" % (va, feature)) for bb in extractor.get_basic_blocks(f): for feature, va in extractor.extract_basic_block_features(f, bb): - if is_global_feature(feature): + if capa.features.common.is_global_feature(feature): continue print("bb : 0x%08x: %s" % (va, feature)) for insn in extractor.get_instructions(f, bb): for feature, va in extractor.extract_insn_features(f, bb, insn): - if is_global_feature(feature): + if capa.features.common.is_global_feature(feature): continue try: From 71d9ebd8594023a29c60ee7de4e13b031d0f262d Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 15:05:57 -0600 Subject: [PATCH 17/52] extractors: ida: extract OS and file format characteristics at all scopes --- capa/features/extractors/ida/extractor.py | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/capa/features/extractors/ida/extractor.py b/capa/features/extractors/ida/extractor.py index dc260adc..c5f3850a 100644 --- a/capa/features/extractors/ida/extractor.py +++ b/capa/features/extractors/ida/extractor.py @@ -5,9 +5,16 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +import logging +import functools +import contextlib +import ida_loader import idaapi +import capa.ida.helpers +import capa.features.extractors.elf +from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_ELF, Characteristic import capa.features.extractors.ida.file import capa.features.extractors.ida.insn import capa.features.extractors.ida.function @@ -15,6 +22,26 @@ import capa.features.extractors.ida.basicblock from capa.features.extractors.base_extractor import FeatureExtractor +def extract_format(): + format_name = ida_loader.get_file_type_name() + + if "PE" in format_name: + yield CHARACTERISTIC_PE, 0x0 + elif "ELF64" in format_name: + yield CHARACTERISTIC_ELF, 0x0 + elif "ELF32" in format_name: + yield CHARACTERISTIC_ELF, 0x0 + else: + raise NotImplementedError("file format: %s", format_name) + + +def extract_os(): + with contextlib.closing(capa.ida.helpers.IDAIO()) as f: + os = capa.features.extractors.elf.detect_elf_os(f) + + yield Characteristic("os/%s" % (os.lower())), 0x0 + + class FunctionHandle: """this acts like an idaapi.func_t but with __int__()""" @@ -57,6 +84,9 @@ class InstructionHandle: class IdaFeatureExtractor(FeatureExtractor): def __init__(self): super(IdaFeatureExtractor, self).__init__() + self.global_features = [] + self.global_features.extend(extract_os()) + self.global_features.extend(extract_format()) def get_base_address(self): return idaapi.get_imagebase() @@ -64,6 +94,7 @@ class IdaFeatureExtractor(FeatureExtractor): def extract_file_features(self): for (feature, ea) in capa.features.extractors.ida.file.extract_features(): yield feature, ea + yield from self.global_features def get_functions(self): import capa.features.extractors.ida.helpers as ida_helpers @@ -86,6 +117,7 @@ class IdaFeatureExtractor(FeatureExtractor): def extract_function_features(self, f): for (feature, ea) in capa.features.extractors.ida.function.extract_features(f): yield feature, ea + yield from self.global_features def get_basic_blocks(self, f): import capa.features.extractors.ida.helpers as ida_helpers @@ -96,6 +128,7 @@ class IdaFeatureExtractor(FeatureExtractor): def extract_basic_block_features(self, f, bb): for (feature, ea) in capa.features.extractors.ida.basicblock.extract_features(f, bb): yield feature, ea + yield from self.global_features def get_instructions(self, f, bb): import capa.features.extractors.ida.helpers as ida_helpers @@ -106,3 +139,4 @@ class IdaFeatureExtractor(FeatureExtractor): def extract_insn_features(self, f, bb, insn): for (feature, ea) in capa.features.extractors.ida.insn.extract_features(f, bb, insn): yield feature, ea + yield from self.global_features From 34819b289d67aac0bf2f80d669fb730da4b86320 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 15:08:31 -0600 Subject: [PATCH 18/52] pep8 --- capa/features/common.py | 9 +++++---- capa/features/extractors/common.py | 1 - capa/features/extractors/elf.py | 2 +- capa/features/extractors/ida/extractor.py | 4 ++-- capa/helpers.py | 2 +- scripts/detect-elf-os.py | 1 + scripts/show-features.py | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/capa/features/common.py b/capa/features/common.py index 59bf5009..c2851c8c 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -285,9 +285,10 @@ def is_global_feature(feature): is this a feature that is extracted at every scope? today, this are OS and file format features. """ - if (isinstance(feature, Characteristic) + if ( + isinstance(feature, Characteristic) and isinstance(feature.value, str) - and (feature.value.startswith("os/") - or feature.value.startswith("format/"))): + and (feature.value.startswith("os/") or feature.value.startswith("format/")) + ): return True - return False \ No newline at end of file + return False diff --git a/capa/features/extractors/common.py b/capa/features/extractors/common.py index fb47f303..b42ebe13 100644 --- a/capa/features/extractors/common.py +++ b/capa/features/extractors/common.py @@ -6,7 +6,6 @@ import contextlib import capa.features.extractors.elf from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_ELF, CHARACTERISTIC_WINDOWS, Characteristic - logger = logging.getLogger(__name__) diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index 45388003..d8d80159 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -221,4 +221,4 @@ def detect_elf_os(f: BinaryIO) -> str: # so we can get the debugging output of subsequent strategies ret = OS.LINUX if ret is None else ret - return ret.value if ret is not None else "unknown" \ No newline at end of file + return ret.value if ret is not None else "unknown" diff --git a/capa/features/extractors/ida/extractor.py b/capa/features/extractors/ida/extractor.py index c5f3850a..6c4747f9 100644 --- a/capa/features/extractors/ida/extractor.py +++ b/capa/features/extractors/ida/extractor.py @@ -9,16 +9,16 @@ import logging import functools import contextlib -import ida_loader import idaapi +import ida_loader import capa.ida.helpers import capa.features.extractors.elf -from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_ELF, Characteristic import capa.features.extractors.ida.file import capa.features.extractors.ida.insn import capa.features.extractors.ida.function import capa.features.extractors.ida.basicblock +from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_ELF, Characteristic from capa.features.extractors.base_extractor import FeatureExtractor diff --git a/capa/helpers.py b/capa/helpers.py index eaee04bd..1bdad425 100644 --- a/capa/helpers.py +++ b/capa/helpers.py @@ -29,4 +29,4 @@ def is_runtime_ida(): except ImportError: return False else: - return True \ No newline at end of file + return True diff --git a/scripts/detect-elf-os.py b/scripts/detect-elf-os.py index dd8c4d74..c01990f4 100644 --- a/scripts/detect-elf-os.py +++ b/scripts/detect-elf-os.py @@ -27,6 +27,7 @@ logger = logging.getLogger("capa.detect-elf-os") def main(argv=None): if capa.helpers.is_runtime_ida(): from capa.ida.helpers import IDAIO + f: BinaryIO = IDAIO() else: diff --git a/scripts/show-features.py b/scripts/show-features.py index 44808aad..6d69ea7f 100644 --- a/scripts/show-features.py +++ b/scripts/show-features.py @@ -75,8 +75,8 @@ import capa.rules import capa.engine import capa.helpers import capa.features -import capa.features.freeze import capa.features.common +import capa.features.freeze logger = logging.getLogger("capa.show-features") From 30d7425b982e8c7fce8f106eed8215f2d21608f4 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 15:10:07 -0600 Subject: [PATCH 19/52] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8078f2e6..20c7e88d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - explorer: enforce max column width Features and Editor panes #691 @mike-hunhoff - explorer: add option to limit features to currently selected disassembly address #692 @mike-hunhoff - all: add support for ELF files #700 @Adir-Shemesh @TcM1911 +- rule format: add characteristic for file format, like `format/pe` @williballenthin +- rule format: add characteristic for operating system, like `os/windows` @701 @williballenthin ### Breaking Changes From d5c9a5cf3c95bc9be27f2b248483a94bc1cbc1df Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 11 Aug 2021 15:15:33 -0600 Subject: [PATCH 20/52] mypy: ignore ida_loader --- .github/mypy/mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/mypy/mypy.ini b/.github/mypy/mypy.ini index e5d862a7..0aded4dc 100644 --- a/.github/mypy/mypy.ini +++ b/.github/mypy/mypy.ini @@ -57,6 +57,9 @@ ignore_missing_imports = True [mypy-ida_funcs.*] ignore_missing_imports = True +[mypy-ida_loader.*] +ignore_missing_imports = True + [mypy-PyQt5.*] ignore_missing_imports = True From f013815b2acccd16ee3af20b95aec75c2c361455 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 16 Aug 2021 12:21:25 -0600 Subject: [PATCH 21/52] features: rename legacy term `arch` to `bitness` makes space for upcoming feature `arch: ` for things like i386/amd64/aarch64 --- CHANGELOG.md | 2 ++ capa/features/common.py | 36 +++++++++++++-------------- capa/features/extractors/ida/insn.py | 22 ++++++++-------- capa/features/extractors/smda/insn.py | 14 +++++------ capa/features/extractors/viv/insn.py | 22 ++++++++-------- capa/features/insn.py | 8 +++--- capa/rules.py | 8 +++--- tests/fixtures.py | 16 ++++++------ tests/test_engine.py | 8 +++--- tests/test_rules.py | 24 +++++++++--------- 10 files changed, 81 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20c7e88d..d51b4bee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ ### Breaking Changes +- legacy term `arch` (i.e., "x32") is now called `bitness` @williballenthin + ### New Rules (20) - collection/webcam/capture-webcam-image johnk3r diff --git a/capa/features/common.py b/capa/features/common.py index c2851c8c..444a856f 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -21,11 +21,6 @@ MAX_BYTES_FEATURE_SIZE = 0x100 # thunks may be chained so we specify a delta to control the depth to which these chains are explored THUNK_CHAIN_DEPTH_DELTA = 5 -# identifiers for supported architectures names that tweak a feature -# for example, offset/x32 -ARCH_X32 = "x32" -ARCH_X64 = "x64" -VALID_ARCH = (ARCH_X32, ARCH_X64) OS_WINDOWS = "os/windows" OS_LINUX = "os/linux" @@ -61,33 +56,33 @@ def escape_string(s: str) -> str: class Feature: - def __init__(self, value: Union[str, int, bytes], arch=None, description=None): + def __init__(self, value: Union[str, int, bytes], bitness=None, description=None): """ Args: value (any): the value of the feature, such as the number or string. - arch (str): one of the VALID_ARCH values, or None. - When None, then the feature applies to any architecture. - Modifies the feature name from `feature` to `feature/arch`, like `offset/x32`. + bitness (str): one of the VALID_BITNESS values, or None. + When None, then the feature applies to any bitness. + Modifies the feature name from `feature` to `feature/bitness`, like `offset/x32`. description (str): a human-readable description that explains the feature value. """ super(Feature, self).__init__() - if arch is not None: - if arch not in VALID_ARCH: - raise ValueError("arch '%s' must be one of %s" % (arch, VALID_ARCH)) - self.name = self.__class__.__name__.lower() + "/" + arch + if bitness is not None: + if bitness not in VALID_BITNESS: + raise ValueError("bitness '%s' must be one of %s" % (bitness, VALID_BITNESS)) + self.name = self.__class__.__name__.lower() + "/" + bitness else: self.name = self.__class__.__name__.lower() self.value = value - self.arch = arch + self.bitness = bitness self.description = description def __hash__(self): - return hash((self.name, self.value, self.arch)) + return hash((self.name, self.value, self.bitness)) def __eq__(self, other): - return self.name == other.name and self.value == other.value and self.arch == other.arch + return self.name == other.name and self.value == other.value and self.bitness == other.bitness def get_value_str(self) -> str: """ @@ -114,8 +109,8 @@ class Feature: return capa.engine.Result(self in ctx, self, [], locations=ctx.get(self, [])) def freeze_serialize(self): - if self.arch is not None: - return (self.__class__.__name__, [self.value, {"arch": self.arch}]) + if self.bitness is not None: + return (self.__class__.__name__, [self.value, {"bitness": self.bitness}]) else: return (self.__class__.__name__, [self.value]) @@ -280,6 +275,11 @@ class Bytes(Feature): return cls(*[codecs.decode(x, "hex") for x in args]) +# identifiers for supported bitness names that tweak a feature +# for example, offset/x32 +BITNESS_X32 = "x32" +BITNESS_X64 = "x64" +VALID_BITNESS = (BITNESS_X32, BITNESS_X64) def is_global_feature(feature): """ is this a feature that is extracted at every scope? diff --git a/capa/features/extractors/ida/insn.py b/capa/features/extractors/ida/insn.py index 82fb4715..8bfcd7fb 100644 --- a/capa/features/extractors/ida/insn.py +++ b/capa/features/extractors/ida/insn.py @@ -14,8 +14,8 @@ import capa.features.extractors.helpers import capa.features.extractors.ida.helpers from capa.features.insn import API, Number, Offset, Mnemonic from capa.features.common import ( - ARCH_X32, - ARCH_X64, + BITNESS_X32, + BITNESS_X64, MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, @@ -28,22 +28,22 @@ from capa.features.common import ( SECURITY_COOKIE_BYTES_DELTA = 0x40 -def get_arch(ctx): +def get_bitness(ctx): """ - fetch the ARCH_* constant for the currently open workspace. + fetch the BITNESS_* constant for the currently open workspace. via Tamir Bahar/@tmr232 https://reverseengineering.stackexchange.com/a/11398/17194 """ - if "arch" not in ctx: + if "bitness" not in ctx: info = idaapi.get_inf_structure() if info.is_64bit(): - ctx["arch"] = ARCH_X64 + ctx["bitness"] = BITNESS_X64 elif info.is_32bit(): - ctx["arch"] = ARCH_X32 + ctx["bitness"] = BITNESS_X32 else: - raise ValueError("unexpected architecture") - return ctx["arch"] + raise ValueError("unexpected bitness") + return ctx["bitness"] def get_imports(ctx): @@ -149,7 +149,7 @@ def extract_insn_number_features(f, bb, insn): const = op.addr yield Number(const), insn.ea - yield Number(const, arch=get_arch(f.ctx)), insn.ea + yield Number(const, bitness=get_bitness(f.ctx)), insn.ea def extract_insn_bytes_features(f, bb, insn): @@ -218,7 +218,7 @@ def extract_insn_offset_features(f, bb, insn): op_off = capa.features.extractors.helpers.twos_complement(op_off, 32) yield Offset(op_off), insn.ea - yield Offset(op_off, arch=get_arch(f.ctx)), insn.ea + yield Offset(op_off, bitness=get_bitness(f.ctx)), insn.ea def contains_stack_cookie_keywords(s): diff --git a/capa/features/extractors/smda/insn.py b/capa/features/extractors/smda/insn.py index da5ebbfa..d666728c 100644 --- a/capa/features/extractors/smda/insn.py +++ b/capa/features/extractors/smda/insn.py @@ -7,8 +7,8 @@ from smda.common.SmdaReport import SmdaReport import capa.features.extractors.helpers from capa.features.insn import API, Number, Offset, Mnemonic from capa.features.common import ( - ARCH_X32, - ARCH_X64, + BITNESS_X32, + BITNESS_X64, MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, @@ -23,12 +23,12 @@ PATTERN_HEXNUM = re.compile(r"[+\-] (?P0x[a-fA-F0-9]+)") PATTERN_SINGLENUM = re.compile(r"[+\-] (?P[0-9])") -def get_arch(smda_report): +def get_bitness(smda_report): if smda_report.architecture == "intel": if smda_report.bitness == 32: - return ARCH_X32 + return BITNESS_X32 elif smda_report.bitness == 64: - return ARCH_X64 + return BITNESS_X64 else: raise NotImplementedError @@ -85,7 +85,7 @@ def extract_insn_number_features(f, bb, insn): for operand in operands: try: yield Number(int(operand, 16)), insn.offset - yield Number(int(operand, 16), arch=get_arch(f.smda_report)), insn.offset + yield Number(int(operand, 16), bitness=get_bitness(f.smda_report)), insn.offset except: continue @@ -228,7 +228,7 @@ def extract_insn_offset_features(f, bb, insn): number = int(number_int.group("num")) number = -1 * number if number_int.group().startswith("-") else number yield Offset(number), insn.offset - yield Offset(number, arch=get_arch(f.smda_report)), insn.offset + yield Offset(number, bitness=get_bitness(f.smda_report)), insn.offset def is_security_cookie(f, bb, insn): diff --git a/capa/features/extractors/viv/insn.py b/capa/features/extractors/viv/insn.py index ffe9e3c9..5157b75b 100644 --- a/capa/features/extractors/viv/insn.py +++ b/capa/features/extractors/viv/insn.py @@ -19,8 +19,8 @@ import capa.features.extractors.helpers import capa.features.extractors.viv.helpers from capa.features.insn import API, Number, Offset, Mnemonic from capa.features.common import ( - ARCH_X32, - ARCH_X64, + BITNESS_X32, + BITNESS_X64, MAX_BYTES_FEATURE_SIZE, THUNK_CHAIN_DEPTH_DELTA, Bytes, @@ -34,12 +34,12 @@ from capa.features.extractors.viv.indirect_calls import NotFoundError, resolve_i SECURITY_COOKIE_BYTES_DELTA = 0x40 -def get_arch(vw): - arch = vw.getMeta("Architecture") - if arch == "i386": - return ARCH_X32 - elif arch == "amd64": - return ARCH_X64 +def get_bitness(vw): + bitness = vw.getMeta("Architecture") + if bitness == "i386": + return BITNESS_X32 + elif bitness == "amd64": + return BITNESS_X64 def interface_extract_instruction_XXX(f, bb, insn): @@ -193,7 +193,7 @@ def extract_insn_number_features(f, bb, insn): return yield Number(v), insn.va - yield Number(v, arch=get_arch(f.vw)), insn.va + yield Number(v, bitness=get_bitness(f.vw)), insn.va def derefs(vw, p): @@ -389,7 +389,7 @@ def extract_insn_offset_features(f, bb, insn): v = oper.disp yield Offset(v), insn.va - yield Offset(v, arch=get_arch(f.vw)), insn.va + yield Offset(v, bitness=get_bitness(f.vw)), insn.va # like: [esi + ecx + 16384] # reg ^ ^ @@ -400,7 +400,7 @@ def extract_insn_offset_features(f, bb, insn): v = oper.disp yield Offset(v), insn.va - yield Offset(v, arch=get_arch(f.vw)), insn.va + yield Offset(v, bitness=get_bitness(f.vw)), insn.va def is_security_cookie(f, bb, insn) -> bool: diff --git a/capa/features/insn.py b/capa/features/insn.py index 6c7e07ff..c5bc727e 100644 --- a/capa/features/insn.py +++ b/capa/features/insn.py @@ -21,16 +21,16 @@ class API(Feature): class Number(Feature): - def __init__(self, value: int, arch=None, description=None): - super(Number, self).__init__(value, arch=arch, description=description) + def __init__(self, value: int, bitness=None, description=None): + super(Number, self).__init__(value, bitness=bitness, description=description) def get_value_str(self): return capa.render.utils.hex(self.value) class Offset(Feature): - def __init__(self, value: int, arch=None, description=None): - super(Offset, self).__init__(value, arch=arch, description=description) + def __init__(self, value: int, bitness=None, description=None): + super(Offset, self).__init__(value, bitness=bitness, description=description) def get_value_str(self): return capa.render.utils.hex(self.value) diff --git a/capa/rules.py b/capa/rules.py index 8e6267bf..c40f992f 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -240,19 +240,19 @@ def parse_feature(key: str): elif key == "number": return capa.features.insn.Number elif key.startswith("number/"): - arch = key.partition("/")[2] + bitness = key.partition("/")[2] # the other handlers here return constructors for features, # and we want to as well, # however, we need to preconfigure one of the arguments (`arch`). # so, instead we return a partially-applied function that # provides `arch` to the feature constructor. # it forwards any other arguments provided to the closure along to the constructor. - return functools.partial(capa.features.insn.Number, arch=arch) + return functools.partial(capa.features.insn.Number, arch=bitness) elif key == "offset": return capa.features.insn.Offset elif key.startswith("offset/"): - arch = key.partition("/")[2] - return functools.partial(capa.features.insn.Offset, arch=arch) + bitness = key.partition("/")[2] + return functools.partial(capa.features.insn.Offset, arch=bitness) elif key == "mnemonic": return capa.features.insn.Mnemonic elif key == "basic blocks": diff --git a/tests/fixtures.py b/tests/fixtures.py index f6e59e25..16d1e0e5 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -22,8 +22,8 @@ import capa.features.insn import capa.features.common import capa.features.basicblock from capa.features.common import ( - ARCH_X32, - ARCH_X64, + BITNESS_X32, + BITNESS_X64, CHARACTERISTIC_PE, CHARACTERISTIC_ELF, CHARACTERISTIC_LINUX, @@ -390,10 +390,10 @@ FEATURE_PRESENCE_TESTS = sorted( # insn/number: stack adjustments ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xC), False), ("mimikatz", "function=0x40105D", capa.features.insn.Number(0x10), False), - # insn/number: arch flavors + # insn/number: bitness flavors ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xFF), True), - ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xFF, arch=ARCH_X32), True), - ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xFF, arch=ARCH_X64), False), + ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xFF, bitness=BITNESS_X32), True), + ("mimikatz", "function=0x40105D", capa.features.insn.Number(0xFF, bitness=BITNESS_X64), False), # insn/offset ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0), True), ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x4), True), @@ -406,10 +406,10 @@ FEATURE_PRESENCE_TESTS = sorted( # insn/offset: negative ("mimikatz", "function=0x4011FB", capa.features.insn.Offset(-0x1), True), ("mimikatz", "function=0x4011FB", capa.features.insn.Offset(-0x2), True), - # insn/offset: arch flavors + # insn/offset: bitness flavors ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0), True), - ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0, arch=ARCH_X32), True), - ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0, arch=ARCH_X64), False), + ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0, bitness=BITNESS_X32), True), + ("mimikatz", "function=0x40105D", capa.features.insn.Offset(0x0, bitness=BITNESS_X64), False), # insn/api ("mimikatz", "function=0x403BAC", capa.features.insn.API("advapi32.CryptAcquireContextW"), True), ("mimikatz", "function=0x403BAC", capa.features.insn.API("advapi32.CryptAcquireContext"), True), diff --git a/tests/test_engine.py b/tests/test_engine.py index 642ddb49..57fffb8e 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -474,11 +474,11 @@ def test_match_namespace(): def test_render_number(): assert str(capa.features.insn.Number(1)) == "number(0x1)" - assert str(capa.features.insn.Number(1, arch=capa.features.common.ARCH_X32)) == "number/x32(0x1)" - assert str(capa.features.insn.Number(1, arch=capa.features.common.ARCH_X64)) == "number/x64(0x1)" + assert str(capa.features.insn.Number(1, bitness=capa.features.common.BITNESS_X32)) == "number/x32(0x1)" + assert str(capa.features.insn.Number(1, bitness=capa.features.common.BITNESS_X64)) == "number/x64(0x1)" def test_render_offset(): assert str(capa.features.insn.Offset(1)) == "offset(0x1)" - assert str(capa.features.insn.Offset(1, arch=capa.features.common.ARCH_X32)) == "offset/x32(0x1)" - assert str(capa.features.insn.Offset(1, arch=capa.features.common.ARCH_X64)) == "offset/x64(0x1)" + assert str(capa.features.insn.Offset(1, bitness=capa.features.common.BITNESS_X32)) == "offset/x32(0x1)" + assert str(capa.features.insn.Offset(1, bitness=capa.features.common.BITNESS_X64)) == "offset/x64(0x1)" diff --git a/tests/test_rules.py b/tests/test_rules.py index 95a39fb1..8e360c22 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -16,8 +16,8 @@ import capa.features.common from capa.features.file import FunctionName from capa.features.insn import Number, Offset from capa.features.common import ( - ARCH_X32, - ARCH_X64, + BITNESS_X32, + BITNESS_X64, FORMAT_PE, OS_WINDOWS, CHARACTERISTIC_PE, @@ -526,7 +526,7 @@ def test_invalid_number(): ) -def test_number_arch(): +def test_number_bitness(): r = capa.rules.Rule.from_yaml( textwrap.dedent( """ @@ -538,13 +538,13 @@ def test_number_arch(): """ ) ) - assert r.evaluate({Number(2, arch=ARCH_X32): {1}}) == True + assert r.evaluate({Number(2, bitness=BITNESS_X32): {1}}) == True assert r.evaluate({Number(2): {1}}) == False - assert r.evaluate({Number(2, arch=ARCH_X64): {1}}) == False + assert r.evaluate({Number(2, bitness=BITNESS_X64): {1}}) == False -def test_number_arch_symbol(): +def test_number_bitness_symbol(): r = capa.rules.Rule.from_yaml( textwrap.dedent( """ @@ -556,7 +556,7 @@ def test_number_arch_symbol(): """ ) ) - assert r.evaluate({Number(2, arch=ARCH_X32, description="some constant"): {1}}) == True + assert r.evaluate({Number(2, bitness=BITNESS_X32, description="some constant"): {1}}) == True def test_offset_symbol(): @@ -604,7 +604,7 @@ def test_count_offset_symbol(): assert r.evaluate({Offset(0x100, description="symbol name"): {1, 2, 3}}) == True -def test_offset_arch(): +def test_offset_bitness(): r = capa.rules.Rule.from_yaml( textwrap.dedent( """ @@ -616,13 +616,13 @@ def test_offset_arch(): """ ) ) - assert r.evaluate({Offset(2, arch=ARCH_X32): {1}}) == True + assert r.evaluate({Offset(2, bitness=BITNESS_X32): {1}}) == True assert r.evaluate({Offset(2): {1}}) == False - assert r.evaluate({Offset(2, arch=ARCH_X64): {1}}) == False + assert r.evaluate({Offset(2, bitness=BITNESS_X64): {1}}) == False -def test_offset_arch_symbol(): +def test_offset_bitness_symbol(): r = capa.rules.Rule.from_yaml( textwrap.dedent( """ @@ -634,7 +634,7 @@ def test_offset_arch_symbol(): """ ) ) - assert r.evaluate({Offset(2, arch=ARCH_X32, description="some constant"): {1}}) == True + assert r.evaluate({Offset(2, bitness=BITNESS_X32, description="some constant"): {1}}) == True def test_invalid_offset(): From ab1326f858625369ad479e83ae87277ed4eb4f4f Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 16 Aug 2021 16:28:26 -0600 Subject: [PATCH 22/52] features: move OS and Format to their own features, not characteristics --- CHANGELOG.md | 4 +- capa/features/common.py | 71 ++++++++++++++--------- capa/features/extractors/common.py | 10 ++-- capa/features/extractors/elf.py | 40 ++++++------- capa/features/extractors/ida/extractor.py | 25 +++++--- capa/features/extractors/pefile.py | 6 +- capa/rules.py | 41 +++++-------- scripts/capa2yara.py | 2 +- tests/fixtures.py | 30 +++++----- tests/test_rules.py | 23 ++++---- 10 files changed, 132 insertions(+), 120 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d51b4bee..a1a39a7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,8 @@ - explorer: enforce max column width Features and Editor panes #691 @mike-hunhoff - explorer: add option to limit features to currently selected disassembly address #692 @mike-hunhoff - all: add support for ELF files #700 @Adir-Shemesh @TcM1911 -- rule format: add characteristic for file format, like `format/pe` @williballenthin -- rule format: add characteristic for operating system, like `os/windows` @701 @williballenthin +- rule format: add feature `format: ` for file format, like `format: pe` @williballenthin +- rule format: add feature `os: ` for operating system, like `os: windows` #701 @williballenthin ### Breaking Changes diff --git a/capa/features/common.py b/capa/features/common.py index 444a856f..a8b6f5ef 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -14,6 +14,7 @@ from typing import Set, Dict, Union import capa.engine import capa.features +import capa.features.extractors.elf logger = logging.getLogger(__name__) MAX_BYTES_FEATURE_SIZE = 0x100 @@ -22,16 +23,6 @@ MAX_BYTES_FEATURE_SIZE = 0x100 THUNK_CHAIN_DEPTH_DELTA = 5 -OS_WINDOWS = "os/windows" -OS_LINUX = "os/linux" -OS_MACOS = "os/macos" -VALID_OS = (OS_WINDOWS, OS_LINUX, OS_MACOS) - -FORMAT_PE = "format/pe" -FORMAT_ELF = "format/elf" -VALID_FORMAT = (FORMAT_PE, FORMAT_ELF) - - def bytes_to_str(b: bytes) -> str: return str(codecs.encode(b, "hex").decode("utf-8")) @@ -139,14 +130,6 @@ class Characteristic(Feature): super(Characteristic, self).__init__(value, description=description) -CHARACTERISTIC_WINDOWS = Characteristic(OS_WINDOWS) -CHARACTERISTIC_LINUX = Characteristic(OS_LINUX) -CHARACTERISTIC_MACOS = Characteristic(OS_MACOS) - -CHARACTERISTIC_PE = Characteristic(FORMAT_PE) -CHARACTERISTIC_ELF = Characteristic(FORMAT_ELF) - - class String(Feature): def __init__(self, value: str, description=None): super(String, self).__init__(value, description=description) @@ -280,15 +263,51 @@ class Bytes(Feature): BITNESS_X32 = "x32" BITNESS_X64 = "x64" VALID_BITNESS = (BITNESS_X32, BITNESS_X64) + + +ARCH_I386 = "i386" +ARCH_AMD64 = "amd64" +VALID_ARCH = (ARCH_I386, ARCH_AMD64) + + +class Arch(Feature): + def __init__(self, value: str, description=None): + assert value in VALID_ARCH + super(Arch, self).__init__(value, description=description) + self.name = "arch" + + +OS_WINDOWS = "windows" +OS_LINUX = "linux" +OS_MACOS = "macos" +VALID_OS = {os.value for os in capa.features.extractors.elf.OS} +VALID_OS.add(OS_WINDOWS) +VALID_OS.add(OS_LINUX) +VALID_OS.add(OS_MACOS) + + +class OS(Feature): + def __init__(self, value: str, description=None): + assert value in (VALID_OS) + super(OS, self).__init__(value, description=description) + self.name = "os" + + +FORMAT_PE = "pe" +FORMAT_ELF = "elf" +VALID_FORMAT = (FORMAT_PE, FORMAT_ELF) + + +class Format(Feature): + def __init__(self, value: str, description=None): + assert value in (VALID_FORMAT) + super(Format, self).__init__(value, description=description) + self.name = "format" + + def is_global_feature(feature): """ is this a feature that is extracted at every scope? - today, this are OS and file format features. + today, this are OS and arch features. """ - if ( - isinstance(feature, Characteristic) - and isinstance(feature.value, str) - and (feature.value.startswith("os/") or feature.value.startswith("format/")) - ): - return True - return False + return isinstance(feature, (OS, Arch)) diff --git a/capa/features/extractors/common.py b/capa/features/extractors/common.py index b42ebe13..ab4d2ef8 100644 --- a/capa/features/extractors/common.py +++ b/capa/features/extractors/common.py @@ -4,27 +4,27 @@ import binascii import contextlib import capa.features.extractors.elf -from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_ELF, CHARACTERISTIC_WINDOWS, Characteristic +from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Format logger = logging.getLogger(__name__) def extract_format(buf): if buf.startswith(b"MZ"): - yield CHARACTERISTIC_PE, 0x0 + yield Format(FORMAT_PE), 0x0 elif buf.startswith(b"\x7fELF"): - yield CHARACTERISTIC_ELF, 0x0 + yield Format(FORMAT_ELF), 0x0 else: raise NotImplementedError("file format: %s", binascii.hexlify(buf[:4]).decode("ascii")) def extract_os(buf): if buf.startswith(b"MZ"): - yield CHARACTERISTIC_WINDOWS, 0x0 + yield OS(OS_WINDOWS), 0x0 elif buf.startswith(b"\x7fELF"): with contextlib.closing(io.BytesIO(buf)) as f: os = capa.features.extractors.elf.detect_elf_os(f) - yield Characteristic("os/%s" % (os.lower())), 0x0 + yield OS(os), 0x0 else: raise NotImplementedError("file format: %s", binascii.hexlify(buf[:4]).decode("ascii")) diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index d8d80159..f11bd642 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -19,27 +19,25 @@ class CorruptElfFile(ValueError): class OS(str, Enum): - HPUX = "HPUX" - NETBSD = "NETBSD" - LINUX = "LINUX" - HURD = "HURD" - _86OPEN = "86OPEN" - SOLARIS = "SOLARIS" - AIX = "AIX" - IRIX = "IRIX" - FREEBSD = "FREEBSD" - TRU64 = "TRU64" - MODESTO = "MODESTO" - OPENBSD = "OPENBSD" - OPENVMS = "OPENVMS" - NSK = "NSK" - AROS = "AROS" - FENIXOS = "FENIXOS" - CLOUD = "CLOUD" - SORTFIX = "SORTFIX" - ARM_AEABI = "ARM_AEABI" - SYLLABLE = "SYLLABLE" - NACL = "NACL" + HPUX = "hpux" + NETBSD = "netbsd" + LINUX = "linux" + HURD = "hurd" + _86OPEN = "86open" + SOLARIS = "solaris" + AIX = "aix" + IRIX = "irix" + FREEBSD = "freebsd" + TRU64 = "tru64" + MODESTO = "modesto" + OPENBSD = "openbsd" + OPENVMS = "openvms" + NSK = "nsk" + AROS = "aros" + FENIXOS = "fenixos" + CLOUD = "cloud" + SYLLABLE = "syllable" + NACL = "nacl" def detect_elf_os(f: BinaryIO) -> str: diff --git a/capa/features/extractors/ida/extractor.py b/capa/features/extractors/ida/extractor.py index 6c4747f9..e91d803f 100644 --- a/capa/features/extractors/ida/extractor.py +++ b/capa/features/extractors/ida/extractor.py @@ -5,8 +5,6 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import logging -import functools import contextlib import idaapi @@ -18,7 +16,7 @@ import capa.features.extractors.ida.file import capa.features.extractors.ida.insn import capa.features.extractors.ida.function import capa.features.extractors.ida.basicblock -from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_ELF, Characteristic +from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Format from capa.features.extractors.base_extractor import FeatureExtractor @@ -26,20 +24,29 @@ def extract_format(): format_name = ida_loader.get_file_type_name() if "PE" in format_name: - yield CHARACTERISTIC_PE, 0x0 + yield Format(FORMAT_PE), 0x0 elif "ELF64" in format_name: - yield CHARACTERISTIC_ELF, 0x0 + yield Format(FORMAT_ELF), 0x0 elif "ELF32" in format_name: - yield CHARACTERISTIC_ELF, 0x0 + yield Format(FORMAT_ELF), 0x0 else: raise NotImplementedError("file format: %s", format_name) def extract_os(): - with contextlib.closing(capa.ida.helpers.IDAIO()) as f: - os = capa.features.extractors.elf.detect_elf_os(f) + format_name = ida_loader.get_file_type_name() - yield Characteristic("os/%s" % (os.lower())), 0x0 + if "PE" in format_name: + yield OS(OS_WINDOWS), 0x0 + + elif "ELF" in format_name: + with contextlib.closing(capa.ida.helpers.IDAIO()) as f: + os = capa.features.extractors.elf.detect_elf_os(f) + + yield OS(os), 0x0 + + else: + raise NotImplementedError("file format: %s", format_name) class FunctionHandle: diff --git a/capa/features/extractors/pefile.py b/capa/features/extractors/pefile.py index b1727574..28ee3797 100644 --- a/capa/features/extractors/pefile.py +++ b/capa/features/extractors/pefile.py @@ -14,7 +14,7 @@ import capa.features.extractors import capa.features.extractors.helpers import capa.features.extractors.strings from capa.features.file import Export, Import, Section -from capa.features.common import CHARACTERISTIC_PE, CHARACTERISTIC_WINDOWS, String, Characteristic +from capa.features.common import OS, Format, String, Characteristic, OS_WINDOWS, FORMAT_PE from capa.features.extractors.base_extractor import FeatureExtractor logger = logging.getLogger(__name__) @@ -114,11 +114,11 @@ def extract_file_function_names(pe, file_path): def extract_os(pe, file_path): # assuming PE -> Windows # though i suppose they're also used by UEFI - yield CHARACTERISTIC_WINDOWS, 0x0 + yield OS(OS_WINDOWS), 0x0 def extract_format(pe, file_path): - yield CHARACTERISTIC_PE, 0x0 + yield Format(FORMAT_PE), 0x0 def extract_file_features(pe, file_path): diff --git a/capa/rules.py b/capa/rules.py index c40f992f..5953154e 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -34,15 +34,7 @@ import capa.features.insn import capa.features.common import capa.features.basicblock from capa.engine import Statement, FeatureSet -from capa.features.common import ( - CHARACTERISTIC_PE, - CHARACTERISTIC_ELF, - CHARACTERISTIC_LINUX, - CHARACTERISTIC_MACOS, - CHARACTERISTIC_WINDOWS, - MAX_BYTES_FEATURE_SIZE, - Feature, -) +from capa.features.common import MAX_BYTES_FEATURE_SIZE, Feature logger = logging.getLogger(__name__) @@ -86,11 +78,8 @@ SUPPORTED_FEATURES = { capa.features.file.FunctionName, capa.features.common.Characteristic("embedded pe"), capa.features.common.String, - CHARACTERISTIC_WINDOWS, - CHARACTERISTIC_LINUX, - CHARACTERISTIC_MACOS, - CHARACTERISTIC_PE, - CHARACTERISTIC_ELF, + capa.features.common.Format, + capa.features.common.OS, }, FUNCTION_SCOPE: { # plus basic block scope features, see below @@ -99,11 +88,7 @@ SUPPORTED_FEATURES = { capa.features.common.Characteristic("calls to"), capa.features.common.Characteristic("loop"), capa.features.common.Characteristic("recursive call"), - CHARACTERISTIC_WINDOWS, - CHARACTERISTIC_LINUX, - CHARACTERISTIC_MACOS, - CHARACTERISTIC_PE, - CHARACTERISTIC_ELF, + capa.features.common.OS, }, BASIC_BLOCK_SCOPE: { capa.features.common.MatchedRule, @@ -121,11 +106,7 @@ SUPPORTED_FEATURES = { capa.features.common.Characteristic("tight loop"), capa.features.common.Characteristic("stack string"), capa.features.common.Characteristic("indirect call"), - CHARACTERISTIC_WINDOWS, - CHARACTERISTIC_LINUX, - CHARACTERISTIC_MACOS, - CHARACTERISTIC_PE, - CHARACTERISTIC_ELF, + capa.features.common.OS, }, } @@ -243,16 +224,16 @@ def parse_feature(key: str): bitness = key.partition("/")[2] # the other handlers here return constructors for features, # and we want to as well, - # however, we need to preconfigure one of the arguments (`arch`). + # however, we need to preconfigure one of the arguments (`bitness`). # so, instead we return a partially-applied function that - # provides `arch` to the feature constructor. + # provides `bitness` to the feature constructor. # it forwards any other arguments provided to the closure along to the constructor. - return functools.partial(capa.features.insn.Number, arch=bitness) + return functools.partial(capa.features.insn.Number, bitness=bitness) elif key == "offset": return capa.features.insn.Offset elif key.startswith("offset/"): bitness = key.partition("/")[2] - return functools.partial(capa.features.insn.Offset, arch=bitness) + return functools.partial(capa.features.insn.Offset, bitness=bitness) elif key == "mnemonic": return capa.features.insn.Mnemonic elif key == "basic blocks": @@ -269,6 +250,10 @@ def parse_feature(key: str): return capa.features.common.MatchedRule elif key == "function-name": return capa.features.file.FunctionName + elif key == "os": + return capa.features.common.OS + elif key == "format": + return capa.features.common.Format else: raise InvalidRule("unexpected statement: %s" % key) diff --git a/scripts/capa2yara.py b/scripts/capa2yara.py index 169b8cc4..5a72c697 100644 --- a/scripts/capa2yara.py +++ b/scripts/capa2yara.py @@ -43,7 +43,7 @@ import capa.rules import capa.engine import capa.features import capa.features.insn -from capa.features.common import ARCH_X32, ARCH_X64, String +from capa.features.common import BITNESS_X32, BITNESS_X64, String logger = logging.getLogger("capa2yara") diff --git a/tests/fixtures.py b/tests/fixtures.py index 16d1e0e5..1d18dfac 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -24,10 +24,12 @@ import capa.features.basicblock from capa.features.common import ( BITNESS_X32, BITNESS_X64, - CHARACTERISTIC_PE, - CHARACTERISTIC_ELF, - CHARACTERISTIC_LINUX, - CHARACTERISTIC_WINDOWS, + FORMAT_ELF, + FORMAT_PE, + Format, + OS, + OS_LINUX, + OS_WINDOWS, ) CD = os.path.dirname(__file__) @@ -511,17 +513,17 @@ FEATURE_PRESENCE_TESTS = sorted( # file/function-name ("pma16-01", "file", capa.features.file.FunctionName("__aulldiv"), True), # os & format - ("pma16-01", "file", CHARACTERISTIC_WINDOWS, True), - ("pma16-01", "file", CHARACTERISTIC_LINUX, False), - ("pma16-01", "function=0x404356", CHARACTERISTIC_WINDOWS, True), - ("pma16-01", "function=0x404356,bb=0x4043B9", CHARACTERISTIC_WINDOWS, True), - ("pma16-01", "file", CHARACTERISTIC_PE, True), - ("pma16-01", "file", CHARACTERISTIC_ELF, False), - ("pma16-01", "function=0x404356", CHARACTERISTIC_PE, True), - ("pma16-01", "function=0x404356,bb=0x4043B9", CHARACTERISTIC_PE, True), + ("pma16-01", "file", OS(OS_WINDOWS), True), + ("pma16-01", "file", OS(OS_LINUX), False), + ("pma16-01", "function=0x404356", OS(OS_WINDOWS), True), + ("pma16-01", "function=0x404356,bb=0x4043B9", OS(OS_WINDOWS), True), + ("pma16-01", "file", Format(FORMAT_PE), True), + ("pma16-01", "file", Format(FORMAT_ELF), False), + ("pma16-01", "function=0x404356", Format(FORMAT_PE), True), + ("pma16-01", "function=0x404356,bb=0x4043B9", Format(FORMAT_PE), True), # elf support - ("7351f.elf", "file", CHARACTERISTIC_LINUX, True), - ("7351f.elf", "file", CHARACTERISTIC_ELF, True), + ("7351f.elf", "file", OS(OS_LINUX), True), + ("7351f.elf", "file", OS(OS_WINDOWS), False), ("7351f.elf", "function=0x408753", capa.features.common.String("/dev/null"), True), ("7351f.elf", "function=0x408753,bb=0x408781", capa.features.insn.API("open"), True), ], diff --git a/tests/test_rules.py b/tests/test_rules.py index 8e360c22..82337f41 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -16,14 +16,15 @@ import capa.features.common from capa.features.file import FunctionName from capa.features.insn import Number, Offset from capa.features.common import ( + FORMAT_PE, + FORMAT_ELF, + OS_WINDOWS, + OS_LINUX, BITNESS_X32, BITNESS_X64, - FORMAT_PE, - OS_WINDOWS, - CHARACTERISTIC_PE, - CHARACTERISTIC_WINDOWS, String, - Characteristic, + OS, + Format ) @@ -964,13 +965,13 @@ def test_os_features(): scope: file features: - and: - - characteristic: os/windows + - os: windows """ ) r = capa.rules.Rule.from_yaml(rule) children = list(r.statement.get_children()) - assert (CHARACTERISTIC_WINDOWS in children) == True - assert (CHARACTERISTIC_LINUX not in children) == True + assert (OS(OS_WINDOWS) in children) == True + assert (OS(OS_LINUX) not in children) == True def test_format_features(): @@ -982,10 +983,10 @@ def test_format_features(): scope: file features: - and: - - characteristic: format/pe + - format: pe """ ) r = capa.rules.Rule.from_yaml(rule) children = list(r.statement.get_children()) - assert (CHARACTERISTIC_PE in children) == True - assert (CHARACTERISTIC_ELF not in children) == True + assert (Format(FORMAT_PE) in children) == True + assert (Format(FORMAT_ELF) not in children) == True From 5405e182c3603ce8582a0f626afa04a2e6156d92 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 16 Aug 2021 16:37:04 -0600 Subject: [PATCH 23/52] features: move Format features to file scope --- capa/features/extractors/ida/extractor.py | 15 +-------------- capa/features/extractors/ida/file.py | 16 ++++++++++++++++ capa/features/extractors/pefile.py | 4 ++-- capa/features/extractors/smda/extractor.py | 1 - capa/features/extractors/smda/file.py | 7 +++++++ capa/features/extractors/viv/extractor.py | 1 - capa/features/extractors/viv/file.py | 6 ++++++ 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/capa/features/extractors/ida/extractor.py b/capa/features/extractors/ida/extractor.py index e91d803f..05e45f57 100644 --- a/capa/features/extractors/ida/extractor.py +++ b/capa/features/extractors/ida/extractor.py @@ -16,23 +16,10 @@ import capa.features.extractors.ida.file import capa.features.extractors.ida.insn import capa.features.extractors.ida.function import capa.features.extractors.ida.basicblock -from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Format +from capa.features.common import OS, OS_WINDOWS from capa.features.extractors.base_extractor import FeatureExtractor -def extract_format(): - format_name = ida_loader.get_file_type_name() - - if "PE" in format_name: - yield Format(FORMAT_PE), 0x0 - elif "ELF64" in format_name: - yield Format(FORMAT_ELF), 0x0 - elif "ELF32" in format_name: - yield Format(FORMAT_ELF), 0x0 - else: - raise NotImplementedError("file format: %s", format_name) - - def extract_os(): format_name = ida_loader.get_file_type_name() diff --git a/capa/features/extractors/ida/file.py b/capa/features/extractors/ida/file.py index d3f6a101..6531c6ad 100644 --- a/capa/features/extractors/ida/file.py +++ b/capa/features/extractors/ida/file.py @@ -11,12 +11,14 @@ import struct import idc import idaapi import idautils +import ida_loader import capa.features.extractors.helpers import capa.features.extractors.strings import capa.features.extractors.ida.helpers from capa.features.file import Export, Import, Section, FunctionName from capa.features.common import String, Characteristic +from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Format def check_segment_for_pe(seg): @@ -153,6 +155,19 @@ def extract_file_function_names(): yield FunctionName(name), ea +def extract_file_format(): + format_name = ida_loader.get_file_type_name() + + if "PE" in format_name: + yield Format(FORMAT_PE), 0x0 + elif "ELF64" in format_name: + yield Format(FORMAT_ELF), 0x0 + elif "ELF32" in format_name: + yield Format(FORMAT_ELF), 0x0 + else: + raise NotImplementedError("file format: %s", format_name) + + def extract_features(): """extract file features""" for file_handler in FILE_HANDLERS: @@ -167,6 +182,7 @@ FILE_HANDLERS = ( extract_file_section_names, extract_file_embedded_pe, extract_file_function_names, + extract_file_format, ) diff --git a/capa/features/extractors/pefile.py b/capa/features/extractors/pefile.py index 28ee3797..a8bcd8bc 100644 --- a/capa/features/extractors/pefile.py +++ b/capa/features/extractors/pefile.py @@ -117,7 +117,7 @@ def extract_os(pe, file_path): yield OS(OS_WINDOWS), 0x0 -def extract_format(pe, file_path): +def extract_file_format(pe, file_path): yield Format(FORMAT_PE), 0x0 @@ -146,7 +146,7 @@ FILE_HANDLERS = ( extract_file_strings, extract_file_function_names, extract_os, - extract_format, + extract_file_format, ) diff --git a/capa/features/extractors/smda/extractor.py b/capa/features/extractors/smda/extractor.py index 2f7443aa..59f64a06 100644 --- a/capa/features/extractors/smda/extractor.py +++ b/capa/features/extractors/smda/extractor.py @@ -18,7 +18,6 @@ class SmdaFeatureExtractor(FeatureExtractor): self.global_features = [] self.global_features.extend(capa.features.extractors.common.extract_os(self.buf)) - self.global_features.extend(capa.features.extractors.common.extract_format(self.buf)) def get_base_address(self): return self.smda_report.base_addr diff --git a/capa/features/extractors/smda/file.py b/capa/features/extractors/smda/file.py index 5250e26b..0be6aa0a 100644 --- a/capa/features/extractors/smda/file.py +++ b/capa/features/extractors/smda/file.py @@ -1,6 +1,7 @@ # if we have SMDA we definitely have lief import lief +import capa.features.extractors.common import capa.features.extractors.helpers import capa.features.extractors.strings from capa.features.file import Export, Import, Section @@ -74,6 +75,11 @@ def extract_file_function_names(smda_report, file_path): return +def extract_file_format(smda_report, file_path): + with open(file_path, "rb") as f: + yield from capa.features.extractors.common.extract_format(f.read()) + + def extract_features(smda_report, file_path): """ extract file features from given workspace @@ -98,4 +104,5 @@ FILE_HANDLERS = ( extract_file_section_names, extract_file_strings, extract_file_function_names, + extract_file_format, ) diff --git a/capa/features/extractors/viv/extractor.py b/capa/features/extractors/viv/extractor.py index f6960390..b922d5dc 100644 --- a/capa/features/extractors/viv/extractor.py +++ b/capa/features/extractors/viv/extractor.py @@ -43,7 +43,6 @@ class VivisectFeatureExtractor(FeatureExtractor): self.global_features = [] self.global_features.extend(capa.features.extractors.common.extract_os(self.buf)) - self.global_features.extend(capa.features.extractors.common.extract_format(self.buf)) def get_base_address(self): # assume there is only one file loaded into the vw diff --git a/capa/features/extractors/viv/file.py b/capa/features/extractors/viv/file.py index 9ec9d672..c92e124e 100644 --- a/capa/features/extractors/viv/file.py +++ b/capa/features/extractors/viv/file.py @@ -11,6 +11,7 @@ import viv_utils import viv_utils.flirt import capa.features.insn +import capa.features.extractors.common import capa.features.extractors.helpers import capa.features.extractors.strings from capa.features.file import Export, Import, Section, FunctionName @@ -87,6 +88,10 @@ def extract_file_function_names(vw, buf): yield FunctionName(name), va +def extract_file_format(vw, buf): + yield from capa.features.extractors.common.extract_format(buf) + + def extract_features(vw, buf: bytes): """ extract file features from given workspace @@ -111,4 +116,5 @@ FILE_HANDLERS = ( extract_file_section_names, extract_file_strings, extract_file_function_names, + extract_file_format, ) From 738fa9150e7fc7400994aa89d7df4caf5d58c4c9 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 16 Aug 2021 16:39:40 -0600 Subject: [PATCH 24/52] fixtures: update tests to account for Format scope --- tests/fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 1d18dfac..3d28c611 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -519,11 +519,11 @@ FEATURE_PRESENCE_TESTS = sorted( ("pma16-01", "function=0x404356,bb=0x4043B9", OS(OS_WINDOWS), True), ("pma16-01", "file", Format(FORMAT_PE), True), ("pma16-01", "file", Format(FORMAT_ELF), False), - ("pma16-01", "function=0x404356", Format(FORMAT_PE), True), - ("pma16-01", "function=0x404356,bb=0x4043B9", Format(FORMAT_PE), True), # elf support ("7351f.elf", "file", OS(OS_LINUX), True), ("7351f.elf", "file", OS(OS_WINDOWS), False), + ("7351f.elf", "file", Format(FORMAT_ELF), True), + ("7351f.elf", "file", Format(FORMAT_PE), False), ("7351f.elf", "function=0x408753", capa.features.common.String("/dev/null"), True), ("7351f.elf", "function=0x408753,bb=0x408781", capa.features.insn.API("open"), True), ], From 8e689c39f48004001c7c5cf0dc38c6f0bba0313f Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 16 Aug 2021 17:06:56 -0600 Subject: [PATCH 25/52] features: add Arch feature at global scope --- CHANGELOG.md | 1 + capa/features/extractors/ida/extractor.py | 3 ++- capa/features/extractors/ida/file.py | 3 +-- capa/features/extractors/pefile.py | 2 +- capa/features/extractors/smda/extractor.py | 3 +++ capa/features/extractors/viv/extractor.py | 3 +++ capa/rules.py | 5 ++++ tests/fixtures.py | 21 ++++++++++++----- tests/test_rules.py | 27 +++++++++++++++++++--- 9 files changed, 55 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1a39a7d..50fccceb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - explorer: add option to limit features to currently selected disassembly address #692 @mike-hunhoff - all: add support for ELF files #700 @Adir-Shemesh @TcM1911 - rule format: add feature `format: ` for file format, like `format: pe` @williballenthin +- rule format: add feature `arch: ` for architecture, like `arch: amd64` @williballenthin - rule format: add feature `os: ` for operating system, like `os: windows` #701 @williballenthin ### Breaking Changes diff --git a/capa/features/extractors/ida/extractor.py b/capa/features/extractors/ida/extractor.py index 05e45f57..93f2ce55 100644 --- a/capa/features/extractors/ida/extractor.py +++ b/capa/features/extractors/ida/extractor.py @@ -14,6 +14,7 @@ import capa.ida.helpers import capa.features.extractors.elf import capa.features.extractors.ida.file import capa.features.extractors.ida.insn +import capa.features.extractors.ida.global_ import capa.features.extractors.ida.function import capa.features.extractors.ida.basicblock from capa.features.common import OS, OS_WINDOWS @@ -80,7 +81,7 @@ class IdaFeatureExtractor(FeatureExtractor): super(IdaFeatureExtractor, self).__init__() self.global_features = [] self.global_features.extend(extract_os()) - self.global_features.extend(extract_format()) + self.global_features.extend(capa.features.extractors.ida.global_.extract_arch()) def get_base_address(self): return idaapi.get_imagebase() diff --git a/capa/features/extractors/ida/file.py b/capa/features/extractors/ida/file.py index 6531c6ad..3fa72351 100644 --- a/capa/features/extractors/ida/file.py +++ b/capa/features/extractors/ida/file.py @@ -17,8 +17,7 @@ import capa.features.extractors.helpers import capa.features.extractors.strings import capa.features.extractors.ida.helpers from capa.features.file import Export, Import, Section, FunctionName -from capa.features.common import String, Characteristic -from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Format +from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Format, String, Characteristic def check_segment_for_pe(seg): diff --git a/capa/features/extractors/pefile.py b/capa/features/extractors/pefile.py index a8bcd8bc..8eb276f2 100644 --- a/capa/features/extractors/pefile.py +++ b/capa/features/extractors/pefile.py @@ -14,7 +14,7 @@ import capa.features.extractors import capa.features.extractors.helpers import capa.features.extractors.strings from capa.features.file import Export, Import, Section -from capa.features.common import OS, Format, String, Characteristic, OS_WINDOWS, FORMAT_PE +from capa.features.common import OS, FORMAT_PE, OS_WINDOWS, Format, String, Characteristic from capa.features.extractors.base_extractor import FeatureExtractor logger = logging.getLogger(__name__) diff --git a/capa/features/extractors/smda/extractor.py b/capa/features/extractors/smda/extractor.py index 59f64a06..1bd7f351 100644 --- a/capa/features/extractors/smda/extractor.py +++ b/capa/features/extractors/smda/extractor.py @@ -3,6 +3,7 @@ from smda.common.SmdaReport import SmdaReport import capa.features.extractors.common import capa.features.extractors.smda.file import capa.features.extractors.smda.insn +import capa.features.extractors.smda.global_ import capa.features.extractors.smda.function import capa.features.extractors.smda.basicblock from capa.features.extractors.base_extractor import FeatureExtractor @@ -16,8 +17,10 @@ class SmdaFeatureExtractor(FeatureExtractor): with open(self.path, "rb") as f: self.buf = f.read() + # pre-compute these because we'll yield them at *every* scope. self.global_features = [] self.global_features.extend(capa.features.extractors.common.extract_os(self.buf)) + self.global_features.extend(capa.features.extractors.smda.global_.extract_arch(self.smda_report)) def get_base_address(self): return self.smda_report.base_addr diff --git a/capa/features/extractors/viv/extractor.py b/capa/features/extractors/viv/extractor.py index b922d5dc..b18d4349 100644 --- a/capa/features/extractors/viv/extractor.py +++ b/capa/features/extractors/viv/extractor.py @@ -13,6 +13,7 @@ import viv_utils.flirt import capa.features.extractors.common import capa.features.extractors.viv.file import capa.features.extractors.viv.insn +import capa.features.extractors.viv.global_ import capa.features.extractors.viv.function import capa.features.extractors.viv.basicblock from capa.features.extractors.base_extractor import FeatureExtractor @@ -41,8 +42,10 @@ class VivisectFeatureExtractor(FeatureExtractor): with open(self.path, "rb") as f: self.buf = f.read() + # pre-compute these because we'll yield them at *every* scope. self.global_features = [] self.global_features.extend(capa.features.extractors.common.extract_os(self.buf)) + self.global_features.extend(capa.features.extractors.viv.global_.extract_arch(self.vw)) def get_base_address(self): # assume there is only one file loaded into the vw diff --git a/capa/rules.py b/capa/rules.py index 5953154e..b8b90faa 100644 --- a/capa/rules.py +++ b/capa/rules.py @@ -80,6 +80,7 @@ SUPPORTED_FEATURES = { capa.features.common.String, capa.features.common.Format, capa.features.common.OS, + capa.features.common.Arch, }, FUNCTION_SCOPE: { # plus basic block scope features, see below @@ -89,6 +90,7 @@ SUPPORTED_FEATURES = { capa.features.common.Characteristic("loop"), capa.features.common.Characteristic("recursive call"), capa.features.common.OS, + capa.features.common.Arch, }, BASIC_BLOCK_SCOPE: { capa.features.common.MatchedRule, @@ -107,6 +109,7 @@ SUPPORTED_FEATURES = { capa.features.common.Characteristic("stack string"), capa.features.common.Characteristic("indirect call"), capa.features.common.OS, + capa.features.common.Arch, }, } @@ -254,6 +257,8 @@ def parse_feature(key: str): return capa.features.common.OS elif key == "format": return capa.features.common.Format + elif key == "arch": + return capa.features.common.Arch else: raise InvalidRule("unexpected statement: %s" % key) diff --git a/tests/fixtures.py b/tests/fixtures.py index 3d28c611..b47e79e2 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -22,14 +22,17 @@ import capa.features.insn import capa.features.common import capa.features.basicblock from capa.features.common import ( - BITNESS_X32, - BITNESS_X64, - FORMAT_ELF, - FORMAT_PE, - Format, OS, OS_LINUX, + ARCH_I386, + FORMAT_PE, + ARCH_AMD64, + FORMAT_ELF, OS_WINDOWS, + BITNESS_X32, + BITNESS_X64, + Arch, + Format, ) CD = os.path.dirname(__file__) @@ -512,11 +515,15 @@ FEATURE_PRESENCE_TESTS = sorted( ("mimikatz", "function=0x456BB9", capa.features.common.Characteristic("calls to"), False), # file/function-name ("pma16-01", "file", capa.features.file.FunctionName("__aulldiv"), True), - # os & format + # os & format & arch ("pma16-01", "file", OS(OS_WINDOWS), True), ("pma16-01", "file", OS(OS_LINUX), False), ("pma16-01", "function=0x404356", OS(OS_WINDOWS), True), ("pma16-01", "function=0x404356,bb=0x4043B9", OS(OS_WINDOWS), True), + ("pma16-01", "file", Arch(ARCH_I386), True), + ("pma16-01", "file", Arch(ARCH_AMD64), False), + ("pma16-01", "function=0x404356", Arch(ARCH_I386), True), + ("pma16-01", "function=0x404356,bb=0x4043B9", Arch(ARCH_I386), True), ("pma16-01", "file", Format(FORMAT_PE), True), ("pma16-01", "file", Format(FORMAT_ELF), False), # elf support @@ -524,6 +531,8 @@ FEATURE_PRESENCE_TESTS = sorted( ("7351f.elf", "file", OS(OS_WINDOWS), False), ("7351f.elf", "file", Format(FORMAT_ELF), True), ("7351f.elf", "file", Format(FORMAT_PE), False), + ("7351f.elf", "file", Arch(ARCH_I386), False), + ("7351f.elf", "file", Arch(ARCH_AMD64), True), ("7351f.elf", "function=0x408753", capa.features.common.String("/dev/null"), True), ("7351f.elf", "function=0x408753,bb=0x408781", capa.features.insn.API("open"), True), ], diff --git a/tests/test_rules.py b/tests/test_rules.py index 82337f41..9362ef91 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -16,15 +16,18 @@ import capa.features.common from capa.features.file import FunctionName from capa.features.insn import Number, Offset from capa.features.common import ( + OS, + OS_LINUX, + ARCH_I386, FORMAT_PE, + ARCH_AMD64, FORMAT_ELF, OS_WINDOWS, - OS_LINUX, BITNESS_X32, BITNESS_X64, + Arch, + Format, String, - OS, - Format ) @@ -990,3 +993,21 @@ def test_format_features(): children = list(r.statement.get_children()) assert (Format(FORMAT_PE) in children) == True assert (Format(FORMAT_ELF) not in children) == True + + +def test_arch_features(): + rule = textwrap.dedent( + """ + rule: + meta: + name: test rule + scope: file + features: + - and: + - arch: amd64 + """ + ) + r = capa.rules.Rule.from_yaml(rule) + children = list(r.statement.get_children()) + assert (Arch(ARCH_AMD64) in children) == True + assert (Arch(ARCH_I386) not in children) == True From fd47b03fac14c018b9960a6190d11242ad09142f Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 16 Aug 2021 17:12:28 -0600 Subject: [PATCH 26/52] render: vverbose: don't render locations of global scope features --- capa/render/vverbose.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/capa/render/vverbose.py b/capa/render/vverbose.py index 95239eab..adecde36 100644 --- a/capa/render/vverbose.py +++ b/capa/render/vverbose.py @@ -113,7 +113,8 @@ def render_feature(ostream, match, feature, indent=0): ostream.write(capa.rules.DESCRIPTION_SEPARATOR) ostream.write(feature["description"]) - render_locations(ostream, match) + if key not in ("os", "arch"): + render_locations(ostream, match) ostream.write("\n") else: # like: From 98c00bd8b1980083b6b8ca3f695a94f399bc6588 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 16 Aug 2021 17:12:45 -0600 Subject: [PATCH 27/52] extractors: add missing global_.py files --- capa/features/extractors/ida/global_.py | 15 +++++++++++++++ capa/features/extractors/smda/global_.py | 11 +++++++++++ capa/features/extractors/viv/global_.py | 15 +++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 capa/features/extractors/ida/global_.py create mode 100644 capa/features/extractors/smda/global_.py create mode 100644 capa/features/extractors/viv/global_.py diff --git a/capa/features/extractors/ida/global_.py b/capa/features/extractors/ida/global_.py new file mode 100644 index 00000000..1919dd5d --- /dev/null +++ b/capa/features/extractors/ida/global_.py @@ -0,0 +1,15 @@ +import idaapi + +from capa.features.common import ARCH_I386, ARCH_AMD64, Arch + + +def extract_arch(): + info = idaapi.get_inf_structure() + if info.procName == "metapc" and info.is_64bit(): + yield Arch(ARCH_AMD64), 0x0 + elif info.procName == "metapc" and info.is_32bit(): + yield Arch(ARCH_I386), 0x0 + elif info.procName == "metapc": + raise NotImplementedError("unsupported architecture: non-32-bit nor non-64-bit intel") + else: + raise NotImplementedError("unsupported architecture: %s" % (info.procName)) diff --git a/capa/features/extractors/smda/global_.py b/capa/features/extractors/smda/global_.py new file mode 100644 index 00000000..9b9bc268 --- /dev/null +++ b/capa/features/extractors/smda/global_.py @@ -0,0 +1,11 @@ +from capa.features.common import ARCH_I386, ARCH_AMD64, Arch + + +def extract_arch(smda_report): + if smda_report.architecture == "intel": + if smda_report.bitness == 32: + yield Arch(ARCH_I386), 0x0 + elif smda_report.bitness == 64: + yield Arch(ARCH_AMD64), 0x0 + else: + raise NotImplementedError(smda_report.architecture) diff --git a/capa/features/extractors/viv/global_.py b/capa/features/extractors/viv/global_.py new file mode 100644 index 00000000..77d1c07d --- /dev/null +++ b/capa/features/extractors/viv/global_.py @@ -0,0 +1,15 @@ +import envi.archs.i386 +import envi.archs.amd64 + +from capa.features.common import ARCH_I386, ARCH_AMD64, Arch + + +def extract_arch(vw): + if isinstance(vw.arch, envi.archs.amd64.Amd64Module): + yield Arch(ARCH_AMD64), 0x0 + + elif isinstance(vw.arch, envi.archs.i386.i386Module): + yield Arch(ARCH_I386), 0x0 + + else: + raise NotImplementedError("unsupported architecture: %s" % (vw.arch.__class__.__name__)) From 0065876702ad2424b2aeaf70f6ef05fd791f774a Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 17 Aug 2021 08:57:27 -0600 Subject: [PATCH 28/52] extractors: ida: move os extraction to global module --- capa/features/extractors/ida/extractor.py | 22 +---------- capa/features/extractors/ida/global_.py | 48 +++++++++++++++++++++-- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/capa/features/extractors/ida/extractor.py b/capa/features/extractors/ida/extractor.py index 93f2ce55..7d00c7ef 100644 --- a/capa/features/extractors/ida/extractor.py +++ b/capa/features/extractors/ida/extractor.py @@ -5,10 +5,7 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import contextlib - import idaapi -import ida_loader import capa.ida.helpers import capa.features.extractors.elf @@ -17,26 +14,9 @@ import capa.features.extractors.ida.insn import capa.features.extractors.ida.global_ import capa.features.extractors.ida.function import capa.features.extractors.ida.basicblock -from capa.features.common import OS, OS_WINDOWS from capa.features.extractors.base_extractor import FeatureExtractor -def extract_os(): - format_name = ida_loader.get_file_type_name() - - if "PE" in format_name: - yield OS(OS_WINDOWS), 0x0 - - elif "ELF" in format_name: - with contextlib.closing(capa.ida.helpers.IDAIO()) as f: - os = capa.features.extractors.elf.detect_elf_os(f) - - yield OS(os), 0x0 - - else: - raise NotImplementedError("file format: %s", format_name) - - class FunctionHandle: """this acts like an idaapi.func_t but with __int__()""" @@ -80,7 +60,7 @@ class IdaFeatureExtractor(FeatureExtractor): def __init__(self): super(IdaFeatureExtractor, self).__init__() self.global_features = [] - self.global_features.extend(extract_os()) + self.global_features.extend(capa.features.extractors.ida.global_.extract_os()) self.global_features.extend(capa.features.extractors.ida.global_.extract_arch()) def get_base_address(self): diff --git a/capa/features/extractors/ida/global_.py b/capa/features/extractors/ida/global_.py index 1919dd5d..b6f525f6 100644 --- a/capa/features/extractors/ida/global_.py +++ b/capa/features/extractors/ida/global_.py @@ -1,7 +1,43 @@ -import idaapi +import logging +import contextlib +import idaapi +import ida_loader + +import capa.ida.helpers +import capa.features.extractors.elf +from capa.features.common import OS, OS_WINDOWS from capa.features.common import ARCH_I386, ARCH_AMD64, Arch +logger = logging.getLogger(__name__) + + +def extract_os(): + format_name = ida_loader.get_file_type_name() + + if "PE" in format_name: + yield OS(OS_WINDOWS), 0x0 + + elif "ELF" in format_name: + with contextlib.closing(capa.ida.helpers.IDAIO()) as f: + os = capa.features.extractors.elf.detect_elf_os(f) + + yield OS(os), 0x0 + + else: + # we likely end up here: + # 1. handling shellcode, or + # 2. handling a new file format (e.g. macho) + # + # for (1) we can't do much - its shellcode and all bets are off. + # we could maybe accept a futher CLI argument to specify the OS, + # but i think this would be rarely used. + # rules that rely on OS conditions will fail to match on shellcode. + # + # for (2), this logic will need to be updated as the format is implemented. + logger.debug("unsupported file format: %s, will not guess OS", format_name) + return + def extract_arch(): info = idaapi.get_inf_structure() @@ -10,6 +46,12 @@ def extract_arch(): elif info.procName == "metapc" and info.is_32bit(): yield Arch(ARCH_I386), 0x0 elif info.procName == "metapc": - raise NotImplementedError("unsupported architecture: non-32-bit nor non-64-bit intel") + logger.debug("unsupported architecture: non-32-bit nor non-64-bit intel") + return else: - raise NotImplementedError("unsupported architecture: %s" % (info.procName)) + # we likely end up here: + # 1. handling a new architecture (e.g. aarch64) + # + # for (1), this logic will need to be updated as the format is implemented. + logger.debug("unsupported architecture: %s", info.procName) + return From 92dfa99059da59f2358aaee01a241cd6ca7f2b0b Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 17 Aug 2021 08:57:42 -0600 Subject: [PATCH 29/52] extractors: log unsupported os/arch/format but don't except --- capa/features/extractors/common.py | 20 ++++++++++++++++++-- capa/features/extractors/smda/global_.py | 11 ++++++++++- capa/features/extractors/viv/global_.py | 11 ++++++++++- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/capa/features/extractors/common.py b/capa/features/extractors/common.py index ab4d2ef8..aadc184a 100644 --- a/capa/features/extractors/common.py +++ b/capa/features/extractors/common.py @@ -15,7 +15,12 @@ def extract_format(buf): elif buf.startswith(b"\x7fELF"): yield Format(FORMAT_ELF), 0x0 else: - raise NotImplementedError("file format: %s", binascii.hexlify(buf[:4]).decode("ascii")) + # we likely end up here: + # 1. handling a file format (e.g. macho) + # + # for (1), this logic will need to be updated as the format is implemented. + logger.debug("unsupported file format: %s", binascii.hexlify(buf[:4]).decode("ascii")) + return def extract_os(buf): @@ -27,4 +32,15 @@ def extract_os(buf): yield OS(os), 0x0 else: - raise NotImplementedError("file format: %s", binascii.hexlify(buf[:4]).decode("ascii")) + # we likely end up here: + # 1. handling shellcode, or + # 2. handling a new file format (e.g. macho) + # + # for (1) we can't do much - its shellcode and all bets are off. + # we could maybe accept a futher CLI argument to specify the OS, + # but i think this would be rarely used. + # rules that rely on OS conditions will fail to match on shellcode. + # + # for (2), this logic will need to be updated as the format is implemented. + logger.debug("unsupported file format: %s, will not guess OS", binascii.hexlify(buf[:4]).decode("ascii")) + return diff --git a/capa/features/extractors/smda/global_.py b/capa/features/extractors/smda/global_.py index 9b9bc268..84befc59 100644 --- a/capa/features/extractors/smda/global_.py +++ b/capa/features/extractors/smda/global_.py @@ -1,5 +1,9 @@ +import logging + from capa.features.common import ARCH_I386, ARCH_AMD64, Arch +logger = logging.getLogger(__name__) + def extract_arch(smda_report): if smda_report.architecture == "intel": @@ -8,4 +12,9 @@ def extract_arch(smda_report): elif smda_report.bitness == 64: yield Arch(ARCH_AMD64), 0x0 else: - raise NotImplementedError(smda_report.architecture) + # we likely end up here: + # 1. handling a new architecture (e.g. aarch64) + # + # for (1), this logic will need to be updated as the format is implemented. + logger.debug("unsupported architecture: %s", smda_report.architecture) + return diff --git a/capa/features/extractors/viv/global_.py b/capa/features/extractors/viv/global_.py index 77d1c07d..8fc08ee2 100644 --- a/capa/features/extractors/viv/global_.py +++ b/capa/features/extractors/viv/global_.py @@ -1,8 +1,12 @@ +import logging + import envi.archs.i386 import envi.archs.amd64 from capa.features.common import ARCH_I386, ARCH_AMD64, Arch +logger = logging.getLogger(__name__) + def extract_arch(vw): if isinstance(vw.arch, envi.archs.amd64.Amd64Module): @@ -12,4 +16,9 @@ def extract_arch(vw): yield Arch(ARCH_I386), 0x0 else: - raise NotImplementedError("unsupported architecture: %s" % (vw.arch.__class__.__name__)) + # we likely end up here: + # 1. handling a new architecture (e.g. aarch64) + # + # for (1), this logic will need to be updated as the format is implemented. + logger.debug("unsupported architecture: %s", vw.arch.__class__.__name__) + return From ac5d163aa0d8ab5e4ec203a1faee5ee330678474 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 17 Aug 2021 09:07:08 -0600 Subject: [PATCH 30/52] pep8 --- capa/features/extractors/ida/global_.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/capa/features/extractors/ida/global_.py b/capa/features/extractors/ida/global_.py index b6f525f6..b3181293 100644 --- a/capa/features/extractors/ida/global_.py +++ b/capa/features/extractors/ida/global_.py @@ -6,8 +6,7 @@ import ida_loader import capa.ida.helpers import capa.features.extractors.elf -from capa.features.common import OS, OS_WINDOWS -from capa.features.common import ARCH_I386, ARCH_AMD64, Arch +from capa.features.common import OS, ARCH_I386, ARCH_AMD64, OS_WINDOWS, Arch logger = logging.getLogger(__name__) From f1df29d27e6813ccc82987d09421ba09c68192b3 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 18 Aug 2021 14:08:36 -0600 Subject: [PATCH 31/52] tests: xfail smda ELF API waiting for #725 --- tests/test_smda_features.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_smda_features.py b/tests/test_smda_features.py index 6614c24d..a868dbb9 100644 --- a/tests/test_smda_features.py +++ b/tests/test_smda_features.py @@ -11,6 +11,7 @@ from fixtures import * from fixtures import parametrize import capa.features.file +import capa.features.insn @parametrize( @@ -22,6 +23,9 @@ def test_smda_features(sample, scope, feature, expected): if scope.__name__ == "file" and isinstance(feature, capa.features.file.FunctionName) and expected is True: pytest.xfail("SMDA has no function ID") + if "elf" in sample and isinstance(feature, capa.features.insn.API): + pytest.xfail("SMDA has no ELF API extraction, see #725") + fixtures.do_test_feature_presence(fixtures.get_smda_extractor, sample, scope, feature, expected) From a35f5a16507ca14f4d25254db1dd35c6d3a31bfa Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 18 Aug 2021 14:21:50 -0600 Subject: [PATCH 32/52] elf: detect FreeBSD via note --- capa/features/extractors/elf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index f11bd642..ae8886cd 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -188,6 +188,9 @@ def detect_elf_os(f: BinaryIO) -> str: elif name == "NetBSD": logger.debug("note owner: %s", "NETBSD") ret = OS.NETBSD if not ret else ret + elif name == "FreeBSD": + logger.debug("note owner: %s", "FREEBSD") + ret = OS.FREEBSD if not ret else ret for i in range(e_phnum): offset = i * e_phentsize From 89603586dacd500a121ad142fa07139731d6ae2d Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 18 Aug 2021 14:23:48 -0600 Subject: [PATCH 33/52] elf: add some doc --- capa/features/extractors/elf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index ae8886cd..baf88e23 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -120,6 +120,8 @@ def detect_elf_os(f: BinaryIO) -> str: logger.warning("failed to read program headers") e_phnum = 0 + # search for PT_NOTE sections that specify an OS + # for example, on Linux there is a GNU section with minimum kernel version for i in range(e_phnum): offset = i * e_phentsize phent = program_headers[offset : offset + e_phentsize] @@ -192,6 +194,8 @@ def detect_elf_os(f: BinaryIO) -> str: logger.debug("note owner: %s", "FREEBSD") ret = OS.FREEBSD if not ret else ret + # search for recognizable dynamic linkers (interpreters) + # for example, on linux, we see file paths like: /lib64/ld-linux-x86-64.so.2 for i in range(e_phnum): offset = i * e_phentsize phent = program_headers[offset : offset + e_phentsize] From 249b8498d9c1ae6ccffde7b3d7e3b328b851022c Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Wed, 18 Aug 2021 16:27:41 -0600 Subject: [PATCH 34/52] pefile: extract Arch --- capa/features/common.py | 1 + capa/features/extractors/pefile.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/capa/features/common.py b/capa/features/common.py index a8b6f5ef..611cd173 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -265,6 +265,7 @@ BITNESS_X64 = "x64" VALID_BITNESS = (BITNESS_X32, BITNESS_X64) +# other candidates here: https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#machine-types ARCH_I386 = "i386" ARCH_AMD64 = "amd64" VALID_ARCH = (ARCH_I386, ARCH_AMD64) diff --git a/capa/features/extractors/pefile.py b/capa/features/extractors/pefile.py index 8eb276f2..2c60499b 100644 --- a/capa/features/extractors/pefile.py +++ b/capa/features/extractors/pefile.py @@ -14,7 +14,7 @@ import capa.features.extractors import capa.features.extractors.helpers import capa.features.extractors.strings from capa.features.file import Export, Import, Section -from capa.features.common import OS, FORMAT_PE, OS_WINDOWS, Format, String, Characteristic +from capa.features.common import OS, ARCH_I386, FORMAT_PE, ARCH_AMD64, OS_WINDOWS, Arch, Format, String, Characteristic from capa.features.extractors.base_extractor import FeatureExtractor logger = logging.getLogger(__name__) @@ -111,7 +111,7 @@ def extract_file_function_names(pe, file_path): return -def extract_os(pe, file_path): +def extract_file_os(pe, file_path): # assuming PE -> Windows # though i suppose they're also used by UEFI yield OS(OS_WINDOWS), 0x0 @@ -121,6 +121,15 @@ def extract_file_format(pe, file_path): yield Format(FORMAT_PE), 0x0 +def extract_file_arch(pe, file_path): + if pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_I386"]: + yield Arch(ARCH_I386), 0x0 + elif pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_AMD64"]: + yield Arch(ARCH_AMD64), 0x0 + else: + logger.warning("unsupported architecture: %s", pefile.MACHINE_TYPE[pe.FILE_HEADER.Machine]) + + def extract_file_features(pe, file_path): """ extract file features from given workspace @@ -145,8 +154,9 @@ FILE_HANDLERS = ( extract_file_section_names, extract_file_strings, extract_file_function_names, - extract_os, + extract_file_os, extract_file_format, + extract_file_arch, ) From 45b6c8dad347d0b6befbd1afb661efade0a46a32 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Thu, 19 Aug 2021 08:01:17 -0600 Subject: [PATCH 35/52] setup: bump SMDA dep ver closes #725 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9c8829d7..5ede5118 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ requirements = [ "networkx==2.5.1", "ruamel.yaml==0.17.10", "vivisect==1.0.3", - "smda==1.5.19", + "smda==1.6.0", "pefile==2021.5.24", "typing==3.7.4.3", ] From a96a5de12d17b401c43bc4342b230ffbb2ea1c7f Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Thu, 19 Aug 2021 08:02:17 -0600 Subject: [PATCH 36/52] tests: re-enable SMDA ELF API tests --- tests/test_smda_features.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_smda_features.py b/tests/test_smda_features.py index a868dbb9..54e519b3 100644 --- a/tests/test_smda_features.py +++ b/tests/test_smda_features.py @@ -23,9 +23,6 @@ def test_smda_features(sample, scope, feature, expected): if scope.__name__ == "file" and isinstance(feature, capa.features.file.FunctionName) and expected is True: pytest.xfail("SMDA has no function ID") - if "elf" in sample and isinstance(feature, capa.features.insn.API): - pytest.xfail("SMDA has no ELF API extraction, see #725") - fixtures.do_test_feature_presence(fixtures.get_smda_extractor, sample, scope, feature, expected) From 3cb7573edb4d93ac9e9499e3a6ec818f91b57de2 Mon Sep 17 00:00:00 2001 From: Michael Hunhoff Date: Thu, 19 Aug 2021 13:06:43 -0600 Subject: [PATCH 37/52] enable os/arch/format for capa explorer --- capa/ida/helpers.py | 15 ++++++++------- capa/ida/plugin/view.py | 6 +++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/capa/ida/helpers.py b/capa/ida/helpers.py index 800d1383..8cc6d894 100644 --- a/capa/ida/helpers.py +++ b/capa/ida/helpers.py @@ -30,11 +30,12 @@ SUPPORTED_IDA_VERSIONS = [ "7.6", ] -# file type names as returned by idaapi.get_file_type_name() +# file type names as returned by idainfo.file_type SUPPORTED_FILE_TYPES = [ - "Portable executable for 80386 (PE)", - "Portable executable for AMD64 (PE)", - "Binary file", # x86/AMD64 shellcode support + idaapi.f_PE, + idaapi.f_ELF, + # idaapi.f_MACHO, + idaapi.f_BIN, ] @@ -55,10 +56,10 @@ def is_supported_ida_version(): def is_supported_file_type(): - file_type = idaapi.get_file_type_name() - if file_type not in SUPPORTED_FILE_TYPES: + file_info = idaapi.get_inf_structure() + if file_info.filetype not in SUPPORTED_FILE_TYPES: logger.error("-" * 80) - logger.error(" Input file does not appear to be a PE file.") + logger.error(" Input file does not appear to be a supported file type.") logger.error(" ") logger.error( " capa currently only supports analyzing PE files (or binary files containing x86/AMD64 shellcode) with IDA." diff --git a/capa/ida/plugin/view.py b/capa/ida/plugin/view.py index 3f13b89d..3986114a 100644 --- a/capa/ida/plugin/view.py +++ b/capa/ida/plugin/view.py @@ -1007,7 +1007,11 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget): self.parent_items[feature], (format_feature(feature), format_address(ea)), feature=feature ) else: - ea = eas.pop() + if eas: + ea = eas.pop() + else: + # some features may not have an address e.g. "format" + ea = "" for (i, v) in enumerate((format_feature(feature), format_address(ea))): self.parent_items[feature].setText(i, v) self.parent_items[feature].setData(0, 0x100, feature) From dae7be076dd97e4fa63f9027337a72b9d83f3b87 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Thu, 19 Aug 2021 14:45:08 -0600 Subject: [PATCH 38/52] elf: fix alignment calculation identified over [here](https://github.com/vivisect/vivisect/pull/436/commits/14f9c972b3bda6b8863600d35b797d04ca38c2dc#r692441396) --- capa/features/extractors/elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index baf88e23..5264b544 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -11,7 +11,7 @@ def align(v, alignment): if remainder == 0: return v else: - return v + remainder + return v + (alignment - remainder) class CorruptElfFile(ValueError): From 04cc94a4503c0f8fbe964ac8d4d3383adb6ae01e Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Fri, 20 Aug 2021 14:56:26 -0600 Subject: [PATCH 39/52] main: detect invalid arch and os --- capa/features/extractors/common.py | 34 +++++++++++++- capa/features/extractors/elf.py | 37 +++++++++++++++ capa/main.py | 73 ++++++++++++++++++++++++------ 3 files changed, 130 insertions(+), 14 deletions(-) diff --git a/capa/features/extractors/common.py b/capa/features/extractors/common.py index aadc184a..b706868f 100644 --- a/capa/features/extractors/common.py +++ b/capa/features/extractors/common.py @@ -3,8 +3,11 @@ import logging import binascii import contextlib +import pefile + import capa.features.extractors.elf -from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Format +import capa.features.extractors.pefile +from capa.features.common import OS, FORMAT_PE, FORMAT_ELF, OS_WINDOWS, Arch, Format logger = logging.getLogger(__name__) @@ -23,6 +26,35 @@ def extract_format(buf): return +def extract_arch(buf): + if buf.startswith(b"MZ"): + yield from capa.features.extractors.pefile.extract_file_arch(pefile.PE(data=buf), "hack: path not provided") + + elif buf.startswith(b"\x7fELF"): + with contextlib.closing(io.BytesIO(buf)) as f: + arch = capa.features.extractors.elf.detect_elf_arch(f) + + if arch == "unknown": + logger.debug("unsupported arch: %s", arch) + return + + yield Arch(arch), 0x0 + + else: + # we likely end up here: + # 1. handling shellcode, or + # 2. handling a new file format (e.g. macho) + # + # for (1) we can't do much - its shellcode and all bets are off. + # we could maybe accept a futher CLI argument to specify the arch, + # but i think this would be rarely used. + # rules that rely on arch conditions will fail to match on shellcode. + # + # for (2), this logic will need to be updated as the format is implemented. + logger.debug("unsupported file format: %s, will not guess Arch", binascii.hexlify(buf[:4]).decode("ascii")) + return + + def extract_os(buf): if buf.startswith(b"MZ"): yield OS(OS_WINDOWS), 0x0 diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index baf88e23..d3c7c0eb 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -227,3 +227,40 @@ def detect_elf_os(f: BinaryIO) -> str: ret = OS.LINUX if ret is None else ret return ret.value if ret is not None else "unknown" + + +class Arch(str, Enum): + I386 = "i386" + AMD64 = "amd64" + + +def detect_elf_arch(f: BinaryIO) -> str: + f.seek(0x0) + file_header = f.read(0x40) + + if not file_header.startswith(b"\x7fELF"): + raise CorruptElfFile("missing magic header") + + ei_data = struct.unpack_from("BB", file_header, 5) + logger.debug("ei_data: 0x%02x", ei_data) + + if ei_data == 1: + endian = "<" + elif ei_data == 2: + endian = ">" + else: + raise CorruptElfFile("not an ELF file: invalid ei_data: 0x%02x" % ei_data) + + (ei_machine,) = struct.unpack_from(endian + "H", file_header, 0x12) + logger.debug("ei_machine: 0x%02x", ei_machine) + + EM_386 = 0x3 + EM_X86_64 = 0x3E + if ei_machine == EM_386: + return Arch.I386 + elif ei_machine == EM_X86_64: + return Arch.AMD64 + else: + # not really unknown, but unsupport at the moment: + # https://github.com/eliben/pyelftools/blob/ab444d982d1849191e910299a985989857466620/elftools/elf/enums.py#L73 + return "unknown" diff --git a/capa/main.py b/capa/main.py index 6f6ce168..b739de09 100644 --- a/capa/main.py +++ b/capa/main.py @@ -21,7 +21,7 @@ import textwrap import itertools import contextlib import collections -from typing import Any, Dict, List, Tuple, Iterable +from typing import Any, Dict, List, Tuple import halo import tqdm @@ -37,6 +37,7 @@ import capa.features.common import capa.features.freeze import capa.render.vverbose import capa.features.extractors +import capa.features.extractors.common import capa.features.extractors.pefile from capa.rules import Rule, RuleSet from capa.engine import FeatureSet, MatchResults @@ -45,7 +46,6 @@ from capa.features.extractors.base_extractor import FunctionHandle, FeatureExtra RULES_PATH_DEFAULT_STRING = "(embedded rules)" SIGNATURES_PATH_DEFAULT_STRING = "(embedded signatures)" -SUPPORTED_FILE_MAGIC = (b"MZ", b"\x7fELF") BACKEND_VIV = "vivisect" BACKEND_SMDA = "smda" EXTENSIONS_SHELLCODE_32 = ("sc32", "raw32") @@ -242,11 +242,23 @@ def is_supported_file_type(sample: str) -> bool: Return if this is a supported file based on magic header values """ with open(sample, "rb") as f: - magic = f.read(4) - if magic.startswith(SUPPORTED_FILE_MAGIC): - return True - else: - return False + taste = f.read(0x100) + + return len(list(capa.features.extractors.common.extract_format(taste))) == 1 + + +def is_supported_arch(sample: str) -> bool: + with open(sample, "rb") as f: + buf = f.read() + + return len(list(capa.features.extractors.common.extract_arch(buf))) == 1 + + +def is_supported_os(sample: str) -> bool: + with open(sample, "rb") as f: + buf = f.read() + + return len(list(capa.features.extractors.common.extract_os(buf))) == 1 SHELLCODE_BASE = 0x690000 @@ -391,6 +403,14 @@ class UnsupportedFormatError(ValueError): pass +class UnsupportedArchError(ValueError): + pass + + +class UnsupportedOSError(ValueError): + pass + + def get_workspace(path, format, sigpaths): """ load the program at the given path into a vivisect workspace using the given format. @@ -445,6 +465,21 @@ def get_extractor( raises: UnsupportedFormatError: """ + if format == "auto" and path.endswith(EXTENSIONS_SHELLCODE_32): + format = "sc32" + elif format == "auto" and path.endswith(EXTENSIONS_SHELLCODE_64): + format = "sc64" + + if format not in ("sc32", "sc64"): + if not is_supported_file_type(path): + raise UnsupportedFormatError() + + if not is_supported_arch(path): + raise UnsupportedArchError() + + if not is_supported_os(path): + raise UnsupportedOSError() + if backend == "smda": from smda.SmdaConfig import SmdaConfig from smda.Disassembler import Disassembler @@ -463,10 +498,6 @@ def get_extractor( import capa.features.extractors.viv.extractor with halo.Halo(text="analyzing program", spinner="simpleDots", stream=sys.stderr, enabled=not disable_progress): - if format == "auto" and path.endswith(EXTENSIONS_SHELLCODE_32): - format = "sc32" - elif format == "auto" and path.endswith(EXTENSIONS_SHELLCODE_64): - format = "sc64" vw = get_workspace(path, format, sigpaths) if should_save_workspace: @@ -917,14 +948,30 @@ def main(argv=None): ) except UnsupportedFormatError: logger.error("-" * 80) - logger.error(" Input file does not appear to be a PE file.") + logger.error(" Input file does not appear to be a PE or ELF file.") logger.error(" ") logger.error( - " capa currently only supports analyzing PE files (or shellcode, when using --format sc32|sc64)." + " capa currently only supports analyzing PE and ELF files (or shellcode, when using --format sc32|sc64)." ) logger.error(" If you don't know the input file type, you can try using the `file` utility to guess it.") logger.error("-" * 80) return -1 + except UnsupportedArchError: + logger.error("-" * 80) + logger.error(" Input file does not appear to be target the x86 architecture.") + logger.error(" ") + logger.error(" capa currently only supports analyzing x86 (32- and 64-bit).") + logger.error("-" * 80) + return -1 + except UnsupportedOSError: + logger.error("-" * 80) + logger.error(" Input file does not appear to target a supported OS.") + logger.error(" ") + logger.error( + " capa currently only supports analyzing executables for some operating systems (including Windows and Linux)." + ) + logger.error("-" * 80) + return -1 meta = collect_metadata(argv, args.sample, args.rules, format, extractor) From aef03b55922a73ea2c80264b9a65305a943944c4 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Fri, 20 Aug 2021 15:00:06 -0600 Subject: [PATCH 40/52] elf: fix type error caught by mypy! --- capa/features/extractors/elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index 2e068b83..0dd02685 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -241,7 +241,7 @@ def detect_elf_arch(f: BinaryIO) -> str: if not file_header.startswith(b"\x7fELF"): raise CorruptElfFile("missing magic header") - ei_data = struct.unpack_from("BB", file_header, 5) + (ei_data,) = struct.unpack_from("B", file_header, 5) logger.debug("ei_data: 0x%02x", ei_data) if ei_data == 1: From 1b9a6c3c59892be91da561beb9e1a1e8845059ba Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Fri, 20 Aug 2021 16:50:40 -0600 Subject: [PATCH 41/52] main: collect os/format/arch into metadata and render it --- capa/main.py | 63 +++++++++++++++++++----- capa/render/default.py | 4 +- capa/render/verbose.py | 6 ++- scripts/bulk-process.py | 2 +- scripts/capa_as_library.py | 2 +- scripts/show-capabilities-by-function.py | 2 +- 6 files changed, 63 insertions(+), 16 deletions(-) diff --git a/capa/main.py b/capa/main.py index b739de09..61fd3f39 100644 --- a/capa/main.py +++ b/capa/main.py @@ -237,7 +237,7 @@ def has_file_limitation(rules: RuleSet, capabilities: MatchResults, is_standalon return False -def is_supported_file_type(sample: str) -> bool: +def is_supported_format(sample: str) -> bool: """ Return if this is a supported file based on magic header values """ @@ -247,6 +247,17 @@ def is_supported_file_type(sample: str) -> bool: return len(list(capa.features.extractors.common.extract_format(taste))) == 1 +def get_format(sample: str) -> str: + with open(sample, "rb") as f: + buf = f.read() + + for feature, _ in capa.features.extractors.common.extract_format(buf): + assert isinstance(feature.value, str) + return feature.value + + return "unknown" + + def is_supported_arch(sample: str) -> bool: with open(sample, "rb") as f: buf = f.read() @@ -254,6 +265,17 @@ def is_supported_arch(sample: str) -> bool: return len(list(capa.features.extractors.common.extract_arch(buf))) == 1 +def get_arch(sample: str) -> str: + with open(sample, "rb") as f: + buf = f.read() + + for feature, _ in capa.features.extractors.common.extract_arch(buf): + assert isinstance(feature.value, str) + return feature.value + + return "unknown" + + def is_supported_os(sample: str) -> bool: with open(sample, "rb") as f: buf = f.read() @@ -261,6 +283,17 @@ def is_supported_os(sample: str) -> bool: return len(list(capa.features.extractors.common.extract_os(buf))) == 1 +def get_os(sample: str) -> str: + with open(sample, "rb") as f: + buf = f.read() + + for feature, _ in capa.features.extractors.common.extract_os(buf): + assert isinstance(feature.value, str) + return feature.value + + return "unknown" + + SHELLCODE_BASE = 0x690000 @@ -431,7 +464,7 @@ def get_workspace(path, format, sigpaths): logger.debug("generating vivisect workspace for: %s", path) if format == "auto": - if not is_supported_file_type(path): + if not is_supported_format(path): raise UnsupportedFormatError() # don't analyze, so that we can add our Flirt function analyzer first. @@ -463,15 +496,12 @@ def get_extractor( ) -> FeatureExtractor: """ raises: - UnsupportedFormatError: + UnsupportedFormatError + UnsupportedArchError + UnsupportedOSError """ - if format == "auto" and path.endswith(EXTENSIONS_SHELLCODE_32): - format = "sc32" - elif format == "auto" and path.endswith(EXTENSIONS_SHELLCODE_64): - format = "sc64" - if format not in ("sc32", "sc64"): - if not is_supported_file_type(path): + if not is_supported_format(path): raise UnsupportedFormatError() if not is_supported_arch(path): @@ -605,7 +635,7 @@ def get_signatures(sigs_path): return paths -def collect_metadata(argv, sample_path, rules_path, format, extractor): +def collect_metadata(argv, sample_path, rules_path, extractor): md5 = hashlib.md5() sha1 = hashlib.sha1() sha256 = hashlib.sha256() @@ -620,6 +650,10 @@ def collect_metadata(argv, sample_path, rules_path, format, extractor): if rules_path != RULES_PATH_DEFAULT_STRING: rules_path = os.path.abspath(os.path.normpath(rules_path)) + format = get_format(sample_path) + arch = get_arch(sample_path) + os_ = get_os(sample_path) + return { "timestamp": datetime.datetime.now().isoformat(), "version": capa.version.__version__, @@ -632,6 +666,8 @@ def collect_metadata(argv, sample_path, rules_path, format, extractor): }, "analysis": { "format": format, + "arch": arch, + "os": os_, "extractor": extractor.__class__.__name__, "rules": rules_path, "base_address": extractor.get_base_address(), @@ -940,6 +976,11 @@ def main(argv=None): extractor = capa.features.freeze.load(f.read()) else: format = args.format + if format == "auto" and args.sample.endswith(EXTENSIONS_SHELLCODE_32): + format = "sc32" + elif format == "auto" and args.sample.endswith(EXTENSIONS_SHELLCODE_64): + format = "sc64" + should_save_workspace = os.environ.get("CAPA_SAVE_WORKSPACE") not in ("0", "no", "NO", "n", None) try: @@ -973,7 +1014,7 @@ def main(argv=None): logger.error("-" * 80) return -1 - meta = collect_metadata(argv, args.sample, args.rules, format, extractor) + meta = collect_metadata(argv, args.sample, args.rules, extractor) capabilities, counts = find_capabilities(rules, extractor, disable_progress=args.quiet) meta["analysis"].update(counts) diff --git a/capa/render/default.py b/capa/render/default.py index 0f3f9ef3..2ed06a66 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -7,7 +7,6 @@ # See the License for the specific language governing permissions and limitations under the License. import collections -from typing import Dict, List import tabulate @@ -33,6 +32,9 @@ def render_meta(doc, ostream: StringIO): (width("md5", 22), width(doc["meta"]["sample"]["md5"], 82)), ("sha1", doc["meta"]["sample"]["sha1"]), ("sha256", doc["meta"]["sample"]["sha256"]), + ("os", doc["meta"]["analysis"]["os"]), + ("format", doc["meta"]["analysis"]["format"]), + ("arch", doc["meta"]["analysis"]["arch"]), ("path", doc["meta"]["sample"]["path"]), ] diff --git a/capa/render/verbose.py b/capa/render/verbose.py index df8a9586..d7908268 100644 --- a/capa/render/verbose.py +++ b/capa/render/verbose.py @@ -41,7 +41,9 @@ def render_meta(ostream, doc): path /tmp/suspicious.dll_ timestamp 2020-07-03T10:17:05.796933 capa version 0.0.0 - format auto + os windows + format pe + arch amd64 extractor VivisectFeatureExtractor base address 0x10000000 rules (embedded rules) @@ -55,7 +57,9 @@ def render_meta(ostream, doc): ("path", doc["meta"]["sample"]["path"]), ("timestamp", doc["meta"]["timestamp"]), ("capa version", doc["meta"]["version"]), + ("os", doc["meta"]["analysis"]["os"]), ("format", doc["meta"]["analysis"]["format"]), + ("arch", doc["meta"]["analysis"]["arch"]), ("extractor", doc["meta"]["analysis"]["extractor"]), ("base address", hex(doc["meta"]["analysis"]["base_address"])), ("rules", doc["meta"]["analysis"]["rules"]), diff --git a/scripts/bulk-process.py b/scripts/bulk-process.py index da80a477..3bb3495e 100644 --- a/scripts/bulk-process.py +++ b/scripts/bulk-process.py @@ -126,7 +126,7 @@ def get_capa_results(args): "error": "unexpected error: %s" % (e), } - meta = capa.main.collect_metadata("", path, "", format, extractor) + meta = capa.main.collect_metadata("", path, "", extractor) capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True) meta["analysis"].update(counts) diff --git a/scripts/capa_as_library.py b/scripts/capa_as_library.py index 449c35b9..012c1bf3 100644 --- a/scripts/capa_as_library.py +++ b/scripts/capa_as_library.py @@ -169,7 +169,7 @@ def capa_details(file_path, output_format="dictionary"): capabilities, counts = capa.main.find_capabilities(rules, extractor, disable_progress=True) # collect metadata (used only to make rendering more complete) - meta = capa.main.collect_metadata("", file_path, RULES_PATH, "auto", extractor) + meta = capa.main.collect_metadata("", file_path, RULES_PATH, extractor) meta["analysis"].update(counts) capa_output = False diff --git a/scripts/show-capabilities-by-function.py b/scripts/show-capabilities-by-function.py index b95481df..11fcdaab 100644 --- a/scripts/show-capabilities-by-function.py +++ b/scripts/show-capabilities-by-function.py @@ -171,7 +171,7 @@ def main(argv=None): logger.error("-" * 80) return -1 - meta = capa.main.collect_metadata(argv, args.sample, args.rules, format, extractor) + meta = capa.main.collect_metadata(argv, args.sample, args.rules, extractor) capabilities, counts = capa.main.find_capabilities(rules, extractor) meta["analysis"].update(counts) From b6ab12d3c11cfb91754a1ff7d310b343247fa2e9 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 23 Aug 2021 08:54:22 -0600 Subject: [PATCH 42/52] Update capa/features/common.py Co-authored-by: Moritz --- capa/features/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/features/common.py b/capa/features/common.py index 611cd173..cda8f4ff 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -309,6 +309,6 @@ class Format(Feature): def is_global_feature(feature): """ is this a feature that is extracted at every scope? - today, this are OS and arch features. + today, these are OS and arch features. """ return isinstance(feature, (OS, Arch)) From a90e93e15010ceeeade9a023f785e454d0359756 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Mon, 23 Aug 2021 08:54:43 -0600 Subject: [PATCH 43/52] Update capa/main.py Co-authored-by: Moritz --- capa/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/main.py b/capa/main.py index 61fd3f39..2b91fe26 100644 --- a/capa/main.py +++ b/capa/main.py @@ -999,7 +999,7 @@ def main(argv=None): return -1 except UnsupportedArchError: logger.error("-" * 80) - logger.error(" Input file does not appear to be target the x86 architecture.") + logger.error(" Input file does not appear to target a supported architecture.") logger.error(" ") logger.error(" capa currently only supports analyzing x86 (32- and 64-bit).") logger.error("-" * 80) From c0fe0420fc673f42d091002dd35e77ad3423009a Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 23 Aug 2021 15:58:32 -0600 Subject: [PATCH 44/52] changelog: tweak PR ref --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48dbd6bd..65c783eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,9 @@ - explorer: enforce max column width Features and Editor panes #691 @mike-hunhoff - explorer: add option to limit features to currently selected disassembly address #692 @mike-hunhoff - all: add support for ELF files #700 @Adir-Shemesh @TcM1911 -- rule format: add feature `format: ` for file format, like `format: pe` @williballenthin -- rule format: add feature `arch: ` for architecture, like `arch: amd64` @williballenthin -- rule format: add feature `os: ` for operating system, like `os: windows` #701 @williballenthin +- rule format: add feature `format: ` for file format, like `format: pe` #723 @williballenthin +- rule format: add feature `arch: ` for architecture, like `arch: amd64` #723 @williballenthin +- rule format: add feature `os: ` for operating system, like `os: windows` #723 @williballenthin ### Breaking Changes From a1bf95ec2c81444071d597e2a5eece346104ed72 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 23 Aug 2021 16:00:57 -0600 Subject: [PATCH 45/52] features: formatting of OS constants --- capa/features/common.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/capa/features/common.py b/capa/features/common.py index cda8f4ff..8f16d6e5 100644 --- a/capa/features/common.py +++ b/capa/features/common.py @@ -282,9 +282,7 @@ OS_WINDOWS = "windows" OS_LINUX = "linux" OS_MACOS = "macos" VALID_OS = {os.value for os in capa.features.extractors.elf.OS} -VALID_OS.add(OS_WINDOWS) -VALID_OS.add(OS_LINUX) -VALID_OS.add(OS_MACOS) +VALID_OS.update({OS_WINDOWS, OS_LINUX, OS_MACOS}) class OS(Feature): From 6482f67a0c22f4ebd3bd3a6813732d41ce2d939d Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 23 Aug 2021 16:06:14 -0600 Subject: [PATCH 46/52] elf: document unused OS constants --- capa/features/extractors/elf.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index 0dd02685..2f7b7b3d 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -83,7 +83,9 @@ def detect_elf_os(f: BinaryIO) -> str: (ei_osabi,) = struct.unpack_from(endian + "B", file_header, 7) OSABI = { # via pyelftools: https://github.com/eliben/pyelftools/blob/0664de05ed2db3d39041e2d51d19622a8ef4fb0f/elftools/elf/enums.py#L35-L58 - # 0: "SYSV", + # some candidates are commented out because the are not useful values, + # at least when guessing OSes + # 0: "SYSV", # too often used when OS is not SYSV 1: OS.HPUX, 2: OS.NETBSD, 3: OS.LINUX, @@ -101,10 +103,10 @@ def detect_elf_os(f: BinaryIO) -> str: 15: OS.AROS, 16: OS.FENIXOS, 17: OS.CLOUD, - # 53: "SORTFIX", - # 64: "ARM_AEABI", - # 97: "ARM", - # 255: "STANDALONE", + # 53: "SORTFIX", # i can't find any reference to this OS, i dont think it exists + # 64: "ARM_AEABI", # not an OS + # 97: "ARM", # not an OS + # 255: "STANDALONE", # not an OS } logger.debug("ei_osabi: 0x%02x (%s)", ei_osabi, OSABI.get(ei_osabi, "unknown")) From dab88e482d9417858adc8ab680260100bd1f8585 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 23 Aug 2021 16:08:01 -0600 Subject: [PATCH 47/52] elf: add more explanation about ei_osabi --- capa/features/extractors/elf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index 2f7b7b3d..90c5d1c5 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -110,6 +110,8 @@ def detect_elf_os(f: BinaryIO) -> str: } logger.debug("ei_osabi: 0x%02x (%s)", ei_osabi, OSABI.get(ei_osabi, "unknown")) + # os_osabi == 0 is commonly set even when the OS is not SYSV. + # other values are unused or unknown. if ei_osabi in OSABI and ei_osabi != 0x0: # update only if not set # so we can get the debugging output of subsequent strategies From a729bdfbe642f978366a87f5840cb72165ed4b6a Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 23 Aug 2021 16:12:07 -0600 Subject: [PATCH 48/52] elf: more clearly set first detected OS --- capa/features/extractors/elf.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/capa/features/extractors/elf.py b/capa/features/extractors/elf.py index 90c5d1c5..ce3fe638 100644 --- a/capa/features/extractors/elf.py +++ b/capa/features/extractors/elf.py @@ -113,9 +113,8 @@ def detect_elf_os(f: BinaryIO) -> str: # os_osabi == 0 is commonly set even when the OS is not SYSV. # other values are unused or unknown. if ei_osabi in OSABI and ei_osabi != 0x0: - # update only if not set - # so we can get the debugging output of subsequent strategies - ret = OSABI[ei_osabi] if not ret else ret + # subsequent strategies may overwrite this value + ret = OSABI[ei_osabi] f.seek(e_phoff) program_header_size = e_phnum * e_phentsize From 30a5493414149da0c0ab20291c83c86e577744b2 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 23 Aug 2021 16:13:01 -0600 Subject: [PATCH 49/52] tests: smda: remove unused import --- tests/test_smda_features.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_smda_features.py b/tests/test_smda_features.py index 54e519b3..6614c24d 100644 --- a/tests/test_smda_features.py +++ b/tests/test_smda_features.py @@ -11,7 +11,6 @@ from fixtures import * from fixtures import parametrize import capa.features.file -import capa.features.insn @parametrize( From fc737878495dda8a49344d74bb0e9ef91320ef2d Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 23 Aug 2021 16:42:16 -0600 Subject: [PATCH 50/52] extractors: file extractor arg consistency via kwargs --- capa/features/extractors/common.py | 2 +- capa/features/extractors/pefile.py | 41 ++++++++++----------- capa/features/extractors/smda/extractor.py | 2 +- capa/features/extractors/smda/file.py | 42 ++++++++++------------ capa/features/extractors/viv/file.py | 20 +++++------ 5 files changed, 49 insertions(+), 58 deletions(-) diff --git a/capa/features/extractors/common.py b/capa/features/extractors/common.py index b706868f..450fb636 100644 --- a/capa/features/extractors/common.py +++ b/capa/features/extractors/common.py @@ -28,7 +28,7 @@ def extract_format(buf): def extract_arch(buf): if buf.startswith(b"MZ"): - yield from capa.features.extractors.pefile.extract_file_arch(pefile.PE(data=buf), "hack: path not provided") + yield from capa.features.extractors.pefile.extract_file_arch(pe=pefile.PE(data=buf)) elif buf.startswith(b"\x7fELF"): with contextlib.closing(io.BytesIO(buf)) as f: diff --git a/capa/features/extractors/pefile.py b/capa/features/extractors/pefile.py index 2c60499b..3e5d97fd 100644 --- a/capa/features/extractors/pefile.py +++ b/capa/features/extractors/pefile.py @@ -20,15 +20,12 @@ from capa.features.extractors.base_extractor import FeatureExtractor logger = logging.getLogger(__name__) -def extract_file_embedded_pe(pe, file_path): - with open(file_path, "rb") as f: - fbytes = f.read() - - for offset, i in capa.features.extractors.helpers.carve_pe(fbytes, 1): +def extract_file_embedded_pe(buf, **kwargs): + for offset, _ in capa.features.extractors.helpers.carve_pe(buf, 1): yield Characteristic("embedded pe"), offset -def extract_file_export_names(pe, file_path): +def extract_file_export_names(pe, **kwargs): base_address = pe.OPTIONAL_HEADER.ImageBase if hasattr(pe, "DIRECTORY_ENTRY_EXPORT"): @@ -43,7 +40,7 @@ def extract_file_export_names(pe, file_path): yield Export(name), va -def extract_file_import_names(pe, file_path): +def extract_file_import_names(pe, **kwargs): """ extract imported function names 1. imports by ordinal: @@ -75,7 +72,7 @@ def extract_file_import_names(pe, file_path): yield Import(name), imp.address -def extract_file_section_names(pe, file_path): +def extract_file_section_names(pe, **kwargs): base_address = pe.OPTIONAL_HEADER.ImageBase for section in pe.sections: @@ -87,21 +84,18 @@ def extract_file_section_names(pe, file_path): yield Section(name), base_address + section.VirtualAddress -def extract_file_strings(pe, file_path): +def extract_file_strings(buf, **kwargs): """ extract ASCII and UTF-16 LE strings from file """ - with open(file_path, "rb") as f: - b = f.read() - - for s in capa.features.extractors.strings.extract_ascii_strings(b): + for s in capa.features.extractors.strings.extract_ascii_strings(buf): yield String(s.s), s.offset - for s in capa.features.extractors.strings.extract_unicode_strings(b): + for s in capa.features.extractors.strings.extract_unicode_strings(buf): yield String(s.s), s.offset -def extract_file_function_names(pe, file_path): +def extract_file_function_names(**kwargs): """ extract the names of statically-linked library functions. """ @@ -111,17 +105,17 @@ def extract_file_function_names(pe, file_path): return -def extract_file_os(pe, file_path): +def extract_file_os(**kwargs): # assuming PE -> Windows # though i suppose they're also used by UEFI yield OS(OS_WINDOWS), 0x0 -def extract_file_format(pe, file_path): +def extract_file_format(**kwargs): yield Format(FORMAT_PE), 0x0 -def extract_file_arch(pe, file_path): +def extract_file_arch(pe, **kwargs): if pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_I386"]: yield Arch(ARCH_I386), 0x0 elif pe.FILE_HEADER.Machine == pefile.MACHINE_TYPE["IMAGE_FILE_MACHINE_AMD64"]: @@ -130,20 +124,20 @@ def extract_file_arch(pe, file_path): logger.warning("unsupported architecture: %s", pefile.MACHINE_TYPE[pe.FILE_HEADER.Machine]) -def extract_file_features(pe, file_path): +def extract_file_features(pe, buf): """ extract file features from given workspace args: pe (pefile.PE): the parsed PE - file_path: path to the input file + buf: the raw sample bytes yields: Tuple[Feature, VA]: a feature and its location. """ for file_handler in FILE_HANDLERS: - for feature, va in file_handler(pe, file_path): + for feature, va in file_handler(pe=pe, buf=buf): yield feature, va @@ -170,7 +164,10 @@ class PefileFeatureExtractor(FeatureExtractor): return self.pe.OPTIONAL_HEADER.ImageBase def extract_file_features(self): - for feature, va in extract_file_features(self.pe, self.path): + with open(self.path, "rb") as f: + buf = f.read() + + for feature, va in extract_file_features(self.pe, buf): yield feature, va def get_functions(self): diff --git a/capa/features/extractors/smda/extractor.py b/capa/features/extractors/smda/extractor.py index 1bd7f351..1a13653b 100644 --- a/capa/features/extractors/smda/extractor.py +++ b/capa/features/extractors/smda/extractor.py @@ -26,7 +26,7 @@ class SmdaFeatureExtractor(FeatureExtractor): return self.smda_report.base_addr def extract_file_features(self): - for feature, va in capa.features.extractors.smda.file.extract_features(self.smda_report, self.path): + for feature, va in capa.features.extractors.smda.file.extract_features(self.smda_report, self.buf): yield feature, va yield from self.global_features diff --git a/capa/features/extractors/smda/file.py b/capa/features/extractors/smda/file.py index 0be6aa0a..fe901bea 100644 --- a/capa/features/extractors/smda/file.py +++ b/capa/features/extractors/smda/file.py @@ -8,24 +8,22 @@ from capa.features.file import Export, Import, Section from capa.features.common import String, Characteristic -def extract_file_embedded_pe(smda_report, file_path): - with open(file_path, "rb") as f: - fbytes = f.read() - - for offset, i in capa.features.extractors.helpers.carve_pe(fbytes, 1): +def extract_file_embedded_pe(buf, **kwargs): + for offset, _ in capa.features.extractors.helpers.carve_pe(buf, 1): yield Characteristic("embedded pe"), offset -def extract_file_export_names(smda_report, file_path): - lief_binary = lief.parse(file_path) +def extract_file_export_names(buf, **kwargs): + lief_binary = lief.parse(buf) + if lief_binary is not None: for function in lief_binary.exported_functions: yield Export(function.name), function.address -def extract_file_import_names(smda_report, file_path): +def extract_file_import_names(smda_report, buf): # extract import table info via LIEF - lief_binary = lief.parse(file_path) + lief_binary = lief.parse(buf) if not isinstance(lief_binary, lief.PE.Binary): return for imported_library in lief_binary.imports: @@ -41,8 +39,8 @@ def extract_file_import_names(smda_report, file_path): yield Import(name), va -def extract_file_section_names(smda_report, file_path): - lief_binary = lief.parse(file_path) +def extract_file_section_names(buf, **kwargs): + lief_binary = lief.parse(buf) if not isinstance(lief_binary, lief.PE.Binary): return if lief_binary and lief_binary.sections: @@ -51,21 +49,18 @@ def extract_file_section_names(smda_report, file_path): yield Section(section.name), base_address + section.virtual_address -def extract_file_strings(smda_report, file_path): +def extract_file_strings(buf, **kwargs): """ extract ASCII and UTF-16 LE strings from file """ - with open(file_path, "rb") as f: - b = f.read() - - for s in capa.features.extractors.strings.extract_ascii_strings(b): + for s in capa.features.extractors.strings.extract_ascii_strings(buf): yield String(s.s), s.offset - for s in capa.features.extractors.strings.extract_unicode_strings(b): + for s in capa.features.extractors.strings.extract_unicode_strings(buf): yield String(s.s), s.offset -def extract_file_function_names(smda_report, file_path): +def extract_file_function_names(smda_report, **kwargs): """ extract the names of statically-linked library functions. """ @@ -75,25 +70,24 @@ def extract_file_function_names(smda_report, file_path): return -def extract_file_format(smda_report, file_path): - with open(file_path, "rb") as f: - yield from capa.features.extractors.common.extract_format(f.read()) +def extract_file_format(buf, **kwargs): + yield from capa.features.extractors.common.extract_format(buf) -def extract_features(smda_report, file_path): +def extract_features(smda_report, buf): """ extract file features from given workspace args: smda_report (smda.common.SmdaReport): a SmdaReport - file_path: path to the input file + buf: the raw bytes of the sample yields: Tuple[Feature, VA]: a feature and its location. """ for file_handler in FILE_HANDLERS: - for feature, va in file_handler(smda_report, file_path): + for feature, va in file_handler(smda_report=smda_report, buf=buf): yield feature, va diff --git a/capa/features/extractors/viv/file.py b/capa/features/extractors/viv/file.py index c92e124e..db3738bd 100644 --- a/capa/features/extractors/viv/file.py +++ b/capa/features/extractors/viv/file.py @@ -18,17 +18,17 @@ from capa.features.file import Export, Import, Section, FunctionName from capa.features.common import String, Characteristic -def extract_file_embedded_pe(vw, buf): - for offset, i in pe_carve.carve(buf, 1): +def extract_file_embedded_pe(buf, **kwargs): + for offset, _ in pe_carve.carve(buf, 1): yield Characteristic("embedded pe"), offset -def extract_file_export_names(vw, buf): - for va, etype, name, _ in vw.getExports(): +def extract_file_export_names(vw, **kwargs): + for va, _, name, _ in vw.getExports(): yield Export(name), va -def extract_file_import_names(vw, buf): +def extract_file_import_names(vw, **kwargs): """ extract imported function names 1. imports by ordinal: @@ -62,12 +62,12 @@ def is_viv_ord_impname(impname: str) -> bool: return True -def extract_file_section_names(vw, buf): +def extract_file_section_names(vw, **kwargs): for va, _, segname, _ in vw.getSegments(): yield Section(segname), va -def extract_file_strings(vw, buf): +def extract_file_strings(buf, **kwargs): """ extract ASCII and UTF-16 LE strings from file """ @@ -78,7 +78,7 @@ def extract_file_strings(vw, buf): yield String(s.s), s.offset -def extract_file_function_names(vw, buf): +def extract_file_function_names(vw, **kwargs): """ extract the names of statically-linked library functions. """ @@ -88,7 +88,7 @@ def extract_file_function_names(vw, buf): yield FunctionName(name), va -def extract_file_format(vw, buf): +def extract_file_format(buf, **kwargs): yield from capa.features.extractors.common.extract_format(buf) @@ -105,7 +105,7 @@ def extract_features(vw, buf: bytes): """ for file_handler in FILE_HANDLERS: - for feature, va in file_handler(vw, buf): + for feature, va in file_handler(vw=vw, buf=buf): yield feature, va From a4b095453254fcb2f503ddcedcf5626227e5d90b Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 23 Aug 2021 16:57:35 -0600 Subject: [PATCH 51/52] viv: ignore mypy FP --- capa/features/extractors/viv/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/features/extractors/viv/file.py b/capa/features/extractors/viv/file.py index db3738bd..4b9cd13f 100644 --- a/capa/features/extractors/viv/file.py +++ b/capa/features/extractors/viv/file.py @@ -105,7 +105,7 @@ def extract_features(vw, buf: bytes): """ for file_handler in FILE_HANDLERS: - for feature, va in file_handler(vw=vw, buf=buf): + for feature, va in file_handler(vw=vw, buf=buf): # type: ignore yield feature, va From 56f9e16a8b08a40d03c2bf26a39fba5e478a2e0b Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Mon, 23 Aug 2021 17:51:28 -0600 Subject: [PATCH 52/52] tests: viv: disable ELF tests due to #735 --- tests/test_viv_features.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_viv_features.py b/tests/test_viv_features.py index fa8bfda3..3989db81 100644 --- a/tests/test_viv_features.py +++ b/tests/test_viv_features.py @@ -15,6 +15,9 @@ from fixtures import * indirect=["sample", "scope"], ) def test_viv_features(sample, scope, feature, expected): + if "elf" in sample: + pytest.xfail("viv ELF parsing is broken (for our test file), see #735") + fixtures.do_test_feature_presence(fixtures.get_viv_extractor, sample, scope, feature, expected) @@ -24,4 +27,7 @@ def test_viv_features(sample, scope, feature, expected): indirect=["sample", "scope"], ) def test_viv_feature_counts(sample, scope, feature, expected): + if "elf" in sample: + pytest.xfail("viv ELF parsing is broken (for our test file), see #735") + fixtures.do_test_feature_count(fixtures.get_viv_extractor, sample, scope, feature, expected)