From 408c5076c6a3b9cce7ec711046c80eed123977c1 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Tue, 15 Aug 2023 08:26:59 +0000 Subject: [PATCH 1/5] tests: ida: don't collect tests as pytest tests closes #1719 --- tests/test_ida_features.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_ida_features.py b/tests/test_ida_features.py index 239f045e..08757573 100644 --- a/tests/test_ida_features.py +++ b/tests/test_ida_features.py @@ -92,6 +92,15 @@ def get_ida_extractor(_path): return capa.features.extractors.ida.extractor.IdaFeatureExtractor() +def nocollect(f): + "don't collect the decorated function as a pytest test" + f.__test__ = False + return f + + +# although these look like pytest tests, they're not, because they doesn't run within pytest +# (the runner is below) and they use `yield`, which is deprecated. +@nocollect @pytest.mark.skip(reason="IDA Pro tests must be run within IDA") def test_ida_features(): # we're guaranteed to be in a function here, so there's a stack frame @@ -118,6 +127,7 @@ def test_ida_features(): yield this_name, id, "pass", None +@nocollect @pytest.mark.skip(reason="IDA Pro tests must be run within IDA") def test_ida_feature_counts(): # we're guaranteed to be in a function here, so there's a stack frame From e6d64ef56148440769810f82925af9533658fd4f Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Tue, 15 Aug 2023 08:41:56 +0000 Subject: [PATCH 2/5] pydantic: remove use of deprecated routines closes #1718 --- capa/features/freeze/__init__.py | 2 +- capa/ida/helpers.py | 3 +-- capa/main.py | 2 +- capa/render/result_document.py | 5 +++++ scripts/import-to-ida.py | 3 ++- scripts/proto-from-results.py | 3 ++- tests/fixtures.py | 4 ++-- tests/test_result_document.py | 6 +++--- 8 files changed, 17 insertions(+), 11 deletions(-) diff --git a/capa/features/freeze/__init__.py b/capa/features/freeze/__init__.py index d29048a9..8eb953b7 100644 --- a/capa/features/freeze/__init__.py +++ b/capa/features/freeze/__init__.py @@ -320,7 +320,7 @@ def loads(s: str) -> capa.features.extractors.base_extractor.FeatureExtractor: """deserialize a set of features (as a NullFeatureExtractor) from a string.""" import capa.features.extractors.null as null - freeze = Freeze.parse_raw(s) + freeze = Freeze.model_validate_json(s) if freeze.version != 2: raise ValueError(f"unsupported freeze format version: {freeze.version}") diff --git a/capa/ida/helpers.py b/capa/ida/helpers.py index 89e12c60..346421b0 100644 --- a/capa/ida/helpers.py +++ b/capa/ida/helpers.py @@ -5,7 +5,6 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import json import logging import datetime import contextlib @@ -223,7 +222,7 @@ def load_and_verify_cached_results() -> Optional[rdoc.ResultDocument]: logger.debug("loading cached capa results from netnode '%s'", CAPA_NETNODE) n = netnode.Netnode(CAPA_NETNODE) - doc = rdoc.ResultDocument.parse_obj(json.loads(n[NETNODE_RESULTS])) + doc = rdoc.ResultDocument.model_validate_json(n[NETNODE_RESULTS]) for rule in rutils.capability_rules(doc): for location_, _ in rule.matches: diff --git a/capa/main.py b/capa/main.py index a7be6cc9..cc921951 100644 --- a/capa/main.py +++ b/capa/main.py @@ -1223,7 +1223,7 @@ def main(argv: Optional[List[str]] = None): if format_ == FORMAT_RESULT: # result document directly parses into meta, capabilities - result_doc = capa.render.result_document.ResultDocument.parse_file(args.sample) + result_doc = capa.render.result_document.ResultDocument.from_file(Path(args.sample)) meta, capabilities = result_doc.to_capa() else: diff --git a/capa/render/result_document.py b/capa/render/result_document.py index ef899fd2..e8c27a49 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -8,6 +8,7 @@ import datetime import collections from typing import Dict, List, Tuple, Union, Literal, Optional +from pathlib import Path from pydantic import Field, BaseModel, ConfigDict @@ -596,3 +597,7 @@ class ResultDocument(FrozenModel): capabilities[rule_name].append((addr.to_capa(), result)) return self.meta, capabilities + + @classmethod + def from_file(cls, path: Path) -> "ResultDocument": + return cls.model_validate_json(path.read_text(encoding="utf-8")) diff --git a/scripts/import-to-ida.py b/scripts/import-to-ida.py index 121d8158..8414cdb8 100644 --- a/scripts/import-to-ida.py +++ b/scripts/import-to-ida.py @@ -30,6 +30,7 @@ See the License for the specific language governing permissions and limitations """ import logging import binascii +from pathlib import Path import ida_nalt import ida_funcs @@ -68,7 +69,7 @@ def main(): if not path: return 0 - result_doc = capa.render.result_document.ResultDocument.parse_file(path) + result_doc = capa.render.result_document.ResultDocument.from_file(Path(path)) meta, capabilities = result_doc.to_capa() # in IDA 7.4, the MD5 hash may be truncated, for example: diff --git a/scripts/proto-from-results.py b/scripts/proto-from-results.py index 6e940366..61df56b6 100644 --- a/scripts/proto-from-results.py +++ b/scripts/proto-from-results.py @@ -31,6 +31,7 @@ Example: import sys import logging import argparse +from pathlib import Path import capa.render.proto import capa.render.result_document @@ -64,7 +65,7 @@ def main(argv=None): logging.basicConfig(level=logging.INFO) logging.getLogger().setLevel(logging.INFO) - rd = capa.render.result_document.ResultDocument.parse_file(args.json) + rd = capa.render.result_document.ResultDocument.from_file(Path(args.json)) pb = capa.render.proto.doc_to_pb2(rd) sys.stdout.buffer.write(pb.SerializeToString(deterministic=True)) diff --git a/tests/fixtures.py b/tests/fixtures.py index a85656a6..0dd6ea59 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1182,8 +1182,8 @@ def _039a6_dotnetfile_extractor(): return get_dnfile_extractor(get_data_path_by_name("_039a6")) -def get_result_doc(path): - return capa.render.result_document.ResultDocument.parse_file(path) +def get_result_doc(path: Path): + return capa.render.result_document.ResultDocument.from_file(path) @pytest.fixture diff --git a/tests/test_result_document.py b/tests/test_result_document.py index 352e126d..10f022d9 100644 --- a/tests/test_result_document.py +++ b/tests/test_result_document.py @@ -237,7 +237,7 @@ def assert_round_trip(rd: rdoc.ResultDocument): one = rd doc = one.model_dump_json(exclude_none=True) - two = rdoc.ResultDocument.parse_raw(doc) + two = rdoc.ResultDocument.model_validate_json(doc) # show the round trip works # first by comparing the objects directly, @@ -272,13 +272,13 @@ def test_round_trip(request, rd_file): def test_json_to_rdoc(): path = fixtures.get_data_path_by_name("pma01-01-rd") - assert isinstance(rdoc.ResultDocument.parse_file(path), rdoc.ResultDocument) + assert isinstance(rdoc.ResultDocument.from_file(path), rdoc.ResultDocument) def test_rdoc_to_capa(): path = fixtures.get_data_path_by_name("pma01-01-rd") - rd = rdoc.ResultDocument.parse_file(path) + rd = rdoc.ResultDocument.from_file(path) meta, capabilites = rd.to_capa() assert isinstance(meta, rdoc.Metadata) From 8118a3f3537e2348b516eb9d674fadbdbd42694d Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Tue, 15 Aug 2023 08:46:18 +0000 Subject: [PATCH 3/5] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a960b46c..87082054 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ - linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin - OS: detect Android ELF files #1705 @williballenthin - ELF: fix parsing of symtab #1704 @williballenthin +- result docment: don't use deprecated pydantic functions #1718 @williballenthin +- pytest: don't mark IDA tests as pytest tests #1719 @williballenthin ### capa explorer IDA Pro plugin - fix unhandled exception when resolving rule path #1693 @mike-hunhoff From a5a1a0bfee122d2549e0135fd913d4d6031f2ee6 Mon Sep 17 00:00:00 2001 From: Yacine <16624109+yelhamer@users.noreply.github.com> Date: Tue, 15 Aug 2023 12:26:02 +0200 Subject: [PATCH 4/5] Update CHANGELOG.md Co-authored-by: Moritz --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87082054..e4f40db0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ - linter: skip native API check for NtProtectVirtualMemory #1675 @williballenthin - OS: detect Android ELF files #1705 @williballenthin - ELF: fix parsing of symtab #1704 @williballenthin -- result docment: don't use deprecated pydantic functions #1718 @williballenthin +- result document: don't use deprecated pydantic functions #1718 @williballenthin - pytest: don't mark IDA tests as pytest tests #1719 @williballenthin ### capa explorer IDA Pro plugin From d71ecc7a79810760581310c9cce2f5b17dfaaa33 Mon Sep 17 00:00:00 2001 From: Yacine <16624109+yelhamer@users.noreply.github.com> Date: Tue, 15 Aug 2023 12:26:19 +0200 Subject: [PATCH 5/5] Update tests/test_ida_features.py Co-authored-by: Moritz --- tests/test_ida_features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ida_features.py b/tests/test_ida_features.py index 08757573..52b3a6a7 100644 --- a/tests/test_ida_features.py +++ b/tests/test_ida_features.py @@ -98,7 +98,7 @@ def nocollect(f): return f -# although these look like pytest tests, they're not, because they doesn't run within pytest +# although these look like pytest tests, they're not, because they don't run within pytest # (the runner is below) and they use `yield`, which is deprecated. @nocollect @pytest.mark.skip(reason="IDA Pro tests must be run within IDA")