diff --git a/.github/flake8.ini b/.github/flake8.ini index 6997d5a4..e7ca4090 100644 --- a/.github/flake8.ini +++ b/.github/flake8.ini @@ -33,8 +33,6 @@ per-file-ignores = scripts/*: T201 # capa.exe is meant to print output capa/main.py: T201 - # IDA tests emit results to output window so need to print - tests/test_ida_features.py: T201 # utility used to find the Binary Ninja API via invoking python.exe capa/features/extractors/binja/find_binja_api.py: T201 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d26911c8..e0cf0bdd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,7 +136,6 @@ repos: - "tests/" - "--ignore=tests/test_binja_features.py" - "--ignore=tests/test_ghidra_features.py" - - "--ignore=tests/test_ida_features.py" - "--ignore=tests/test_viv_features.py" - "--ignore=tests/test_idalib_features.py" - "--ignore=tests/test_main.py" diff --git a/tests/test_ida_features.py b/tests/test_ida_features.py deleted file mode 100644 index 5e2f7380..00000000 --- a/tests/test_ida_features.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -run this script from within IDA to test the IDA feature extractor. -you must have loaded a file referenced by a test case in order -for this to do anything meaningful. for example, mimikatz.exe from testfiles. - -you can invoke from the command line like this: - - & 'C:\\Program Files\\IDA Pro 8.2\\idat.exe' \ - -S"C:\\Exclusions\\code\\capa\\tests\\test_ida_features.py --CAPA_AUTOEXIT=true" \ - -A \ - -Lidalog \ - 'C:\\Exclusions\\code\\capa\\tests\\data\\mimikatz.exe_' - -if you invoke from the command line, and provide the script argument `--CAPA_AUTOEXIT=true`, -then the script will exit IDA after running the tests. - -the output (in idalog) will look like this: - -``` -Loading processor module C:\\Program Files\\IDA Pro 8.2\\procs\\pc.dll for metapc...Initializing processor module metapc...OK -Loading type libraries... -Autoanalysis subsystem has been initialized. -Database for file 'mimikatz.exe_' has been loaded. --------------------------------------------------------------------------------- -PASS: test_ida_feature_counts/mimikatz-function=0x40E5C2-basic block-7 -PASS: test_ida_feature_counts/mimikatz-function=0x4702FD-characteristic(calls from)-0 -SKIP: test_ida_features/294b8d...-function=0x404970,bb=0x404970,insn=0x40499F-string(\r\n\x00:ht)-False -SKIP: test_ida_features/64d9f-function=0x10001510,bb=0x100015B0-offset(0x4000)-True -... -SKIP: test_ida_features/pma16-01-function=0x404356,bb=0x4043B9-arch(i386)-True -PASS: test_ida_features/mimikatz-file-import(cabinet.FCIAddFile)-True -DONE -C:\\Exclusions\\code\\capa\\tests\\test_ida_features.py: Traceback (most recent call last): - File "C:\\Program Files\\IDA Pro 8.2\\python\\3\\ida_idaapi.py", line 588, in IDAPython_ExecScript - exec(code, g) - File "C:/Exclusions/code/capa/tests/test_ida_features.py", line 120, in - sys.exit(0) -SystemExit: 0 - -> OK -Flushing buffers, please wait...ok -``` - -Look for lines that start with "FAIL" to identify test failures. -""" -import io -import sys -import inspect -import logging -import traceback -from pathlib import Path - -import pytest - -try: - sys.path.append(str(Path(__file__).parent)) - import fixtures -finally: - sys.path.pop() - - -logger = logging.getLogger("test_ida_features") - - -def check_input_file(wanted): - import idautils - - # some versions (7.4) of IDA return a truncated version of the MD5. - # https://github.com/idapython/bin/issues/11 - try: - found = idautils.GetInputFileMD5()[:31].decode("ascii").lower() - except UnicodeDecodeError: - # in IDA 7.5 or so, GetInputFileMD5 started returning raw binary - # rather than the hex digest - found = bytes.hex(idautils.GetInputFileMD5()[:15]).lower() - - if not wanted.startswith(found): - raise RuntimeError(f"please run the tests against sample with MD5: `{wanted}`") - - -def get_ida_extractor(_path): - # have to import this inline so pytest doesn't bail outside of IDA - import capa.features.extractors.ida.extractor - - return capa.features.extractors.ida.extractor.IdaFeatureExtractor() - - -def nocollect(f): - "don't collect the decorated function as a pytest test" - f.__test__ = False - return f - - -# although these look like pytest tests, they're not, because they don't run within pytest -# (the runner is below) and they use `yield`, which is deprecated. -@nocollect -@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") -def test_ida_features(): - # we're guaranteed to be in a function here, so there's a stack frame - this_name = inspect.currentframe().f_code.co_name # type: ignore - for sample, scope, feature, expected in fixtures.FEATURE_PRESENCE_TESTS + fixtures.FEATURE_PRESENCE_TESTS_IDA: - id = fixtures.make_test_id((sample, scope, feature, expected)) - - try: - check_input_file(fixtures.get_sample_md5_by_name(sample)) - except RuntimeError: - yield this_name, id, "skip", None - continue - - scope = fixtures.resolve_scope(scope) - sample = fixtures.resolve_sample(sample) - - try: - fixtures.do_test_feature_presence(get_ida_extractor, sample, scope, feature, expected) - except Exception: - f = io.StringIO() - traceback.print_exc(file=f) - yield this_name, id, "fail", f.getvalue() - else: - yield this_name, id, "pass", None - - -@nocollect -@pytest.mark.skip(reason="IDA Pro tests must be run within IDA") -def test_ida_feature_counts(): - # we're guaranteed to be in a function here, so there's a stack frame - this_name = inspect.currentframe().f_code.co_name # type: ignore - for sample, scope, feature, expected in fixtures.FEATURE_COUNT_TESTS: - id = fixtures.make_test_id((sample, scope, feature, expected)) - - try: - check_input_file(fixtures.get_sample_md5_by_name(sample)) - except RuntimeError: - yield this_name, id, "skip", None - continue - - scope = fixtures.resolve_scope(scope) - sample = fixtures.resolve_sample(sample) - - try: - fixtures.do_test_feature_count(get_ida_extractor, sample, scope, feature, expected) - except Exception: - f = io.StringIO() - traceback.print_exc(file=f) - yield this_name, id, "fail", f.getvalue() - else: - yield this_name, id, "pass", None - - -if __name__ == "__main__": - import idc - import ida_auto - - ida_auto.auto_wait() - - print("-" * 80) - - # invoke all functions in this module that start with `test_` - for name in dir(sys.modules[__name__]): - if not name.startswith("test_"): - continue - - test = getattr(sys.modules[__name__], name) - logger.debug("invoking test: %s", name) - sys.stderr.flush() - for name, id, state, info in test(): - print(f"{state.upper()}: {name}/{id}") - if info: - print(info) - - print("DONE") - - if "--CAPA_AUTOEXIT=true" in idc.ARGV: - sys.exit(0)