From 22d0f94cea22c7227d33ec7b53a08bb0fc1e4513 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 7 Jul 2020 12:28:47 -0600 Subject: [PATCH 1/4] move ci files to .github directory since this is where we host everything closes #113 --- .../pyinstaller}/hooks/hook-vivisect.py | 0 {ci => .github/pyinstaller}/logo.ico | Bin {ci => .github/pyinstaller}/logo.png | Bin {ci => .github}/tox.ini | 0 ci/pyinstaller.spec | 193 ------------------ 5 files changed, 193 deletions(-) rename {ci => .github/pyinstaller}/hooks/hook-vivisect.py (100%) rename {ci => .github/pyinstaller}/logo.ico (100%) rename {ci => .github/pyinstaller}/logo.png (100%) rename {ci => .github}/tox.ini (100%) delete mode 100644 ci/pyinstaller.spec diff --git a/ci/hooks/hook-vivisect.py b/.github/pyinstaller/hooks/hook-vivisect.py similarity index 100% rename from ci/hooks/hook-vivisect.py rename to .github/pyinstaller/hooks/hook-vivisect.py diff --git a/ci/logo.ico b/.github/pyinstaller/logo.ico similarity index 100% rename from ci/logo.ico rename to .github/pyinstaller/logo.ico diff --git a/ci/logo.png b/.github/pyinstaller/logo.png similarity index 100% rename from ci/logo.png rename to .github/pyinstaller/logo.png diff --git a/ci/tox.ini b/.github/tox.ini similarity index 100% rename from ci/tox.ini rename to .github/tox.ini diff --git a/ci/pyinstaller.spec b/ci/pyinstaller.spec deleted file mode 100644 index 785db528..00000000 --- a/ci/pyinstaller.spec +++ /dev/null @@ -1,193 +0,0 @@ -# -*- mode: python -*- -import os.path -import subprocess - -import wcwidth - - -with open('./capa/version.py', 'wb') as f: - f.write("__version__ = '%s'" - % subprocess.check_output(["git", "describe", "--always"]).strip()) - -a = Analysis( - ['../capa/main.py'], - pathex=['capa'], - binaries=None, - datas=[ - ('../rules', 'rules'), - # capa.render.default uses tabulate that depends on wcwidth. - # it seems wcwidth uses a json file `version.json` - # and this doesn't get picked up by pyinstaller automatically. - # so we manually embed the wcwidth resources here. - # - # ref: https://stackoverflow.com/a/62278462/87207 - (os.path.dirname(wcwidth.__file__), 'wcwidth') - ], - hiddenimports=[ - # vivisect does manual/runtime importing of its modules, - # so declare the things that could be imported here. - "pycparser", - "vivisect", - "vivisect.analysis", - "vivisect.analysis.amd64", - "vivisect.analysis.amd64", - "vivisect.analysis.amd64.emulation", - "vivisect.analysis.amd64.golang", - "vivisect.analysis.crypto", - "vivisect.analysis.crypto", - "vivisect.analysis.crypto.constants", - "vivisect.analysis.elf", - "vivisect.analysis.elf", - "vivisect.analysis.elf.elfplt", - "vivisect.analysis.elf.libc_start_main", - "vivisect.analysis.generic", - "vivisect.analysis.generic", - "vivisect.analysis.generic.codeblocks", - "vivisect.analysis.generic.emucode", - "vivisect.analysis.generic.entrypoints", - "vivisect.analysis.generic.funcentries", - "vivisect.analysis.generic.impapi", - "vivisect.analysis.generic.mkpointers", - "vivisect.analysis.generic.pointers", - "vivisect.analysis.generic.pointertables", - "vivisect.analysis.generic.relocations", - "vivisect.analysis.generic.strconst", - "vivisect.analysis.generic.switchcase", - "vivisect.analysis.generic.thunks", - "vivisect.analysis.i386", - "vivisect.analysis.i386", - "vivisect.analysis.i386.calling", - "vivisect.analysis.i386.golang", - "vivisect.analysis.i386.importcalls", - "vivisect.analysis.i386.instrhook", - "vivisect.analysis.i386.thunk_bx", - "vivisect.analysis.ms", - "vivisect.analysis.ms", - "vivisect.analysis.ms.hotpatch", - "vivisect.analysis.ms.localhints", - "vivisect.analysis.ms.msvc", - "vivisect.analysis.ms.msvcfunc", - "vivisect.analysis.ms.vftables", - "vivisect.analysis.pe", - "vivisect.impapi.posix.amd64", - "vivisect.impapi.posix.i386", - "vivisect.impapi.windows", - "vivisect.impapi.windows.amd64", - "vivisect.impapi.windows.i386", - "vivisect.parsers.blob", - "vivisect.parsers.elf", - "vivisect.parsers.ihex", - "vivisect.parsers.macho", - "vivisect.parsers.parse_pe", - "vivisect.parsers.utils", - "vivisect.storage", - "vivisect.storage.basicfile", - "vstruct.constants", - "vstruct.constants.ntstatus", - "vstruct.defs", - "vstruct.defs.arm7", - "vstruct.defs.bmp", - "vstruct.defs.dns", - "vstruct.defs.elf", - "vstruct.defs.gif", - "vstruct.defs.ihex", - "vstruct.defs.inet", - "vstruct.defs.java", - "vstruct.defs.kdcom", - "vstruct.defs.macho", - "vstruct.defs.macho.const", - "vstruct.defs.macho.fat", - "vstruct.defs.macho.loader", - "vstruct.defs.macho.stabs", - "vstruct.defs.minidump", - "vstruct.defs.pcap", - "vstruct.defs.pe", - "vstruct.defs.pptp", - "vstruct.defs.rar", - "vstruct.defs.swf", - "vstruct.defs.win32", - "vstruct.defs.windows", - "vstruct.defs.windows.win_5_1_i386", - "vstruct.defs.windows.win_5_1_i386.ntdll", - "vstruct.defs.windows.win_5_1_i386.ntoskrnl", - "vstruct.defs.windows.win_5_1_i386.win32k", - "vstruct.defs.windows.win_5_2_i386", - "vstruct.defs.windows.win_5_2_i386.ntdll", - "vstruct.defs.windows.win_5_2_i386.ntoskrnl", - "vstruct.defs.windows.win_5_2_i386.win32k", - "vstruct.defs.windows.win_6_1_amd64", - "vstruct.defs.windows.win_6_1_amd64.ntdll", - "vstruct.defs.windows.win_6_1_amd64.ntoskrnl", - "vstruct.defs.windows.win_6_1_amd64.win32k", - "vstruct.defs.windows.win_6_1_i386", - "vstruct.defs.windows.win_6_1_i386.ntdll", - "vstruct.defs.windows.win_6_1_i386.ntoskrnl", - "vstruct.defs.windows.win_6_1_i386.win32k", - "vstruct.defs.windows.win_6_1_wow64", - "vstruct.defs.windows.win_6_1_wow64.ntdll", - "vstruct.defs.windows.win_6_2_amd64", - "vstruct.defs.windows.win_6_2_amd64.ntdll", - "vstruct.defs.windows.win_6_2_amd64.ntoskrnl", - "vstruct.defs.windows.win_6_2_amd64.win32k", - "vstruct.defs.windows.win_6_2_i386", - "vstruct.defs.windows.win_6_2_i386.ntdll", - "vstruct.defs.windows.win_6_2_i386.ntoskrnl", - "vstruct.defs.windows.win_6_2_i386.win32k", - "vstruct.defs.windows.win_6_2_wow64", - "vstruct.defs.windows.win_6_2_wow64.ntdll", - "vstruct.defs.windows.win_6_3_amd64", - "vstruct.defs.windows.win_6_3_amd64.ntdll", - "vstruct.defs.windows.win_6_3_amd64.ntoskrnl", - "vstruct.defs.windows.win_6_3_i386", - "vstruct.defs.windows.win_6_3_i386.ntdll", - "vstruct.defs.windows.win_6_3_i386.ntoskrnl", - "vstruct.defs.windows.win_6_3_wow64", - "vstruct.defs.windows.win_6_3_wow64.ntdll", - ], - hookspath=['ci/hooks'], - runtime_hooks=None, - excludes=[ - # ignore packages that would otherwise be bundled with the .exe. - # review: build/pyinstaller/xref-pyinstaller.html - - # we don't do any GUI stuff, so ignore these modules - "tkinter", - "_tkinter", - "Tkinter", - # tqdm provides renderers for ipython, - # however, this drags in a lot of dependencies. - # since we don't spawn a notebook, we can safely remove these. - "IPython", - "ipywidgets", - ]) - -a.binaries = a.binaries - TOC([ - ('tcl85.dll', None, None), - ('tk85.dll', None, None), - ('_tkinter', None, None)]) - -pyz = PYZ(a.pure, a.zipped_data) - -exe = EXE(pyz, - a.scripts, - a.binaries, - a.zipfiles, - a.datas, - exclude_binaries=False, - name='capa', - icon='logo.ico', - debug=False, - strip=None, - upx=True, - console=True ) - -# enable the following to debug the contents of the .exe -# -#coll = COLLECT(exe, -# a.binaries, -# a.zipfiles, -# a.datas, -# strip=None, -# upx=True, -# name='capa-dat') - From d55e1cd16bca5fe791da6c7ef3348fcd16d62d8a Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 7 Jul 2020 12:29:17 -0600 Subject: [PATCH 2/4] pep8 --- capa/features/extractors/ida/__init__.py | 1 - capa/features/extractors/ida/basicblock.py | 1 - capa/features/extractors/ida/file.py | 1 - capa/features/extractors/ida/function.py | 1 - capa/features/extractors/ida/insn.py | 1 - capa/ida/ida_capa_explorer.py | 3 +-- 6 files changed, 1 insertion(+), 7 deletions(-) diff --git a/capa/features/extractors/ida/__init__.py b/capa/features/extractors/ida/__init__.py index c994faad..b40310d9 100644 --- a/capa/features/extractors/ida/__init__.py +++ b/capa/features/extractors/ida/__init__.py @@ -7,7 +7,6 @@ import capa.features.extractors.ida.file import capa.features.extractors.ida.insn import capa.features.extractors.ida.function import capa.features.extractors.ida.basicblock - from capa.features.extractors import FeatureExtractor diff --git a/capa/features/extractors/ida/basicblock.py b/capa/features/extractors/ida/basicblock.py index 00ff9859..c9eb601e 100644 --- a/capa/features/extractors/ida/basicblock.py +++ b/capa/features/extractors/ida/basicblock.py @@ -5,7 +5,6 @@ import struct import idaapi import capa.features.extractors.ida.helpers - from capa.features import Characteristic from capa.features.basicblock import BasicBlock from capa.features.extractors.ida import helpers diff --git a/capa/features/extractors/ida/file.py b/capa/features/extractors/ida/file.py index a227c82e..650ec775 100644 --- a/capa/features/extractors/ida/file.py +++ b/capa/features/extractors/ida/file.py @@ -7,7 +7,6 @@ import idautils import capa.features.extractors.helpers import capa.features.extractors.strings import capa.features.extractors.ida.helpers - from capa.features import String, Characteristic from capa.features.file import Export, Import, Section diff --git a/capa/features/extractors/ida/function.py b/capa/features/extractors/ida/function.py index fd5d91b1..31d12920 100644 --- a/capa/features/extractors/ida/function.py +++ b/capa/features/extractors/ida/function.py @@ -2,7 +2,6 @@ import idaapi import idautils import capa.features.extractors.ida.helpers - from capa.features import Characteristic from capa.features.extractors import loops diff --git a/capa/features/extractors/ida/insn.py b/capa/features/extractors/ida/insn.py index f6fa2dac..ba237766 100644 --- a/capa/features/extractors/ida/insn.py +++ b/capa/features/extractors/ida/insn.py @@ -4,7 +4,6 @@ import idautils import capa.features.extractors.helpers import capa.features.extractors.ida.helpers - from capa.features import MAX_BYTES_FEATURE_SIZE, Bytes, String, Characteristic from capa.features.insn import Number, Offset, Mnemonic diff --git a/capa/ida/ida_capa_explorer.py b/capa/ida/ida_capa_explorer.py index c5468b31..ad1c7a7f 100644 --- a/capa/ida/ida_capa_explorer.py +++ b/capa/ida/ida_capa_explorer.py @@ -1,16 +1,15 @@ import os import logging import collections -from PyQt5 import QtGui, QtCore, QtWidgets import idaapi +from PyQt5 import QtGui, QtCore, QtWidgets import capa.main import capa.rules import capa.ida.helpers import capa.render.utils as rutils import capa.features.extractors.ida - from capa.ida.explorer.view import CapaExplorerQtreeView from capa.ida.explorer.model import CapaExplorerDataModel from capa.ida.explorer.proxy import CapaExplorerSortFilterProxyModel From 56b9462d01e29a7cbfc7b164e1c504b30ef3f499 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 7 Jul 2020 12:30:54 -0600 Subject: [PATCH 3/4] git hooks: fix references to tox config --- scripts/hooks/post-commit | 2 +- scripts/hooks/pre-push | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/hooks/post-commit b/scripts/hooks/post-commit index 42953c4f..e3334be5 100755 --- a/scripts/hooks/post-commit +++ b/scripts/hooks/post-commit @@ -12,7 +12,7 @@ if [[ "$STASH_LIST" == *"$MSG"* ]]; then fi # Run style checker and print state (it doesn't block the commit) -pycodestyle --config=./ci/tox.ini ./capa/ > style-checker-output.log 2>&1; +pycodestyle --config=./.github/tox.ini ./capa/ > style-checker-output.log 2>&1; if [ $? == 0 ]; then echo 'Style checker succeeds!! 💘'; else diff --git a/scripts/hooks/pre-push b/scripts/hooks/pre-push index edab4689..620c5a78 100755 --- a/scripts/hooks/pre-push +++ b/scripts/hooks/pre-push @@ -19,7 +19,7 @@ restore_stashed() { } # Run style checker and print state -pycodestyle --config=./ci/tox.ini ./capa/ > style-checker-output.log 2>&1; +pycodestyle --config=./.github/tox.ini ./capa/ > style-checker-output.log 2>&1; if [ $? == 0 ]; then echo 'Style checker succeeds!! 💘'; else From f99fe8aacbfe4b8253d454dafbbf331eec74c285 Mon Sep 17 00:00:00 2001 From: William Ballenthin Date: Tue, 7 Jul 2020 12:32:39 -0600 Subject: [PATCH 4/4] re-add missing pyinstaller spec --- .github/pyinstaller/pyinstaller.spec | 204 +++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 .github/pyinstaller/pyinstaller.spec diff --git a/.github/pyinstaller/pyinstaller.spec b/.github/pyinstaller/pyinstaller.spec new file mode 100644 index 00000000..10fa2b13 --- /dev/null +++ b/.github/pyinstaller/pyinstaller.spec @@ -0,0 +1,204 @@ +# -*- mode: python -*- +import os.path +import subprocess + +import wcwidth + + +# when invoking pyinstaller from the project root, +# this gets run from the project root. +with open('./capa/version.py', 'wb') as f: + f.write("__version__ = '%s'" + % subprocess.check_output(["git", "describe", "--always"]).strip()) + +a = Analysis( + # when invoking pyinstaller from the project root, + # this gets invoked from the directory of the spec file, + # i.e. ./.github/pyinstaller + ['../../capa/main.py'], + pathex=['capa'], + binaries=None, + datas=[ + # when invoking pyinstaller from the project root, + # this gets invoked from the directory of the spec file, + # i.e. ./.github/pyinstaller + ('../../rules', 'rules'), + + # capa.render.default uses tabulate that depends on wcwidth. + # it seems wcwidth uses a json file `version.json` + # and this doesn't get picked up by pyinstaller automatically. + # so we manually embed the wcwidth resources here. + # + # ref: https://stackoverflow.com/a/62278462/87207 + (os.path.dirname(wcwidth.__file__), 'wcwidth') + ], + hiddenimports=[ + # vivisect does manual/runtime importing of its modules, + # so declare the things that could be imported here. + "pycparser", + "vivisect", + "vivisect.analysis", + "vivisect.analysis.amd64", + "vivisect.analysis.amd64", + "vivisect.analysis.amd64.emulation", + "vivisect.analysis.amd64.golang", + "vivisect.analysis.crypto", + "vivisect.analysis.crypto", + "vivisect.analysis.crypto.constants", + "vivisect.analysis.elf", + "vivisect.analysis.elf", + "vivisect.analysis.elf.elfplt", + "vivisect.analysis.elf.libc_start_main", + "vivisect.analysis.generic", + "vivisect.analysis.generic", + "vivisect.analysis.generic.codeblocks", + "vivisect.analysis.generic.emucode", + "vivisect.analysis.generic.entrypoints", + "vivisect.analysis.generic.funcentries", + "vivisect.analysis.generic.impapi", + "vivisect.analysis.generic.mkpointers", + "vivisect.analysis.generic.pointers", + "vivisect.analysis.generic.pointertables", + "vivisect.analysis.generic.relocations", + "vivisect.analysis.generic.strconst", + "vivisect.analysis.generic.switchcase", + "vivisect.analysis.generic.thunks", + "vivisect.analysis.i386", + "vivisect.analysis.i386", + "vivisect.analysis.i386.calling", + "vivisect.analysis.i386.golang", + "vivisect.analysis.i386.importcalls", + "vivisect.analysis.i386.instrhook", + "vivisect.analysis.i386.thunk_bx", + "vivisect.analysis.ms", + "vivisect.analysis.ms", + "vivisect.analysis.ms.hotpatch", + "vivisect.analysis.ms.localhints", + "vivisect.analysis.ms.msvc", + "vivisect.analysis.ms.msvcfunc", + "vivisect.analysis.ms.vftables", + "vivisect.analysis.pe", + "vivisect.impapi.posix.amd64", + "vivisect.impapi.posix.i386", + "vivisect.impapi.windows", + "vivisect.impapi.windows.amd64", + "vivisect.impapi.windows.i386", + "vivisect.parsers.blob", + "vivisect.parsers.elf", + "vivisect.parsers.ihex", + "vivisect.parsers.macho", + "vivisect.parsers.parse_pe", + "vivisect.parsers.utils", + "vivisect.storage", + "vivisect.storage.basicfile", + "vstruct.constants", + "vstruct.constants.ntstatus", + "vstruct.defs", + "vstruct.defs.arm7", + "vstruct.defs.bmp", + "vstruct.defs.dns", + "vstruct.defs.elf", + "vstruct.defs.gif", + "vstruct.defs.ihex", + "vstruct.defs.inet", + "vstruct.defs.java", + "vstruct.defs.kdcom", + "vstruct.defs.macho", + "vstruct.defs.macho.const", + "vstruct.defs.macho.fat", + "vstruct.defs.macho.loader", + "vstruct.defs.macho.stabs", + "vstruct.defs.minidump", + "vstruct.defs.pcap", + "vstruct.defs.pe", + "vstruct.defs.pptp", + "vstruct.defs.rar", + "vstruct.defs.swf", + "vstruct.defs.win32", + "vstruct.defs.windows", + "vstruct.defs.windows.win_5_1_i386", + "vstruct.defs.windows.win_5_1_i386.ntdll", + "vstruct.defs.windows.win_5_1_i386.ntoskrnl", + "vstruct.defs.windows.win_5_1_i386.win32k", + "vstruct.defs.windows.win_5_2_i386", + "vstruct.defs.windows.win_5_2_i386.ntdll", + "vstruct.defs.windows.win_5_2_i386.ntoskrnl", + "vstruct.defs.windows.win_5_2_i386.win32k", + "vstruct.defs.windows.win_6_1_amd64", + "vstruct.defs.windows.win_6_1_amd64.ntdll", + "vstruct.defs.windows.win_6_1_amd64.ntoskrnl", + "vstruct.defs.windows.win_6_1_amd64.win32k", + "vstruct.defs.windows.win_6_1_i386", + "vstruct.defs.windows.win_6_1_i386.ntdll", + "vstruct.defs.windows.win_6_1_i386.ntoskrnl", + "vstruct.defs.windows.win_6_1_i386.win32k", + "vstruct.defs.windows.win_6_1_wow64", + "vstruct.defs.windows.win_6_1_wow64.ntdll", + "vstruct.defs.windows.win_6_2_amd64", + "vstruct.defs.windows.win_6_2_amd64.ntdll", + "vstruct.defs.windows.win_6_2_amd64.ntoskrnl", + "vstruct.defs.windows.win_6_2_amd64.win32k", + "vstruct.defs.windows.win_6_2_i386", + "vstruct.defs.windows.win_6_2_i386.ntdll", + "vstruct.defs.windows.win_6_2_i386.ntoskrnl", + "vstruct.defs.windows.win_6_2_i386.win32k", + "vstruct.defs.windows.win_6_2_wow64", + "vstruct.defs.windows.win_6_2_wow64.ntdll", + "vstruct.defs.windows.win_6_3_amd64", + "vstruct.defs.windows.win_6_3_amd64.ntdll", + "vstruct.defs.windows.win_6_3_amd64.ntoskrnl", + "vstruct.defs.windows.win_6_3_i386", + "vstruct.defs.windows.win_6_3_i386.ntdll", + "vstruct.defs.windows.win_6_3_i386.ntoskrnl", + "vstruct.defs.windows.win_6_3_wow64", + "vstruct.defs.windows.win_6_3_wow64.ntdll", + ], + # when invoking pyinstaller from the project root, + # this gets run from the project root. + hookspath=['.github/pyinstaller/hooks'], + runtime_hooks=None, + excludes=[ + # ignore packages that would otherwise be bundled with the .exe. + # review: build/pyinstaller/xref-pyinstaller.html + + # we don't do any GUI stuff, so ignore these modules + "tkinter", + "_tkinter", + "Tkinter", + # tqdm provides renderers for ipython, + # however, this drags in a lot of dependencies. + # since we don't spawn a notebook, we can safely remove these. + "IPython", + "ipywidgets", + ]) + +a.binaries = a.binaries - TOC([ + ('tcl85.dll', None, None), + ('tk85.dll', None, None), + ('_tkinter', None, None)]) + +pyz = PYZ(a.pure, a.zipped_data) + +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + exclude_binaries=False, + name='capa', + icon='logo.ico', + debug=False, + strip=None, + upx=True, + console=True ) + +# enable the following to debug the contents of the .exe +# +#coll = COLLECT(exe, +# a.binaries, +# a.zipfiles, +# a.datas, +# strip=None, +# upx=True, +# name='capa-dat') +