diff --git a/.github/pyinstaller/hooks/hook-smda.py b/.github/pyinstaller/hooks/hook-smda.py new file mode 100644 index 00000000..23fe4b82 --- /dev/null +++ b/.github/pyinstaller/hooks/hook-smda.py @@ -0,0 +1,5 @@ +# Copyright (C) 2020 FireEye, Inc. All Rights Reserved. +import PyInstaller.utils.hooks + +# ref: https://groups.google.com/g/pyinstaller/c/amWi0-66uZI/m/miPoKfWjBAAJ +binaries = PyInstaller.utils.hooks.collect_dynamic_libs("capstone") diff --git a/.github/pyinstaller/hooks/hook-vivisect.py b/.github/pyinstaller/hooks/hook-vivisect.py index 6e6504de..b98ce4f1 100644 --- a/.github/pyinstaller/hooks/hook-vivisect.py +++ b/.github/pyinstaller/hooks/hook-vivisect.py @@ -13,3 +13,144 @@ from PyInstaller.utils.hooks import copy_metadata # # ref: https://github.com/pyinstaller/pyinstaller/issues/1713#issuecomment-162682084 datas = copy_metadata("vivisect") + +excludedimports = [ + # viv gui requires these heavy libraries, + # but viv as a library doesn't. + # they shouldn't be installed in our configuration, + # but we'll ensure they don't slip in here (such as on developers' systems). + "PyQt5", + "qt5", + "pyqtwebengine", + # the above are imported by these viv modules. + # so really, we'd want to exclude these submodules of viv. + # but i dont think this works. + "vqt", + "vdb.qt", + "envi.qt", + # unused by capa + "pyasn1", +] + +hiddenimports = [ + # vivisect does manual/runtime importing of its modules, + # so declare the things that could be imported here. + "vivisect", + "vivisect.analysis", + "vivisect.analysis.amd64", + "vivisect.analysis.amd64", + "vivisect.analysis.amd64.emulation", + "vivisect.analysis.amd64.golang", + "vivisect.analysis.crypto", + "vivisect.analysis.crypto", + "vivisect.analysis.crypto.constants", + "vivisect.analysis.elf", + "vivisect.analysis.elf", + "vivisect.analysis.elf.elfplt", + "vivisect.analysis.elf.libc_start_main", + "vivisect.analysis.generic", + "vivisect.analysis.generic", + "vivisect.analysis.generic.codeblocks", + "vivisect.analysis.generic.emucode", + "vivisect.analysis.generic.entrypoints", + "vivisect.analysis.generic.funcentries", + "vivisect.analysis.generic.impapi", + "vivisect.analysis.generic.mkpointers", + "vivisect.analysis.generic.pointers", + "vivisect.analysis.generic.pointertables", + "vivisect.analysis.generic.relocations", + "vivisect.analysis.generic.strconst", + "vivisect.analysis.generic.switchcase", + "vivisect.analysis.generic.thunks", + "vivisect.analysis.generic.noret", + "vivisect.analysis.i386", + "vivisect.analysis.i386", + "vivisect.analysis.i386.calling", + "vivisect.analysis.i386.golang", + "vivisect.analysis.i386.importcalls", + "vivisect.analysis.i386.instrhook", + "vivisect.analysis.i386.thunk_bx", + "vivisect.analysis.ms", + "vivisect.analysis.ms", + "vivisect.analysis.ms.hotpatch", + "vivisect.analysis.ms.localhints", + "vivisect.analysis.ms.msvc", + "vivisect.analysis.ms.msvcfunc", + "vivisect.analysis.ms.vftables", + "vivisect.analysis.pe", + "vivisect.impapi.posix.amd64", + "vivisect.impapi.posix.i386", + "vivisect.impapi.windows", + "vivisect.impapi.windows.amd64", + "vivisect.impapi.windows.i386", + "vivisect.impapi.winkern.i386", + "vivisect.impapi.winkern.amd64", + "vivisect.parsers.blob", + "vivisect.parsers.elf", + "vivisect.parsers.ihex", + "vivisect.parsers.macho", + "vivisect.parsers.pe", + "vivisect.storage", + "vivisect.storage.basicfile", + "vstruct.constants", + "vstruct.constants.ntstatus", + "vstruct.defs", + "vstruct.defs.arm7", + "vstruct.defs.bmp", + "vstruct.defs.dns", + "vstruct.defs.elf", + "vstruct.defs.gif", + "vstruct.defs.ihex", + "vstruct.defs.inet", + "vstruct.defs.java", + "vstruct.defs.kdcom", + "vstruct.defs.macho", + "vstruct.defs.macho.const", + "vstruct.defs.macho.fat", + "vstruct.defs.macho.loader", + "vstruct.defs.macho.stabs", + "vstruct.defs.minidump", + "vstruct.defs.pcap", + "vstruct.defs.pe", + "vstruct.defs.pptp", + "vstruct.defs.rar", + "vstruct.defs.swf", + "vstruct.defs.win32", + "vstruct.defs.windows", + "vstruct.defs.windows.win_5_1_i386", + "vstruct.defs.windows.win_5_1_i386.ntdll", + "vstruct.defs.windows.win_5_1_i386.ntoskrnl", + "vstruct.defs.windows.win_5_1_i386.win32k", + "vstruct.defs.windows.win_5_2_i386", + "vstruct.defs.windows.win_5_2_i386.ntdll", + "vstruct.defs.windows.win_5_2_i386.ntoskrnl", + "vstruct.defs.windows.win_5_2_i386.win32k", + "vstruct.defs.windows.win_6_1_amd64", + "vstruct.defs.windows.win_6_1_amd64.ntdll", + "vstruct.defs.windows.win_6_1_amd64.ntoskrnl", + "vstruct.defs.windows.win_6_1_amd64.win32k", + "vstruct.defs.windows.win_6_1_i386", + "vstruct.defs.windows.win_6_1_i386.ntdll", + "vstruct.defs.windows.win_6_1_i386.ntoskrnl", + "vstruct.defs.windows.win_6_1_i386.win32k", + "vstruct.defs.windows.win_6_1_wow64", + "vstruct.defs.windows.win_6_1_wow64.ntdll", + "vstruct.defs.windows.win_6_2_amd64", + "vstruct.defs.windows.win_6_2_amd64.ntdll", + "vstruct.defs.windows.win_6_2_amd64.ntoskrnl", + "vstruct.defs.windows.win_6_2_amd64.win32k", + "vstruct.defs.windows.win_6_2_i386", + "vstruct.defs.windows.win_6_2_i386.ntdll", + "vstruct.defs.windows.win_6_2_i386.ntoskrnl", + "vstruct.defs.windows.win_6_2_i386.win32k", + "vstruct.defs.windows.win_6_2_wow64", + "vstruct.defs.windows.win_6_2_wow64.ntdll", + "vstruct.defs.windows.win_6_3_amd64", + "vstruct.defs.windows.win_6_3_amd64.ntdll", + "vstruct.defs.windows.win_6_3_amd64.ntoskrnl", + "vstruct.defs.windows.win_6_3_i386", + "vstruct.defs.windows.win_6_3_i386.ntdll", + "vstruct.defs.windows.win_6_3_i386.ntoskrnl", + "vstruct.defs.windows.win_6_3_wow64", + "vstruct.defs.windows.win_6_3_wow64.ntdll", +] diff --git a/.github/pyinstaller/pyinstaller.spec b/.github/pyinstaller/pyinstaller.spec index 58925289..b8967751 100644 --- a/.github/pyinstaller/pyinstaller.spec +++ b/.github/pyinstaller/pyinstaller.spec @@ -16,9 +16,10 @@ with open('./capa/version.py', 'wb') as f: # - commits since # g------- git hash fragment version = (subprocess.check_output(["git", "describe", "--always", "--tags", "--long"]) + .decode("utf-8") .strip() .replace("tags/", "")) - f.write("__version__ = '%s'" % version) + f.write(("__version__ = '%s'" % version).encode("utf-8")) a = Analysis( # when invoking pyinstaller from the project root, @@ -41,128 +42,6 @@ a = Analysis( # ref: https://stackoverflow.com/a/62278462/87207 (os.path.dirname(wcwidth.__file__), 'wcwidth') ], - hiddenimports=[ - # vivisect does manual/runtime importing of its modules, - # so declare the things that could be imported here. - "vivisect", - "vivisect.analysis", - "vivisect.analysis.amd64", - "vivisect.analysis.amd64", - "vivisect.analysis.amd64.emulation", - "vivisect.analysis.amd64.golang", - "vivisect.analysis.crypto", - "vivisect.analysis.crypto", - "vivisect.analysis.crypto.constants", - "vivisect.analysis.elf", - "vivisect.analysis.elf", - "vivisect.analysis.elf.elfplt", - "vivisect.analysis.elf.libc_start_main", - "vivisect.analysis.generic", - "vivisect.analysis.generic", - "vivisect.analysis.generic.codeblocks", - "vivisect.analysis.generic.emucode", - "vivisect.analysis.generic.entrypoints", - "vivisect.analysis.generic.funcentries", - "vivisect.analysis.generic.impapi", - "vivisect.analysis.generic.mkpointers", - "vivisect.analysis.generic.pointers", - "vivisect.analysis.generic.pointertables", - "vivisect.analysis.generic.relocations", - "vivisect.analysis.generic.strconst", - "vivisect.analysis.generic.switchcase", - "vivisect.analysis.generic.thunks", - "vivisect.analysis.i386", - "vivisect.analysis.i386", - "vivisect.analysis.i386.calling", - "vivisect.analysis.i386.golang", - "vivisect.analysis.i386.importcalls", - "vivisect.analysis.i386.instrhook", - "vivisect.analysis.i386.thunk_bx", - "vivisect.analysis.ms", - "vivisect.analysis.ms", - "vivisect.analysis.ms.hotpatch", - "vivisect.analysis.ms.localhints", - "vivisect.analysis.ms.msvc", - "vivisect.analysis.ms.msvcfunc", - "vivisect.analysis.ms.vftables", - "vivisect.analysis.pe", - "vivisect.impapi.posix.amd64", - "vivisect.impapi.posix.i386", - "vivisect.impapi.windows", - "vivisect.impapi.windows.amd64", - "vivisect.impapi.windows.i386", - "vivisect.impapi.winkern.i386", - "vivisect.impapi.winkern.amd64", - "vivisect.parsers.blob", - "vivisect.parsers.elf", - "vivisect.parsers.ihex", - "vivisect.parsers.macho", - "vivisect.parsers.pe", - "vivisect.parsers.utils", - "vivisect.storage", - "vivisect.storage.basicfile", - "vstruct.constants", - "vstruct.constants.ntstatus", - "vstruct.defs", - "vstruct.defs.arm7", - "vstruct.defs.bmp", - "vstruct.defs.dns", - "vstruct.defs.elf", - "vstruct.defs.gif", - "vstruct.defs.ihex", - "vstruct.defs.inet", - "vstruct.defs.java", - "vstruct.defs.kdcom", - "vstruct.defs.macho", - "vstruct.defs.macho.const", - "vstruct.defs.macho.fat", - "vstruct.defs.macho.loader", - "vstruct.defs.macho.stabs", - "vstruct.defs.minidump", - "vstruct.defs.pcap", - "vstruct.defs.pe", - "vstruct.defs.pptp", - "vstruct.defs.rar", - "vstruct.defs.swf", - "vstruct.defs.win32", - "vstruct.defs.windows", - "vstruct.defs.windows.win_5_1_i386", - "vstruct.defs.windows.win_5_1_i386.ntdll", - "vstruct.defs.windows.win_5_1_i386.ntoskrnl", - "vstruct.defs.windows.win_5_1_i386.win32k", - "vstruct.defs.windows.win_5_2_i386", - "vstruct.defs.windows.win_5_2_i386.ntdll", - "vstruct.defs.windows.win_5_2_i386.ntoskrnl", - "vstruct.defs.windows.win_5_2_i386.win32k", - "vstruct.defs.windows.win_6_1_amd64", - "vstruct.defs.windows.win_6_1_amd64.ntdll", - "vstruct.defs.windows.win_6_1_amd64.ntoskrnl", - "vstruct.defs.windows.win_6_1_amd64.win32k", - "vstruct.defs.windows.win_6_1_i386", - "vstruct.defs.windows.win_6_1_i386.ntdll", - "vstruct.defs.windows.win_6_1_i386.ntoskrnl", - "vstruct.defs.windows.win_6_1_i386.win32k", - "vstruct.defs.windows.win_6_1_wow64", - "vstruct.defs.windows.win_6_1_wow64.ntdll", - "vstruct.defs.windows.win_6_2_amd64", - "vstruct.defs.windows.win_6_2_amd64.ntdll", - "vstruct.defs.windows.win_6_2_amd64.ntoskrnl", - "vstruct.defs.windows.win_6_2_amd64.win32k", - "vstruct.defs.windows.win_6_2_i386", - "vstruct.defs.windows.win_6_2_i386.ntdll", - "vstruct.defs.windows.win_6_2_i386.ntoskrnl", - "vstruct.defs.windows.win_6_2_i386.win32k", - "vstruct.defs.windows.win_6_2_wow64", - "vstruct.defs.windows.win_6_2_wow64.ntdll", - "vstruct.defs.windows.win_6_3_amd64", - "vstruct.defs.windows.win_6_3_amd64.ntdll", - "vstruct.defs.windows.win_6_3_amd64.ntoskrnl", - "vstruct.defs.windows.win_6_3_i386", - "vstruct.defs.windows.win_6_3_i386.ntdll", - "vstruct.defs.windows.win_6_3_i386.ntoskrnl", - "vstruct.defs.windows.win_6_3_wow64", - "vstruct.defs.windows.win_6_3_wow64.ntdll", - ], # when invoking pyinstaller from the project root, # this gets run from the project root. hookspath=['.github/pyinstaller/hooks'], @@ -180,6 +59,25 @@ a = Analysis( # since we don't spawn a notebook, we can safely remove these. "IPython", "ipywidgets", + + # these are pulled in by networkx + # but we don't need to compute the strongly connected components. + "numpy", + "scipy", + "matplotlib", + "pandas", + "pytest", + + # deps from viv that we don't use. + # this duplicates the entries in `hook-vivisect`, + # but works better this way. + "vqt", + "vdb.qt", + "envi.qt", + "PyQt5", + "qt5", + "pyqtwebengine", + "pyasn1" ]) a.binaries = a.binaries - TOC([ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 474d9a2e..62d8ae56 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,19 +26,14 @@ jobs: uses: actions/checkout@v2 with: submodules: true - - name: Set up Python 2.7 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 2.7 + python-version: 3.9 - if: matrix.os == 'ubuntu-latest' run: sudo apt-get install -y libyaml-dev - - if: matrix.os == 'windows-latest' - run: | - choco install vcredist2008 - choco install --ignore-dependencies vcpython27 - name: Install PyInstaller - # pyinstaller 4 doesn't support Python 2.7 - run: pip install 'pyinstaller==3.*' + run: pip install 'pyinstaller==4.2' - name: Install capa run: pip install -e . - name: Build standalone executable