Merge pull request #515 from Ana06/v1-6-1

Add placeholder for master (unreleased changes) in CHANGELOG. Document
this in the release checklist.

.gitattributes

@@ -0,0 +1,9 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto
+
+# Explicitly declare text files you want to always be normalized and converted
+# to native line endings on checkout.
+*.py text
+*.yml text
+*.md text
+*.txt text

.github/ISSUE_TEMPLATE/bug_report.md

@@ -5,7 +5,7 @@ about: Create a report to help us improve
 ---
 <!--
 # Is your bug report related to capa rules (for example a false positive)?
-We use sybmodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/fireeye/capa-rules/issues.
+We use submodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/fireeye/capa-rules/issues.
 
 # Have you checked that your issue isn't already filed?
 Please search if there is a similar issue at https://github.com/fireeye/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -5,7 +5,7 @@ about: Suggest an idea for capa
 ---
 <!--
 # Is your issue related to capa rules (for example an idea for a new rule)?
-We use sybmodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/fireeye/capa-rules/issues.
+We use submodules to separate code, rules and test data. If your issue is related to capa rules, please report it at https://github.com/fireeye/capa-rules/issues.
 
 # Have you checked that your issue isn't already filed?
 Please search if there is a similar issue at https://github.com/fireeye/capa/issues. If there is already a similar issue, please add more details there instead of opening a new one.

.github/pull_request_template.md

@@ -0,0 +1,32 @@
+<!--
+Thank you for contributing to capa! :heart:
+
+IMPORTANT NOTE
+It's most important that you submit your improvements. So even if you don't use this complete template we look forward to collaborating!
+
+Please read capa's CONTRIBUTING guide if you haven't done so already.
+It contains helpful information about how to contribute to capa. Check https://github.com/fireeye/capa/blob/master/.github/CONTRIBUTING.md
+
+PR template based on https://embeddedartistry.com/blog/2017/08/04/a-github-pull-request-template-for-your-projects/
+-->
+
+### Description
+
+<!-- Please describe the changes in this PR. Including your motivation and context helps us to review. -->
+
+closes # (issue)
+
+### Type of change
+
+Please update the [CHANGELOG.md](/CHANGELOG.md)
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+  - [ ] I have made the corresponding changes to the documentation
+
+### Tests
+
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] No new tests needed

.github/workflows/build.yml

@@ -15,10 +15,10 @@ jobs:
             # use old linux so that the shared library versioning is more portable
             artifact_name: capa
             asset_name: linux
-          - os: windows-latest
+          - os: windows-2019
             artifact_name: capa.exe
             asset_name: windows
-          - os: macos-latest
+          - os: macos-10.15
             artifact_name: capa
             asset_name: macos
     steps:
@@ -30,7 +30,7 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
-      - if: matrix.os == 'ubuntu-latest'
+      - if: matrix.os == 'ubuntu-16.04'
         run: sudo apt-get install -y libyaml-dev
       - name: Install PyInstaller
         run: pip install 'pyinstaller==4.2'
@@ -47,7 +47,7 @@ jobs:
 
   zip:
     name: zip ${{ matrix.asset_name }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs: build
     strategy:
       matrix:

.github/workflows/publish.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   deploy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python

.github/workflows/tag.yml

@@ -0,0 +1,24 @@
+name: tag
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  tag:
+    name: Tag capa rules
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout capa-rules
+      uses: actions/checkout@v2
+      with:
+        repository: fireeye/capa-rules
+        token: ${{ secrets.CAPA_TOKEN }}
+    - name: Tag capa-rules
+      run: git tag ${{ github.event.release.tag_name }}
+    - name: Push tag to capa-rules
+      uses: ad-m/github-push-action@master
+      with:
+        repository: fireeye/capa-rules
+        github_token: ${{ secrets.CAPA_TOKEN }}
+        tags: true

.github/workflows/tests.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   code_style:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa
       uses: actions/checkout@v2
@@ -24,7 +24,7 @@ jobs:
       run: black -l 120 --check .
 
   rule_linter:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - name: Checkout capa with rules submodule
       uses: actions/checkout@v2
@@ -41,30 +41,39 @@ jobs:
       run: python scripts/lint.py rules/
 
   tests:
-    name: Tests in ${{ matrix.python }}
+    name: Tests in ${{ matrix.python-version }} on ${{ matrix.os }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     needs: [code_style, rule_linter]
     strategy:
       fail-fast: false
       matrix:
+        os: [ubuntu-20.04, windows-2019, macos-10.15]
+        # across all operating systems
+        python-version: [3.6, 3.9]
         include:
-          - python: 2.7
+          # on Ubuntu run these as well
-          - python: 3.7
+          - os: ubuntu-20.04
-          - python: 3.8
+            python-version: 2.7
-          - python: 3.9.1
+          - os: ubuntu-20.04
+            python-version: 3.7
+          - os: ubuntu-20.04
+            python-version: 3.8
     steps:
     - name: Checkout capa with submodules
       uses: actions/checkout@v2
       with:
         submodules: true
-    - name: Set up Python ${{ matrix.python }}
+    - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
-        python-version: ${{ matrix.python }}
+        python-version: ${{ matrix.python-version }}
     - name: Install pyyaml
+      if: matrix.os == 'ubuntu-20.04'
       run: sudo apt-get install -y libyaml-dev
+    - name: Install Microsoft Visual C++ 9.0
+      if: matrix.os == 'windows-2019' && matrix.python-version == '2.7'
+      run: choco install vcpython27
     - name: Install capa
       run: pip install -e .[dev]
     - name: Run tests
       run: pytest tests/
-

CHANGELOG.md

@@ -1,5 +1,91 @@
 # Change Log
 
+## master (unreleased)
+
+### New Features
+
+### New Rules
+
+### Bug Fixes
+
+### Changes
+
+### Development
+
+### Raw diffs
+- [capa v1.6.1...master](https://github.com/fireeye/capa/compare/v1.6.1...master)
+- [capa-rules v1.6.1...master](https://github.com/fireeye/capa-rules/compare/v1.6.1...master)
+
+
+## v1.6.1 (2021-04-07)
+
+This release includes several bug fixes, such as a vivisect issue that prevented capa from working on Windows with Python 3. It also adds 17 new rules and a bunch of improvements in the rules and IDA rule generator. We appreciate everyone who opened issues, provided feedback, and contributed code and rules.
+
+### Upcoming changes
+
+**This is the very last capa release that supports Python 2.** The next release will be v2.0 and will have breaking changes, including the removal of Python 2 support.
+
+### New features
+
+- explorer: add support for multi-line tab and SHIFT + Tab #474 @mike-hunhoff
+
+![multi-line tab in rule generator](doc/img/changelog/tab.gif)
+
+### New Rules (17)
+
+- encrypt data using RC4 with custom key via WinAPI @MalwareMechanic
+- encrypt data using Curve25519 @dandonov
+- packaged as an IExpress self-extracting archive @recvfrom
+- create registry key via offline registry library @johnk3r
+- open registry key via offline registry library @johnk3r
+- query registry key via offline registry library @johnk3r
+- set registry key via offline registry library @johnk3r
+- delete registry key via offline registry library @johnk3r
+- enumerate PE sections @Ana06
+- inject DLL reflectively @Ana06
+- inspect section memory permissions @Ana06
+- parse PE exports @Ana06
+- rebuild import table @Ana06
+- compare security identifiers @mike-hunhoff
+- get user security identifier @mike-hunhoff
+- listen for remote procedure calls @mike-hunhoff
+- query remote server for available data @mike-hunhoff
+
+### Bug Fixes
+
+- vivisect: update to v1.0.1 which includes bug fix for #459 (capa failed in Windows with Python 3 and vivisect) #512 @williballenthin
+- explorer: fix initialize rules directory #464 @mike-hunhoff
+- explorer: support subscope rules #493 @mike-hunhoff
+- explorer: add checks to validate matched data when searching #500 @mike-hunhoff
+- features, explorer: add support for string features with special characters e.g. '\n' #468 @mike-hunhoff
+
+### Changes
+
+- vivisect: raises `IncompatibleVivVersion` instead of `UnicodeDecodeError` when using incompatible Python 2 `.viv` files with Python3 #479 @Ana06
+- explorer: improve settings modification #465 @mike-hunhoff
+- rules: improvements @mr-tz, @re-fox, @mike-hunhoff
+- rules, lint: enforce string with double quotes formatting in rules #468 @mike-hunhoff
+- lint: ensure LF end of line #485 #486 @mr-tz
+- setup: pin dependencies #513 #504 @Ana06 @mr-tz
+
+### Development
+
+- ci: test on Windows, Ubuntu, macOS across Python versions #470 @mr-tz @Ana06
+- ci: pin OS versions #491 @williballenthin
+- ci: tag capa-rules on release #476 @Ana06
+- doc: document release process #476 @Ana06
+- doc: Improve README badges #477 #478 @ana06 @mr-tz
+- doc: update capa explorer documentation #503 @mike-hunhoff
+- doc: add PR template #495 @mr-tz
+- changelog: document incompatibility of viv files #475 @Ana06
+- rule loading: ignore files starting with .git #492 @mr-tz
+
+### Raw diffs
+
+  - [capa v1.6.0...v1.6.1](https://github.com/fireeye/capa/compare/v1.6.0...v1.6.1)
+  - [capa-rules v1.6.0...v1.6.1](https://github.com/fireeye/capa-rules/compare/v1.6.0...v1.6.1)
+
+
 ## v1.6.0 (2021-03-09)
 
 This release adds the capa explorer rule generator plugin for IDA Pro, vivisect support for Python 3 and 12 new rules. We appreciate everyone who opened issues, provided feedback, and contributed code and rules. Thank you also to the vivisect development team (@rakuy0, @atlas0fd00m) for the Python 3 support (`vivisect==1.0.0`) and the fixes for Python 2 (`vivisect==0.2.1`).
@@ -10,6 +96,15 @@ The capa explorer IDA plugin now helps you quickly build new capa rules using fe
 
 ![](doc/img/rulegen_expanded.png)
 
+### Python 2/3 vivisect workspace compatibility
+
+This version of capa adds Python 3 support in vivisect. Note that `.viv` files (generated by vivisect) are not compatible between Python 2 and Python 3. When updating to Python 3 you need to delete all the `.viv` files for capa to work.
+
+If you get the following error (or a similar one), you most likely need to delete `.viv` files:
+```
+UnicodeDecodeError: 'ascii' codec can't decode byte 0x90 in position 2: ordinal not in range(128)
+```
+
 ### Upcoming changes
 
 **This is the last capa release that supports Python 2.** The next release will be v2.0 and will have breaking changes, including the removal of Python 2 support.

README.md

@@ -1,7 +1,10 @@
 ![capa](.github/logo.png)
 
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
+[![Last release](https://img.shields.io/github/v/release/fireeye/capa)](https://github.com/fireeye/capa/releases)
+[![Number of rules](https://img.shields.io/badge/rules-485-blue.svg)](https://github.com/fireeye/capa-rules)
 [![CI status](https://github.com/fireeye/capa/workflows/CI/badge.svg)](https://github.com/fireeye/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
-[![Number of rules](https://img.shields.io/badge/rules-469-blue.svg)](https://github.com/fireeye/capa-rules)
+[![Downloads](https://img.shields.io/github/downloads/fireeye/capa/total)](https://github.com/fireeye/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
 
 capa detects capabilities in executable files.
@@ -146,8 +149,8 @@ rule:
 The [github.com/fireeye/capa-rules](https://github.com/fireeye/capa-rules) repository contains hundreds of standard library rules that are distributed with capa.
 Please learn to write rules and contribute new entries as you find interesting techniques in malware.
 
-If you use IDA Pro, then you can use the [capa explorer plugin](capa/ida/plugin/).
+If you use IDA Pro, then you can use the [capa explorer](capa/ida/plugin/) plugin.
-capa explorer lets you quickly identify and navigate to interesting areas of a program and help you build new capa rules out of the features extracted directly from your IDB.
+capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted directly from your IDA Pro database.
 
 ![capa + IDA Pro integration](doc/img/explorer_expanded.png)
 

capa/features/__init__.py

@@ -38,6 +38,20 @@ def hex_string(h):
     return " ".join(h[i : i + 2] for i in range(0, len(h), 2)).upper()
 
 
+def escape_string(s):
+    """escape special characters"""
+    s = repr(s)
+    if not s.startswith(('"', "'")):
+        # u'hello\r\nworld' -> hello\\r\\nworld
+        s = s[2:-1]
+    else:
+        # 'hello\r\nworld' -> hello\\r\\nworld
+        s = s[1:-1]
+    s = s.replace("\\'", "'")  # repr() may escape "'" in some edge cases, remove
+    s = s.replace('"', '\\"')  # repr() does not escape '"', add
+    return s
+
+
 class Feature(object):
     def __init__(self, value, arch=None, description=None):
         """

capa/ida/plugin/README.md

@@ -1,13 +1,12 @@
 ![capa explorer](../../../.github/capa-explorer-logo.png)
 
-capa explorer is an IDA Pro plugin written in Python that integrates the FLARE team's open-source framework, capa, with IDA. capa is a framework that uses a well-defined collection of rules to 
+capa explorer is an IDAPython plugin that integrates the FLARE team's open-source framework, capa, with IDA Pro. capa is a framework that uses a well-defined collection of rules to 
 identify capabilities in a program. You can run capa against a PE file or shellcode and it tells you what it thinks the program can do. For example, it might suggest that 
-the program is a backdoor, can install services, or relies on HTTP to communicate. You can use capa explorer to run capa directly on an IDA database without requiring access
+the program is a backdoor, can install services, or relies on HTTP to communicate. capa explorer runs capa directly against your IDA Pro database (IDB) without requiring access
-to or execution of the source binary. Once a database has been analyzed, capa explorer can be used to quickly identify and navigate to interesting areas of a program and manually build new capa rules out
+to the original binary file. Once a database has been analyzed, capa explorer helps you identify interesting areas of a program and build new capa rules using features extracted from your IDB.
-of the features extracted directly from your IDB.
 
 We love using capa explorer during malware analysis because it teaches us what parts of a program suggest a behavior. As we click on rows, capa explorer jumps directly 
-to important addresses in the IDA Pro database and highlights key features in the Disassembly view so they stand out visually. To illustrate, we use capa explorer to 
+to important addresses in the IDB and highlights key features in the Disassembly view so they stand out visually. To illustrate, we use capa explorer to 
 analyze Lab 14-02 from [Practical Malware Analysis](https://nostarch.com/malware) (PMA) available [here](https://practicalmalwareanalysis.com/labs/). Our goal is to understand 
 the program's functionality.
 
@@ -15,16 +14,15 @@ After loading Lab 14-02 into IDA and analyzing the database with capa explorer,
 
 ![](../../../doc/img/explorer_condensed.png)
 
-We can use capa explorer to navigate the IDA Disassembly view directly to the suspect function and get an assembly-level breakdown of why capa matched `self delete via COMSPEC environment variable` 
+We can use capa explorer to navigate our Disassembly view directly to the suspect function and get an assembly-level breakdown of why capa matched `self delete via COMSPEC environment variable`.
-for this particular function.
 
 ![](../../../doc/img/explorer_expanded.png)
 
 Using the `Rule Information` and `Details` columns capa explorer shows us that the suspect function matched `self delete via COMSPEC environment variable` because it contains capa rule matches for `create process`, `get COMSPEC environment variable`,
-and `query environment variable`, references to the strings `COMSPEC`, ` > nul`, and `/c del`, and calls to the Windows API functions `GetEnvironmentVariableA` and `ShellExecuteEx`.
+and `query environment variable`, references to the strings `COMSPEC`, ` > nul`, and `/c del `, and calls to the Windows API functions `GetEnvironmentVariableA` and `ShellExecuteEx`.
 
-capa explorer also helps you build new capa rules. To start select the `Rule Generator` tab, navigate to a function in the IDA `Disassembly` view,
+capa explorer also helps you build new capa rules. To start select the `Rule Generator` tab, navigate to a function in your Disassembly view,
-and click `Analyze`. capa explorer will extract features from this function and display them in the `Function Features` pane. You can add features listed in this pane to the `Editor` pane
+and click `Analyze`. capa explorer will extract features from the function and display them in the `Features` pane. You can add features listed in this pane to the `Editor` pane
 by either double-clicking a feature or using multi-select + right-click to add multiple features at once. The `Preview` and `Editor` panes help edit your rule. Use the `Preview` pane
 to modify the rule text directly and the `Editor` pane to construct and rearrange your hierarchy of statements and features. When you finish a rule you can save it directly to a file by clicking `Save`.
 
@@ -44,7 +42,7 @@ If you encounter issues with your specific setup, please open a new [Issue](http
 
 ### Supported File Types
 
-capa explorer is limited to the file types supported by capa, which includes:
+capa explorer is limited to the file types supported by capa, which include:
 
 * Windows 32-bit and 64-bit PE files
 * Windows 32-bit and 64-bit shellcode
@@ -62,50 +60,48 @@ You can install capa explorer using the following steps:
 
 ### Usage
 
-1. Run IDA and analyze a supported file type (select the `Manual Load` and `Load Resources` options in IDA for best results)
+1. Open IDA and analyze a supported file type (select the `Manual Load` and `Load Resources` options in IDA for best results)
 2. Open capa explorer in IDA by navigating to `Edit > Plugins > FLARE capa explorer` or using the keyboard shortcut `Alt+F5`
 3. Select the `Program Analysis` tab
 4. Click the `Analyze` button
 
 When running capa explorer for the first time you are prompted to select a file directory containing capa rules. The plugin conveniently
-remembers your selection for future runs; you can change this selection by navigating to `Settings > Change default rules directory...`. We recommend 
+remembers your selection for future runs; you can change this selection and other default settings by clicking `Settings`. We recommend 
 downloading and using the [standard collection of capa rules](https://github.com/fireeye/capa-rules) when getting started with the plugin.
 
 #### Tips for Program Analysis
 
 * Start analysis by clicking the `Analyze` button
-* Reset the plugin user interface and remove highlighting from IDA disassembly view by clicking the `Reset` button
+* Reset the plugin user interface and remove highlighting from your Disassembly view by clicking the `Reset` button
-* Change your capa rules directory by navigating to `Settings > Change default rules directory...` from the plugin menu
+* Change your capa rules directory and other default settings by clicking `Settings`
 * Hover your cursor over a rule match to view the source content of the rule
-* Double-click the `Address` column to navigate the IDA Disassembly view to the associated feature
+* Double-click the `Address` column to navigate your Disassembly view to the address of the associated feature
 * Double-click a result in the `Rule Information` column to expand its children
-* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in the IDA Dissasembly view
+* Select a checkbox in the `Rule Information` column to highlight the address of the associated feature in your Dissasembly view
 
 #### Tips for Rule Generator
 
-* Navigate to a function in the `Disassembly` view and click`Analyze` to get started
+* Navigate to a function in your Disassembly view and click`Analyze` to get started
-* Double-click or multi-select + right-click in the `Function Features` pane to add features to the `Editor` pane
+* Double-click or use multi-select + right-click to add features from the `Features` pane to the `Editor` pane
-* Right-click features in the `Editor` pane to make modifications
+* Right-click features in the `Editor` pane to make context-specific modifications
-* Drag-and-drop (single click + multi-select support) features in the `Editor` pane to quickly build a hierarchy of statements and features
+* Drag-and-drop (single click + multi-select support) features in the `Editor` pane to construct your hierarchy of statements and features
-* Right-click anywhere in the `Editor` pane not on a feature to quickly remove all features
+* Right-click anywhere in the `Editor` pane not on a feature to remove all features
-* Add descriptions/comments by placing editing the appropriate column in the `Editor` pane
+* Add descriptions or comments to a feature by editing the corresponding column in the `Editor` pane
-* Directly edit rule text, including rule metadata fields using the `Preview` pane
+* Directly edit rule text and metadata fields using the `Preview` pane
-* Change the default rule author and default scope displayed in the `Preview` pane by navigating to `Settings`
+* Change the default rule author and default rule scope displayed in the `Preview` pane by clicking `Settings`
 
 ## Development
 
-Because capa explorer is packaged with capa you will need to install capa locally for development.
+capa explorer is packaged with capa so you will need to install capa locally for development. You can install capa locally by following the steps outlined in `Method 3: Inspecting the capa source code` of the [capa 
-
-You can install capa locally by following the steps outlined in `Method 3: Inspecting the capa source code` of the [capa 
 installation guide](https://github.com/fireeye/capa/blob/master/doc/installation.md#method-3-inspecting-the-capa-source-code). Once installed, copy [capa_explorer.py](https://raw.githubusercontent.com/fireeye/capa/master/capa/ida/plugin/capa_explorer.py) 
-to your IDA plugins directory to run the plugin in IDA.
+to your plugins directory to install capa explorer in IDA.
 
 ### Components
 
 capa explorer consists of two main components:
 
-* An IDA [feature extractor](https://github.com/fireeye/capa/tree/master/capa/features/extractors/ida) built on top of IDA's binary analysis engine
+* An [feature extractor](https://github.com/fireeye/capa/tree/master/capa/features/extractors/ida) built on top of IDA's binary analysis engine
-  * This component uses IDAPython to extract [capa features](https://github.com/fireeye/capa-rules/blob/master/doc/format.md#extracted-features) from the IDA database such as strings, 
+  * This component uses IDAPython to extract [capa features](https://github.com/fireeye/capa-rules/blob/master/doc/format.md#extracted-features) from your IDBs such as strings, 
 disassembly, and control flow; these extracted features are used by capa to find feature combinations that result in a rule match
 * An [interactive user interface](https://github.com/fireeye/capa/tree/master/capa/ida/plugin) for displaying and exploring capa rule matches
-  * This component integrates the IDA feature extractor and capa, providing an interactive user interface to dissect rule matches found by capa using features extracted by the IDA feature extractor
+  * This component integrates the feature extractor and capa, providing an interactive user interface to dissect rule matches found by capa using features extracted directly from your IDBs

capa/ida/plugin/form.py

@@ -37,6 +37,10 @@ from capa.ida.plugin.proxy import CapaExplorerRangeProxyModel, CapaExplorerSearc
 logger = logging.getLogger(__name__)
 settings = ida_settings.IDASettings("capa")
 
+CAPA_SETTINGS_RULE_PATH = "rule_path"
+CAPA_SETTINGS_RULEGEN_AUTHOR = "rulegen_author"
+CAPA_SETTINGS_RULEGEN_SCOPE = "rulegen_scope"
+
 
 def write_file(path, data):
     """ """
@@ -166,6 +170,60 @@ class CapaExplorerFeatureExtractor(capa.features.extractors.ida.IdaFeatureExtrac
         return super(CapaExplorerFeatureExtractor, self).extract_function_features(f)
 
 
+class QLineEditClicked(QtWidgets.QLineEdit):
+    def __init__(self, content, parent=None):
+        """ """
+        super(QLineEditClicked, self).__init__(content, parent)
+
+    def mouseReleaseEvent(self, e):
+        """ """
+        old = self.text()
+        new = str(
+            QtWidgets.QFileDialog.getExistingDirectory(
+                self.parent(), "Please select a capa rules directory", settings.user.get(CAPA_SETTINGS_RULE_PATH, "")
+            )
+        )
+        if new:
+            self.setText(new)
+        else:
+            self.setText(old)
+
+
+class CapaSettingsInputDialog(QtWidgets.QDialog):
+    def __init__(self, title, parent=None):
+        """ """
+        super(CapaSettingsInputDialog, self).__init__(parent)
+
+        self.setWindowTitle(title)
+        self.setMinimumWidth(500)
+        self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowContextHelpButtonHint)
+
+        self.edit_rule_path = QLineEditClicked(settings.user.get(CAPA_SETTINGS_RULE_PATH, ""))
+        self.edit_rule_author = QtWidgets.QLineEdit(settings.user.get(CAPA_SETTINGS_RULEGEN_AUTHOR, ""))
+        self.edit_rule_scope = QtWidgets.QComboBox()
+
+        scopes = ("file", "function", "basic block")
+
+        self.edit_rule_scope.addItems(scopes)
+        self.edit_rule_scope.setCurrentIndex(scopes.index(settings.user.get(CAPA_SETTINGS_RULEGEN_SCOPE, "function")))
+
+        buttons = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Cancel, self)
+
+        layout = QtWidgets.QFormLayout(self)
+        layout.addRow("capa rules path", self.edit_rule_path)
+        layout.addRow("Default rule author", self.edit_rule_author)
+        layout.addRow("Default rule scope", self.edit_rule_scope)
+
+        layout.addWidget(buttons)
+
+        buttons.accepted.connect(self.accept)
+        buttons.rejected.connect(self.reject)
+
+    def get_values(self):
+        """ """
+        return self.edit_rule_path.text(), self.edit_rule_author.text(), self.edit_rule_scope.currentText()
+
+
 class CapaExplorerForm(idaapi.PluginForm):
     """form element for plugin interface"""
 
@@ -197,11 +255,11 @@ class CapaExplorerForm(idaapi.PluginForm):
         self.view_rulegen = None
         self.view_tabs = None
         self.view_tab_rulegen = None
-        self.view_menu_bar = None
         self.view_status_label = None
         self.view_buttons = None
         self.view_analyze_button = None
         self.view_reset_button = None
+        self.view_settings_button = None
         self.view_save_button = None
 
         self.view_rulegen_preview = None
@@ -273,10 +331,6 @@ class CapaExplorerForm(idaapi.PluginForm):
         self.load_view_status_label()
         self.load_view_buttons()
 
-        # load menu bar and sub menus
-        self.load_view_menu_bar()
-        self.load_configure_menu()
-
         # load parent view
         self.load_view_parent()
 
@@ -285,11 +339,6 @@ class CapaExplorerForm(idaapi.PluginForm):
         tabs = QtWidgets.QTabWidget()
         self.view_tabs = tabs
 
-    def load_view_menu_bar(self):
-        """load menu bar"""
-        bar = QtWidgets.QMenuBar()
-        self.view_menu_bar = bar
-
     def load_view_checkbox_limit_by(self):
         """load limit results by function checkbox"""
         check = QtWidgets.QCheckBox("Limit results to current function")
@@ -319,19 +368,23 @@ class CapaExplorerForm(idaapi.PluginForm):
         analyze_button = QtWidgets.QPushButton("Analyze")
         reset_button = QtWidgets.QPushButton("Reset")
         save_button = QtWidgets.QPushButton("Save")
+        settings_button = QtWidgets.QPushButton("Settings")
 
         analyze_button.clicked.connect(self.slot_analyze)
         reset_button.clicked.connect(self.slot_reset)
         save_button.clicked.connect(self.slot_save)
+        settings_button.clicked.connect(self.slot_settings)
 
         layout = QtWidgets.QHBoxLayout()
         layout.addWidget(analyze_button)
         layout.addWidget(reset_button)
-        layout.addStretch(2)
+        layout.addWidget(settings_button)
+        layout.addStretch(3)
         layout.addWidget(save_button, alignment=QtCore.Qt.AlignRight)
 
         self.view_analyze_button = analyze_button
         self.view_reset_button = reset_button
+        self.view_settings_button = settings_button
         self.view_save_button = save_button
         self.view_buttons = layout
 
@@ -350,7 +403,6 @@ class CapaExplorerForm(idaapi.PluginForm):
         layout.addWidget(self.view_tabs)
         layout.addLayout(self.view_buttons)
         layout.addWidget(self.view_status_label)
-        layout.setMenuBar(self.view_menu_bar)
 
         self.parent.setLayout(layout)
 
@@ -450,27 +502,6 @@ class CapaExplorerForm(idaapi.PluginForm):
 
         self.view_tabs.addTab(tab, "Rule Generator")
 
-    def load_configure_menu(self):
-        """ """
-        actions = (
-            ("Change default rules directory...", "Set default rules directory", self.slot_change_rules_dir),
-            ("Change default rule author...", "Set default rule author", self.slot_change_rule_author),
-            ("Change default rule scope...", "Set default rule scope", self.slot_change_rule_scope),
-        )
-        self.load_menu("Settings", actions)
-
-    def load_menu(self, title, actions):
-        """load menu actions
-
-        @param title: menu name displayed in UI
-        @param actions: tuple of tuples containing action name, tooltip, and slot function
-        """
-        menu = self.view_menu_bar.addMenu(title)
-        for (name, _, slot) in actions:
-            action = QtWidgets.QAction(name, self.parent)
-            action.triggered.connect(slot)
-            menu.addAction(action)
-
     def load_ida_hooks(self):
         """load IDA UI hooks"""
         # map named action (defined in idagui.cfg) to Python function
@@ -567,7 +598,7 @@ class CapaExplorerForm(idaapi.PluginForm):
 
         try:
             # resolve rules directory - check self and settings first, then ask user
-            if not os.path.exists(settings.user.get("rule_path", "")):
+            if not os.path.exists(settings.user.get(CAPA_SETTINGS_RULE_PATH, "")):
                 idaapi.info("Please select a file directory containing capa rules.")
                 path = self.ask_user_directory()
                 if not path:
@@ -575,7 +606,7 @@ class CapaExplorerForm(idaapi.PluginForm):
                         "You must select a file directory containing capa rules before analysis can be run. The standard collection of capa rules can be downloaded from https://github.com/fireeye/capa-rules."
                     )
                     return False
-                settings.user["rule_path"] = path
+                settings.user[CAPA_SETTINGS_RULE_PATH] = path
         except Exception as e:
             logger.error("Failed to load capa rules (error: %s).", e)
             return False
@@ -584,8 +615,9 @@ class CapaExplorerForm(idaapi.PluginForm):
             logger.info("User cancelled analysis.")
             return False
 
-        rule_path = settings.user["rule_path"]
+        rule_path = settings.user[CAPA_SETTINGS_RULE_PATH]
         try:
+            # TODO refactor: this first part is identical to capa.main.get_rules
             if not os.path.exists(rule_path):
                 raise IOError("rule path %s does not exist or cannot be accessed" % rule_path)
 
@@ -601,8 +633,8 @@ class CapaExplorerForm(idaapi.PluginForm):
                         continue
                     for file in files:
                         if not file.endswith(".yml"):
-                            if not (file.endswith(".md") or file.endswith(".git") or file.endswith(".txt")):
+                            if not (file.startswith(".git") or file.endswith((".git", ".md", ".txt"))):
-                                # expect to see readme.md, format.md, and maybe a .git directory
+                                # expect to see .git* files, readme.md, format.md, and maybe a .git directory
                                 # other things maybe are rules, but are mis-named.
                                 logger.warning("skipping non-.yml file: %s", file)
                             continue
@@ -613,7 +645,8 @@ class CapaExplorerForm(idaapi.PluginForm):
             total_paths = len(rule_paths)
             for (i, rule_path) in enumerate(rule_paths):
                 update_wait_box(
-                    "loading capa rules from %s (%d of %d)" % (settings.user["rule_path"], i + 1, total_paths)
+                    "loading capa rules from %s (%d of %d)"
+                    % (settings.user[CAPA_SETTINGS_RULE_PATH], i + 1, total_paths)
                 )
                 if ida_kernwin.user_cancelled():
                     raise UserCancelledError("user cancelled")
@@ -632,12 +665,14 @@ class CapaExplorerForm(idaapi.PluginForm):
             logger.info("User cancelled analysis.")
             return False
         except Exception as e:
-            capa.ida.helpers.inform_user_ida_ui("Failed to load capa rules from %s" % settings.user["rule_path"])
+            capa.ida.helpers.inform_user_ida_ui(
-            logger.error("Failed to load rules from %s (error: %s).", settings.user["rule_path"], e)
+                "Failed to load capa rules from %s" % settings.user[CAPA_SETTINGS_RULE_PATH]
+            )
+            logger.error("Failed to load rules from %s (error: %s).", settings.user[CAPA_SETTINGS_RULE_PATH], e)
             logger.error(
                 "Make sure your file directory contains properly formatted capa rules. You can download the standard collection of capa rules from https://github.com/fireeye/capa-rules."
             )
-            settings.user["rule_path"] = ""
+            settings.user[CAPA_SETTINGS_RULE_PATH] = ""
             return False
 
         self.ruleset_cache = ruleset
@@ -743,7 +778,7 @@ class CapaExplorerForm(idaapi.PluginForm):
         try:
             self.model_data.render_capa_doc(self.doc, self.view_show_results_by_function.isChecked())
             self.set_view_status_label(
-                "capa rules directory: %s (%d rules)" % (settings.user["rule_path"], len(self.rules_cache))
+                "capa rules directory: %s (%d rules)" % (settings.user[CAPA_SETTINGS_RULE_PATH], len(self.rules_cache))
             )
         except Exception as e:
             logger.error("Failed to render results (error: %s)", e)
@@ -881,14 +916,14 @@ class CapaExplorerForm(idaapi.PluginForm):
             # load preview and feature tree
             self.view_rulegen_preview.load_preview_meta(
                 f.start_ea if f else None,
-                settings.user.get("rulegen_author", "<insert_author>"),
+                settings.user.get(CAPA_SETTINGS_RULEGEN_AUTHOR, "<insert_author>"),
-                settings.user.get("rulegen_scope", "function"),
+                settings.user.get(CAPA_SETTINGS_RULEGEN_SCOPE, "function"),
             )
             self.view_rulegen_features.load_features(file_features, func_features)
 
             # self.view_rulegen_header_label.setText("Function Features (%s)" % trim_function_name(f))
             self.set_view_status_label(
-                "capa rules directory: %s (%d rules)" % (settings.user["rule_path"], len(self.rules_cache))
+                "capa rules directory: %s (%d rules)" % (settings.user[CAPA_SETTINGS_RULE_PATH], len(self.rules_cache))
             )
         except Exception as e:
             logger.error("Failed to render views (error: %s)" % e)
@@ -985,6 +1020,12 @@ class CapaExplorerForm(idaapi.PluginForm):
 
         # create deep copy of current rules, add our new rule
         rules = copy.copy(self.rules_cache)
+
+        # ensure subscope rules are included
+        for sub in rule.extract_subscope_rules():
+            rules.append(sub)
+
+        # include our new rule in the list
         rules.append(rule)
 
         try:
@@ -1066,6 +1107,16 @@ class CapaExplorerForm(idaapi.PluginForm):
         elif self.view_tabs.currentIndex() == 1:
             self.save_function_analysis()
 
+    def slot_settings(self):
+        """ """
+        dialog = CapaSettingsInputDialog("capa explorer settings", parent=self.parent)
+        if dialog.exec_():
+            (
+                settings.user[CAPA_SETTINGS_RULE_PATH],
+                settings.user[CAPA_SETTINGS_RULEGEN_AUTHOR],
+                settings.user[CAPA_SETTINGS_RULEGEN_SCOPE],
+            ) = dialog.get_values()
+
     def save_program_analysis(self):
         """ """
         if not self.doc:
@@ -1143,42 +1194,16 @@ class CapaExplorerForm(idaapi.PluginForm):
         """create Qt dialog to ask user for a directory"""
         return str(
             QtWidgets.QFileDialog.getExistingDirectory(
-                self.parent, "Please select a capa rules directory", settings.user["rule_path"]
+                self.parent, "Please select a capa rules directory", settings.user.get(CAPA_SETTINGS_RULE_PATH, "")
             )
         )
 
     def ask_user_capa_rule_file(self):
         """ """
         return QtWidgets.QFileDialog.getSaveFileName(
-            None, "Please select a capa rule to edit", settings.user["rule_path"], "*.yml"
+            None, "Please select a capa rule to edit", settings.user.get(CAPA_SETTINGS_RULE_PATH, ""), "*.yml"
         )[0]
 
-    def slot_change_rule_scope(self):
-        """ """
-        scope = idaapi.ask_str(str(settings.user.get("rulegen_scope", "function")), 0, "Enter default rule scope")
-        if scope:
-            settings.user["rulegen_scope"] = scope
-            idaapi.info("Run analysis again for your changes to take effect.")
-
-    def slot_change_rule_author(self):
-        """ """
-        author = idaapi.ask_str(str(settings.user.get("rulegen_author", "")), 0, "Enter default rule author")
-        if author:
-            settings.user["rulegen_author"] = author
-            idaapi.info("Run analysis again for your changes to take effect.")
-
-    def slot_change_rules_dir(self):
-        """allow user to change rules directory
-
-        user selection stored in settings for future runs
-        """
-        path = self.ask_user_directory()
-        if path:
-            settings.user["rule_path"] = path
-            self.rules_cache = None
-            self.ruleset_cache = None
-            idaapi.info("Run analysis again for your changes to take effect.")
-
     def set_view_status_label(self, text):
         """update status label control
 

capa/ida/plugin/model.py

@@ -488,13 +488,17 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
         @param feature: capa feature read from doc
         """
-        if feature[feature["type"]]:
+        key = feature["type"]
+        value = feature[feature["type"]]
+        if value:
+            if key == "string":
+                value = '"%s"' % capa.features.escape_string(value)
             if feature.get("description", ""):
-                return "%s(%s = %s)" % (feature["type"], feature[feature["type"]], feature["description"])
+                return "%s(%s = %s)" % (key, value, feature["description"])
             else:
-                return "%s(%s)" % (feature["type"], feature[feature["type"]])
+                return "%s(%s)" % (key, value)
         else:
-            return "%s" % feature["type"]
+            return "%s" % key
 
     def render_capa_doc_feature_node(self, parent, feature, locations, doc):
         """process capa doc feature node
@@ -551,7 +555,9 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
             )
 
         if feature["type"] == "regex":
-            return CapaExplorerStringViewItem(parent, display, location, feature["match"])
+            return CapaExplorerStringViewItem(
+                parent, display, location, '"%s"' % capa.features.escape_string(feature["match"])
+            )
 
         if feature["type"] == "basicblock":
             return CapaExplorerBlockItem(parent, location)
@@ -576,7 +582,9 @@ class CapaExplorerDataModel(QtCore.QAbstractItemModel):
 
         if feature["type"] in ("string",):
             # display string preview
-            return CapaExplorerStringViewItem(parent, display, location, feature[feature["type"]])
+            return CapaExplorerStringViewItem(
+                parent, display, location, '"%s"' % capa.features.escape_string(feature[feature["type"]])
+            )
 
         if feature["type"] in ("import", "export"):
             # display no preview

capa/ida/plugin/view.py

@@ -178,6 +178,9 @@ def build_context_menu(o, actions):
 
 
 class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
+
+    INDENT = " " * 2
+
     def __init__(self, parent=None):
         """ """
         super(CapaExplorerRulgenPreview, self).__init__(parent)
@@ -210,12 +213,99 @@ class CapaExplorerRulgenPreview(QtWidgets.QTextEdit):
         self.setText("\n".join(metadata_default))
 
     def keyPressEvent(self, e):
-        """ """
+        """intercept key press events"""
+        if e.key() in (QtCore.Qt.Key_Tab, QtCore.Qt.Key_Backtab):
+            # apparently it's not easy to implement tabs as spaces, or multi-line tab or SHIFT + Tab
+            # so we need to implement it ourselves so we can retain properly formatted capa rules
+            # when a user uses the Tab key
+            if self.textCursor().selection().isEmpty():
+                # single line, only worry about Tab
                 if e.key() == QtCore.Qt.Key_Tab:
-            self.insertPlainText(" " * 2)
+                    self.insertPlainText(self.INDENT)
+            else:
+                # multi-line tab or SHIFT + Tab
+                cur = self.textCursor()
+                select_start_ppos = cur.selectionStart()
+                select_end_ppos = cur.selectionEnd()
+
+                scroll_ppos = self.verticalScrollBar().sliderPosition()
+
+                # determine lineno for first selected line, and column
+                cur.setPosition(select_start_ppos)
+                start_lineno = self.count_previous_lines_from_block(cur.block())
+                start_lineco = cur.columnNumber()
+
+                # determine lineno for last selected line
+                cur.setPosition(select_end_ppos)
+                end_lineno = self.count_previous_lines_from_block(cur.block())
+
+                # now we need to indent or dedent the selected lines. for now, we read the text, modify
+                # the lines between start_lineno and end_lineno accordingly, and then reset the view
+                # this might not be the best solution, but it avoids messing around with cursor positions
+                # to determine the beginning of lines
+
+                plain = self.toPlainText().splitlines()
+
+                if e.key() == QtCore.Qt.Key_Tab:
+                    # user Tab, indent selected lines
+                    lines_modified = end_lineno - start_lineno
+                    first_modified = True
+                    change = [self.INDENT + line for line in plain[start_lineno : end_lineno + 1]]
+                else:
+                    # user SHIFT + Tab, dedent selected lines
+                    lines_modified = 0
+                    first_modified = False
+                    change = []
+                    for (lineno, line) in enumerate(plain[start_lineno : end_lineno + 1]):
+                        if line.startswith(self.INDENT):
+                            if lineno == 0:
+                                # keep track if first line is modified, so we can properly display
+                                # the text selection later
+                                first_modified = True
+                            lines_modified += 1
+                            line = line[len(self.INDENT) :]
+                        change.append(line)
+
+                # apply modifications, and reset view
+                plain[start_lineno : end_lineno + 1] = change
+                self.setPlainText("\n".join(plain) + "\n")
+
+                # now we need to properly adjust the selection positions, so users don't have to
+                # re-select when indenting or dedenting the same lines repeatedly
+                if e.key() == QtCore.Qt.Key_Tab:
+                    # user Tab, increase increment selection positions
+                    select_start_ppos += len(self.INDENT)
+                    select_end_ppos += (lines_modified * len(self.INDENT)) + len(self.INDENT)
+                elif lines_modified:
+                    # user SHIFT + Tab, decrease selection positions
+                    if start_lineco not in (0, 1) and first_modified:
+                        # only decrease start position if not in first column
+                        select_start_ppos -= len(self.INDENT)
+                    select_end_ppos -= lines_modified * len(self.INDENT)
+
+                # apply updated selection and restore previous scroll position
+                self.set_selection(select_start_ppos, select_end_ppos, len(self.toPlainText()))
+                self.verticalScrollBar().setSliderPosition(scroll_ppos)
         else:
             super(CapaExplorerRulgenPreview, self).keyPressEvent(e)
 
+    def count_previous_lines_from_block(self, block):
+        """calculate number of lines preceding block"""
+        count = 0
+        while True:
+            block = block.previous()
+            if not block.isValid():
+                break
+            count += block.lineCount()
+        return count
+
+    def set_selection(self, start, end, max):
+        """set text selection"""
+        cursor = self.textCursor()
+        cursor.setPosition(start)
+        cursor.setPosition(end if end < max else max, QtGui.QTextCursor.KeepAnchor)
+        self.setTextCursor(cursor)
+
 
 class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
 
@@ -325,6 +415,11 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
         # create a new parent under root node, by default; new node added last position in tree
         new_parent = self.new_expression_node(self.root, (action.data()[0], ""))
 
+        if "basic block" in action.data()[0]:
+            # add default child expression when nesting under basic block
+            new_parent.setExpanded(True)
+            new_parent = self.new_expression_node(new_parent, ("- or:", ""))
+
         for o in self.get_features(selected=True):
             # take child from its parent by index, add to new parent
             new_parent.addChild(o.parent().takeChild(o.parent().indexOfChild(o)))
@@ -335,6 +430,15 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
     def slot_edit_expression(self, action):
         """ """
         expression, o = action.data()
+        if "basic block" in expression and "basic block" not in o.text(
+            CapaExplorerRulgenEditor.get_column_feature_index()
+        ):
+            # current expression is "basic block", and not changing to "basic block" expression
+            children = o.takeChildren()
+            new_parent = self.new_expression_node(o, ("- or:", ""))
+            for child in children:
+                new_parent.addChild(child)
+            new_parent.setExpanded(True)
         o.setText(CapaExplorerRulgenEditor.get_column_feature_index(), expression)
 
     def slot_clear_all(self, action):
@@ -520,11 +624,23 @@ class CapaExplorerRulgenEditor(QtWidgets.QTreeWidget):
 
         # single features
         for (k, v) in filter(lambda t: t[1] == 1, counted):
-            self.new_feature_node(self.root, ("- %s: %s" % (k.name.lower(), k.get_value_str()), ""))
+            if isinstance(k, (capa.features.String,)):
+                value = '"%s"' % capa.features.escape_string(k.get_value_str())
+            else:
+                value = k.get_value_str()
+            self.new_feature_node(self.root, ("- %s: %s" % (k.name.lower(), value), ""))
 
         # n > 1 features
         for (k, v) in filter(lambda t: t[1] > 1, counted):
-            self.new_feature_node(self.root, ("- count(%s): %d" % (str(k), v), ""))
+            if k.value:
+                if isinstance(k, (capa.features.String,)):
+                    value = '"%s"' % capa.features.escape_string(k.get_value_str())
+                else:
+                    value = k.get_value_str()
+                display = "- count(%s(%s)): %d" % (k.name.lower(), value, v)
+            else:
+                display = "- count(%s): %d" % (k.name.lower(), v)
+            self.new_feature_node(self.root, (display, ""))
 
         self.expandAll()
         self.update_preview()
@@ -699,7 +815,9 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
         if text:
             for o in iterate_tree(self):
                 data = o.data(0, 0x100)
-                if data and text.lower() not in data.get_value_str().lower():
+                if data:
+                    to_match = data.get_value_str()
+                    if not to_match or text.lower() not in to_match.lower():
                         o.setHidden(True)
                         continue
                 o.setHidden(False)
@@ -776,16 +894,20 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
         def format_address(e):
             return "%X" % e if e else ""
 
+        def format_feature(feature):
+            """ """
+            name = feature.name.lower()
+            value = feature.get_value_str()
+            if isinstance(feature, (capa.features.String,)):
+                value = '"%s"' % capa.features.escape_string(value)
+            return "%s(%s)" % (name, value)
+
         for (feature, eas) in sorted(features.items(), key=lambda k: sorted(k[1])):
             if isinstance(feature, capa.features.basicblock.BasicBlock):
                 # filter basic blocks for now, we may want to add these back in some time
                 # in the future
                 continue
 
-            if isinstance(feature, capa.features.String):
-                # strip string for display
-                feature.value = feature.value.strip()
-
             # level 0
             if type(feature) not in self.parent_items:
                 self.parent_items[type(feature)] = self.new_parent_node(parent, (feature.name.lower(),))
@@ -794,20 +916,22 @@ class CapaExplorerRulegenFeatures(QtWidgets.QTreeWidget):
             if feature not in self.parent_items:
                 if len(eas) > 1:
                     self.parent_items[feature] = self.new_parent_node(
-                        self.parent_items[type(feature)], (str(feature),), feature=feature
+                        self.parent_items[type(feature)], (format_feature(feature),), feature=feature
                     )
                 else:
                     self.parent_items[feature] = self.new_leaf_node(
-                        self.parent_items[type(feature)], (str(feature),), feature=feature
+                        self.parent_items[type(feature)], (format_feature(feature),), feature=feature
                     )
 
             # level n > 1
             if len(eas) > 1:
                 for ea in sorted(eas):
-                    self.new_leaf_node(self.parent_items[feature], (str(feature), format_address(ea)), feature=feature)
+                    self.new_leaf_node(
+                        self.parent_items[feature], (format_feature(feature), format_address(ea)), feature=feature
+                    )
             else:
                 ea = eas.pop()
-                for (i, v) in enumerate((str(feature), format_address(ea))):
+                for (i, v) in enumerate((format_feature(feature), format_address(ea))):
                     self.parent_items[feature].setText(i, v)
                 self.parent_items[feature].setData(0, 0x100, feature)
 

capa/main.py

@@ -379,8 +379,8 @@ def get_rules(rule_path, disable_progress=False):
 
             for file in files:
                 if not file.endswith(".yml"):
-                    if not (file.endswith(".md") or file.endswith(".git") or file.endswith(".txt")):
+                    if not (file.startswith(".git") or file.endswith((".git", ".md", ".txt"))):
-                        # expect to see readme.md, format.md, and maybe a .git directory
+                        # expect to see .git* files, readme.md, format.md, and maybe a .git directory
                         # other things maybe are rules, but are mis-named.
                         logger.warning("skipping non-.yml file: %s", file)
                     continue

capa/render/vverbose.py

@@ -56,7 +56,11 @@ def render_statement(ostream, match, statement, indent=0):
         child = statement["child"]
 
         if child[child["type"]]:
-            value = rutils.bold2(child[child["type"]])
+            if child["type"] == "string":
+                value = '"%s"' % capa.features.escape_string(child[child["type"]])
+            else:
+                value = child[child["type"]]
+            value = rutils.bold2(value)
             if child.get("description"):
                 ostream.write("count(%s(%s = %s)): " % (child["type"], value, child["description"]))
             else:
@@ -90,6 +94,9 @@ def render_feature(ostream, match, feature, indent=0):
         key = "string"  # render string for regex to mirror the rule source
         value = feature["match"]  # the match provides more information than the value for regex
 
+    if key == "string":
+        value = '"%s"' % capa.features.escape_string(value)
+
     ostream.write(key)
     ostream.write(": ")
 

capa/rules.py

@@ -736,6 +736,8 @@ class Rule(object):
         # the below regex makes these adjustments and while ugly, we don't have to explore the ruamel.yaml insides
         doc = re.sub(r"!!int '0x-([0-9a-fA-F]+)'", r"-0x\1", doc)
 
+        # normalize CRLF to LF
+        doc = doc.replace("\r\n", "\n")
         return doc
 
 

capa/version.py

@@ -1 +1 @@
-__version__ = "1.6.0"
+__version__ = "1.6.1"

doc/release.md

@@ -0,0 +1,44 @@
+# Release checklist
+
+- [ ] Ensure all [milestoned issues/PRs](https://github.com/fireeye/capa/milestones) are addressed, or reassign to a new milestone.
+- [ ] Add the `dont merge` label to all PRs that are close to be ready to merge (or merge them if they are ready) in [capa](https://github.com/fireeye/capa/pulls) and [capa-rules](https://github.com/fireeye/capa-rules/pulls).
+- [ ] Ensure the [CI workflow succeeds in master](https://github.com/fireeye/capa/actions/workflows/tests.yml?query=branch%3Amaster).
+- [ ] Ensure that `python scripts/lint.py rules/ --thorough` succeeds (only `missing examples`  offenses are allowed in the nursery).
+- [ ] Review changes
+  - capa https://github.com/fireeye/capa/compare/\<last-release\>...master
+  - capa-rules https://github.com/fireeye/capa-rules/compare/\<last-release>\...master
+- [ ] Update [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md)
+  - Do not forget to add a nice introduction thanking contributors
+  - Remember that we need a major release if we introduce breaking changes
+  - Sections
+    - New Features
+    - New Rules
+    - Bug Fixes
+    - Changes
+    - Development
+    - Raw diffs
+  - Update `Raw diffs` links
+  - Create placeholder for `master (unreleased)` section
+    ```
+    ## master (unreleased)
+
+    ### New Features
+
+    ### New Rules
+
+    ### Bug Fixes
+
+    ### Changes
+
+    ### Development
+
+    ### Raw diffs
+    - [capa <release>...master](https://github.com/fireeye/capa/compare/<release>...master)
+    - [capa-rules <release>...master](https://github.com/fireeye/capa-rules/compare/<release>...master)
+    ```
+- [ ] Update [capa/version.py](https://github.com/fireeye/capa/blob/master/capa/version.py)
+- [ ] Create a PR with the updated [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md) and [capa/version.py](https://github.com/fireeye/capa/blob/master/capa/version.py). Copy this checklist in the PR description.
+- [ ] After PR review, merge the PR and [create the release in GH](https://github.com/fireeye/capa/releases/new) using text from the [CHANGELOG.md](https://github.com/fireeye/capa/blob/master/CHANGELOG.md).
+- [ ] Verify GH actions [upload artifacts](https://github.com/fireeye/capa/releases), [publish to PyPI](https://pypi.org/project/flare-capa) and [create a tag in capa rules](https://github.com/fireeye/capa-rules/tags) upon completion.
+- [ ] [Spread the word](https://twitter.com)
+

scripts/capafmt.py

@@ -65,6 +65,8 @@ def main(argv=None):
             return 0
         else:
             logger.info("rule requires reformatting (%s)", rule.name)
+            if "\r\n" in rule.definition:
+                logger.info("please make sure that the file uses LF (\\n) line endings only")
             return 1
 
     if args.in_place:

scripts/lint.py

@@ -25,6 +25,8 @@ import argparse
 import itertools
 import posixpath
 
+import ruamel.yaml
+
 import capa.main
 import capa.rules
 import capa.engine
@@ -301,6 +303,16 @@ class FeatureNtdllNtoskrnlApi(Lint):
         return False
 
 
+class FormatLineFeedEOL(Lint):
+    name = "line(s) end with CRLF (\\r\\n)"
+    recommendation = "convert line endings to LF (\\n) for example using dos2unix"
+
+    def check_rule(self, ctx, rule):
+        if len(rule.definition.split("\r\n")) > 0:
+            return False
+        return True
+
+
 class FormatSingleEmptyLineEOF(Lint):
     name = "EOF format"
     recommendation = "end file with a single empty line"
@@ -321,12 +333,43 @@ class FormatIncorrect(Lint):
 
         if actual != expected:
             diff = difflib.ndiff(actual.splitlines(1), expected.splitlines(True))
-            self.recommendation = self.recommendation_template.format("".join(diff))
+            recommendation_template = self.recommendation_template
+            if "\r\n" in actual:
+                recommendation_template = (
+                    self.recommendation_template + "\nplease make sure that the file uses LF (\\n) line endings only"
+                )
+            self.recommendation = recommendation_template.format("".join(diff))
             return True
 
         return False
 
 
+class FormatStringQuotesIncorrect(Lint):
+    name = "rule string quotes incorrect"
+
+    def check_rule(self, ctx, rule):
+        events = capa.rules.Rule._get_ruamel_yaml_parser().parse(rule.definition)
+        for key in events:
+            if not (isinstance(key, ruamel.yaml.ScalarEvent) and key.value == "string"):
+                continue
+            value = next(events)  # assume value is next event
+            if not isinstance(value, ruamel.yaml.ScalarEvent):
+                # ignore non-scalar
+                continue
+            if value.value.startswith("/") and value.value.endswith(("/", "/i")):
+                # ignore regex for now
+                continue
+            if value.style is None:
+                # no quotes
+                self.recommendation = 'add double quotes to "%s"' % value.value
+                return True
+            if value.style == "'":
+                # single quote
+                self.recommendation = 'change single quotes to double quotes for "%s"' % value.value
+                return True
+        return False
+
+
 def run_lints(lints, ctx, rule):
     for lint in lints:
         if lint.check_rule(ctx, rule):
@@ -385,7 +428,9 @@ def lint_features(ctx, rule):
 
 
 FORMAT_LINTS = (
+    FormatLineFeedEOL(),
     FormatSingleEmptyLineEOF(),
+    FormatStringQuotesIncorrect(),
     FormatIncorrect(),
 )
 
@@ -554,7 +599,7 @@ def main(argv=None):
 
     samples_path = os.path.join(os.path.dirname(__file__), "..", "tests", "data")
 
-    parser = argparse.ArgumentParser(description="A program.")
+    parser = argparse.ArgumentParser(description="Lint capa rules.")
     capa.main.install_common_args(parser, wanted={"tag"})
     parser.add_argument("rules", type=str, help="Path to rules")
     parser.add_argument("--samples", type=str, default=samples_path, help="Path to samples")
@@ -566,8 +611,12 @@ def main(argv=None):
     args = parser.parse_args(args=argv)
     capa.main.handle_common_args(args)
 
-    logging.getLogger("capa").setLevel(logging.CRITICAL)
+    if args.debug:
-    logging.getLogger("viv_utils").setLevel(logging.CRITICAL)
+        logging.getLogger("capa").setLevel(logging.DEBUG)
+        logging.getLogger("viv_utils").setLevel(logging.DEBUG)
+    else:
+        logging.getLogger("capa").setLevel(logging.ERROR)
+        logging.getLogger("viv_utils").setLevel(logging.ERROR)
 
     time0 = time.time()
 

setup.py

@@ -12,32 +12,32 @@ import sys
 import setuptools
 
 requirements = [
-    "six",
+    "six==1.15.0",
-    "tqdm",
+    "tqdm==4.60.0",
-    "pyyaml",
+    "pyyaml==5.4.1",
-    "tabulate",
+    "tabulate==0.8.9",
-    "colorama",
+    "colorama==0.4.4",
-    "termcolor",
+    "termcolor==1.1.0",
-    "ruamel.yaml",
+    "wcwidth==0.2.5",
-    "wcwidth",
     "ida-settings==2.1.0",
+    "viv-utils==0.6.0",
 ]
 
 if sys.version_info >= (3, 0):
     # py3
-    requirements.append("halo")
+    requirements.append("halo==0.0.31")
-    requirements.append("networkx")
+    requirements.append("networkx==2.5.1")
-    requirements.append("vivisect==1.0.0")
+    requirements.append("ruamel.yaml==0.17.0")
-    requirements.append("viv-utils==0.3.19")
+    requirements.append("vivisect==1.0.1")
     requirements.append("smda==1.5.13")
 else:
     # py2
     requirements.append("enum34==1.1.6")  # v1.1.6 is needed by halo 0.0.30 / spinners 0.0.24
     requirements.append("halo==0.0.30")  # halo==0.0.30 is the last version to support py2.7
     requirements.append("vivisect==0.2.1")
-    requirements.append("viv-utils==0.3.19")
     requirements.append("networkx==2.2")  # v2.2 is last version supported by Python 2.7
-    requirements.append("backports.functools-lru-cache")
+    requirements.append("ruamel.yaml==0.16.13")  # last version tested with Python 2.7
+    requirements.append("backports.functools-lru-cache==1.6.1")
 
 # this sets __version__
 # via: http://stackoverflow.com/a/7071358/87207
@@ -77,13 +77,13 @@ setuptools.setup(
     install_requires=requirements,
     extras_require={
         "dev": [
-            "pytest",
+            "pytest==4.6.11",  # TODO: Change to 6.2.3 when removing py2
-            "pytest-sugar",
+            "pytest-sugar==0.9.4",
-            "pytest-instafail",
+            "pytest-instafail==0.4.2",
-            "pytest-cov",
+            "pytest-cov==2.11.1",
-            "pycodestyle",
+            "pycodestyle==2.7.0",
-            "black ; python_version>'3.0'",
+            "black==20.8b1 ; python_version>'3.0'",
-            "isort",
+            "isort==4.3.21",  # TODO: Change to 5.8.0 when removing py2
         ]
     },
     zip_safe=False,

tests/test_rules.py

@@ -681,6 +681,25 @@ def test_explicit_string_values_int():
     assert (String("0x123") in children) == True
 
 
+def test_string_values_special_characters():
+    rule = textwrap.dedent(
+        """
+        rule:
+            meta:
+                name: test rule
+            features:
+                - or:
+                    - string: "hello\\r\\nworld"
+                    - string: "bye\\nbye"
+                      description: "test description"
+        """
+    )
+    r = capa.rules.Rule.from_yaml(rule)
+    children = list(r.statement.get_children())
+    assert (String("hello\r\nworld") in children) == True
+    assert (String("bye\nbye") in children) == True
+
+
 def test_regex_values_always_string():
     rules = [
         capa.rules.Rule.from_yaml(

tests/test_smda_features.py

@@ -15,8 +15,9 @@ from fixtures import *
     FEATURE_PRESENCE_TESTS,
     indirect=["sample", "scope"],
 )
+@pytest.mark.xfail(sys.version_info < (3, 0), reason="SMDA only works on py3")
+@pytest.mark.xfail(sys.platform == "win32", reason="SMDA bug: https://github.com/danielplohmann/smda/issues/20")
 def test_smda_features(sample, scope, feature, expected):
-    with xfail(sys.version_info < (3, 0), reason="SMDA only works on py3"):
     do_test_feature_presence(get_smda_extractor, sample, scope, feature, expected)