From 256611bef5372295004d3aa10e94c841ac57c1e0 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Thu, 27 Apr 2023 06:00:38 +0530
Subject: [PATCH 01/32] Create detect_duplicate_features.py

Fixes #1451
Python script to detect feature overlap between new and existing CAPA rules. Checks if the a feature in new rules exists in an existing rule
---
 scripts/detect_duplicate_features.py | 67 ++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 scripts/detect_duplicate_features.py

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
new file mode 100644
index 00000000..7da0655e
--- /dev/null
+++ b/scripts/detect_duplicate_features.py
@@ -0,0 +1,67 @@
+import os
+import yaml
+
+def findall_features(features):
+    feature_list = []
+    for feature in features:
+        if 'and' in feature:
+            and_list = findall_features(feature['and'])
+            for x in and_list:
+                feature_list.append(x)
+        elif 'or' in feature:
+            or_list = findall_features(feature['or'])
+            for y in or_list:
+                feature_list.append(y)
+        else:
+            feature_list.append(feature)
+    return feature_list
+
+def find_overlapping_rules(new_rule_path, rules_path):
+    if not new_rule_path.endswith('.yml'):
+        return 'ERROR ! New rule path file name incorrect'
+    
+    count = 0
+
+    with open(new_rule_path, 'r') as f:
+        new_rule = yaml.safe_load(f)
+    if 'rule' not in new_rule:
+        return "ERROR ! given new rule path isn't a rule"
+    
+    new_rule_features = findall_features(new_rule['rule']['features'])
+
+    overlapping_rules = []
+    
+    for dirpath, dirnames, filenames in os.walk(rules_path):
+        for filename in filenames:
+            if filename.endswith('.yml'):
+                rule_path = os.path.join(dirpath, filename)
+                with open(rule_path, 'r') as f:
+                    rule = yaml.safe_load(f)
+                    if 'rule' not in rule:
+                        continue
+                    rule_features = findall_features(rule['rule']['features'])
+                    count+=1
+                if any([feature in rule_features for feature in new_rule_features]):
+                    overlapping_rules.append(rule_path)
+    result = {'overlapping_rules': overlapping_rules,
+              'count': count}
+    
+    return result
+
+# usage
+base_dir = ''
+new_rule_path = base_dir + 'rules\\anti-analysis\\reference-analysis-tools-strings.yml'
+rules_path = base_dir + 'rules'
+
+try:
+    result = find_overlapping_rules(new_rule_path, rules_path)
+    print('New rule path : %s' % new_rule_path)
+    print('Number of rules checked : %s ' % result['count'])
+    print('Paths to overlapping rules : ', result['overlapping_rules'])
+    print('Number of rules containing same features : %s' % len(result['overlapping_rules']))
+except Exception as e:
+    print(e)
+    try:
+        print(result,'')
+    except:
+        pass

From 09865ccd9be97ef26e892cab6d751325e2e7d169 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Thu, 27 Apr 2023 06:11:58 +0530
Subject: [PATCH 02/32] Fixes Linting Issues

Update detect_duplicate_features.py
---
 scripts/detect_duplicate_features.py | 62 +++++++++++++++-------------
 1 file changed, 34 insertions(+), 28 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 7da0655e..f852e178 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,67 +1,73 @@
 import os
+
 import yaml
 
+
 def findall_features(features):
     feature_list = []
     for feature in features:
-        if 'and' in feature:
-            and_list = findall_features(feature['and'])
+        if "and" in feature:
+            and_list = findall_features(feature["and"])
             for x in and_list:
                 feature_list.append(x)
-        elif 'or' in feature:
-            or_list = findall_features(feature['or'])
+        elif "or" in feature:
+            or_list = findall_features(feature["or"])
             for y in or_list:
                 feature_list.append(y)
         else:
             feature_list.append(feature)
+
     return feature_list
 
+
 def find_overlapping_rules(new_rule_path, rules_path):
-    if not new_rule_path.endswith('.yml'):
-        return 'ERROR ! New rule path file name incorrect'
-    
+    if not new_rule_path.endswith(".yml"):
+        return "ERROR ! New rule path file name incorrect"
+
     count = 0
 
-    with open(new_rule_path, 'r') as f:
+    with open(new_rule_path, "r") as f:
         new_rule = yaml.safe_load(f)
-    if 'rule' not in new_rule:
+
+    if "rule" not in new_rule:
         return "ERROR ! given new rule path isn't a rule"
-    
-    new_rule_features = findall_features(new_rule['rule']['features'])
+
+    new_rule_features = findall_features(new_rule["rule"]["features"])
 
     overlapping_rules = []
-    
+
     for dirpath, dirnames, filenames in os.walk(rules_path):
         for filename in filenames:
-            if filename.endswith('.yml'):
+            if filename.endswith(".yml"):
                 rule_path = os.path.join(dirpath, filename)
-                with open(rule_path, 'r') as f:
+                with open(rule_path, "r") as f:
                     rule = yaml.safe_load(f)
-                    if 'rule' not in rule:
+                    if "rule" not in rule:
                         continue
-                    rule_features = findall_features(rule['rule']['features'])
-                    count+=1
+                    rule_features = findall_features(rule["rule"]["features"])
+                    count += 1
                 if any([feature in rule_features for feature in new_rule_features]):
                     overlapping_rules.append(rule_path)
-    result = {'overlapping_rules': overlapping_rules,
-              'count': count}
-    
+
+    result = {"overlapping_rules": overlapping_rules, "count": count}
+
     return result
 
+
 # usage
-base_dir = ''
-new_rule_path = base_dir + 'rules\\anti-analysis\\reference-analysis-tools-strings.yml'
-rules_path = base_dir + 'rules'
+base_dir = ""
+new_rule_path = base_dir + "rules\\anti-analysis\\reference-analysis-tools-strings.yml"
+rules_path = base_dir + "rules"
 
 try:
     result = find_overlapping_rules(new_rule_path, rules_path)
-    print('New rule path : %s' % new_rule_path)
-    print('Number of rules checked : %s ' % result['count'])
-    print('Paths to overlapping rules : ', result['overlapping_rules'])
-    print('Number of rules containing same features : %s' % len(result['overlapping_rules']))
+    print("New rule path : %s" % new_rule_path)
+    print("Number of rules checked : %s " % result["count"])
+    print("Paths to overlapping rules : ", result["overlapping_rules"])
+    print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
 except Exception as e:
     print(e)
     try:
-        print(result,'')
+        print(result, "")
     except:
         pass

From 1c558a203d64a7c3669d20707bf747e8d1b6d372 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Wed, 3 May 2023 22:00:50 +0530
Subject: [PATCH 03/32] Update detect_duplicate_features.py

Added a main routine and using argparse to retrieve these from the command line
---
 scripts/detect_duplicate_features.py | 46 +++++++++++++++++++---------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index f852e178..8174c1ff 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,4 +1,5 @@
 import os
+import argparse
 
 import yaml
 
@@ -54,20 +55,35 @@ def find_overlapping_rules(new_rule_path, rules_path):
     return result
 
 
-# usage
-base_dir = ""
-new_rule_path = base_dir + "rules\\anti-analysis\\reference-analysis-tools-strings.yml"
-rules_path = base_dir + "rules"
+# python script.py --base-dir /path/to/capa/rules rules/anti-analysis/reference-analysis-tools-strings.yml rules
+
+
+def main():
+    # usage
+
+    parser = argparse.ArgumentParser(description="Find overlapping rules in Capa rules.")
+    parser.add_argument("-b", "--base-dir", default="", help="Base directory for Capa rules.")
+    parser.add_argument("-f", "--new_rule_path", required=True, help="Path to the new Capa rule.")
+    parser.add_argument("-d", "--rules_path", required=True, help="Path to the directory containing Capa rules.")
+    args = parser.parse_args()
+
+    base_dir = args.base_dir
+    new_rule_path = os.path.join(base_dir, args.new_rule_path)
+    rules_path = os.path.join(base_dir, args.rules_path)
 
-try:
-    result = find_overlapping_rules(new_rule_path, rules_path)
-    print("New rule path : %s" % new_rule_path)
-    print("Number of rules checked : %s " % result["count"])
-    print("Paths to overlapping rules : ", result["overlapping_rules"])
-    print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
-except Exception as e:
-    print(e)
     try:
-        print(result, "")
-    except:
-        pass
+        result = find_overlapping_rules(new_rule_path, rules_path)
+        print("New rule path : %s" % new_rule_path)
+        print("Number of rules checked : %s " % result["count"])
+        print("Paths to overlapping rules : ", result["overlapping_rules"])
+        print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
+    except Exception as e:
+        print(e)
+        try:
+            print(result, "")
+        except:
+            pass
+
+
+if __name__ == "__main__":
+    main()

From 0945d9aea2e5e24a36e9263f7fa68297653dccc8 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Thu, 4 May 2023 19:55:17 +0530
Subject: [PATCH 04/32] Update CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6412f40..ab7457a1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## master (unreleased)
 
 ### New Features
+- Utility script to detect feature overlap between new and existing CAPA rules #1451 @Aayush-Goel-04
 
 ### Breaking Changes
 

From 30516c33b75c9bb5b127ca592bdc89b2de4dbe06 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Fri, 5 May 2023 14:04:47 +0530
Subject: [PATCH 05/32] Update detect_duplicate_features.py

Improved parse routine based on suggestions.

Co-Authored-By: Moritz <mr-tz@users.noreply.github.com>
---
 scripts/detect_duplicate_features.py | 46 +++++++++++++---------------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 8174c1ff..9aba66ad 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -23,7 +23,7 @@ def findall_features(features):
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
-        return "ERROR ! New rule path file name incorrect"
+        raise ValueError("ERROR ! New rule path file name incorrect")
 
     count = 0
 
@@ -31,24 +31,24 @@ def find_overlapping_rules(new_rule_path, rules_path):
         new_rule = yaml.safe_load(f)
 
     if "rule" not in new_rule:
-        return "ERROR ! given new rule path isn't a rule"
+        raise ValueError("ERROR ! given new rule path isn't a rule")
 
     new_rule_features = findall_features(new_rule["rule"]["features"])
 
     overlapping_rules = []
-
-    for dirpath, dirnames, filenames in os.walk(rules_path):
-        for filename in filenames:
-            if filename.endswith(".yml"):
-                rule_path = os.path.join(dirpath, filename)
-                with open(rule_path, "r") as f:
-                    rule = yaml.safe_load(f)
-                    if "rule" not in rule:
-                        continue
-                    rule_features = findall_features(rule["rule"]["features"])
-                    count += 1
-                if any([feature in rule_features for feature in new_rule_features]):
-                    overlapping_rules.append(rule_path)
+    for rules in rules_path:
+        for dirpath, dirnames, filenames in os.walk(rules):
+            for filename in filenames:
+                if filename.endswith(".yml"):
+                    rule_path = os.path.join(dirpath, filename)
+                    with open(rule_path, "r") as f:
+                        rule = yaml.safe_load(f)
+                        if "rule" not in rule:
+                            continue
+                        rule_features = findall_features(rule["rule"]["features"])
+                        count += 1
+                    if any([feature in rule_features for feature in new_rule_features]):
+                        overlapping_rules.append(rule_path)
 
     result = {"overlapping_rules": overlapping_rules, "count": count}
 
@@ -59,17 +59,15 @@ def find_overlapping_rules(new_rule_path, rules_path):
 
 
 def main():
-    # usage
+    parser = argparse.ArgumentParser(description="Find overlapping features in Capa rules.")
+
+    parser.add_argument("rules", type=str, action="append", help="Path to rules")
+    parser.add_argument("new_rule", type=str, help="Path to new rule")
 
-    parser = argparse.ArgumentParser(description="Find overlapping rules in Capa rules.")
-    parser.add_argument("-b", "--base-dir", default="", help="Base directory for Capa rules.")
-    parser.add_argument("-f", "--new_rule_path", required=True, help="Path to the new Capa rule.")
-    parser.add_argument("-d", "--rules_path", required=True, help="Path to the directory containing Capa rules.")
     args = parser.parse_args()
 
-    base_dir = args.base_dir
-    new_rule_path = os.path.join(base_dir, args.new_rule_path)
-    rules_path = os.path.join(base_dir, args.rules_path)
+    new_rule_path = args.new_rule
+    rules_path = args.rules
 
     try:
         result = find_overlapping_rules(new_rule_path, rules_path)
@@ -80,7 +78,7 @@ def main():
     except Exception as e:
         print(e)
         try:
-            print(result, "")
+            print(result)
         except:
             pass
 

From 9eacf72366a167267d28fe243a8fa0202e4099b7 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Sat, 6 May 2023 17:19:57 +0530
Subject: [PATCH 06/32] Update detect_duplicate_features.py

loading yaml file using capa.rule.Rule.from_yaml.
Returning any exception/errors occuring while checking the files.
---
 scripts/detect_duplicate_features.py | 81 ++++++++++++++--------------
 1 file changed, 39 insertions(+), 42 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 9aba66ad..b61d36f9 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,63 +1,61 @@
 import os
 import argparse
 
-import yaml
+import capa.rules
+import capa.engine as ceng
 
 
-def findall_features(features):
-    feature_list = []
-    for feature in features:
-        if "and" in feature:
-            and_list = findall_features(feature["and"])
-            for x in and_list:
-                feature_list.append(x)
-        elif "or" in feature:
-            or_list = findall_features(feature["or"])
-            for y in or_list:
-                feature_list.append(y)
-        else:
-            feature_list.append(feature)
+def get_child_features(feature):
+    children = []
 
-    return feature_list
+    if isinstance(feature, (ceng.And, ceng.Or, ceng.Some)):
+        for child in feature.children:
+            children.extend(get_child_features(child))
+    elif isinstance(feature, (ceng.Subscope, ceng.Range, ceng.Not)):
+        children.extend(get_child_features(feature.child))
+    else:
+        children.append(feature)
+    return children
+
+
+def get_features(rule_path, errors):
+    with open(rule_path, "r") as f:
+        feature_list = []
+        try:
+            new_rule = capa.rules.Rule.from_yaml(f.read())
+            feature_list = get_child_features(new_rule.statement)
+        except Exception as e:
+            errors.append("rule :" + rule_path + " " + str(type(e)) + " " + str(e))
+    return feature_list, errors
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
-        raise ValueError("ERROR ! New rule path file name incorrect")
+        raise FileNotFoundError("FileNotFoundError ! New rule file name doesn't end with yml")
 
+    new_rule_features, error = get_features(new_rule_path, [])
+    if error:
+        raise Warning(error[0])
+
+    errors: list = []
     count = 0
-
-    with open(new_rule_path, "r") as f:
-        new_rule = yaml.safe_load(f)
-
-    if "rule" not in new_rule:
-        raise ValueError("ERROR ! given new rule path isn't a rule")
-
-    new_rule_features = findall_features(new_rule["rule"]["features"])
-
     overlapping_rules = []
     for rules in rules_path:
         for dirpath, dirnames, filenames in os.walk(rules):
             for filename in filenames:
                 if filename.endswith(".yml"):
                     rule_path = os.path.join(dirpath, filename)
-                    with open(rule_path, "r") as f:
-                        rule = yaml.safe_load(f)
-                        if "rule" not in rule:
-                            continue
-                        rule_features = findall_features(rule["rule"]["features"])
-                        count += 1
+                    rule_features, errors = get_features(rule_path, errors)
+                    if not len(rule_features):
+                        continue
+                    count += 1
                     if any([feature in rule_features for feature in new_rule_features]):
                         overlapping_rules.append(rule_path)
 
-    result = {"overlapping_rules": overlapping_rules, "count": count}
-
+    result = {"overlapping_rules": overlapping_rules, "count": count, "errors": errors}
     return result
 
 
-# python script.py --base-dir /path/to/capa/rules rules/anti-analysis/reference-analysis-tools-strings.yml rules
-
-
 def main():
     parser = argparse.ArgumentParser(description="Find overlapping features in Capa rules.")
 
@@ -68,19 +66,18 @@ def main():
 
     new_rule_path = args.new_rule
     rules_path = args.rules
-
     try:
         result = find_overlapping_rules(new_rule_path, rules_path)
-        print("New rule path : %s" % new_rule_path)
+        print("\nNew rule path : %s" % new_rule_path)
         print("Number of rules checked : %s " % result["count"])
         print("Paths to overlapping rules : ", result["overlapping_rules"])
         print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
+
+        print("\nWhile checking following .yml files error occured:")
+        for error in result["errors"]:
+            print(error)
     except Exception as e:
         print(e)
-        try:
-            print(result)
-        except:
-            pass
 
 
 if __name__ == "__main__":

From ec6b6a22666df81c01545be6d1545204e8a9b3f8 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 8 May 2023 14:58:30 +0530
Subject: [PATCH 07/32] Update detect_duplicate_features.py

---
 scripts/detect_duplicate_features.py | 30 +++++++++++++++++-----------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index b61d36f9..4f1c7198 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -18,24 +18,25 @@ def get_child_features(feature):
     return children
 
 
-def get_features(rule_path, errors):
+def get_features(rule_path):
+    error = ""
+    feature_list = []
     with open(rule_path, "r") as f:
-        feature_list = []
         try:
             new_rule = capa.rules.Rule.from_yaml(f.read())
             feature_list = get_child_features(new_rule.statement)
         except Exception as e:
-            errors.append("rule :" + rule_path + " " + str(type(e)) + " " + str(e))
-    return feature_list, errors
+            error = "rule :" + rule_path + " " + str(type(e)) + " " + str(e)
+    return feature_list, error
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
         raise FileNotFoundError("FileNotFoundError ! New rule file name doesn't end with yml")
 
-    new_rule_features, error = get_features(new_rule_path, [])
+    new_rule_features, error = get_features(new_rule_path)
     if error:
-        raise Warning(error[0])
+        raise Warning(error)
 
     errors: list = []
     count = 0
@@ -45,7 +46,9 @@ def find_overlapping_rules(new_rule_path, rules_path):
             for filename in filenames:
                 if filename.endswith(".yml"):
                     rule_path = os.path.join(dirpath, filename)
-                    rule_features, errors = get_features(rule_path, errors)
+                    rule_features, error = get_features(rule_path)
+                    if error:
+                        errors.append(error)
                     if not len(rule_features):
                         continue
                     count += 1
@@ -70,12 +73,15 @@ def main():
         result = find_overlapping_rules(new_rule_path, rules_path)
         print("\nNew rule path : %s" % new_rule_path)
         print("Number of rules checked : %s " % result["count"])
-        print("Paths to overlapping rules : ", result["overlapping_rules"])
+        print("Paths to overlapping rules : ")
+        for r in result["overlapping_rules"]:
+            print(r)
         print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
-
-        print("\nWhile checking following .yml files error occured:")
-        for error in result["errors"]:
-            print(error)
+        if result["errors"]:
+            print("\nWhile checking following .yml files error occured:")
+            for error in result["errors"]:
+                print(error)
+            print("\n")
     except Exception as e:
         print(e)
 

From 39d2a70679b01f7a7cd0f08e22221cd0d5bd6511 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 8 May 2023 17:29:01 +0530
Subject: [PATCH 08/32] Update detect_duplicate_features.py

Using get_rules menthod to get set of all existing rules.
---
 scripts/detect_duplicate_features.py | 68 +++++++++++++++-------------
 1 file changed, 37 insertions(+), 31 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 4f1c7198..b44a30ad 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,11 +1,16 @@
-import os
 import argparse
 
+import capa.main
 import capa.rules
 import capa.engine as ceng
 
 
-def get_child_features(feature):
+def get_child_features(feature) -> list:
+    """
+    args:
+    \tfeature : capa.rule.Rule.statement containing feature statements
+    returns a list containg all the features in the rule
+    """
     children = []
 
     if isinstance(feature, (ceng.And, ceng.Or, ceng.Some)):
@@ -19,43 +24,45 @@ def get_child_features(feature):
 
 
 def get_features(rule_path):
-    error = ""
+    """
+    args:
+    \tfeature : rule path
+    returns a list containg all the features in the rule
+    """
     feature_list = []
     with open(rule_path, "r") as f:
         try:
             new_rule = capa.rules.Rule.from_yaml(f.read())
             feature_list = get_child_features(new_rule.statement)
         except Exception as e:
-            error = "rule :" + rule_path + " " + str(type(e)) + " " + str(e)
-    return feature_list, error
+            raise Warning("Error: " + rule_path + " " + str(type(e)) + " " + str(e))
+    return feature_list
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
         raise FileNotFoundError("FileNotFoundError ! New rule file name doesn't end with yml")
 
-    new_rule_features, error = get_features(new_rule_path)
-    if error:
-        raise Warning(error)
+    # Loads features of new rule in a list.
+    new_rule_features = get_features(new_rule_path)
 
-    errors: list = []
     count = 0
     overlapping_rules = []
-    for rules in rules_path:
-        for dirpath, dirnames, filenames in os.walk(rules):
-            for filename in filenames:
-                if filename.endswith(".yml"):
-                    rule_path = os.path.join(dirpath, filename)
-                    rule_features, error = get_features(rule_path)
-                    if error:
-                        errors.append(error)
-                    if not len(rule_features):
-                        continue
-                    count += 1
-                    if any([feature in rule_features for feature in new_rule_features]):
-                        overlapping_rules.append(rule_path)
 
-    result = {"overlapping_rules": overlapping_rules, "count": count, "errors": errors}
+    # capa.rules.RuleSet stores all rules in given paths
+    ruleset = capa.main.get_rules(rules_path)
+
+    for rule_name, rule in ruleset.rules.items():
+        rule_features = get_child_features(rule.statement)
+
+        if not len(rule_features):
+            continue
+        count += 1
+        # Checks if any features match between existing and new rule.
+        if any([feature in rule_features for feature in new_rule_features]):
+            overlapping_rules.append(rule_name)
+
+    result = {"overlapping_rules": overlapping_rules, "count": count}
     return result
 
 
@@ -73,15 +80,14 @@ def main():
         result = find_overlapping_rules(new_rule_path, rules_path)
         print("\nNew rule path : %s" % new_rule_path)
         print("Number of rules checked : %s " % result["count"])
-        print("Paths to overlapping rules : ")
-        for r in result["overlapping_rules"]:
-            print(r)
+        if result["overlapping_rules"]:
+            print("Paths to overlapping rules : ")
+            for r in result["overlapping_rules"]:
+                print("- %s" % r)
+        else:
+            print("Paths to overlapping rules : None")
         print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
-        if result["errors"]:
-            print("\nWhile checking following .yml files error occured:")
-            for error in result["errors"]:
-                print(error)
-            print("\n")
+        print("\n")
     except Exception as e:
         print(e)
 

From d91070c116d5215e61ddfc2e9da7802aa5ba93e7 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 8 May 2023 20:17:29 +0530
Subject: [PATCH 09/32] Update detect_duplicate_features.py

---
 scripts/detect_duplicate_features.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index b44a30ad..0ce8ff3a 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -5,11 +5,15 @@ import capa.rules
 import capa.engine as ceng
 
 
-def get_child_features(feature) -> list:
+def get_child_features(feature: ceng.Statement) -> list:
     """
-    args:
-    \tfeature : capa.rule.Rule.statement containing feature statements
-    returns a list containg all the features in the rule
+    Recursively extracts all feature statements from a given rule statement.
+
+    Args:
+        feature (capa.engine.Statement): The feature statement to extract features from.
+
+    Returns:
+        list: A list of all feature statements contained within the given feature statement.
     """
     children = []
 
@@ -23,11 +27,15 @@ def get_child_features(feature) -> list:
     return children
 
 
-def get_features(rule_path):
+def get_features(rule_path: str) -> list:
     """
-    args:
-    \tfeature : rule path
-    returns a list containg all the features in the rule
+    Extracts all features from a given rule file.
+
+    Args:
+        rule_path (str): The path to the rule file to extract features from.
+
+    Returns:
+        list: A list of all feature statements contained within the rule file.
     """
     feature_list = []
     with open(rule_path, "r") as f:

From 187a4712cb858a1fdb06ffc692ffa7398f453552 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Wed, 10 May 2023 14:39:16 +0530
Subject: [PATCH 10/32] Update test_scripts.py

Here new_rule_path and expected_overlaps will be changed based on the new test rule designed.
Adding tests to check if the code works fine
---
 scripts/detect_duplicate_features.py | 41 ++++++++++++++++------------
 tests/test_scripts.py                | 12 ++++++++
 2 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 0ce8ff3a..a72688c0 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,9 +1,13 @@
+import sys
+import logging
 import argparse
 
 import capa.main
 import capa.rules
 import capa.engine as ceng
 
+logger = logging.getLogger("detect_duplicate_features")
+
 
 def get_child_features(feature: ceng.Statement) -> list:
     """
@@ -43,13 +47,15 @@ def get_features(rule_path: str) -> list:
             new_rule = capa.rules.Rule.from_yaml(f.read())
             feature_list = get_child_features(new_rule.statement)
         except Exception as e:
-            raise Warning("Error: " + rule_path + " " + str(type(e)) + " " + str(e))
+            logger.error("Error: New rule " + rule_path + " " + str(type(e)) + " " + str(e))
+            sys.exit(1)
     return feature_list
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
-        raise FileNotFoundError("FileNotFoundError ! New rule file name doesn't end with yml")
+        logger.error("FileNotFoundError ! New rule file name doesn't end with .yml")
+        sys.exit(1)
 
     # Loads features of new rule in a list.
     new_rule_features = get_features(new_rule_path)
@@ -84,21 +90,22 @@ def main():
 
     new_rule_path = args.new_rule
     rules_path = args.rules
-    try:
-        result = find_overlapping_rules(new_rule_path, rules_path)
-        print("\nNew rule path : %s" % new_rule_path)
-        print("Number of rules checked : %s " % result["count"])
-        if result["overlapping_rules"]:
-            print("Paths to overlapping rules : ")
-            for r in result["overlapping_rules"]:
-                print("- %s" % r)
-        else:
-            print("Paths to overlapping rules : None")
-        print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
-        print("\n")
-    except Exception as e:
-        print(e)
+
+    result = find_overlapping_rules(new_rule_path, rules_path)
+
+    print("\nNew rule path : %s" % new_rule_path)
+    print("Number of rules checked : %s " % result["count"])
+    if result["overlapping_rules"]:
+        print("Paths to overlapping rules : ")
+        for r in result["overlapping_rules"]:
+            print("- %s" % r)
+    else:
+        print("Paths to overlapping rules : None")
+    print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
+    print("\n")
+
+    return len(result["overlapping_rules"])
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 3b299a36..eca6a1f4 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -82,3 +82,15 @@ def test_proto_conversion(tmpdir):
     assert p.returncode == 0
 
     assert p.stdout.startswith(b'{\n  "meta": ') or p.stdout.startswith(b'{\r\n  "meta": ')
+
+
+def test_detect_duplicate_features():
+    new_rule_path = "collection/credit-card/parse-credit-card-information.yml"
+    args = [
+        get_rules_path(),
+        os.path.join(get_rules_path(), new_rule_path),
+    ]
+    expected_overlaps = 49
+    script_path = get_script_path("detect_duplicate_features.py")
+    p = run_program(script_path, args)
+    assert p.returncode == expected_overlaps

From eca86470c670eb7c04b7746dbba04acf0281e9e1 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Thu, 11 May 2023 14:12:52 +0530
Subject: [PATCH 11/32] Update test_scripts.py

RULE_CONTENT can be modified as required
---
 tests/test_scripts.py | 51 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 9 deletions(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index eca6a1f4..00792620 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -8,9 +8,13 @@
 
 import os
 import sys
+import textwrap
 import subprocess
 
 import pytest
+from fixtures import *
+
+import capa.main
 
 CD = os.path.dirname(__file__)
 
@@ -84,13 +88,42 @@ def test_proto_conversion(tmpdir):
     assert p.stdout.startswith(b'{\n  "meta": ') or p.stdout.startswith(b'{\r\n  "meta": ')
 
 
-def test_detect_duplicate_features():
-    new_rule_path = "collection/credit-card/parse-credit-card-information.yml"
-    args = [
-        get_rules_path(),
-        os.path.join(get_rules_path(), new_rule_path),
-    ]
-    expected_overlaps = 49
+def run_detect_duplicate_features(rule_path):
+    # rule_path = "collection/credit-card/parse-credit-card-information.yml"
+    args = [get_rules_path(), rule_path]
     script_path = get_script_path("detect_duplicate_features.py")
-    p = run_program(script_path, args)
-    assert p.returncode == expected_overlaps
+    args = [sys.executable] + [script_path] + args
+    print(f"running: '{args}'")
+    return subprocess.run(args)
+
+
+def test_detect_duplicate_features(z9324d_extractor, tmpdir):
+    # tests a single rule can be loaded successfully
+    RULE_CONTENT = textwrap.dedent(
+        """
+        rule:
+          meta:
+            name: Test Rule
+            scope: function
+          features:
+            - string: "sites.ini"
+        """
+    )
+    expected_overlaps = 3
+    path = z9324d_extractor.path
+    rule_file = tmpdir.mkdir("capa").join("rule.yml")
+    rule_file.write(RULE_CONTENT)
+    assert (
+        capa.main.main(
+            [
+                path,
+                "-v",
+                "-r",
+                rule_file.strpath,
+            ]
+        )
+        == 0
+    )
+    # tests if number of overlaps found are correct.
+    overlaps_found = run_detect_duplicate_features(rule_file.strpath).returncode
+    assert overlaps_found == expected_overlaps

From 41ff457d652a006066c48680bb09243045dd4b32 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Fri, 12 May 2023 16:53:44 +0530
Subject: [PATCH 12/32] Update tests/test_scripts.py

Co-authored-by: Moritz <mr-tz@users.noreply.github.com>
---
 tests/test_scripts.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 00792620..ce8f01c3 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -98,7 +98,6 @@ def run_detect_duplicate_features(rule_path):
 
 
 def test_detect_duplicate_features(z9324d_extractor, tmpdir):
-    # tests a single rule can be loaded successfully
     RULE_CONTENT = textwrap.dedent(
         """
         rule:

From 807efec40fa0e7849c15bfd2f22a13c674738d13 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Fri, 12 May 2023 22:32:37 +0530
Subject: [PATCH 13/32] Create RuleSet to test overlap script

---
 scripts/detect_duplicate_features.py |  4 +-
 tests/test_scripts.py                | 96 ++++++++++++++++++----------
 2 files changed, 65 insertions(+), 35 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index a72688c0..ff21fd0b 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -48,14 +48,14 @@ def get_features(rule_path: str) -> list:
             feature_list = get_child_features(new_rule.statement)
         except Exception as e:
             logger.error("Error: New rule " + rule_path + " " + str(type(e)) + " " + str(e))
-            sys.exit(1)
+            sys.exit(-1)
     return feature_list
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
         logger.error("FileNotFoundError ! New rule file name doesn't end with .yml")
-        sys.exit(1)
+        sys.exit(-1)
 
     # Loads features of new rule in a list.
     new_rule_features = get_features(new_rule_path)
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index ce8f01c3..3936ed59 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -14,8 +14,6 @@ import subprocess
 import pytest
 from fixtures import *
 
-import capa.main
-
 CD = os.path.dirname(__file__)
 
 
@@ -88,41 +86,73 @@ def test_proto_conversion(tmpdir):
     assert p.stdout.startswith(b'{\n  "meta": ') or p.stdout.startswith(b'{\r\n  "meta": ')
 
 
-def run_detect_duplicate_features(rule_path):
+def run_detect_duplicate_features(rule_dir, rule_path):
     # rule_path = "collection/credit-card/parse-credit-card-information.yml"
-    args = [get_rules_path(), rule_path]
+    args = [rule_dir, rule_path]
     script_path = get_script_path("detect_duplicate_features.py")
     args = [sys.executable] + [script_path] + args
     print(f"running: '{args}'")
     return subprocess.run(args)
 
 
-def test_detect_duplicate_features(z9324d_extractor, tmpdir):
-    RULE_CONTENT = textwrap.dedent(
-        """
-        rule:
-          meta:
-            name: Test Rule
-            scope: function
-          features:
-            - string: "sites.ini"
-        """
-    )
-    expected_overlaps = 3
-    path = z9324d_extractor.path
-    rule_file = tmpdir.mkdir("capa").join("rule.yml")
-    rule_file.write(RULE_CONTENT)
-    assert (
-        capa.main.main(
-            [
-                path,
-                "-v",
-                "-r",
-                rule_file.strpath,
-            ]
-        )
-        == 0
-    )
-    # tests if number of overlaps found are correct.
-    overlaps_found = run_detect_duplicate_features(rule_file.strpath).returncode
-    assert overlaps_found == expected_overlaps
+def test_detect_duplicate_features(tmpdir):
+    RULESET = {
+        "rule_1": textwrap.dedent(
+            """
+                rule:
+                    meta:
+                        name: Test Rule 1
+                        scope: function
+                    features:
+                      - or:
+                        - string: "sites.ini"
+                        - number: 0xEDB88320
+            """
+        ),
+        "rule_2": textwrap.dedent(
+            """
+                rule:
+                    meta:
+                        name: Test Rule 2
+                        scope: function
+                    features:
+                      - and:
+                        - string: "sites.ini"
+                        - number: 8
+            """
+        ),
+        "rule_3": textwrap.dedent(
+            """
+                rule:
+                    meta:
+                        name: Test Rule 3
+                        scope: function
+                    features:
+                      - not:
+                        - number: 0xEDB88320
+            """
+        ),
+        "rule_4": textwrap.dedent(
+            """
+                rule:
+                    meta:
+                        name: Test Rule 4
+                        scope: function
+                    features:
+                      - not:
+                        - number: 4
+            """
+        ),
+    }
+
+    rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
+    rule_overlaps = [3, 2, 2, 1]
+    rule_paths = []
+    for rule_name, RULE_CONTENT in RULESET.items():
+        rule_file = rule_dir.join("%s.yml" % rule_name)
+        rule_file.write(RULE_CONTENT)
+        rule_paths.append(rule_file.strpath)
+    # tests if number of overlaps for rules in RULESET found are correct.
+    for expected_overlaps, rule_path in zip(rule_overlaps, rule_paths):
+        overlaps_found = run_detect_duplicate_features(rule_dir.strpath, rule_path)
+        assert overlaps_found.returncode == expected_overlaps

From d861b0798e96d815dc530d7afda80e1fb2d9c544 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 May 2023 14:58:08 +0000
Subject: [PATCH 14/32] build(deps-dev): bump types-protobuf from 4.22.0.2 to
 4.23.0.1

Bumps [types-protobuf](https://github.com/python/typeshed) from 4.22.0.2 to 4.23.0.1.
- [Commits](https://github.com/python/typeshed/commits)

---
updated-dependencies:
- dependency-name: types-protobuf
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 0737b0a7..0157c50a 100644
--- a/setup.py
+++ b/setup.py
@@ -90,7 +90,7 @@ setuptools.setup(
             "types-termcolor==1.1.4",
             "types-psutil==5.8.23",
             "types_requests==2.28.1",
-            "types-protobuf==4.22.0.2",
+            "types-protobuf==4.23.0.1",
         ],
         "build": [
             "pyinstaller==5.10.1",

From 12c191582fef94a0b5e0a9b7780bfe1a97189ad3 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 15 May 2023 22:58:19 +0530
Subject: [PATCH 15/32] Update tests/test_scripts.py

Co-authored-by: Moritz <mr-tz@users.noreply.github.com>
---
 tests/test_scripts.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 3936ed59..308968b8 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -87,7 +87,6 @@ def test_proto_conversion(tmpdir):
 
 
 def run_detect_duplicate_features(rule_dir, rule_path):
-    # rule_path = "collection/credit-card/parse-credit-card-information.yml"
     args = [rule_dir, rule_path]
     script_path = get_script_path("detect_duplicate_features.py")
     args = [sys.executable] + [script_path] + args

From 931dcb1dc50d693bb5e9389d97e1ebf325100972 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 15 May 2023 23:35:11 +0530
Subject: [PATCH 16/32] Update test_scripts.py

---
 tests/test_scripts.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 308968b8..a4f7114a 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -144,8 +144,18 @@ def test_detect_duplicate_features(tmpdir):
         ),
     }
 
-    rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_overlaps = [3, 2, 2, 1]
+    """
+        The rule_overlaps list represents the number of overlaps between each rule in the RULESET.
+        An overlap includes a rule overlap with itself.
+        The overlaps are like:
+        - Rule 1 overlaps with 3 other rules in RULESET
+        - Rule 4 overlaps with itself in RULESET
+        These overlap values indicate the number of rules with which
+        each rule in RULESET has overlapping features.
+    """
+
+    rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_paths = []
     for rule_name, RULE_CONTENT in RULESET.items():
         rule_file = rule_dir.join("%s.yml" % rule_name)

From 0afc16fd02ec966be7a50555f89df8753c1b6794 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Wed, 17 May 2023 23:31:37 +0530
Subject: [PATCH 17/32] Update test rules to test script

---
 CHANGELOG.md          |  2 +-
 tests/test_scripts.py | 53 +++++++++++++++++++++++++++++++++++++------
 2 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6fa5813f..a7b48114 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,7 @@
 ## master (unreleased)
 
 ### New Features
-- Utility script to detect feature overlap between new and existing CAPA rules #1451 @Aayush-Goel-04
+- Utility script to detect feature overlap between new and existing CAPA rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 
 ### Breaking Changes
 
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index a4f7114a..e67ab003 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -95,7 +95,22 @@ def run_detect_duplicate_features(rule_dir, rule_path):
 
 
 def test_detect_duplicate_features(tmpdir):
-    RULESET = {
+    TEST_RULE_0 = textwrap.dedent(
+        """
+        rule:
+            meta:
+                name: Test Rule 0
+                scope: function
+            features:
+              - and:
+                - number: 2
+                - or:
+                  - mnemonic: shr
+                  - api: connect
+        """
+    )
+
+    TEST_RULESET = {
         "rule_1": textwrap.dedent(
             """
                 rule:
@@ -106,6 +121,14 @@ def test_detect_duplicate_features(tmpdir):
                       - or:
                         - string: "sites.ini"
                         - number: 0xEDB88320
+                        - and:
+                          - or:
+                            - arch: i386
+                            - number: 4
+                            - not:
+                              - count(mnemonic(xor)): 5
+                          - not:
+                            - os: linux
             """
         ),
         "rule_2": textwrap.dedent(
@@ -117,7 +140,11 @@ def test_detect_duplicate_features(tmpdir):
                     features:
                       - and:
                         - string: "sites.ini"
-                        - number: 8
+                        - arch: i386
+                        - basic block:
+                          - and:
+                            - api: setsockopt
+                            - count(mnemonic(mov)): 3
             """
         ),
         "rule_3": textwrap.dedent(
@@ -127,8 +154,13 @@ def test_detect_duplicate_features(tmpdir):
                         name: Test Rule 3
                         scope: function
                     features:
-                      - not:
-                        - number: 0xEDB88320
+                      - or:
+                        - not:
+                          - os: linux
+                        - basic block:
+                          - and:
+                            - api: bind
+                            - count(mnemonic(mov)): 3
             """
         ),
         "rule_4": textwrap.dedent(
@@ -139,28 +171,35 @@ def test_detect_duplicate_features(tmpdir):
                         scope: function
                     features:
                       - not:
-                        - number: 4
+                        - string: "expa"
             """
         ),
     }
 
-    rule_overlaps = [3, 2, 2, 1]
     """
         The rule_overlaps list represents the number of overlaps between each rule in the RULESET.
         An overlap includes a rule overlap with itself.
         The overlaps are like:
+        - Rule 0 has zero overlaps in RULESET
         - Rule 1 overlaps with 3 other rules in RULESET
         - Rule 4 overlaps with itself in RULESET
         These overlap values indicate the number of rules with which
         each rule in RULESET has overlapping features.
     """
+    rule_overlaps = [0, 3, 4, 4, 1]
 
     rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_paths = []
-    for rule_name, RULE_CONTENT in RULESET.items():
+
+    rule_file = tmpdir.join("%s.yml" % "rule_0")
+    rule_file.write(TEST_RULE_0)
+    rule_paths.append(rule_file.strpath)
+
+    for rule_name, RULE_CONTENT in TEST_RULESET.items():
         rule_file = rule_dir.join("%s.yml" % rule_name)
         rule_file.write(RULE_CONTENT)
         rule_paths.append(rule_file.strpath)
+
     # tests if number of overlaps for rules in RULESET found are correct.
     for expected_overlaps, rule_path in zip(rule_overlaps, rule_paths):
         overlaps_found = run_detect_duplicate_features(rule_dir.strpath, rule_path)

From 275386806dc46890cacb0736e9ce1082a376cabf Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Wed, 17 May 2023 23:57:52 +0530
Subject: [PATCH 18/32] Add redirect print to capa main

---
 CHANGELOG.md    |  1 +
 capa/helpers.py | 37 +++++++++++++++++++++++++++++++
 capa/main.py    | 58 +++++++++++++++++++++++++------------------------
 scripts/lint.py | 35 +----------------------------
 4 files changed, 69 insertions(+), 62 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9c008f93..d078c5b0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@
 - improve ELF strtab and needed parsing @mr-tz
 - better handle exceptional cases when parsing ELF files [#1458](https://github.com/mandiant/capa/issues/1458) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 - Improved testing coverage for Binary Ninja Backend [#1446](https://github.com/mandiant/capa/issues/1446) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
+- Add redirect print to tqdm for capa main [#749](https://github.com/mandiant/capa/issues/749) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 - extractor: fix binja installation path detection does not work with Python 3.11
 
 ### capa explorer IDA Pro plugin
diff --git a/capa/helpers.py b/capa/helpers.py
index 25d9efe0..c03e0553 100644
--- a/capa/helpers.py
+++ b/capa/helpers.py
@@ -6,9 +6,13 @@
 #  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import os
+import inspect
 import logging
+import contextlib
 from typing import NoReturn
 
+import tqdm
+
 from capa.exceptions import UnsupportedFormatError
 from capa.features.common import FORMAT_PE, FORMAT_SC32, FORMAT_SC64, FORMAT_DOTNET, FORMAT_UNKNOWN, Format
 
@@ -85,6 +89,39 @@ def get_format(sample: str) -> str:
     return FORMAT_UNKNOWN
 
 
+@contextlib.contextmanager
+def redirecting_print_to_tqdm(disable_progress):
+    """
+    tqdm (progress bar) expects to have fairly tight control over console output.
+    so calls to `print()` will break the progress bar and make things look bad.
+    so, this context manager temporarily replaces the `print` implementation
+    with one that is compatible with tqdm.
+    via: https://stackoverflow.com/a/42424890/87207
+    """
+    old_print = print
+
+    def new_print(*args, **kwargs):
+        # If tqdm.tqdm.write raises error, use builtin print
+        if disable_progress:
+            old_print(*args, **kwargs)
+        else:
+            try:
+                tqdm.tqdm.write(*args, **kwargs)
+            except:
+                old_print(*args, **kwargs)
+
+    try:
+        # Globally replace print with new_print.
+        # Verified this works manually on Python 3.11:
+        #     >>> import inspect
+        #     >>> inspect.builtins
+        #     <module 'builtins' (built-in)>
+        inspect.builtins.print = new_print  # type: ignore
+        yield
+    finally:
+        inspect.builtins.print = old_print  # type: ignore
+
+
 def log_unsupported_format_error():
     logger.error("-" * 80)
     logger.error(" Input file does not appear to be a PE or ELF file.")
diff --git a/capa/main.py b/capa/main.py
index 14411b0e..fc71a89c 100644
--- a/capa/main.py
+++ b/capa/main.py
@@ -53,6 +53,7 @@ from capa.helpers import (
     get_file_taste,
     get_auto_format,
     log_unsupported_os_error,
+    redirecting_print_to_tqdm,
     log_unsupported_arch_error,
     log_unsupported_format_error,
 )
@@ -251,38 +252,39 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
         "library_functions": {},
     }  # type: Dict[str, Any]
 
-    pbar = tqdm.tqdm
-    if disable_progress:
-        # do not use tqdm to avoid unnecessary side effects when caller intends
-        # to disable progress completely
-        def pbar(s, *args, **kwargs):
-            return s
+    with redirecting_print_to_tqdm(disable_progress):
+        pbar = tqdm.tqdm
+        if disable_progress:
+            # do not use tqdm to avoid unnecessary side effects when caller intends
+            # to disable progress completely
+            def pbar(s, *args, **kwargs):
+                return s
 
-    functions = list(extractor.get_functions())
-    n_funcs = len(functions)
+        functions = list(extractor.get_functions())
+        n_funcs = len(functions)
 
-    pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions")
-    for f in pb:
-        if extractor.is_library_function(f.address):
-            function_name = extractor.get_function_name(f.address)
-            logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
-            meta["library_functions"][f.address] = function_name
-            n_libs = len(meta["library_functions"])
-            percentage = round(100 * (n_libs / n_funcs))
-            if isinstance(pb, tqdm.tqdm):
-                pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
-            continue
+        pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions")
+        for f in pb:
+            if extractor.is_library_function(f.address):
+                function_name = extractor.get_function_name(f.address)
+                logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
+                meta["library_functions"][f.address] = function_name
+                n_libs = len(meta["library_functions"])
+                percentage = round(100 * (n_libs / n_funcs))
+                if isinstance(pb, tqdm.tqdm):
+                    pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
+                continue
 
-        function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(ruleset, extractor, f)
-        meta["feature_counts"]["functions"][f.address] = feature_count
-        logger.debug("analyzed function 0x%x and extracted %d features", f.address, feature_count)
+            function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(ruleset, extractor, f)
+            meta["feature_counts"]["functions"][f.address] = feature_count
+            logger.debug("analyzed function 0x%x and extracted %d features", f.address, feature_count)
 
-        for rule_name, res in function_matches.items():
-            all_function_matches[rule_name].extend(res)
-        for rule_name, res in bb_matches.items():
-            all_bb_matches[rule_name].extend(res)
-        for rule_name, res in insn_matches.items():
-            all_insn_matches[rule_name].extend(res)
+            for rule_name, res in function_matches.items():
+                all_function_matches[rule_name].extend(res)
+            for rule_name, res in bb_matches.items():
+                all_bb_matches[rule_name].extend(res)
+            for rule_name, res in insn_matches.items():
+                all_insn_matches[rule_name].extend(res)
 
     # collection of features that captures the rule matches within function, BB, and instruction scopes.
     # mapping from feature (matched rule) to set of addresses at which it matched.
diff --git a/scripts/lint.py b/scripts/lint.py
index 92c7fbcf..a80d3e12 100644
--- a/scripts/lint.py
+++ b/scripts/lint.py
@@ -22,13 +22,11 @@ import time
 import string
 import difflib
 import hashlib
-import inspect
 import logging
 import pathlib
 import argparse
 import itertools
 import posixpath
-import contextlib
 from typing import Set, Dict, List
 from pathlib import Path
 from dataclasses import field, dataclass
@@ -866,37 +864,6 @@ def width(s, count):
         return s.ljust(count)
 
 
-@contextlib.contextmanager
-def redirecting_print_to_tqdm():
-    """
-    tqdm (progress bar) expects to have fairly tight control over console output.
-    so calls to `print()` will break the progress bar and make things look bad.
-    so, this context manager temporarily replaces the `print` implementation
-    with one that is compatible with tqdm.
-
-    via: https://stackoverflow.com/a/42424890/87207
-    """
-    old_print = print
-
-    def new_print(*args, **kwargs):
-        # If tqdm.tqdm.write raises error, use builtin print
-        try:
-            tqdm.tqdm.write(*args, **kwargs)
-        except:
-            old_print(*args, **kwargs)
-
-    try:
-        # Globally replace print with new_print.
-        # Verified this works manually on Python 3.11:
-        #     >>> import inspect
-        #     >>> inspect.builtins
-        #     <module 'builtins' (built-in)>
-        inspect.builtins.print = new_print  # type: ignore
-        yield
-    finally:
-        inspect.builtins.print = old_print  # type: ignore
-
-
 def lint(ctx: Context):
     """
     Returns: Dict[string, Tuple(int, int)]
@@ -907,7 +874,7 @@ def lint(ctx: Context):
 
     source_rules = [rule for rule in ctx.rules.rules.values() if not rule.is_subscope_rule()]
     with tqdm.contrib.logging.tqdm_logging_redirect(source_rules, unit="rule") as pbar:
-        with redirecting_print_to_tqdm():
+        with capa.helpers.redirecting_print_to_tqdm(False):
             for rule in pbar:
                 name = rule.name
                 pbar.set_description(width(f"linting rule: {name}", 48))

From 932066bc0ed73151cd2995127d2969d4f42fa5d5 Mon Sep 17 00:00:00 2001
From: Capa Bot <capa-dev@mandiant.com>
Date: Fri, 19 May 2023 08:22:32 +0000
Subject: [PATCH 19/32] Sync capa rules submodule

---
 rules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rules b/rules
index 312d4cad..cc41fac5 160000
--- a/rules
+++ b/rules
@@ -1 +1 @@
-Subproject commit 312d4cad891498e1d360dffcc98f669b63869c94
+Subproject commit cc41fac571e6e022716b4d76e92e906bdd8ed178

From acdaeb26d3782dd7ee98f9cc0a73582d0f16f3e5 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Sat, 20 May 2023 13:09:48 +0530
Subject: [PATCH 20/32] Update test_scripts.py

---
 tests/test_scripts.py | 39 ++++++++++++++-------------------------
 1 file changed, 14 insertions(+), 25 deletions(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index e67ab003..fdd065f7 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -86,14 +86,6 @@ def test_proto_conversion(tmpdir):
     assert p.stdout.startswith(b'{\n  "meta": ') or p.stdout.startswith(b'{\r\n  "meta": ')
 
 
-def run_detect_duplicate_features(rule_dir, rule_path):
-    args = [rule_dir, rule_path]
-    script_path = get_script_path("detect_duplicate_features.py")
-    args = [sys.executable] + [script_path] + args
-    print(f"running: '{args}'")
-    return subprocess.run(args)
-
-
 def test_detect_duplicate_features(tmpdir):
     TEST_RULE_0 = textwrap.dedent(
         """
@@ -103,10 +95,9 @@ def test_detect_duplicate_features(tmpdir):
                 scope: function
             features:
               - and:
-                - number: 2
-                - or:
-                  - mnemonic: shr
-                  - api: connect
+                - number: 1
+                - not:
+                  - string: process
         """
     )
 
@@ -116,11 +107,10 @@ def test_detect_duplicate_features(tmpdir):
                 rule:
                     meta:
                         name: Test Rule 1
-                        scope: function
                     features:
                       - or:
-                        - string: "sites.ini"
-                        - number: 0xEDB88320
+                        - string: unique
+                        - number: 2
                         - and:
                           - or:
                             - arch: i386
@@ -136,15 +126,13 @@ def test_detect_duplicate_features(tmpdir):
                 rule:
                     meta:
                         name: Test Rule 2
-                        scope: function
                     features:
                       - and:
                         - string: "sites.ini"
-                        - arch: i386
                         - basic block:
                           - and:
-                            - api: setsockopt
-                            - count(mnemonic(mov)): 3
+                            - api: CreateFile
+                            - mnemonic: xor
             """
         ),
         "rule_3": textwrap.dedent(
@@ -152,15 +140,14 @@ def test_detect_duplicate_features(tmpdir):
                 rule:
                     meta:
                         name: Test Rule 3
-                        scope: function
                     features:
                       - or:
                         - not:
-                          - os: linux
+                          - number: 4
                         - basic block:
                           - and:
                             - api: bind
-                            - count(mnemonic(mov)): 3
+                            - number: 2
             """
         ),
         "rule_4": textwrap.dedent(
@@ -168,7 +155,6 @@ def test_detect_duplicate_features(tmpdir):
                 rule:
                     meta:
                         name: Test Rule 4
-                        scope: function
                     features:
                       - not:
                         - string: "expa"
@@ -179,6 +165,7 @@ def test_detect_duplicate_features(tmpdir):
     """
         The rule_overlaps list represents the number of overlaps between each rule in the RULESET.
         An overlap includes a rule overlap with itself.
+        The scripts 
         The overlaps are like:
         - Rule 0 has zero overlaps in RULESET
         - Rule 1 overlaps with 3 other rules in RULESET
@@ -186,7 +173,7 @@ def test_detect_duplicate_features(tmpdir):
         These overlap values indicate the number of rules with which
         each rule in RULESET has overlapping features.
     """
-    rule_overlaps = [0, 3, 4, 4, 1]
+    rule_overlaps = [0, 4, 3, 3, 1]
 
     rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_paths = []
@@ -201,6 +188,8 @@ def test_detect_duplicate_features(tmpdir):
         rule_paths.append(rule_file.strpath)
 
     # tests if number of overlaps for rules in RULESET found are correct.
+    script_path = get_script_path("detect_duplicate_features.py")
     for expected_overlaps, rule_path in zip(rule_overlaps, rule_paths):
-        overlaps_found = run_detect_duplicate_features(rule_dir.strpath, rule_path)
+        args = [rule_dir.strpath, rule_path]
+        overlaps_found = run_program(script_path, args)
         assert overlaps_found.returncode == expected_overlaps

From 52c3ea733bef93029e8e93f324512d08c6f22f40 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Wed, 24 May 2023 15:39:24 +0530
Subject: [PATCH 21/32] Update tests/test_scripts.py

Co-authored-by: Moritz <mr-tz@users.noreply.github.com>
---
 tests/test_scripts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index fdd065f7..2d8fefac 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -178,7 +178,7 @@ def test_detect_duplicate_features(tmpdir):
     rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_paths = []
 
-    rule_file = tmpdir.join("%s.yml" % "rule_0")
+    rule_file = tmpdir.join("rule_0.yml")
     rule_file.write(TEST_RULE_0)
     rule_paths.append(rule_file.strpath)
 

From 456d4272abb35e8e5c6fc392037411cce2efeb1a Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Thu, 25 May 2023 12:40:51 +0530
Subject: [PATCH 22/32] Add logging redirect to capa main

---
 CHANGELOG.md |  2 +-
 capa/main.py | 60 ++++++++++++++++++++++++++++------------------------
 2 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d078c5b0..22575f72 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,7 +22,7 @@
 - improve ELF strtab and needed parsing @mr-tz
 - better handle exceptional cases when parsing ELF files [#1458](https://github.com/mandiant/capa/issues/1458) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 - Improved testing coverage for Binary Ninja Backend [#1446](https://github.com/mandiant/capa/issues/1446) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
-- Add redirect print to tqdm for capa main [#749](https://github.com/mandiant/capa/issues/749) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
+- Add logging and print redirect to tqdm for capa main [#749](https://github.com/mandiant/capa/issues/749) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 - extractor: fix binja installation path detection does not work with Python 3.11
 
 ### capa explorer IDA Pro plugin
diff --git a/capa/main.py b/capa/main.py
index fc71a89c..b305673c 100644
--- a/capa/main.py
+++ b/capa/main.py
@@ -25,6 +25,7 @@ from typing import Any, Dict, List, Tuple, Callable
 import halo
 import tqdm
 import colorama
+import tqdm.contrib.logging
 from pefile import PEFormatError
 from elftools.common.exceptions import ELFError
 
@@ -253,38 +254,41 @@ def find_capabilities(ruleset: RuleSet, extractor: FeatureExtractor, disable_pro
     }  # type: Dict[str, Any]
 
     with redirecting_print_to_tqdm(disable_progress):
-        pbar = tqdm.tqdm
-        if disable_progress:
-            # do not use tqdm to avoid unnecessary side effects when caller intends
-            # to disable progress completely
-            def pbar(s, *args, **kwargs):
-                return s
+        with tqdm.contrib.logging.logging_redirect_tqdm():
+            pbar = tqdm.tqdm
+            if disable_progress:
+                # do not use tqdm to avoid unnecessary side effects when caller intends
+                # to disable progress completely
+                def pbar(s, *args, **kwargs):
+                    return s
 
-        functions = list(extractor.get_functions())
-        n_funcs = len(functions)
+            functions = list(extractor.get_functions())
+            n_funcs = len(functions)
 
-        pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions")
-        for f in pb:
-            if extractor.is_library_function(f.address):
-                function_name = extractor.get_function_name(f.address)
-                logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
-                meta["library_functions"][f.address] = function_name
-                n_libs = len(meta["library_functions"])
-                percentage = round(100 * (n_libs / n_funcs))
-                if isinstance(pb, tqdm.tqdm):
-                    pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
-                continue
+            pb = pbar(functions, desc="matching", unit=" functions", postfix="skipped 0 library functions")
+            for f in pb:
+                if extractor.is_library_function(f.address):
+                    function_name = extractor.get_function_name(f.address)
+                    logger.debug("skipping library function 0x%x (%s)", f.address, function_name)
+                    meta["library_functions"][f.address] = function_name
+                    n_libs = len(meta["library_functions"])
+                    percentage = round(100 * (n_libs / n_funcs))
+                    if isinstance(pb, tqdm.tqdm):
+                        pb.set_postfix_str(f"skipped {n_libs} library functions ({percentage}%)")
+                    continue
 
-            function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(ruleset, extractor, f)
-            meta["feature_counts"]["functions"][f.address] = feature_count
-            logger.debug("analyzed function 0x%x and extracted %d features", f.address, feature_count)
+                function_matches, bb_matches, insn_matches, feature_count = find_code_capabilities(
+                    ruleset, extractor, f
+                )
+                meta["feature_counts"]["functions"][f.address] = feature_count
+                logger.debug("analyzed function 0x%x and extracted %d features", f.address, feature_count)
 
-            for rule_name, res in function_matches.items():
-                all_function_matches[rule_name].extend(res)
-            for rule_name, res in bb_matches.items():
-                all_bb_matches[rule_name].extend(res)
-            for rule_name, res in insn_matches.items():
-                all_insn_matches[rule_name].extend(res)
+                for rule_name, res in function_matches.items():
+                    all_function_matches[rule_name].extend(res)
+                for rule_name, res in bb_matches.items():
+                    all_bb_matches[rule_name].extend(res)
+                for rule_name, res in insn_matches.items():
+                    all_insn_matches[rule_name].extend(res)
 
     # collection of features that captures the rule matches within function, BB, and instruction scopes.
     # mapping from feature (matched rule) to set of addresses at which it matched.

From e068ce7bc9239ff06b8f7ff565b4a248df9e9692 Mon Sep 17 00:00:00 2001
From: Capa Bot <capa-dev@mandiant.com>
Date: Fri, 26 May 2023 08:34:57 +0000
Subject: [PATCH 23/32] Sync capa rules submodule

---
 README.md | 2 +-
 rules     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index a700f894..8d1a8715 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-797-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-796-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
diff --git a/rules b/rules
index cc41fac5..11b9f291 160000
--- a/rules
+++ b/rules
@@ -1 +1 @@
-Subproject commit cc41fac571e6e022716b4d76e92e906bdd8ed178
+Subproject commit 11b9f2910abcfa8c113c13b7894610186c2172c7

From f789e144fdd5b9ab46b895d031fc2f1c5a854f03 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 May 2023 14:58:10 +0000
Subject: [PATCH 24/32] build(deps): bump protobuf from 4.22.3 to 4.23.2

Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 4.22.3 to 4.23.2.
- [Release notes](https://github.com/protocolbuffers/protobuf/releases)
- [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py)
- [Commits](https://github.com/protocolbuffers/protobuf/compare/v4.22.3...v4.23.2)

---
updated-dependencies:
- dependency-name: protobuf
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6e36bb21..62bbd218 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ requirements = [
     "dnfile==0.13.0",
     "dncil==1.0.2",
     "pydantic==1.10.7",
-    "protobuf==4.22.3",
+    "protobuf==4.23.2",
 ]
 
 # this sets __version__

From a07b47c84541c65f62d5f0c344ca9e8f0cafc3aa Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 May 2023 14:59:02 +0000
Subject: [PATCH 25/32] build(deps-dev): bump ruff from 0.0.265 to 0.0.270

Bumps [ruff](https://github.com/charliermarsh/ruff) from 0.0.265 to 0.0.270.
- [Release notes](https://github.com/charliermarsh/ruff/releases)
- [Changelog](https://github.com/charliermarsh/ruff/blob/main/BREAKING_CHANGES.md)
- [Commits](https://github.com/charliermarsh/ruff/compare/v0.0.265...v0.0.270)

---
updated-dependencies:
- dependency-name: ruff
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6e36bb21..59872a55 100644
--- a/setup.py
+++ b/setup.py
@@ -74,7 +74,7 @@ setuptools.setup(
             "pytest-instafail==0.5.0",
             "pytest-cov==4.0.0",
             "pycodestyle==2.10.0",
-            "ruff==0.0.265",
+            "ruff==0.0.270",
             "black==23.3.0",
             "isort==5.11.4",
             "mypy==1.3.0",

From c113a3b5b8a7183444ab51e1adcd89baf84822e0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 May 2023 14:59:13 +0000
Subject: [PATCH 26/32] build(deps): bump ruamel-yaml from 0.17.21 to 0.17.28

Bumps [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) from 0.17.21 to 0.17.28.

---
updated-dependencies:
- dependency-name: ruamel-yaml
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6e36bb21..bc9af0dd 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@ requirements = [
     "viv-utils[flirt]==0.7.9",
     "halo==0.0.31",
     "networkx==2.5.1",  # newer versions no longer support py3.7.
-    "ruamel.yaml==0.17.21",
+    "ruamel.yaml==0.17.28",
     "vivisect==1.1.1",
     "pefile==2023.2.7",
     "pyelftools==0.29",

From 1696a9ad2d23ad5f6c99263e3453a3a3750e7e27 Mon Sep 17 00:00:00 2001
From: Capa Bot <capa-dev@mandiant.com>
Date: Tue, 30 May 2023 14:28:43 +0000
Subject: [PATCH 27/32] Sync capa-testfiles submodule

---
 tests/data | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data b/tests/data
index 91e2796c..00db2da0 160000
--- a/tests/data
+++ b/tests/data
@@ -1 +1 @@
-Subproject commit 91e2796c9950eb1942b6b910bc42714110d07f8b
+Subproject commit 00db2da0eb45a5efea109a6a338cae8f0e0b409e

From 99e81e1d8fa5ce905cea284ba3ce0e23842c90f5 Mon Sep 17 00:00:00 2001
From: Capa Bot <capa-dev@mandiant.com>
Date: Tue, 30 May 2023 14:31:43 +0000
Subject: [PATCH 28/32] Sync capa rules submodule

---
 CHANGELOG.md | 3 ++-
 README.md    | 2 +-
 rules        | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 22575f72..18b951c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,12 +6,13 @@
 
 ### Breaking Changes
 
-### New Rules (4)
+### New Rules (5)
 
 - load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
 - nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
 - data-manipulation/encryption/aes/encrypt-data-using-aes-mixcolumns-step @mr-tz
 - linking/static/aplib/linked-against-aplib still@teamt5.org
+- communication/mailslot/read-from-mailslot nick.simonian@mandiant.com
 -
 
 ### Bug Fixes
diff --git a/README.md b/README.md
index 8d1a8715..a700f894 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-796-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-797-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
diff --git a/rules b/rules
index 11b9f291..d032ec7e 160000
--- a/rules
+++ b/rules
@@ -1 +1 @@
-Subproject commit 11b9f2910abcfa8c113c13b7894610186c2172c7
+Subproject commit d032ec7eda055e865f3aa425c0fdb8cb4468b6eb

From 9caea57cde888253e2bec7f757813f7c24692090 Mon Sep 17 00:00:00 2001
From: Capa Bot <capa-dev@mandiant.com>
Date: Tue, 30 May 2023 14:37:56 +0000
Subject: [PATCH 29/32] Sync capa rules submodule

---
 rules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rules b/rules
index d032ec7e..0554a8be 160000
--- a/rules
+++ b/rules
@@ -1 +1 @@
-Subproject commit d032ec7eda055e865f3aa425c0fdb8cb4468b6eb
+Subproject commit 0554a8be12b3ee6eac32817d5df9da34cba28fc8

From d3c44a82633ca2ee5cae2182197dc3706bd7c0be Mon Sep 17 00:00:00 2001
From: Capa Bot <capa-dev@mandiant.com>
Date: Wed, 31 May 2023 18:16:12 +0000
Subject: [PATCH 30/32] Sync capa rules submodule

---
 CHANGELOG.md | 3 ++-
 README.md    | 2 +-
 rules        | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 18b951c9..0d2ca4ec 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,13 +6,14 @@
 
 ### Breaking Changes
 
-### New Rules (5)
+### New Rules (6)
 
 - load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
 - nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
 - data-manipulation/encryption/aes/encrypt-data-using-aes-mixcolumns-step @mr-tz
 - linking/static/aplib/linked-against-aplib still@teamt5.org
 - communication/mailslot/read-from-mailslot nick.simonian@mandiant.com
+- nursery/hash-data-using-sha512managed-in-dotnet jonathanlepore@google.com
 -
 
 ### Bug Fixes
diff --git a/README.md b/README.md
index a700f894..b6d3936f 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-797-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-798-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
diff --git a/rules b/rules
index 0554a8be..0db65de1 160000
--- a/rules
+++ b/rules
@@ -1 +1 @@
-Subproject commit 0554a8be12b3ee6eac32817d5df9da34cba28fc8
+Subproject commit 0db65de1614ee81f9a3baf923423e015bfa78c16

From 0f54a6f67ee898fc09ff073a1b1dbf89e4a64a48 Mon Sep 17 00:00:00 2001
From: Capa Bot <capa-dev@mandiant.com>
Date: Fri, 2 Jun 2023 07:13:58 +0000
Subject: [PATCH 31/32] Sync capa rules submodule

---
 CHANGELOG.md | 3 ++-
 README.md    | 2 +-
 rules        | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0d2ca4ec..7ed43031 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,7 @@
 
 ### Breaking Changes
 
-### New Rules (6)
+### New Rules (7)
 
 - load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
 - nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
@@ -14,6 +14,7 @@
 - linking/static/aplib/linked-against-aplib still@teamt5.org
 - communication/mailslot/read-from-mailslot nick.simonian@mandiant.com
 - nursery/hash-data-using-sha512managed-in-dotnet jonathanlepore@google.com
+- nursery/compiled-with-exescript jonathanlepore@google.com
 -
 
 ### Bug Fixes
diff --git a/README.md b/README.md
index b6d3936f..16d561bc 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-798-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-799-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
diff --git a/rules b/rules
index 0db65de1..188e6552 160000
--- a/rules
+++ b/rules
@@ -1 +1 @@
-Subproject commit 0db65de1614ee81f9a3baf923423e015bfa78c16
+Subproject commit 188e65528ec496eaaa792c3470cb4ab680a1b156

From 0cbe4618e15fa635c19eb1ec2340bc1df3a5c303 Mon Sep 17 00:00:00 2001
From: Capa Bot <capa-dev@mandiant.com>
Date: Fri, 2 Jun 2023 09:20:23 +0000
Subject: [PATCH 32/32] Sync capa-testfiles submodule

---
 tests/data | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data b/tests/data
index 00db2da0..a37873c8 160000
--- a/tests/data
+++ b/tests/data
@@ -1 +1 @@
-Subproject commit 00db2da0eb45a5efea109a6a338cae8f0e0b409e
+Subproject commit a37873c8a571b515f2baaf19bfcfaff5c7ef5342