From 256611bef5372295004d3aa10e94c841ac57c1e0 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Thu, 27 Apr 2023 06:00:38 +0530
Subject: [PATCH 01/20] Create detect_duplicate_features.py

Fixes #1451
Python script to detect feature overlap between new and existing CAPA rules. Checks if the a feature in new rules exists in an existing rule
---
 scripts/detect_duplicate_features.py | 67 ++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 scripts/detect_duplicate_features.py

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
new file mode 100644
index 00000000..7da0655e
--- /dev/null
+++ b/scripts/detect_duplicate_features.py
@@ -0,0 +1,67 @@
+import os
+import yaml
+
+def findall_features(features):
+    feature_list = []
+    for feature in features:
+        if 'and' in feature:
+            and_list = findall_features(feature['and'])
+            for x in and_list:
+                feature_list.append(x)
+        elif 'or' in feature:
+            or_list = findall_features(feature['or'])
+            for y in or_list:
+                feature_list.append(y)
+        else:
+            feature_list.append(feature)
+    return feature_list
+
+def find_overlapping_rules(new_rule_path, rules_path):
+    if not new_rule_path.endswith('.yml'):
+        return 'ERROR ! New rule path file name incorrect'
+    
+    count = 0
+
+    with open(new_rule_path, 'r') as f:
+        new_rule = yaml.safe_load(f)
+    if 'rule' not in new_rule:
+        return "ERROR ! given new rule path isn't a rule"
+    
+    new_rule_features = findall_features(new_rule['rule']['features'])
+
+    overlapping_rules = []
+    
+    for dirpath, dirnames, filenames in os.walk(rules_path):
+        for filename in filenames:
+            if filename.endswith('.yml'):
+                rule_path = os.path.join(dirpath, filename)
+                with open(rule_path, 'r') as f:
+                    rule = yaml.safe_load(f)
+                    if 'rule' not in rule:
+                        continue
+                    rule_features = findall_features(rule['rule']['features'])
+                    count+=1
+                if any([feature in rule_features for feature in new_rule_features]):
+                    overlapping_rules.append(rule_path)
+    result = {'overlapping_rules': overlapping_rules,
+              'count': count}
+    
+    return result
+
+# usage
+base_dir = ''
+new_rule_path = base_dir + 'rules\\anti-analysis\\reference-analysis-tools-strings.yml'
+rules_path = base_dir + 'rules'
+
+try:
+    result = find_overlapping_rules(new_rule_path, rules_path)
+    print('New rule path : %s' % new_rule_path)
+    print('Number of rules checked : %s ' % result['count'])
+    print('Paths to overlapping rules : ', result['overlapping_rules'])
+    print('Number of rules containing same features : %s' % len(result['overlapping_rules']))
+except Exception as e:
+    print(e)
+    try:
+        print(result,'')
+    except:
+        pass

From 09865ccd9be97ef26e892cab6d751325e2e7d169 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Thu, 27 Apr 2023 06:11:58 +0530
Subject: [PATCH 02/20] Fixes Linting Issues

Update detect_duplicate_features.py
---
 scripts/detect_duplicate_features.py | 62 +++++++++++++++-------------
 1 file changed, 34 insertions(+), 28 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 7da0655e..f852e178 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,67 +1,73 @@
 import os
+
 import yaml
 
+
 def findall_features(features):
     feature_list = []
     for feature in features:
-        if 'and' in feature:
-            and_list = findall_features(feature['and'])
+        if "and" in feature:
+            and_list = findall_features(feature["and"])
             for x in and_list:
                 feature_list.append(x)
-        elif 'or' in feature:
-            or_list = findall_features(feature['or'])
+        elif "or" in feature:
+            or_list = findall_features(feature["or"])
             for y in or_list:
                 feature_list.append(y)
         else:
             feature_list.append(feature)
+
     return feature_list
 
+
 def find_overlapping_rules(new_rule_path, rules_path):
-    if not new_rule_path.endswith('.yml'):
-        return 'ERROR ! New rule path file name incorrect'
-    
+    if not new_rule_path.endswith(".yml"):
+        return "ERROR ! New rule path file name incorrect"
+
     count = 0
 
-    with open(new_rule_path, 'r') as f:
+    with open(new_rule_path, "r") as f:
         new_rule = yaml.safe_load(f)
-    if 'rule' not in new_rule:
+
+    if "rule" not in new_rule:
         return "ERROR ! given new rule path isn't a rule"
-    
-    new_rule_features = findall_features(new_rule['rule']['features'])
+
+    new_rule_features = findall_features(new_rule["rule"]["features"])
 
     overlapping_rules = []
-    
+
     for dirpath, dirnames, filenames in os.walk(rules_path):
         for filename in filenames:
-            if filename.endswith('.yml'):
+            if filename.endswith(".yml"):
                 rule_path = os.path.join(dirpath, filename)
-                with open(rule_path, 'r') as f:
+                with open(rule_path, "r") as f:
                     rule = yaml.safe_load(f)
-                    if 'rule' not in rule:
+                    if "rule" not in rule:
                         continue
-                    rule_features = findall_features(rule['rule']['features'])
-                    count+=1
+                    rule_features = findall_features(rule["rule"]["features"])
+                    count += 1
                 if any([feature in rule_features for feature in new_rule_features]):
                     overlapping_rules.append(rule_path)
-    result = {'overlapping_rules': overlapping_rules,
-              'count': count}
-    
+
+    result = {"overlapping_rules": overlapping_rules, "count": count}
+
     return result
 
+
 # usage
-base_dir = ''
-new_rule_path = base_dir + 'rules\\anti-analysis\\reference-analysis-tools-strings.yml'
-rules_path = base_dir + 'rules'
+base_dir = ""
+new_rule_path = base_dir + "rules\\anti-analysis\\reference-analysis-tools-strings.yml"
+rules_path = base_dir + "rules"
 
 try:
     result = find_overlapping_rules(new_rule_path, rules_path)
-    print('New rule path : %s' % new_rule_path)
-    print('Number of rules checked : %s ' % result['count'])
-    print('Paths to overlapping rules : ', result['overlapping_rules'])
-    print('Number of rules containing same features : %s' % len(result['overlapping_rules']))
+    print("New rule path : %s" % new_rule_path)
+    print("Number of rules checked : %s " % result["count"])
+    print("Paths to overlapping rules : ", result["overlapping_rules"])
+    print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
 except Exception as e:
     print(e)
     try:
-        print(result,'')
+        print(result, "")
     except:
         pass

From 1c558a203d64a7c3669d20707bf747e8d1b6d372 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Wed, 3 May 2023 22:00:50 +0530
Subject: [PATCH 03/20] Update detect_duplicate_features.py

Added a main routine and using argparse to retrieve these from the command line
---
 scripts/detect_duplicate_features.py | 46 +++++++++++++++++++---------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index f852e178..8174c1ff 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,4 +1,5 @@
 import os
+import argparse
 
 import yaml
 
@@ -54,20 +55,35 @@ def find_overlapping_rules(new_rule_path, rules_path):
     return result
 
 
-# usage
-base_dir = ""
-new_rule_path = base_dir + "rules\\anti-analysis\\reference-analysis-tools-strings.yml"
-rules_path = base_dir + "rules"
+# python script.py --base-dir /path/to/capa/rules rules/anti-analysis/reference-analysis-tools-strings.yml rules
+
+
+def main():
+    # usage
+
+    parser = argparse.ArgumentParser(description="Find overlapping rules in Capa rules.")
+    parser.add_argument("-b", "--base-dir", default="", help="Base directory for Capa rules.")
+    parser.add_argument("-f", "--new_rule_path", required=True, help="Path to the new Capa rule.")
+    parser.add_argument("-d", "--rules_path", required=True, help="Path to the directory containing Capa rules.")
+    args = parser.parse_args()
+
+    base_dir = args.base_dir
+    new_rule_path = os.path.join(base_dir, args.new_rule_path)
+    rules_path = os.path.join(base_dir, args.rules_path)
 
-try:
-    result = find_overlapping_rules(new_rule_path, rules_path)
-    print("New rule path : %s" % new_rule_path)
-    print("Number of rules checked : %s " % result["count"])
-    print("Paths to overlapping rules : ", result["overlapping_rules"])
-    print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
-except Exception as e:
-    print(e)
     try:
-        print(result, "")
-    except:
-        pass
+        result = find_overlapping_rules(new_rule_path, rules_path)
+        print("New rule path : %s" % new_rule_path)
+        print("Number of rules checked : %s " % result["count"])
+        print("Paths to overlapping rules : ", result["overlapping_rules"])
+        print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
+    except Exception as e:
+        print(e)
+        try:
+            print(result, "")
+        except:
+            pass
+
+
+if __name__ == "__main__":
+    main()

From 0945d9aea2e5e24a36e9263f7fa68297653dccc8 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Thu, 4 May 2023 19:55:17 +0530
Subject: [PATCH 04/20] Update CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b6412f40..ab7457a1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## master (unreleased)
 
 ### New Features
+- Utility script to detect feature overlap between new and existing CAPA rules #1451 @Aayush-Goel-04
 
 ### Breaking Changes
 

From 30516c33b75c9bb5b127ca592bdc89b2de4dbe06 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Fri, 5 May 2023 14:04:47 +0530
Subject: [PATCH 05/20] Update detect_duplicate_features.py

Improved parse routine based on suggestions.

Co-Authored-By: Moritz <mr-tz@users.noreply.github.com>
---
 scripts/detect_duplicate_features.py | 46 +++++++++++++---------------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 8174c1ff..9aba66ad 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -23,7 +23,7 @@ def findall_features(features):
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
-        return "ERROR ! New rule path file name incorrect"
+        raise ValueError("ERROR ! New rule path file name incorrect")
 
     count = 0
 
@@ -31,24 +31,24 @@ def find_overlapping_rules(new_rule_path, rules_path):
         new_rule = yaml.safe_load(f)
 
     if "rule" not in new_rule:
-        return "ERROR ! given new rule path isn't a rule"
+        raise ValueError("ERROR ! given new rule path isn't a rule")
 
     new_rule_features = findall_features(new_rule["rule"]["features"])
 
     overlapping_rules = []
-
-    for dirpath, dirnames, filenames in os.walk(rules_path):
-        for filename in filenames:
-            if filename.endswith(".yml"):
-                rule_path = os.path.join(dirpath, filename)
-                with open(rule_path, "r") as f:
-                    rule = yaml.safe_load(f)
-                    if "rule" not in rule:
-                        continue
-                    rule_features = findall_features(rule["rule"]["features"])
-                    count += 1
-                if any([feature in rule_features for feature in new_rule_features]):
-                    overlapping_rules.append(rule_path)
+    for rules in rules_path:
+        for dirpath, dirnames, filenames in os.walk(rules):
+            for filename in filenames:
+                if filename.endswith(".yml"):
+                    rule_path = os.path.join(dirpath, filename)
+                    with open(rule_path, "r") as f:
+                        rule = yaml.safe_load(f)
+                        if "rule" not in rule:
+                            continue
+                        rule_features = findall_features(rule["rule"]["features"])
+                        count += 1
+                    if any([feature in rule_features for feature in new_rule_features]):
+                        overlapping_rules.append(rule_path)
 
     result = {"overlapping_rules": overlapping_rules, "count": count}
 
@@ -59,17 +59,15 @@ def find_overlapping_rules(new_rule_path, rules_path):
 
 
 def main():
-    # usage
+    parser = argparse.ArgumentParser(description="Find overlapping features in Capa rules.")
+
+    parser.add_argument("rules", type=str, action="append", help="Path to rules")
+    parser.add_argument("new_rule", type=str, help="Path to new rule")
 
-    parser = argparse.ArgumentParser(description="Find overlapping rules in Capa rules.")
-    parser.add_argument("-b", "--base-dir", default="", help="Base directory for Capa rules.")
-    parser.add_argument("-f", "--new_rule_path", required=True, help="Path to the new Capa rule.")
-    parser.add_argument("-d", "--rules_path", required=True, help="Path to the directory containing Capa rules.")
     args = parser.parse_args()
 
-    base_dir = args.base_dir
-    new_rule_path = os.path.join(base_dir, args.new_rule_path)
-    rules_path = os.path.join(base_dir, args.rules_path)
+    new_rule_path = args.new_rule
+    rules_path = args.rules
 
     try:
         result = find_overlapping_rules(new_rule_path, rules_path)
@@ -80,7 +78,7 @@ def main():
     except Exception as e:
         print(e)
         try:
-            print(result, "")
+            print(result)
         except:
             pass
 

From 9eacf72366a167267d28fe243a8fa0202e4099b7 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Sat, 6 May 2023 17:19:57 +0530
Subject: [PATCH 06/20] Update detect_duplicate_features.py

loading yaml file using capa.rule.Rule.from_yaml.
Returning any exception/errors occuring while checking the files.
---
 scripts/detect_duplicate_features.py | 81 ++++++++++++++--------------
 1 file changed, 39 insertions(+), 42 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 9aba66ad..b61d36f9 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,63 +1,61 @@
 import os
 import argparse
 
-import yaml
+import capa.rules
+import capa.engine as ceng
 
 
-def findall_features(features):
-    feature_list = []
-    for feature in features:
-        if "and" in feature:
-            and_list = findall_features(feature["and"])
-            for x in and_list:
-                feature_list.append(x)
-        elif "or" in feature:
-            or_list = findall_features(feature["or"])
-            for y in or_list:
-                feature_list.append(y)
-        else:
-            feature_list.append(feature)
+def get_child_features(feature):
+    children = []
 
-    return feature_list
+    if isinstance(feature, (ceng.And, ceng.Or, ceng.Some)):
+        for child in feature.children:
+            children.extend(get_child_features(child))
+    elif isinstance(feature, (ceng.Subscope, ceng.Range, ceng.Not)):
+        children.extend(get_child_features(feature.child))
+    else:
+        children.append(feature)
+    return children
+
+
+def get_features(rule_path, errors):
+    with open(rule_path, "r") as f:
+        feature_list = []
+        try:
+            new_rule = capa.rules.Rule.from_yaml(f.read())
+            feature_list = get_child_features(new_rule.statement)
+        except Exception as e:
+            errors.append("rule :" + rule_path + " " + str(type(e)) + " " + str(e))
+    return feature_list, errors
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
-        raise ValueError("ERROR ! New rule path file name incorrect")
+        raise FileNotFoundError("FileNotFoundError ! New rule file name doesn't end with yml")
 
+    new_rule_features, error = get_features(new_rule_path, [])
+    if error:
+        raise Warning(error[0])
+
+    errors: list = []
     count = 0
-
-    with open(new_rule_path, "r") as f:
-        new_rule = yaml.safe_load(f)
-
-    if "rule" not in new_rule:
-        raise ValueError("ERROR ! given new rule path isn't a rule")
-
-    new_rule_features = findall_features(new_rule["rule"]["features"])
-
     overlapping_rules = []
     for rules in rules_path:
         for dirpath, dirnames, filenames in os.walk(rules):
             for filename in filenames:
                 if filename.endswith(".yml"):
                     rule_path = os.path.join(dirpath, filename)
-                    with open(rule_path, "r") as f:
-                        rule = yaml.safe_load(f)
-                        if "rule" not in rule:
-                            continue
-                        rule_features = findall_features(rule["rule"]["features"])
-                        count += 1
+                    rule_features, errors = get_features(rule_path, errors)
+                    if not len(rule_features):
+                        continue
+                    count += 1
                     if any([feature in rule_features for feature in new_rule_features]):
                         overlapping_rules.append(rule_path)
 
-    result = {"overlapping_rules": overlapping_rules, "count": count}
-
+    result = {"overlapping_rules": overlapping_rules, "count": count, "errors": errors}
     return result
 
 
-# python script.py --base-dir /path/to/capa/rules rules/anti-analysis/reference-analysis-tools-strings.yml rules
-
-
 def main():
     parser = argparse.ArgumentParser(description="Find overlapping features in Capa rules.")
 
@@ -68,19 +66,18 @@ def main():
 
     new_rule_path = args.new_rule
     rules_path = args.rules
-
     try:
         result = find_overlapping_rules(new_rule_path, rules_path)
-        print("New rule path : %s" % new_rule_path)
+        print("\nNew rule path : %s" % new_rule_path)
         print("Number of rules checked : %s " % result["count"])
         print("Paths to overlapping rules : ", result["overlapping_rules"])
         print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
+
+        print("\nWhile checking following .yml files error occured:")
+        for error in result["errors"]:
+            print(error)
     except Exception as e:
         print(e)
-        try:
-            print(result)
-        except:
-            pass
 
 
 if __name__ == "__main__":

From ec6b6a22666df81c01545be6d1545204e8a9b3f8 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 8 May 2023 14:58:30 +0530
Subject: [PATCH 07/20] Update detect_duplicate_features.py

---
 scripts/detect_duplicate_features.py | 30 +++++++++++++++++-----------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index b61d36f9..4f1c7198 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -18,24 +18,25 @@ def get_child_features(feature):
     return children
 
 
-def get_features(rule_path, errors):
+def get_features(rule_path):
+    error = ""
+    feature_list = []
     with open(rule_path, "r") as f:
-        feature_list = []
         try:
             new_rule = capa.rules.Rule.from_yaml(f.read())
             feature_list = get_child_features(new_rule.statement)
         except Exception as e:
-            errors.append("rule :" + rule_path + " " + str(type(e)) + " " + str(e))
-    return feature_list, errors
+            error = "rule :" + rule_path + " " + str(type(e)) + " " + str(e)
+    return feature_list, error
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
         raise FileNotFoundError("FileNotFoundError ! New rule file name doesn't end with yml")
 
-    new_rule_features, error = get_features(new_rule_path, [])
+    new_rule_features, error = get_features(new_rule_path)
     if error:
-        raise Warning(error[0])
+        raise Warning(error)
 
     errors: list = []
     count = 0
@@ -45,7 +46,9 @@ def find_overlapping_rules(new_rule_path, rules_path):
             for filename in filenames:
                 if filename.endswith(".yml"):
                     rule_path = os.path.join(dirpath, filename)
-                    rule_features, errors = get_features(rule_path, errors)
+                    rule_features, error = get_features(rule_path)
+                    if error:
+                        errors.append(error)
                     if not len(rule_features):
                         continue
                     count += 1
@@ -70,12 +73,15 @@ def main():
         result = find_overlapping_rules(new_rule_path, rules_path)
         print("\nNew rule path : %s" % new_rule_path)
         print("Number of rules checked : %s " % result["count"])
-        print("Paths to overlapping rules : ", result["overlapping_rules"])
+        print("Paths to overlapping rules : ")
+        for r in result["overlapping_rules"]:
+            print(r)
         print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
-
-        print("\nWhile checking following .yml files error occured:")
-        for error in result["errors"]:
-            print(error)
+        if result["errors"]:
+            print("\nWhile checking following .yml files error occured:")
+            for error in result["errors"]:
+                print(error)
+            print("\n")
     except Exception as e:
         print(e)
 

From 39d2a70679b01f7a7cd0f08e22221cd0d5bd6511 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 8 May 2023 17:29:01 +0530
Subject: [PATCH 08/20] Update detect_duplicate_features.py

Using get_rules menthod to get set of all existing rules.
---
 scripts/detect_duplicate_features.py | 68 +++++++++++++++-------------
 1 file changed, 37 insertions(+), 31 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 4f1c7198..b44a30ad 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,11 +1,16 @@
-import os
 import argparse
 
+import capa.main
 import capa.rules
 import capa.engine as ceng
 
 
-def get_child_features(feature):
+def get_child_features(feature) -> list:
+    """
+    args:
+    \tfeature : capa.rule.Rule.statement containing feature statements
+    returns a list containg all the features in the rule
+    """
     children = []
 
     if isinstance(feature, (ceng.And, ceng.Or, ceng.Some)):
@@ -19,43 +24,45 @@ def get_child_features(feature):
 
 
 def get_features(rule_path):
-    error = ""
+    """
+    args:
+    \tfeature : rule path
+    returns a list containg all the features in the rule
+    """
     feature_list = []
     with open(rule_path, "r") as f:
         try:
             new_rule = capa.rules.Rule.from_yaml(f.read())
             feature_list = get_child_features(new_rule.statement)
         except Exception as e:
-            error = "rule :" + rule_path + " " + str(type(e)) + " " + str(e)
-    return feature_list, error
+            raise Warning("Error: " + rule_path + " " + str(type(e)) + " " + str(e))
+    return feature_list
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
         raise FileNotFoundError("FileNotFoundError ! New rule file name doesn't end with yml")
 
-    new_rule_features, error = get_features(new_rule_path)
-    if error:
-        raise Warning(error)
+    # Loads features of new rule in a list.
+    new_rule_features = get_features(new_rule_path)
 
-    errors: list = []
     count = 0
     overlapping_rules = []
-    for rules in rules_path:
-        for dirpath, dirnames, filenames in os.walk(rules):
-            for filename in filenames:
-                if filename.endswith(".yml"):
-                    rule_path = os.path.join(dirpath, filename)
-                    rule_features, error = get_features(rule_path)
-                    if error:
-                        errors.append(error)
-                    if not len(rule_features):
-                        continue
-                    count += 1
-                    if any([feature in rule_features for feature in new_rule_features]):
-                        overlapping_rules.append(rule_path)
 
-    result = {"overlapping_rules": overlapping_rules, "count": count, "errors": errors}
+    # capa.rules.RuleSet stores all rules in given paths
+    ruleset = capa.main.get_rules(rules_path)
+
+    for rule_name, rule in ruleset.rules.items():
+        rule_features = get_child_features(rule.statement)
+
+        if not len(rule_features):
+            continue
+        count += 1
+        # Checks if any features match between existing and new rule.
+        if any([feature in rule_features for feature in new_rule_features]):
+            overlapping_rules.append(rule_name)
+
+    result = {"overlapping_rules": overlapping_rules, "count": count}
     return result
 
 
@@ -73,15 +80,14 @@ def main():
         result = find_overlapping_rules(new_rule_path, rules_path)
         print("\nNew rule path : %s" % new_rule_path)
         print("Number of rules checked : %s " % result["count"])
-        print("Paths to overlapping rules : ")
-        for r in result["overlapping_rules"]:
-            print(r)
+        if result["overlapping_rules"]:
+            print("Paths to overlapping rules : ")
+            for r in result["overlapping_rules"]:
+                print("- %s" % r)
+        else:
+            print("Paths to overlapping rules : None")
         print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
-        if result["errors"]:
-            print("\nWhile checking following .yml files error occured:")
-            for error in result["errors"]:
-                print(error)
-            print("\n")
+        print("\n")
     except Exception as e:
         print(e)
 

From d91070c116d5215e61ddfc2e9da7802aa5ba93e7 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 8 May 2023 20:17:29 +0530
Subject: [PATCH 09/20] Update detect_duplicate_features.py

---
 scripts/detect_duplicate_features.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index b44a30ad..0ce8ff3a 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -5,11 +5,15 @@ import capa.rules
 import capa.engine as ceng
 
 
-def get_child_features(feature) -> list:
+def get_child_features(feature: ceng.Statement) -> list:
     """
-    args:
-    \tfeature : capa.rule.Rule.statement containing feature statements
-    returns a list containg all the features in the rule
+    Recursively extracts all feature statements from a given rule statement.
+
+    Args:
+        feature (capa.engine.Statement): The feature statement to extract features from.
+
+    Returns:
+        list: A list of all feature statements contained within the given feature statement.
     """
     children = []
 
@@ -23,11 +27,15 @@ def get_child_features(feature) -> list:
     return children
 
 
-def get_features(rule_path):
+def get_features(rule_path: str) -> list:
     """
-    args:
-    \tfeature : rule path
-    returns a list containg all the features in the rule
+    Extracts all features from a given rule file.
+
+    Args:
+        rule_path (str): The path to the rule file to extract features from.
+
+    Returns:
+        list: A list of all feature statements contained within the rule file.
     """
     feature_list = []
     with open(rule_path, "r") as f:

From 187a4712cb858a1fdb06ffc692ffa7398f453552 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Wed, 10 May 2023 14:39:16 +0530
Subject: [PATCH 10/20] Update test_scripts.py

Here new_rule_path and expected_overlaps will be changed based on the new test rule designed.
Adding tests to check if the code works fine
---
 scripts/detect_duplicate_features.py | 41 ++++++++++++++++------------
 tests/test_scripts.py                | 12 ++++++++
 2 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index 0ce8ff3a..a72688c0 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -1,9 +1,13 @@
+import sys
+import logging
 import argparse
 
 import capa.main
 import capa.rules
 import capa.engine as ceng
 
+logger = logging.getLogger("detect_duplicate_features")
+
 
 def get_child_features(feature: ceng.Statement) -> list:
     """
@@ -43,13 +47,15 @@ def get_features(rule_path: str) -> list:
             new_rule = capa.rules.Rule.from_yaml(f.read())
             feature_list = get_child_features(new_rule.statement)
         except Exception as e:
-            raise Warning("Error: " + rule_path + " " + str(type(e)) + " " + str(e))
+            logger.error("Error: New rule " + rule_path + " " + str(type(e)) + " " + str(e))
+            sys.exit(1)
     return feature_list
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
-        raise FileNotFoundError("FileNotFoundError ! New rule file name doesn't end with yml")
+        logger.error("FileNotFoundError ! New rule file name doesn't end with .yml")
+        sys.exit(1)
 
     # Loads features of new rule in a list.
     new_rule_features = get_features(new_rule_path)
@@ -84,21 +90,22 @@ def main():
 
     new_rule_path = args.new_rule
     rules_path = args.rules
-    try:
-        result = find_overlapping_rules(new_rule_path, rules_path)
-        print("\nNew rule path : %s" % new_rule_path)
-        print("Number of rules checked : %s " % result["count"])
-        if result["overlapping_rules"]:
-            print("Paths to overlapping rules : ")
-            for r in result["overlapping_rules"]:
-                print("- %s" % r)
-        else:
-            print("Paths to overlapping rules : None")
-        print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
-        print("\n")
-    except Exception as e:
-        print(e)
+
+    result = find_overlapping_rules(new_rule_path, rules_path)
+
+    print("\nNew rule path : %s" % new_rule_path)
+    print("Number of rules checked : %s " % result["count"])
+    if result["overlapping_rules"]:
+        print("Paths to overlapping rules : ")
+        for r in result["overlapping_rules"]:
+            print("- %s" % r)
+    else:
+        print("Paths to overlapping rules : None")
+    print("Number of rules containing same features : %s" % len(result["overlapping_rules"]))
+    print("\n")
+
+    return len(result["overlapping_rules"])
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 3b299a36..eca6a1f4 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -82,3 +82,15 @@ def test_proto_conversion(tmpdir):
     assert p.returncode == 0
 
     assert p.stdout.startswith(b'{\n  "meta": ') or p.stdout.startswith(b'{\r\n  "meta": ')
+
+
+def test_detect_duplicate_features():
+    new_rule_path = "collection/credit-card/parse-credit-card-information.yml"
+    args = [
+        get_rules_path(),
+        os.path.join(get_rules_path(), new_rule_path),
+    ]
+    expected_overlaps = 49
+    script_path = get_script_path("detect_duplicate_features.py")
+    p = run_program(script_path, args)
+    assert p.returncode == expected_overlaps

From eca86470c670eb7c04b7746dbba04acf0281e9e1 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Thu, 11 May 2023 14:12:52 +0530
Subject: [PATCH 11/20] Update test_scripts.py

RULE_CONTENT can be modified as required
---
 tests/test_scripts.py | 51 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 9 deletions(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index eca6a1f4..00792620 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -8,9 +8,13 @@
 
 import os
 import sys
+import textwrap
 import subprocess
 
 import pytest
+from fixtures import *
+
+import capa.main
 
 CD = os.path.dirname(__file__)
 
@@ -84,13 +88,42 @@ def test_proto_conversion(tmpdir):
     assert p.stdout.startswith(b'{\n  "meta": ') or p.stdout.startswith(b'{\r\n  "meta": ')
 
 
-def test_detect_duplicate_features():
-    new_rule_path = "collection/credit-card/parse-credit-card-information.yml"
-    args = [
-        get_rules_path(),
-        os.path.join(get_rules_path(), new_rule_path),
-    ]
-    expected_overlaps = 49
+def run_detect_duplicate_features(rule_path):
+    # rule_path = "collection/credit-card/parse-credit-card-information.yml"
+    args = [get_rules_path(), rule_path]
     script_path = get_script_path("detect_duplicate_features.py")
-    p = run_program(script_path, args)
-    assert p.returncode == expected_overlaps
+    args = [sys.executable] + [script_path] + args
+    print(f"running: '{args}'")
+    return subprocess.run(args)
+
+
+def test_detect_duplicate_features(z9324d_extractor, tmpdir):
+    # tests a single rule can be loaded successfully
+    RULE_CONTENT = textwrap.dedent(
+        """
+        rule:
+          meta:
+            name: Test Rule
+            scope: function
+          features:
+            - string: "sites.ini"
+        """
+    )
+    expected_overlaps = 3
+    path = z9324d_extractor.path
+    rule_file = tmpdir.mkdir("capa").join("rule.yml")
+    rule_file.write(RULE_CONTENT)
+    assert (
+        capa.main.main(
+            [
+                path,
+                "-v",
+                "-r",
+                rule_file.strpath,
+            ]
+        )
+        == 0
+    )
+    # tests if number of overlaps found are correct.
+    overlaps_found = run_detect_duplicate_features(rule_file.strpath).returncode
+    assert overlaps_found == expected_overlaps

From 41ff457d652a006066c48680bb09243045dd4b32 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Fri, 12 May 2023 16:53:44 +0530
Subject: [PATCH 12/20] Update tests/test_scripts.py

Co-authored-by: Moritz <mr-tz@users.noreply.github.com>
---
 tests/test_scripts.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 00792620..ce8f01c3 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -98,7 +98,6 @@ def run_detect_duplicate_features(rule_path):
 
 
 def test_detect_duplicate_features(z9324d_extractor, tmpdir):
-    # tests a single rule can be loaded successfully
     RULE_CONTENT = textwrap.dedent(
         """
         rule:

From 807efec40fa0e7849c15bfd2f22a13c674738d13 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Fri, 12 May 2023 22:32:37 +0530
Subject: [PATCH 13/20] Create RuleSet to test overlap script

---
 scripts/detect_duplicate_features.py |  4 +-
 tests/test_scripts.py                | 96 ++++++++++++++++++----------
 2 files changed, 65 insertions(+), 35 deletions(-)

diff --git a/scripts/detect_duplicate_features.py b/scripts/detect_duplicate_features.py
index a72688c0..ff21fd0b 100644
--- a/scripts/detect_duplicate_features.py
+++ b/scripts/detect_duplicate_features.py
@@ -48,14 +48,14 @@ def get_features(rule_path: str) -> list:
             feature_list = get_child_features(new_rule.statement)
         except Exception as e:
             logger.error("Error: New rule " + rule_path + " " + str(type(e)) + " " + str(e))
-            sys.exit(1)
+            sys.exit(-1)
     return feature_list
 
 
 def find_overlapping_rules(new_rule_path, rules_path):
     if not new_rule_path.endswith(".yml"):
         logger.error("FileNotFoundError ! New rule file name doesn't end with .yml")
-        sys.exit(1)
+        sys.exit(-1)
 
     # Loads features of new rule in a list.
     new_rule_features = get_features(new_rule_path)
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index ce8f01c3..3936ed59 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -14,8 +14,6 @@ import subprocess
 import pytest
 from fixtures import *
 
-import capa.main
-
 CD = os.path.dirname(__file__)
 
 
@@ -88,41 +86,73 @@ def test_proto_conversion(tmpdir):
     assert p.stdout.startswith(b'{\n  "meta": ') or p.stdout.startswith(b'{\r\n  "meta": ')
 
 
-def run_detect_duplicate_features(rule_path):
+def run_detect_duplicate_features(rule_dir, rule_path):
     # rule_path = "collection/credit-card/parse-credit-card-information.yml"
-    args = [get_rules_path(), rule_path]
+    args = [rule_dir, rule_path]
     script_path = get_script_path("detect_duplicate_features.py")
     args = [sys.executable] + [script_path] + args
     print(f"running: '{args}'")
     return subprocess.run(args)
 
 
-def test_detect_duplicate_features(z9324d_extractor, tmpdir):
-    RULE_CONTENT = textwrap.dedent(
-        """
-        rule:
-          meta:
-            name: Test Rule
-            scope: function
-          features:
-            - string: "sites.ini"
-        """
-    )
-    expected_overlaps = 3
-    path = z9324d_extractor.path
-    rule_file = tmpdir.mkdir("capa").join("rule.yml")
-    rule_file.write(RULE_CONTENT)
-    assert (
-        capa.main.main(
-            [
-                path,
-                "-v",
-                "-r",
-                rule_file.strpath,
-            ]
-        )
-        == 0
-    )
-    # tests if number of overlaps found are correct.
-    overlaps_found = run_detect_duplicate_features(rule_file.strpath).returncode
-    assert overlaps_found == expected_overlaps
+def test_detect_duplicate_features(tmpdir):
+    RULESET = {
+        "rule_1": textwrap.dedent(
+            """
+                rule:
+                    meta:
+                        name: Test Rule 1
+                        scope: function
+                    features:
+                      - or:
+                        - string: "sites.ini"
+                        - number: 0xEDB88320
+            """
+        ),
+        "rule_2": textwrap.dedent(
+            """
+                rule:
+                    meta:
+                        name: Test Rule 2
+                        scope: function
+                    features:
+                      - and:
+                        - string: "sites.ini"
+                        - number: 8
+            """
+        ),
+        "rule_3": textwrap.dedent(
+            """
+                rule:
+                    meta:
+                        name: Test Rule 3
+                        scope: function
+                    features:
+                      - not:
+                        - number: 0xEDB88320
+            """
+        ),
+        "rule_4": textwrap.dedent(
+            """
+                rule:
+                    meta:
+                        name: Test Rule 4
+                        scope: function
+                    features:
+                      - not:
+                        - number: 4
+            """
+        ),
+    }
+
+    rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
+    rule_overlaps = [3, 2, 2, 1]
+    rule_paths = []
+    for rule_name, RULE_CONTENT in RULESET.items():
+        rule_file = rule_dir.join("%s.yml" % rule_name)
+        rule_file.write(RULE_CONTENT)
+        rule_paths.append(rule_file.strpath)
+    # tests if number of overlaps for rules in RULESET found are correct.
+    for expected_overlaps, rule_path in zip(rule_overlaps, rule_paths):
+        overlaps_found = run_detect_duplicate_features(rule_dir.strpath, rule_path)
+        assert overlaps_found.returncode == expected_overlaps

From 12c191582fef94a0b5e0a9b7780bfe1a97189ad3 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 15 May 2023 22:58:19 +0530
Subject: [PATCH 14/20] Update tests/test_scripts.py

Co-authored-by: Moritz <mr-tz@users.noreply.github.com>
---
 tests/test_scripts.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 3936ed59..308968b8 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -87,7 +87,6 @@ def test_proto_conversion(tmpdir):
 
 
 def run_detect_duplicate_features(rule_dir, rule_path):
-    # rule_path = "collection/credit-card/parse-credit-card-information.yml"
     args = [rule_dir, rule_path]
     script_path = get_script_path("detect_duplicate_features.py")
     args = [sys.executable] + [script_path] + args

From 931dcb1dc50d693bb5e9389d97e1ebf325100972 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Mon, 15 May 2023 23:35:11 +0530
Subject: [PATCH 15/20] Update test_scripts.py

---
 tests/test_scripts.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 308968b8..a4f7114a 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -144,8 +144,18 @@ def test_detect_duplicate_features(tmpdir):
         ),
     }
 
-    rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_overlaps = [3, 2, 2, 1]
+    """
+        The rule_overlaps list represents the number of overlaps between each rule in the RULESET.
+        An overlap includes a rule overlap with itself.
+        The overlaps are like:
+        - Rule 1 overlaps with 3 other rules in RULESET
+        - Rule 4 overlaps with itself in RULESET
+        These overlap values indicate the number of rules with which
+        each rule in RULESET has overlapping features.
+    """
+
+    rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_paths = []
     for rule_name, RULE_CONTENT in RULESET.items():
         rule_file = rule_dir.join("%s.yml" % rule_name)

From 0afc16fd02ec966be7a50555f89df8753c1b6794 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Wed, 17 May 2023 23:31:37 +0530
Subject: [PATCH 16/20] Update test rules to test script

---
 CHANGELOG.md          |  2 +-
 tests/test_scripts.py | 53 +++++++++++++++++++++++++++++++++++++------
 2 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6fa5813f..a7b48114 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,7 +3,7 @@
 ## master (unreleased)
 
 ### New Features
-- Utility script to detect feature overlap between new and existing CAPA rules #1451 @Aayush-Goel-04
+- Utility script to detect feature overlap between new and existing CAPA rules [#1451](https://github.com/mandiant/capa/issues/1451) [@Aayush-Goel-04](https://github.com/aayush-goel-04)
 
 ### Breaking Changes
 
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index a4f7114a..e67ab003 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -95,7 +95,22 @@ def run_detect_duplicate_features(rule_dir, rule_path):
 
 
 def test_detect_duplicate_features(tmpdir):
-    RULESET = {
+    TEST_RULE_0 = textwrap.dedent(
+        """
+        rule:
+            meta:
+                name: Test Rule 0
+                scope: function
+            features:
+              - and:
+                - number: 2
+                - or:
+                  - mnemonic: shr
+                  - api: connect
+        """
+    )
+
+    TEST_RULESET = {
         "rule_1": textwrap.dedent(
             """
                 rule:
@@ -106,6 +121,14 @@ def test_detect_duplicate_features(tmpdir):
                       - or:
                         - string: "sites.ini"
                         - number: 0xEDB88320
+                        - and:
+                          - or:
+                            - arch: i386
+                            - number: 4
+                            - not:
+                              - count(mnemonic(xor)): 5
+                          - not:
+                            - os: linux
             """
         ),
         "rule_2": textwrap.dedent(
@@ -117,7 +140,11 @@ def test_detect_duplicate_features(tmpdir):
                     features:
                       - and:
                         - string: "sites.ini"
-                        - number: 8
+                        - arch: i386
+                        - basic block:
+                          - and:
+                            - api: setsockopt
+                            - count(mnemonic(mov)): 3
             """
         ),
         "rule_3": textwrap.dedent(
@@ -127,8 +154,13 @@ def test_detect_duplicate_features(tmpdir):
                         name: Test Rule 3
                         scope: function
                     features:
-                      - not:
-                        - number: 0xEDB88320
+                      - or:
+                        - not:
+                          - os: linux
+                        - basic block:
+                          - and:
+                            - api: bind
+                            - count(mnemonic(mov)): 3
             """
         ),
         "rule_4": textwrap.dedent(
@@ -139,28 +171,35 @@ def test_detect_duplicate_features(tmpdir):
                         scope: function
                     features:
                       - not:
-                        - number: 4
+                        - string: "expa"
             """
         ),
     }
 
-    rule_overlaps = [3, 2, 2, 1]
     """
         The rule_overlaps list represents the number of overlaps between each rule in the RULESET.
         An overlap includes a rule overlap with itself.
         The overlaps are like:
+        - Rule 0 has zero overlaps in RULESET
         - Rule 1 overlaps with 3 other rules in RULESET
         - Rule 4 overlaps with itself in RULESET
         These overlap values indicate the number of rules with which
         each rule in RULESET has overlapping features.
     """
+    rule_overlaps = [0, 3, 4, 4, 1]
 
     rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_paths = []
-    for rule_name, RULE_CONTENT in RULESET.items():
+
+    rule_file = tmpdir.join("%s.yml" % "rule_0")
+    rule_file.write(TEST_RULE_0)
+    rule_paths.append(rule_file.strpath)
+
+    for rule_name, RULE_CONTENT in TEST_RULESET.items():
         rule_file = rule_dir.join("%s.yml" % rule_name)
         rule_file.write(RULE_CONTENT)
         rule_paths.append(rule_file.strpath)
+
     # tests if number of overlaps for rules in RULESET found are correct.
     for expected_overlaps, rule_path in zip(rule_overlaps, rule_paths):
         overlaps_found = run_detect_duplicate_features(rule_dir.strpath, rule_path)

From acdaeb26d3782dd7ee98f9cc0a73582d0f16f3e5 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Sat, 20 May 2023 13:09:48 +0530
Subject: [PATCH 17/20] Update test_scripts.py

---
 tests/test_scripts.py | 39 ++++++++++++++-------------------------
 1 file changed, 14 insertions(+), 25 deletions(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index e67ab003..fdd065f7 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -86,14 +86,6 @@ def test_proto_conversion(tmpdir):
     assert p.stdout.startswith(b'{\n  "meta": ') or p.stdout.startswith(b'{\r\n  "meta": ')
 
 
-def run_detect_duplicate_features(rule_dir, rule_path):
-    args = [rule_dir, rule_path]
-    script_path = get_script_path("detect_duplicate_features.py")
-    args = [sys.executable] + [script_path] + args
-    print(f"running: '{args}'")
-    return subprocess.run(args)
-
-
 def test_detect_duplicate_features(tmpdir):
     TEST_RULE_0 = textwrap.dedent(
         """
@@ -103,10 +95,9 @@ def test_detect_duplicate_features(tmpdir):
                 scope: function
             features:
               - and:
-                - number: 2
-                - or:
-                  - mnemonic: shr
-                  - api: connect
+                - number: 1
+                - not:
+                  - string: process
         """
     )
 
@@ -116,11 +107,10 @@ def test_detect_duplicate_features(tmpdir):
                 rule:
                     meta:
                         name: Test Rule 1
-                        scope: function
                     features:
                       - or:
-                        - string: "sites.ini"
-                        - number: 0xEDB88320
+                        - string: unique
+                        - number: 2
                         - and:
                           - or:
                             - arch: i386
@@ -136,15 +126,13 @@ def test_detect_duplicate_features(tmpdir):
                 rule:
                     meta:
                         name: Test Rule 2
-                        scope: function
                     features:
                       - and:
                         - string: "sites.ini"
-                        - arch: i386
                         - basic block:
                           - and:
-                            - api: setsockopt
-                            - count(mnemonic(mov)): 3
+                            - api: CreateFile
+                            - mnemonic: xor
             """
         ),
         "rule_3": textwrap.dedent(
@@ -152,15 +140,14 @@ def test_detect_duplicate_features(tmpdir):
                 rule:
                     meta:
                         name: Test Rule 3
-                        scope: function
                     features:
                       - or:
                         - not:
-                          - os: linux
+                          - number: 4
                         - basic block:
                           - and:
                             - api: bind
-                            - count(mnemonic(mov)): 3
+                            - number: 2
             """
         ),
         "rule_4": textwrap.dedent(
@@ -168,7 +155,6 @@ def test_detect_duplicate_features(tmpdir):
                 rule:
                     meta:
                         name: Test Rule 4
-                        scope: function
                     features:
                       - not:
                         - string: "expa"
@@ -179,6 +165,7 @@ def test_detect_duplicate_features(tmpdir):
     """
         The rule_overlaps list represents the number of overlaps between each rule in the RULESET.
         An overlap includes a rule overlap with itself.
+        The scripts 
         The overlaps are like:
         - Rule 0 has zero overlaps in RULESET
         - Rule 1 overlaps with 3 other rules in RULESET
@@ -186,7 +173,7 @@ def test_detect_duplicate_features(tmpdir):
         These overlap values indicate the number of rules with which
         each rule in RULESET has overlapping features.
     """
-    rule_overlaps = [0, 3, 4, 4, 1]
+    rule_overlaps = [0, 4, 3, 3, 1]
 
     rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_paths = []
@@ -201,6 +188,8 @@ def test_detect_duplicate_features(tmpdir):
         rule_paths.append(rule_file.strpath)
 
     # tests if number of overlaps for rules in RULESET found are correct.
+    script_path = get_script_path("detect_duplicate_features.py")
     for expected_overlaps, rule_path in zip(rule_overlaps, rule_paths):
-        overlaps_found = run_detect_duplicate_features(rule_dir.strpath, rule_path)
+        args = [rule_dir.strpath, rule_path]
+        overlaps_found = run_program(script_path, args)
         assert overlaps_found.returncode == expected_overlaps

From 52c3ea733bef93029e8e93f324512d08c6f22f40 Mon Sep 17 00:00:00 2001
From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com>
Date: Wed, 24 May 2023 15:39:24 +0530
Subject: [PATCH 18/20] Update tests/test_scripts.py

Co-authored-by: Moritz <mr-tz@users.noreply.github.com>
---
 tests/test_scripts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index fdd065f7..2d8fefac 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -178,7 +178,7 @@ def test_detect_duplicate_features(tmpdir):
     rule_dir = tmpdir.mkdir("capa_rule_overlap_test")
     rule_paths = []
 
-    rule_file = tmpdir.join("%s.yml" % "rule_0")
+    rule_file = tmpdir.join("rule_0.yml")
     rule_file.write(TEST_RULE_0)
     rule_paths.append(rule_file.strpath)
 

From c113a3b5b8a7183444ab51e1adcd89baf84822e0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 May 2023 14:59:13 +0000
Subject: [PATCH 19/20] build(deps): bump ruamel-yaml from 0.17.21 to 0.17.28

Bumps [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) from 0.17.21 to 0.17.28.

---
updated-dependencies:
- dependency-name: ruamel-yaml
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6e36bb21..bc9af0dd 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@ requirements = [
     "viv-utils[flirt]==0.7.9",
     "halo==0.0.31",
     "networkx==2.5.1",  # newer versions no longer support py3.7.
-    "ruamel.yaml==0.17.21",
+    "ruamel.yaml==0.17.28",
     "vivisect==1.1.1",
     "pefile==2023.2.7",
     "pyelftools==0.29",

From 0f54a6f67ee898fc09ff073a1b1dbf89e4a64a48 Mon Sep 17 00:00:00 2001
From: Capa Bot <capa-dev@mandiant.com>
Date: Fri, 2 Jun 2023 07:13:58 +0000
Subject: [PATCH 20/20] Sync capa rules submodule

---
 CHANGELOG.md | 3 ++-
 README.md    | 2 +-
 rules        | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0d2ca4ec..7ed43031 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,7 @@
 
 ### Breaking Changes
 
-### New Rules (6)
+### New Rules (7)
 
 - load-code/shellcode/execute-shellcode-via-windows-callback-function ervin.ocampo@mandiant.com jakub.jozwiak@mandiant.com
 - nursery/execute-shellcode-via-indirect-call ronnie.salomonsen@mandiant.com
@@ -14,6 +14,7 @@
 - linking/static/aplib/linked-against-aplib still@teamt5.org
 - communication/mailslot/read-from-mailslot nick.simonian@mandiant.com
 - nursery/hash-data-using-sha512managed-in-dotnet jonathanlepore@google.com
+- nursery/compiled-with-exescript jonathanlepore@google.com
 -
 
 ### Bug Fixes
diff --git a/README.md b/README.md
index b6d3936f..16d561bc 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/flare-capa)](https://pypi.org/project/flare-capa)
 [![Last release](https://img.shields.io/github/v/release/mandiant/capa)](https://github.com/mandiant/capa/releases)
-[![Number of rules](https://img.shields.io/badge/rules-798-blue.svg)](https://github.com/mandiant/capa-rules)
+[![Number of rules](https://img.shields.io/badge/rules-799-blue.svg)](https://github.com/mandiant/capa-rules)
 [![CI status](https://github.com/mandiant/capa/workflows/CI/badge.svg)](https://github.com/mandiant/capa/actions?query=workflow%3ACI+event%3Apush+branch%3Amaster)
 [![Downloads](https://img.shields.io/github/downloads/mandiant/capa/total)](https://github.com/mandiant/capa/releases)
 [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](LICENSE.txt)
diff --git a/rules b/rules
index 0db65de1..188e6552 160000
--- a/rules
+++ b/rules
@@ -1 +1 @@
-Subproject commit 0db65de1614ee81f9a3baf923423e015bfa78c16
+Subproject commit 188e65528ec496eaaa792c3470cb4ab680a1b156