Merge branch 'test/pipal-e2e': pipal parsing/shell fix + e2e tests, picker rename (v2.14.2)

CHANGELOG.md

@@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 Dates are omitted for releases predating this file; see the git tags for exact timing.
 
+## [2.14.2] - 2026-07-25
+
+### Fixed
+
+- **Pipal base-word parsing.** `pipal()` built one rigid regex that required
+  *exactly* `pipal_count` consecutive base-word lines, so any cracked set with
+  fewer unique base words than `pipal_count` (default 10 — the common case on
+  small cracks) matched nothing and returned no base words. The `Top N base
+  words` section is now parsed line by line, returning up to `pipal_count`
+  words and stopping at the end of the section.
+- **Shell-safe pipal invocation.** The pipal subprocess is now spawned with
+  list-form arguments instead of a `shell=True` formatted string, so hash-file
+  paths containing shell metacharacters can no longer be interpreted as
+  commands.
+
+### Changed
+
+- Renamed the internal `_omen_pick_training_wordlist` helper to
+  `_pick_training_wordlist`, since it is shared by the OMEN, Markov-adjacent,
+  and LLM (wordlist mode) attacks rather than being OMEN-specific.
+
 ## [2.14.1] - 2026-07-25
 
 ### Fixed

hate_crack/attacks.py

@@ -592,7 +592,7 @@ def ollama_attack(ctx: Any) -> None:
             )
             return
         elif choice == "2":
-            path = _omen_pick_training_wordlist(ctx, title="LLM Sample Wordlists")
+            path = _pick_training_wordlist(ctx, title="LLM Sample Wordlists")
             if not path:
                 return
             ctx.hcatOllama(ctx.hcatHashType, ctx.hcatHashFile, "wordlist", path)
@@ -606,7 +606,7 @@ def ollama_attack(ctx: Any) -> None:
             print("\t[!] Invalid selection.")
 
 
-def _omen_pick_training_wordlist(ctx: Any, title: str = "Training Wordlists"):
+def _pick_training_wordlist(ctx: Any, title: str = "Training Wordlists"):
     """Show wordlist picker. Returns path or None (user cancelled with 'q')."""
     wordlist_files = ctx.list_wordlist_files(ctx.hcatWordlists)
     # Print the grid once, outside the retry loop: a wordlists directory can
@@ -682,7 +682,7 @@ def omen_attack(ctx: Any) -> None:
         print("\n\tNo valid OMEN model found. Training is required.")
 
     if need_training:
-        training_file = _omen_pick_training_wordlist(ctx)
+        training_file = _pick_training_wordlist(ctx)
         if not training_file:
             return
         if not ctx.hcatOmenTrain(training_file):
@@ -709,7 +709,7 @@ def _markov_pick_training_source(ctx: Any):
     has_cracked = os.path.isfile(out_path) and os.path.getsize(out_path) > 0
 
     wordlist_files = ctx.list_wordlist_files(ctx.hcatWordlists)
-    # Print the grid once, outside the retry loop — see _omen_pick_training_wordlist.
+    # Print the grid once, outside the retry loop — see _pick_training_wordlist.
     entries = []
     if has_cracked:
         entries.append("0) Cracked passwords (current session)")

hate_crack/main.py

@@ -4443,15 +4443,18 @@ def pipal():
                     pipalFile.write(clearTextPass)
                 pipalFile.close()
 
-            pipalProcess = subprocess.Popen(
-                "{pipal_path} {pipal_file} -t {pipal_count} --output {pipal_out}".format(
-                    pipal_path=pipalPath,
-                    pipal_file=hcatHashFilePipal + ".passwords",
-                    pipal_out=hcatHashFilePipal + ".pipal",
-                    pipal_count=pipal_count,
-                ),
-                shell=True,
-            )
+            # List-form Popen (no shell=True) so paths/filenames containing
+            # shell metacharacters can't be interpreted as commands. shlex.split
+            # on pipalPath still allows an interpreter prefix (e.g. "ruby
+            # /opt/pipal/pipal.rb") to be configured.
+            pipal_cmd = shlex.split(pipalPath) + [
+                hcatHashFilePipal + ".passwords",
+                "-t",
+                str(pipal_count),
+                "--output",
+                hcatHashFilePipal + ".pipal",
+            ]
+            pipalProcess = subprocess.Popen(pipal_cmd)
             try:
                 pipalProcess.wait()
             except KeyboardInterrupt:
@@ -4474,25 +4477,31 @@ def pipal():
                     print(pipalfile.read())
                 print("\n--- Pipal Output End ---\n")
             with open(hcatHashFilePipal + ".pipal") as pipalfile:
-                pipal_content = pipalfile.readlines()
-                raw_pipal = "\n".join(pipal_content)
-                raw_pipal = re.sub("\n+", "\n", raw_pipal)
-                raw_regex = r"Top [0-9]+ base words\n"
-                for word in range(pipal_count):
-                    raw_regex += r"(\S+).*\n"
-                basewords_re = re.compile(raw_regex)
-                results = re.search(basewords_re, raw_pipal)
+                pipal_content = pipalfile.read()
+                # Parse the "Top N base words" section line by line rather than
+                # with one rigid regex.  The old approach required *exactly*
+                # pipal_count baseword lines, so any cracked set with fewer
+                # unique base words than pipal_count (the common case on small
+                # cracks) matched nothing and returned []. Collect up to
+                # pipal_count base words and stop at the end of the section.
                 top_basewords = []
-                if results:
-                    if results.lastindex is not None:
-                        for i in range(1, results.lastindex + 1):
-                            if i is not None:
-                                top_basewords.append(results.group(i))
-                    else:
-                        pass
-                    return top_basewords
-                else:
-                    return []
+                in_section = False
+                for line in pipal_content.splitlines():
+                    if re.match(r"\s*Top\s+[0-9]+\s+base words", line):
+                        in_section = True
+                        continue
+                    if in_section:
+                        if not line.strip():
+                            # blank line terminates the base words section
+                            break
+                        # Capture the base word (first token); tolerate both
+                        # "word = 5 (5%)" and "word 5" separators.
+                        match = re.match(r"\s*(\S+)", line)
+                        if match:
+                            top_basewords.append(match.group(1))
+                            if len(top_basewords) >= pipal_count:
+                                break
+                return top_basewords
         else:
             print("No hashes were cracked :(")
             return []

tests/test_attacks_behavior.py

@@ -532,7 +532,7 @@ class TestOllamaAttack:
 
 
 class TestOmenPickTrainingWordlistReprompt:
-    """_omen_pick_training_wordlist re-prompts on invalid input instead of aborting."""
+    """_pick_training_wordlist re-prompts on invalid input instead of aborting."""
 
     def _make_ctx(self, wordlist_files=None):
         ctx = MagicMock()
@@ -542,29 +542,29 @@ class TestOmenPickTrainingWordlistReprompt:
         return ctx
 
     def test_invalid_input_reprompts_then_valid_pick(self) -> None:
-        from hate_crack.attacks import _omen_pick_training_wordlist
+        from hate_crack.attacks import _pick_training_wordlist
 
         ctx = self._make_ctx(["rockyou.txt"])
         # First input is invalid, second is valid
         with patch("builtins.input", side_effect=["bad", "1"]):
-            result = _omen_pick_training_wordlist(ctx)
+            result = _pick_training_wordlist(ctx)
         assert result is not None
         assert "rockyou.txt" in result
 
     def test_cancel_with_q_returns_none(self) -> None:
-        from hate_crack.attacks import _omen_pick_training_wordlist
+        from hate_crack.attacks import _pick_training_wordlist
 
         ctx = self._make_ctx(["rockyou.txt"])
         with patch("builtins.input", return_value="q"):
-            result = _omen_pick_training_wordlist(ctx)
+            result = _pick_training_wordlist(ctx)
         assert result is None
 
     def test_multiple_invalid_inputs_then_cancel(self) -> None:
-        from hate_crack.attacks import _omen_pick_training_wordlist
+        from hate_crack.attacks import _pick_training_wordlist
 
         ctx = self._make_ctx(["rockyou.txt"])
         with patch("builtins.input", side_effect=["99", "abc", "q"]):
-            result = _omen_pick_training_wordlist(ctx)
+            result = _pick_training_wordlist(ctx)
         assert result is None
 
 

tests/test_custom_path_autocomplete.py

@@ -41,7 +41,7 @@ class TestOmenCustomPath:
         ctx = _make_ctx()
         ctx.select_file_with_autocomplete.return_value = "/data/custom.txt"
         with patch("builtins.input", side_effect=["p"]):
-            result = attacks._omen_pick_training_wordlist(ctx)
+            result = attacks._pick_training_wordlist(ctx)
         ctx.select_file_with_autocomplete.assert_called_once()
         assert result == "/data/custom.txt"
 
@@ -50,7 +50,7 @@ class TestOmenCustomPath:
         ctx = _make_ctx()
         ctx.select_file_with_autocomplete.return_value = None
         with patch("builtins.input", side_effect=["p"]):
-            result = attacks._omen_pick_training_wordlist(ctx)
+            result = attacks._pick_training_wordlist(ctx)
         assert result is None
 
 

tests/test_pipal_e2e.py

@@ -0,0 +1,211 @@
+"""End-to-end tests for ``hate_crack.main.pipal``.
+
+These are hermetic e2e tests: instead of requiring a real Ruby + pipal.rb
+install, they drop a small fake ``pipal`` executable on disk that consumes the
+same CLI (``<passwords> -t <N> --output <file>``) and emits a pipal-shaped
+report.  ``pipal()`` is then driven through its real code path — writing the
+``.passwords`` file (including ``$HEX[...]`` decoding), spawning the subprocess
+via ``subprocess.Popen(..., shell=True)``, and parsing the ``Top N base words``
+section back out.
+
+A real-tool variant is gated behind ``HATE_CRACK_PIPAL_REAL=1`` +
+``HATE_CRACK_PIPAL_PATH`` for anyone who wants to run against genuine pipal.rb.
+"""
+
+import os
+import stat
+import sys
+import textwrap
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _get_main_module():
+    if str(PROJECT_ROOT) not in sys.path:
+        sys.path.insert(0, str(PROJECT_ROOT))
+    os.environ["HATE_CRACK_SKIP_INIT"] = "1"
+    import hate_crack.main as m  # noqa: PLC0415
+
+    return m
+
+
+# A fake pipal that mimics the real tool closely enough for the parser:
+# base word = password lowercased with trailing non-alphabetic characters
+# stripped, ranked by descending frequency, and only the top -t entries are
+# emitted under the "Top N base words" header.
+_FAKE_PIPAL = textwrap.dedent(
+    """\
+    #!/usr/bin/env python3
+    import re
+    import sys
+
+    args = sys.argv[1:]
+    pw_file = args[0]
+    top = 10
+    out = None
+    i = 1
+    while i < len(args):
+        if args[i] in ("-t", "--top"):
+            top = int(args[i + 1]); i += 2
+        elif args[i] in ("-o", "--output"):
+            out = args[i + 1]; i += 2
+        elif args[i] == "--output":
+            out = args[i + 1]; i += 2
+        else:
+            i += 1
+
+    counts = {}
+    with open(pw_file, encoding="utf-8", errors="replace") as fh:
+        for line in fh:
+            pw = line.rstrip("\\n")
+            if not pw:
+                continue
+            base = re.sub(r"[^a-zA-Z]+$", "", pw).lower()
+            if not base:
+                continue
+            counts[base] = counts.get(base, 0) + 1
+
+    ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
+    total = sum(counts.values())
+
+    lines = []
+    lines.append("Basic Results")
+    lines.append("")
+    lines.append("Total entries = %d" % total)
+    lines.append("")
+    lines.append("Top %d base words" % top)
+    for word, c in ranked[:top]:
+        pct = 100.0 * c / total if total else 0.0
+        lines.append("%s = %d (%.2f%%)" % (word, c, pct))
+    lines.append("")
+
+    with open(out, "w", encoding="utf-8") as fh:
+        fh.write("\\n".join(lines) + "\\n")
+    """
+)
+
+
+def _install_fake_pipal(tmp_path: Path) -> Path:
+    fake = tmp_path / "pipal"
+    fake.write_text(_FAKE_PIPAL)
+    fake.chmod(fake.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+    return fake
+
+
+def _run_pipal(monkeypatch, m, tmp_path, cracked_lines, pipal_count):
+    """Wire up main-module globals and run the real ``pipal()``."""
+    fake = _install_fake_pipal(tmp_path)
+    hashfile = tmp_path / "hashes.txt"
+    hashfile.write_text("dummy\n")
+    (tmp_path / "hashes.txt.out").write_text("".join(cracked_lines))
+
+    monkeypatch.setattr(m, "pipalPath", str(fake))
+    monkeypatch.setattr(m, "pipal_count", pipal_count)
+    # hcatHashFile / hcatHashType only exist after main() runs; create them.
+    monkeypatch.setattr(m, "hcatHashFile", str(hashfile), raising=False)
+    monkeypatch.setattr(m, "hcatHashType", "0", raising=False)  # not NTLM
+    return m.pipal(), hashfile
+
+
+class TestPipalE2E:
+    def test_returns_top_basewords(self, monkeypatch, tmp_path, capsys):
+        m = _get_main_module()
+        cracked = [
+            "hash1:password1\n",
+            "hash2:password2\n",
+            "hash3:Password!\n",
+            "hash4:summer2021\n",
+            "hash5:summer99\n",
+            "hash6:winter1\n",
+        ]
+        result, hashfile = _run_pipal(monkeypatch, m, tmp_path, cracked, pipal_count=3)
+
+        assert result == ["password", "summer", "winter"]
+        # pipal report and intermediate passwords file were produced
+        assert (Path(str(hashfile) + ".pipal")).is_file()
+        assert (Path(str(hashfile) + ".passwords")).is_file()
+
+    def test_passwords_file_decodes_hex(self, monkeypatch, tmp_path):
+        m = _get_main_module()
+        # 70617373776f7264 == "password"
+        cracked = [
+            "hash1:$HEX[70617373776f7264]\n",
+            "hash2:hunter2\n",
+        ]
+        _run_pipal(monkeypatch, m, tmp_path, cracked, pipal_count=2)
+
+        pw_path = tmp_path / "hashes.txt.passwords"
+        contents = pw_path.read_text(encoding="utf-8").splitlines()
+        assert "password" in contents  # $HEX decoded, not written literally
+        assert "hunter2" in contents
+        assert not any(line.startswith("$HEX[") for line in contents)
+
+    def test_no_cracked_output_returns_empty(self, monkeypatch, tmp_path):
+        m = _get_main_module()
+        fake = _install_fake_pipal(tmp_path)
+        hashfile = tmp_path / "hashes.txt"
+        hashfile.write_text("dummy\n")  # note: no .out file created
+
+        monkeypatch.setattr(m, "pipalPath", str(fake))
+        monkeypatch.setattr(m, "pipal_count", 3)
+        monkeypatch.setattr(m, "hcatHashFile", str(hashfile), raising=False)
+        monkeypatch.setattr(m, "hcatHashType", "0", raising=False)
+        assert m.pipal() == []
+
+    def test_missing_pipal_path_returns_none(self, monkeypatch, tmp_path):
+        m = _get_main_module()
+        hashfile = tmp_path / "hashes.txt"
+        hashfile.write_text("dummy\n")
+        (tmp_path / "hashes.txt.out").write_text("hash1:password1\n")
+
+        monkeypatch.setattr(m, "pipalPath", str(tmp_path / "does-not-exist"))
+        monkeypatch.setattr(m, "pipal_count", 3)
+        monkeypatch.setattr(m, "hcatHashFile", str(hashfile), raising=False)
+        monkeypatch.setattr(m, "hcatHashType", "0", raising=False)
+        assert m.pipal() is None
+
+    def test_handles_shell_metacharacters_in_path(self, monkeypatch, tmp_path):
+        """The subprocess call must not use a shell (no command injection).
+
+        A hash-file path containing shell metacharacters would, under the old
+        ``shell=True`` string command, either break or execute the injected
+        fragment.  With list-form Popen it is handled as a literal path.
+        """
+        m = _get_main_module()
+        weird_dir = tmp_path / "a b; touch INJECTED"
+        weird_dir.mkdir()
+        cracked = ["hash1:password1\n", "hash2:summer2021\n"]
+
+        fake = _install_fake_pipal(tmp_path)
+        hashfile = weird_dir / "hashes.txt"
+        hashfile.write_text("dummy\n")
+        (weird_dir / "hashes.txt.out").write_text("".join(cracked))
+
+        monkeypatch.setattr(m, "pipalPath", str(fake))
+        monkeypatch.setattr(m, "pipal_count", 3)
+        monkeypatch.setattr(m, "hcatHashFile", str(hashfile), raising=False)
+        monkeypatch.setattr(m, "hcatHashType", "0", raising=False)
+
+        result = m.pipal()
+
+        assert result == ["password", "summer"]
+        # the injected `touch INJECTED` must NOT have run
+        assert not (tmp_path / "INJECTED").exists()
+        assert not Path("INJECTED").exists()
+
+    def test_fewer_basewords_than_count(self, monkeypatch, tmp_path):
+        """Real-world case: fewer unique base words than ``pipal_count``.
+
+        A small cracked set should still surface the base words it *does*
+        have, rather than silently returning nothing.
+        """
+        m = _get_main_module()
+        cracked = [
+            "hash1:password1\n",
+            "hash2:summer2021\n",
+            "hash3:winter1\n",
+        ]
+        # default pipal_count (10) is larger than the 3 available base words
+        result, _ = _run_pipal(monkeypatch, m, tmp_path, cracked, pipal_count=10)
+        assert result == ["password", "summer", "winter"]