From 60ae6656ad3d060fda88fc1c1574f47b6c8c0ed2 Mon Sep 17 00:00:00 2001 From: Justin Bollinger Date: Mon, 4 May 2026 21:26:47 -0400 Subject: [PATCH] fix(wordlist-optimize): use rli.bin (hash-set) instead of rli2.bin (sorted merge-join) wordlist_optimize worker called wordlist_subtract_single (rli2.bin), which requires both files to be lexicographically sorted. splitlen.bin preserves insertion order, so rli2.bin silently missed duplicates. Switch to wordlist_subtract (rli.bin), which loads the remove-list into a hash set and is order-independent. Also add "(comma-separated)" to the input paths prompt. Update worker tests to patch wordlist_subtract instead of wordlist_subtract_single. Co-Authored-By: Claude Sonnet 4.6 --- hate_crack/attacks.py | 2 +- hate_crack/main.py | 2 +- tests/test_wordlist_tools.py | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/hate_crack/attacks.py b/hate_crack/attacks.py index 3f3d415..11919c5 100644 --- a/hate_crack/attacks.py +++ b/hate_crack/attacks.py @@ -1195,7 +1195,7 @@ def wordlist_shard(ctx: Any) -> None: def wordlist_optimize(ctx: Any) -> None: """Prompt for input wordlists and output directory, then optimize.""" raw = ctx.select_file_with_autocomplete( - "\n[*] Enter input wordlist paths", + "\n[*] Enter input wordlist paths (comma-separated)", base_dir=ctx.hcatWordlists, ).strip() inputs = [p.strip() for p in raw.split(",") if p.strip()] diff --git a/hate_crack/main.py b/hate_crack/main.py index cb2c3af..a4d8681 100755 --- a/hate_crack/main.py +++ b/hate_crack/main.py @@ -4036,7 +4036,7 @@ def wordlist_optimize(input_wordlists: list[str], outdir: str) -> bool: ) as out_fh: out_path = out_fh.name try: - if not wordlist_subtract_single(src, dst, out_path): + if not wordlist_subtract(src, out_path, dst): return False if os.path.getsize(out_path) > 0: with open(dst, "ab") as df, open(out_path, "rb") as sf: diff --git a/tests/test_wordlist_tools.py b/tests/test_wordlist_tools.py index 30e12fe..52710f7 100644 --- a/tests/test_wordlist_tools.py +++ b/tests/test_wordlist_tools.py @@ -416,7 +416,7 @@ class TestWordlistOptimizeWorker: outdir.mkdir() with patch("hate_crack.main.wordlist_splitlen", return_value=True) as mock_split, \ - patch("hate_crack.main.wordlist_subtract_single") as mock_sub: + patch("hate_crack.main.wordlist_subtract") as mock_sub: result = worker([str(wl)], str(outdir)) assert result is True @@ -424,7 +424,7 @@ class TestWordlistOptimizeWorker: mock_sub.assert_not_called() # ------------------------------------------------------------------ - # (b) merge-path: existing per-length file → subtract_single + append + # (b) merge-path: existing per-length file → wordlist_subtract + append # ------------------------------------------------------------------ def test_merge_path_existing_length_file(self, tmp_path): worker = self._get_worker() @@ -441,7 +441,7 @@ class TestWordlistOptimizeWorker: # The second call goes through the merge path for the tmp subdir. # wordlist_splitlen for wl_b produces "len5.txt" in a temp dir. - # wordlist_subtract_single produces a non-empty output → append. + # wordlist_subtract produces a non-empty output → append. new_words = b"word2\n" def fake_splitlen(infile: str, outdir_arg: str) -> bool: @@ -449,14 +449,14 @@ class TestWordlistOptimizeWorker: (Path(outdir_arg) / "len5.txt").write_bytes(b"word2\n") return True - def fake_subtract(src: str, dst: str, out: str) -> bool: - # Simulate: the diff is "word2\n" + def fake_subtract(src: str, out: str, *remove_files: str) -> bool: + # Simulate: the diff is "word2\n" — write to outfile (second arg) with open(out, "wb") as f: f.write(new_words) return True with patch("hate_crack.main.wordlist_splitlen", side_effect=fake_splitlen), \ - patch("hate_crack.main.wordlist_subtract_single", side_effect=fake_subtract): + patch("hate_crack.main.wordlist_subtract", side_effect=fake_subtract): # outdir is already non-empty (len_file exists), so wl_b goes to merge path result = worker([str(wl_b)], str(outdir)) @@ -484,7 +484,7 @@ class TestWordlistOptimizeWorker: return True with patch("hate_crack.main.wordlist_splitlen", side_effect=fake_splitlen), \ - patch("hate_crack.main.wordlist_subtract_single") as mock_sub: + patch("hate_crack.main.wordlist_subtract") as mock_sub: result = worker([str(wl_b)], str(outdir)) assert result is True @@ -508,7 +508,7 @@ class TestWordlistOptimizeWorker: assert result is False # ------------------------------------------------------------------ - # (e) wordlist_subtract_single failure returns False + # (e) wordlist_subtract failure returns False # ------------------------------------------------------------------ def test_subtract_failure_returns_false(self, tmp_path): worker = self._get_worker() @@ -525,7 +525,7 @@ class TestWordlistOptimizeWorker: return True with patch("hate_crack.main.wordlist_splitlen", side_effect=fake_splitlen), \ - patch("hate_crack.main.wordlist_subtract_single", return_value=False): + patch("hate_crack.main.wordlist_subtract", return_value=False): result = worker([str(wl_b)], str(outdir)) assert result is False