feat: e2e tests for non-interactive CLI subcommands (#164)

* feat: add e2e test suite skeleton and shared fixtures

New tests/e2e/ subpackage for a real-subprocess, real-hashcat CLI
end-to-end suite, gated behind HATE_CRACK_HASHCAT_REAL=1 (never runs in
standard CI — ubuntu-latest CI runners have no hashcat installed).
Adds shared fixtures: e2e_home (HOME-isolated config, since
_candidate_roots() doesn't search cwd), e2e_hash_file (NTLM hashes for
three known plaintexts), e2e_wordlist/e2e_wordlist_no_target,
e2e_rules_dir, plus a preflight fixture that skips with an actionable
message on missing binaries or an unisolatable local config.json.

* feat: e2e tests for the 4 non-interactive CLI subcommands

Real subprocess + real hashcat coverage for quick/dict/brute/topmask,
plus a harness smoke test proving the suite can tell "ran clean, zero
cracks" apart from "actually cracked something" (test_harness_smoke_*).

* fix: pure-Python MD4 for e2e NTLM fixture (hashlib md4 unavailable)

hashlib.new("md4", ...) raises ValueError/UnsupportedDigestmodError on
OpenSSL 3.x builds that don't load the legacy provider (confirmed on
real developer hardware — uv's managed Python, backed by Homebrew's
OpenSSL 3.6.2, has no MD4 at all; only the platform's separately
bundled /usr/bin/python3 happened to work, which nothing in this
suite's subprocess calls actually uses). Replaced with a small,
dependency-free pure-Python MD4 (RFC 1320) verified against the
"password"/empty-string NTLM test vectors and cross-checked by having
real hashcat crack a hash this helper computes.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Justin Bollinger
2026-07-28 22:36:22 -04:00
committed by GitHub
co-authored by Claude Sonnet 5
parent 95d6f410e0
commit c04c111af1
6 changed files with 469 additions and 0 deletions
+1
View File
@@ -112,4 +112,5 @@ dev = [
"pytest==9.0.3",
"pytest-cov==7.1.0",
"pytest-timeout>=2.4.0",
"pexpect>=4.9.0",
]
View File
+240
View File
@@ -0,0 +1,240 @@
"""Shared fixtures for the real-hashcat, real-subprocess CLI e2e suite.
Gated entirely behind HATE_CRACK_HASHCAT_REAL=1 — see the module-level
pytestmark in each tests/e2e/test_e2e_*.py file. Never runs in standard CI
(ubuntu-latest CI runners have no hashcat installed).
"""
import json
import os
import shutil
import struct
import pytest
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
E2E_PLAINTEXTS = ("changeme123", "e2e2026", "notarealpassword")
# The adhoc_mask_crack test targets exactly "changeme123": 8 lowercase
# letters ("changeme") + 3 digits ("123") = 11 mask positions.
E2E_MASK = "?l?l?l?l?l?l?l?l?d?d?d"
def _md4(data: bytes) -> bytes:
"""Pure-Python MD4 (RFC 1320). No external dependency, works regardless
of whether the platform's OpenSSL build exposes MD4 (OpenSSL 3.x's
default provider does not — only the legacy provider does, and
hashlib.new("md4", ...) raises ValueError/UnsupportedDigestmodError on
such builds, which this suite hit on real developer hardware)."""
def left_rotate(x, n):
return ((x << n) | (x >> (32 - n))) & 0xFFFFFFFF
def F(x, y, z):
return (x & y) | (~x & z)
def G(x, y, z):
return (x & y) | (x & z) | (y & z)
def H(x, y, z):
return x ^ y ^ z
msg = bytearray(data)
orig_len_bits = (len(data) * 8) & 0xFFFFFFFFFFFFFFFF
msg.append(0x80)
while len(msg) % 64 != 56:
msg.append(0)
msg += struct.pack("<Q", orig_len_bits)
a0, b0, c0, d0 = 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476
for chunk_offset in range(0, len(msg), 64):
chunk = msg[chunk_offset:chunk_offset + 64]
X = list(struct.unpack("<16I", chunk))
A, B, C, D = a0, b0, c0, d0
# Round 1
s = [3, 7, 11, 19]
for i in range(16):
k = i
if i % 4 == 0:
A = left_rotate((A + F(B, C, D) + X[k]) & 0xFFFFFFFF, s[0])
elif i % 4 == 1:
D = left_rotate((D + F(A, B, C) + X[k]) & 0xFFFFFFFF, s[1])
elif i % 4 == 2:
C = left_rotate((C + F(D, A, B) + X[k]) & 0xFFFFFFFF, s[2])
else:
B = left_rotate((B + F(C, D, A) + X[k]) & 0xFFFFFFFF, s[3])
# Round 2
s = [3, 5, 9, 13]
order = [0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]
for i, k in enumerate(order):
if i % 4 == 0:
A = left_rotate((A + G(B, C, D) + X[k] + 0x5A827999) & 0xFFFFFFFF, s[0])
elif i % 4 == 1:
D = left_rotate((D + G(A, B, C) + X[k] + 0x5A827999) & 0xFFFFFFFF, s[1])
elif i % 4 == 2:
C = left_rotate((C + G(D, A, B) + X[k] + 0x5A827999) & 0xFFFFFFFF, s[2])
else:
B = left_rotate((B + G(C, D, A) + X[k] + 0x5A827999) & 0xFFFFFFFF, s[3])
# Round 3
s = [3, 9, 11, 15]
order = [0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15]
for i, k in enumerate(order):
if i % 4 == 0:
A = left_rotate((A + H(B, C, D) + X[k] + 0x6ED9EBA1) & 0xFFFFFFFF, s[0])
elif i % 4 == 1:
D = left_rotate((D + H(A, B, C) + X[k] + 0x6ED9EBA1) & 0xFFFFFFFF, s[1])
elif i % 4 == 2:
C = left_rotate((C + H(D, A, B) + X[k] + 0x6ED9EBA1) & 0xFFFFFFFF, s[2])
else:
B = left_rotate((B + H(C, D, A) + X[k] + 0x6ED9EBA1) & 0xFFFFFFFF, s[3])
a0 = (a0 + A) & 0xFFFFFFFF
b0 = (b0 + B) & 0xFFFFFFFF
c0 = (c0 + C) & 0xFFFFFFFF
d0 = (d0 + D) & 0xFFFFFFFF
return struct.pack("<4I", a0, b0, c0, d0)
def _ntlm(password: str) -> str:
return _md4(password.encode("utf-16-le")).hex()
def _missing_required_binaries() -> list[str]:
missing = []
if not shutil.which("hashcat"):
missing.append("hashcat (not on PATH)")
hate_path_candidates = [
os.path.join(REPO_ROOT, "hate_crack"),
REPO_ROOT,
]
hashcat_utils_ok = any(
os.path.isdir(os.path.join(c, "hashcat-utils", "bin")) for c in hate_path_candidates
)
if not hashcat_utils_ok:
missing.append("hashcat-utils/bin (run `make submodules`)")
pcfg_ok = any(
os.path.isfile(os.path.join(c, "pcfg_cracker", "pcfg_guesser.py"))
for c in hate_path_candidates
)
if not pcfg_ok:
missing.append("pcfg_cracker/pcfg_guesser.py (run `make submodules`)")
return missing
def _isolation_hazard() -> str | None:
for candidate in (REPO_ROOT, os.path.join(REPO_ROOT, "hate_crack")):
if os.path.isfile(os.path.join(candidate, "config.json")):
return (
f"a config.json exists at {candidate}; move it aside before "
"running HATE_CRACK_HASHCAT_REAL tests — these tests set HOME "
"to isolate config resolution, but _candidate_roots() checks "
"the repo root and package directory before ~/.hate_crack, so "
"a config.json there always wins regardless of HOME."
)
return None
@pytest.fixture(scope="session", autouse=True)
def _e2e_preflight():
if os.environ.get("HATE_CRACK_HASHCAT_REAL") != "1":
yield
return
hazard = _isolation_hazard()
if hazard:
pytest.skip(hazard)
missing = _missing_required_binaries()
if missing:
pytest.skip("Missing required e2e binaries: " + "; ".join(missing))
yield
@pytest.fixture
def e2e_home(tmp_path):
"""Fresh HOME dir with ~/.hate_crack/config.json isolating config
resolution for the subprocess (see Global Constraints: _candidate_roots()
doesn't search cwd, only repo paths then ~/.hate_crack)."""
home_dir = tmp_path / "home"
hate_crack_dir = home_dir / ".hate_crack"
hate_crack_dir.mkdir(parents=True)
with open(os.path.join(REPO_ROOT, "config.json.example")) as f:
config = json.load(f)
wordlists_dir = tmp_path / "wordlists"
optimized_dir = tmp_path / "optimized_wordlists"
rules_dir = tmp_path / "rules"
wordlists_dir.mkdir()
optimized_dir.mkdir()
rules_dir.mkdir()
config["notify_enabled"] = False
config["check_for_updates"] = False
config["hcatWordlists"] = str(wordlists_dir)
config["hcatOptimizedWordlists"] = str(optimized_dir)
config["rules_directory"] = str(rules_dir)
(hate_crack_dir / "config.json").write_text(json.dumps(config))
return home_dir
@pytest.fixture
def e2e_hash_file(tmp_path):
lines = [
f"user{i}:{_ntlm(pw)}" for i, pw in enumerate(E2E_PLAINTEXTS, start=1)
]
hash_file = tmp_path / "hashes.ntlm"
hash_file.write_text("\n".join(lines) + "\n")
return hash_file
@pytest.fixture
def e2e_wordlist(e2e_home):
"""~30-line wordlist containing all three E2E_PLAINTEXTS plus decoys,
placed under e2e_home's configured wordlists dir."""
wordlists_dir = json.loads(
(e2e_home / ".hate_crack" / "config.json").read_text()
)["hcatWordlists"]
decoys = [
"password", "letmein", "qwerty123", "dragon", "monkey", "football",
"baseball", "sunshine", "princess", "welcome", "shadow", "master",
"abc123", "trustno1", "iloveyou", "starwars", "whatever", "freedom",
"hunter2", "cheese", "computer", "internet", "superman", "batman",
"flower", "hockey", "soccer", "tiger",
]
lines = list(E2E_PLAINTEXTS) + decoys
wordlist_path = os.path.join(wordlists_dir, "e2e.txt")
with open(wordlist_path, "w") as f:
f.write("\n".join(lines) + "\n")
return wordlist_path
@pytest.fixture
def e2e_wordlist_no_target(e2e_home):
"""A wordlist deliberately NOT containing any E2E_PLAINTEXTS, for the
harness smoke test (proves 'ran clean, zero cracks' is distinguishable
from 'actually cracked something')."""
wordlists_dir = json.loads(
(e2e_home / ".hate_crack" / "config.json").read_text()
)["hcatWordlists"]
decoys = ["nope1", "nope2", "wrongpassword", "notitherer"]
wordlist_path = os.path.join(wordlists_dir, "no_target.txt")
with open(wordlist_path, "w") as f:
f.write("\n".join(decoys) + "\n")
return wordlist_path
@pytest.fixture
def e2e_rules_dir(e2e_home):
rules_dir = json.loads(
(e2e_home / ".hate_crack" / "config.json").read_text()
)["rules_directory"]
rule_path = os.path.join(rules_dir, "e2e.rule")
with open(rule_path, "w") as f:
f.write(":\n")
return rules_dir
+45
View File
@@ -0,0 +1,45 @@
"""Subprocess harness for hate_crack.py's non-interactive CLI subcommands
(quick, dict, brute, topmask). See
docs/superpowers/specs/2026-07-28-cli-e2e-testing-design.md.
"""
import os
import signal
import subprocess
import sys
HATE_CRACK_SCRIPT = os.path.join(
os.path.dirname(__file__), "..", "..", "hate_crack.py"
)
def run_noninteractive(args, home_dir, timeout=90):
"""Run hate_crack.py <args> as a real subprocess. Returns CompletedProcess.
Runs the child in its own process group (``start_new_session=True``) and,
on timeout, kills the whole group instead of just the direct child.
hate_crack.py's ``args`` (quick/dict/brute/topmask) launch ``hashcat`` as
a *grandchild* via their own internal ``subprocess.Popen`` +
``.wait()``; plain ``subprocess.run(..., timeout=...)`` only kills the
immediate ``hate_crack.py`` process on timeout, leaving that hashcat
grandchild running indefinitely as an orphan that keeps the GPU busy and
starves every subsequent test. Observed live: a too-tight timeout left 3
orphaned hashcat processes running well after their tests had already
been reported as failed/timed-out by pytest.
"""
cmd = [sys.executable, HATE_CRACK_SCRIPT] + args
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
env={**os.environ, "PYTHONUNBUFFERED": "1", "HOME": str(home_dir)},
start_new_session=True,
)
try:
stdout, stderr = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
stdout, stderr = proc.communicate()
raise subprocess.TimeoutExpired(cmd, timeout, output=stdout, stderr=stderr)
return subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr)
+146
View File
@@ -0,0 +1,146 @@
"""Real-subprocess, real-hashcat e2e tests for the 4 non-interactive CLI
subcommands: quick, dict, brute, topmask.
"""
import json
import os
import pytest
from tests.e2e.conftest import E2E_PLAINTEXTS, _ntlm
from tests.e2e.noninteractive_harness import run_noninteractive
pytestmark = pytest.mark.skipif(
os.environ.get("HATE_CRACK_HASHCAT_REAL") != "1",
reason="Set HATE_CRACK_HASHCAT_REAL=1 to run real-hashcat e2e tests.",
)
# hcatDictionary() (the "dict" methodology) hard-codes 3 rule filenames that
# ship with neither this repo nor Task 1's e2e_home fixture: best66.rule,
# d3ad0ne.rule, T0XlC.rule (see hate_crack/main.py's hcatDictionary). A real
# run against a bare e2e_home fixture fails with FileNotFoundError before
# hashcat is even invoked. This fixture is scoped to this test file (not
# conftest.py, which belongs to Task 1) and drops in trivial identity rules
# (":") under the same names so "dict" can run end-to-end without needing
# the real community rule files installed.
_DICT_REQUIRED_RULES = ("best66.rule", "d3ad0ne.rule", "T0XlC.rule")
@pytest.fixture
def e2e_dict_rules(e2e_home):
rules_dir = json.loads(
(e2e_home / ".hate_crack" / "config.json").read_text()
)["rules_directory"]
for name in _DICT_REQUIRED_RULES:
with open(os.path.join(rules_dir, name), "w") as f:
f.write(":\n")
return rules_dir
def _read_cracked(out_path):
if not os.path.isfile(out_path):
return ""
with open(out_path) as f:
return f.read()
def test_quick_cracks_all_plaintexts(e2e_home, e2e_hash_file, e2e_wordlist):
result = run_noninteractive(
["quick", str(e2e_hash_file), "1000", "--wordlist", str(e2e_wordlist)],
home_dir=e2e_home,
)
assert result.returncode == 0, result.stdout + result.stderr
cracked = _read_cracked(f"{e2e_hash_file}.nt.out")
for plaintext in E2E_PLAINTEXTS:
assert plaintext in cracked
def test_dict_cracks_all_plaintexts(
e2e_home, e2e_hash_file, e2e_wordlist, e2e_dict_rules
):
# "dict" uses the configured wordlists dir (e2e_home's config.json
# points hcatWordlists at the same dir e2e_wordlist writes into), no
# --wordlist flag.
result = run_noninteractive(
["dict", str(e2e_hash_file), "1000"],
home_dir=e2e_home,
)
assert result.returncode == 0, result.stdout + result.stderr
cracked = _read_cracked(f"{e2e_hash_file}.nt.out")
for plaintext in E2E_PLAINTEXTS:
assert plaintext in cracked
_BRUTE_PLAINTEXT = "hc2026" # 6 chars — see fixture below for why.
@pytest.fixture
def e2e_brute_hash_file(tmp_path):
"""Dedicated single-hash file for the brute test, deliberately NOT using
e2e_hash_file/E2E_PLAINTEXTS (Task 1's shared fixtures — see conftest.py).
hcatBruteForce() hard-codes the full ?a charset (?l?u?d?s, 95 chars) at
every mask position, so keyspace is 95**N regardless of what --min/--max
is passed. None of E2E_PLAINTEXTS is short enough for that to be a fast,
*deterministic* test: measured live on this hardware (Apple M3 Max,
OpenCL backend, hashcat -b -m 1000 -> ~14 GH/s), 95**7 (e2e2026's length)
is a ~70 trillion-key space taking ~77 minutes to exhaust, and hashcat's
GPU candidate-generator does not enumerate in simple first-character
order, so there is no reliable early-exit fraction to bound wall time by
— a live run got to 5% progress in ~4 minutes with no crack yet. A
6-char password bounds the keyspace to 95**6 (~7.35e11 keys), which at
the same measured throughput exhausts in ~52 seconds *even in the
worst case* (full keyspace scan, no lucky early hit needed) — the
property this test actually needs is a guaranteed upper bound, not an
expected-value guess about candidate ordering.
"""
hash_file = tmp_path / "hashes_brute.ntlm"
hash_file.write_text(f"user1:{_ntlm(_BRUTE_PLAINTEXT)}\n")
return hash_file
def test_brute_cracks_single_length_target(e2e_home, e2e_brute_hash_file):
result = run_noninteractive(
["brute", str(e2e_brute_hash_file), "1000", "--min", "6", "--max", "6"],
home_dir=e2e_home,
timeout=180,
)
assert result.returncode == 0, result.stdout + result.stderr
cracked = _read_cracked(f"{e2e_brute_hash_file}.nt.out")
assert _BRUTE_PLAINTEXT in cracked
def test_topmask_cracks_with_zero_target_time(e2e_home, e2e_hash_file):
result = run_noninteractive(
["topmask", str(e2e_hash_file), "1000", "--target-time", "0"],
home_dir=e2e_home,
timeout=180,
)
assert result.returncode == 0, result.stdout + result.stderr
# Observed on a live run: with no optimized-wordlist stats available,
# --target-time 0 drives PACK/maskgen.py's mask selection down to zero
# candidate masks (it hits a ZeroDivisionError internally computing mask
# coverage %, which hate_crack does not propagate as a failure), so
# hashcat receives an empty mask list, logs "Invalid mask.", and exits
# cleanly. hate_crack still creates the (empty) `.nt.out` file via
# `-o` before hashcat gives up, so its existence is the correct
# observable signal here, not zero-byte content or any specific crack.
assert os.path.isfile(f"{e2e_hash_file}.nt.out")
def test_harness_smoke_no_crack_when_wordlist_lacks_target(
e2e_home, e2e_hash_file, e2e_wordlist_no_target
):
"""Proves the harness distinguishes 'ran clean, zero cracks' from
'actually cracked something' — every other test's positive assertion
depends on this being true."""
result = run_noninteractive(
[
"quick", str(e2e_hash_file), "1000",
"--wordlist", str(e2e_wordlist_no_target),
],
home_dir=e2e_home,
)
assert result.returncode == 0, result.stdout + result.stderr
cracked = _read_cracked(f"{e2e_hash_file}.nt.out")
for plaintext in E2E_PLAINTEXTS:
assert plaintext not in cracked
+37
View File
@@ -0,0 +1,37 @@
from tests.e2e.conftest import _ntlm
def test_ntlm_known_vector_password():
assert _ntlm("password") == "8846f7eaee8fb117ad06bdd830b7586c"
def test_ntlm_known_vector_empty_string():
assert _ntlm("") == "31d6cfe0d16ae931b73c59d7e0c089c0"
def test_ntlm_matches_hashcat_cracking_a_known_hash(tmp_path):
"""Cross-check against real hashcat itself, not just known vectors:
write the NTLM hash this helper computes for a password, ask hashcat
to crack it against a one-line wordlist containing that password, and
confirm hashcat reports a crack. This is the actual thing this whole
suite depends on being correct."""
import subprocess
import shutil
if not shutil.which("hashcat"):
import pytest
pytest.skip("hashcat not on PATH")
from tests.e2e.conftest import _ntlm
pw = "e2etestvector99"
hash_file = tmp_path / "test.ntlm"
hash_file.write_text(_ntlm(pw) + "\n")
wordlist = tmp_path / "wl.txt"
wordlist.write_text(pw + "\n")
out_file = tmp_path / "test.ntlm.out"
result = subprocess.run(
["hashcat", "-m", "1000", str(hash_file), str(wordlist),
"-o", str(out_file), "--potfile-disable"],
capture_output=True, text=True, timeout=60,
)
assert result.returncode in (0, 1), result.stdout + result.stderr
assert out_file.is_file()
assert pw in out_file.read_text()