Merge branch 'feat/hashview-live-tests-local' into integrate/hashview-routes

# Conflicts:
#	hate_crack/api.py
#	tests/test_hashview.py

README.md

@@ -588,6 +588,31 @@ environment variable and provide valid credentials in `config.json`:
 HATE_CRACK_RUN_LIVE_TESTS=1 uv run pytest tests/test_upload_cracked_hashes.py -v
 ```
 
+### Live Hashview Tests Against a Local Docker Stack
+
+Instead of pointing the live tests at a remote Hashview server, you can have
+the suite spin up a local [Hashview](https://github.com/hashview/hashview)
+Docker stack, seed it, run the live tests against it, and tear it down. Set
+`HASHVIEW_TEST_LOCAL=1` and point `HASHVIEW_REPO` at a Hashview checkout:
+
+```bash
+HASHVIEW_TEST_LOCAL=1 HASHVIEW_REPO=~/projects/hashview \
+  HATE_CRACK_SKIP_INIT=1 uv run pytest tests/test_hashview_cli_subcommands_subprocess.py -v
+```
+
+This brings up `docker compose` in the Hashview repo, seeds an admin API key,
+a customer, a hashfile, and cracked "effective task" data, then exports the
+`HASHVIEW_*` env vars the tests read. Useful env vars:
+
+- `HASHVIEW_TEST_LOCAL=1` — enable the local stack (no-op otherwise)
+- `HASHVIEW_REPO=<path>` — Hashview checkout (default `~/projects/hashview`)
+- `HASHVIEW_KEEP=1` — leave containers running after the session (faster re-runs)
+- `HASHVIEW_LOCAL_PORT=5000` — host port the app is published on
+
+The hate_crack CLI honours the `HASHVIEW_URL` / `HASHVIEW_API_KEY` environment
+variables (overriding `config.json`), which is what lets the suite point the
+CLI at the local stack without editing your persisted config.
+
 ### End-to-End Install Tests (Local + Docker)
 
 Local uv tool install + script execution (uses a temporary HOME):

hate_crack/api.py

@@ -1417,7 +1417,8 @@ class HashviewAPI:
         import sys
 
         # Hashview's GET /v1/hashfiles/<id> returns exactly the uncracked
-        # ("left") ciphertexts for the hashfile (see v1_api_get_hashfile).
+        # ("left") ciphertexts for the hashfile (see v1_api_get_hashfile). The
+        # older /v1/hashfiles/<id>/left route no longer exists and 404s.
         url = f"{self.base_url}/v1/hashfiles/{hashfile_id}"
         resp = self.session.get(url, headers=self._auth_headers(), stream=True)
         resp.raise_for_status()

hate_crack/main.py

@@ -272,8 +272,14 @@ if _missing_keys:
     print(f"[config] Added {len(_missing_keys)} missing key(s) to {_config_path}")
     print(f"         Keys: {', '.join(_missing_keys)}")
 
-hashview_url = config_parser["hashview_url"]
-hashview_api_key = config_parser["hashview_api_key"]
+# Environment variables override config.json so the CLI can be pointed at a
+# different Hashview instance (e.g. a local docker stack for the live test
+# suite) without editing the persisted config. Empty/unset env vars fall back
+# to config.json.
+hashview_url = os.environ.get("HASHVIEW_URL") or config_parser["hashview_url"]
+hashview_api_key = (
+    os.environ.get("HASHVIEW_API_KEY") or config_parser["hashview_api_key"]
+)
 
 SKIP_INIT = os.environ.get("HATE_CRACK_SKIP_INIT") == "1"
 

tests/_hashview_local.py

@@ -0,0 +1,231 @@
+"""Bring up / tear down a local Hashview docker stack for the live test suite.
+
+Used by the session-scoped ``hashview_local_stack`` fixture in ``conftest.py``.
+Activated by ``HASHVIEW_TEST_LOCAL=1``; otherwise everything here is a no-op and
+the live tests skip as before.
+
+Lifecycle (when enabled):
+  1. write ``hashview/config.conf`` with ``SERVER_NAME = 127.0.0.1:5000`` so the
+     app is reachable from the host (a stale ``app:5000`` SERVER_NAME makes every
+     request 404 via Flask host matching)
+  2. ``docker compose up -d --build``
+  3. poll until the app answers on the host
+  4. seed the DB (admin api_key, customer, hashfile, cracked effective-task data)
+  5. export ``HASHVIEW_*`` env vars so both the test helpers and the hate_crack
+     CLI (which now honours ``HASHVIEW_URL`` / ``HASHVIEW_API_KEY``) target local
+  6. on teardown ``docker compose down -v`` unless ``HASHVIEW_KEEP=1``
+
+Config via env:
+  HASHVIEW_TEST_LOCAL=1     enable the stack
+  HASHVIEW_REPO=<path>      hashview checkout (default ~/projects/hashview)
+  HASHVIEW_KEEP=1           leave containers running after the session
+  HASHVIEW_LOCAL_PORT=5000  host port the app is published on
+"""
+import os
+import shutil
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+LOCAL_API_KEY = "3eac1ab7-e525-4bb7-b565-2c9f045dfc56"
+CUSTOMER_ID = "1"
+HASHFILE_ID = "1"
+# md5 is hashcat mode 0; the suite's pwdump test uploads NTLM (1000). Seed both.
+HASH_TYPE = "0"
+SEED_HASH_TYPES = "0,1000"
+
+_CONFIG_CONF = """[SERVER]
+SERVER_NAME = 127.0.0.1:{port}
+SECRET_KEY = hate-crack-local-test-secret
+
+[database]
+host = db
+username = hashview
+password = hashview
+
+[SMTP]
+server = smtp.example.com
+port = 25
+use_tls = False
+username =
+password =
+default_sender =
+"""
+
+
+def enabled() -> bool:
+    return os.environ.get("HASHVIEW_TEST_LOCAL", "").lower() in ("1", "true", "yes")
+
+
+def _keep() -> bool:
+    return os.environ.get("HASHVIEW_KEEP", "").lower() in ("1", "true", "yes")
+
+
+def _repo() -> Path:
+    return Path(
+        os.environ.get("HASHVIEW_REPO", os.path.expanduser("~/projects/hashview"))
+    ).resolve()
+
+
+def _port() -> str:
+    return os.environ.get("HASHVIEW_LOCAL_PORT", "5000")
+
+
+def _base_url() -> str:
+    return f"http://127.0.0.1:{_port()}"
+
+
+def _compose(repo: Path, *args: str, check: bool = True, capture: bool = False):
+    env = {**os.environ, "DOCKER_PLATFORM": os.environ.get("DOCKER_PLATFORM", "linux/amd64")}
+    return subprocess.run(
+        ["docker", "compose", *args],
+        cwd=str(repo),
+        env=env,
+        check=check,
+        text=True,
+        stdout=subprocess.PIPE if capture else None,
+        stderr=subprocess.STDOUT if capture else None,
+    )
+
+
+def _http_status(url: str, cookie: str | None = None):
+    """Return the HTTP status for ``url`` (following no redirects), or None."""
+
+    class _NoRedirect(urllib.request.HTTPRedirectHandler):
+        def redirect_request(self, *args, **kwargs):
+            return None
+
+    opener = urllib.request.build_opener(_NoRedirect)
+    req = urllib.request.Request(url)
+    if cookie:
+        req.add_header("Cookie", f"uuid={cookie}")
+    try:
+        with opener.open(req, timeout=3) as resp:
+            return resp.status
+    except urllib.error.HTTPError as exc:
+        return exc.code
+    except Exception:
+        return None
+
+
+# Statuses that mean "Flask is up and routing" — i.e. SERVER_NAME matches and
+# the app is serving. ``/login`` answers 200 or 302 (redirect) depending on
+# session/setup state; either is fine. A 404 means the route didn't match
+# (e.g. a stale ``app:5000`` SERVER_NAME) and a connection error (None) means
+# the app isn't listening yet.
+_ROUTING_STATUSES = {200, 301, 302, 303, 403}
+
+
+def _wait_ready(timeout: float = 240.0) -> bool:
+    """Poll until the app is routing requests on ``/login``.
+
+    On a cold start the DB volume is fresh, so the app boots and runs
+    migrations before it stops connection-refusing. Routing (not auth/seed)
+    readiness is the gate here; auth/seed readiness is checked separately in
+    :func:`_wait_authenticated`.
+    """
+    url = f"{_base_url()}/login"
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if _http_status(url) in _ROUTING_STATUSES:
+            return True
+        time.sleep(2)
+    return False
+
+
+def _wait_authenticated(timeout: float = 60.0) -> bool:
+    """Poll an authenticated endpoint until the seeded api_key is accepted.
+
+    Closes the race where ``/login`` is up but the seed's admin api_key isn't
+    effective yet: an unauthenticated ``/v1/customers`` 302-redirects (to an
+    HTML page), while a recognised cookie returns 200 JSON.
+    """
+    url = f"{_base_url()}/v1/customers"
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if _http_status(url, cookie=LOCAL_API_KEY) == 200:
+            return True
+        time.sleep(2)
+    return False
+
+
+def _seed(repo: Path, attempts: int = 6) -> None:
+    seeder = Path(__file__).with_name("hashview_local_seed.py")
+    _compose(repo, "cp", str(seeder), "app:/tmp/hashview_local_seed.py")
+    seed_env = [
+        "-e", "PYTHONPATH=/",
+        "-e", f"HASHVIEW_API_KEY={LOCAL_API_KEY}",
+        "-e", f"HASHVIEW_CUSTOMER_ID={CUSTOMER_ID}",
+        "-e", f"HASHVIEW_HASHFILE_ID={HASHFILE_ID}",
+        "-e", f"HASHVIEW_SEED_HASH_TYPES={SEED_HASH_TYPES}",
+    ]
+    # The admin user (id=1) and default tasks are created during the app's
+    # first-request boot, which can lag a few seconds behind the app routing.
+    # Retry so a cold start doesn't fail on "Admin user not found".
+    last = None
+    for i in range(attempts):
+        result = _compose(
+            repo,
+            "exec",
+            "-T",
+            *seed_env,
+            "-w",
+            "/",
+            "app",
+            "python",
+            "/tmp/hashview_local_seed.py",
+            check=False,
+            capture=True,
+        )
+        if result.returncode == 0:
+            return
+        last = result.stdout
+        time.sleep(5)
+    raise RuntimeError(f"seed did not succeed after {attempts} attempts: {last}")
+
+
+def _export_env() -> None:
+    os.environ["HASHVIEW_TEST_REAL"] = "1"
+    os.environ["HASHVIEW_URL"] = _base_url()
+    os.environ["HASHVIEW_API_KEY"] = LOCAL_API_KEY
+    os.environ["HASHVIEW_CUSTOMER_ID"] = CUSTOMER_ID
+    os.environ["HASHVIEW_HASHFILE_ID"] = HASHFILE_ID
+    os.environ.setdefault("HASHVIEW_HASH_TYPE", HASH_TYPE)
+
+
+def setup():
+    """Bring the stack up + seed. Returns a skip reason string, or None on success."""
+    if shutil.which("docker") is None:
+        return "docker not available"
+    repo = _repo()
+    if not repo.is_dir():
+        return f"hashview repo not found at {repo} (set HASHVIEW_REPO)"
+    if not (repo / "docker-compose.yml").is_file():
+        return f"no docker-compose.yml in hashview repo {repo}"
+
+    (repo / "hashview" / "config.conf").write_text(_CONFIG_CONF.format(port=_port()))
+    try:
+        _compose(repo, "up", "-d", "--build")
+    except subprocess.CalledProcessError as exc:
+        return f"docker compose up failed: {exc}"
+    if not _wait_ready():
+        return f"hashview app did not become ready at {_base_url()}"
+    try:
+        _seed(repo)
+    except (subprocess.CalledProcessError, RuntimeError) as exc:
+        return f"hashview DB seed failed: {exc}"
+    if not _wait_authenticated():
+        return "seeded api_key not accepted by Hashview (auth/seed race)"
+
+    _export_env()
+    return None
+
+
+def teardown():
+    if _keep():
+        return
+    repo = _repo()
+    if repo.is_dir() and (repo / "docker-compose.yml").is_file():
+        _compose(repo, "down", "-v", check=False, capture=True)

tests/conftest.py

@@ -42,3 +42,40 @@ def _isolate_notify_state():
     notify.clear_state_for_tests()
     yield
     notify.clear_state_for_tests()
+
+
+def pytest_configure(config):
+    """Spin up + seed a local Hashview docker stack for the live test suite.
+
+    No-op unless ``HASHVIEW_TEST_LOCAL=1``. When enabled, brings up the stack
+    from the hashview repo, seeds the DB, and exports the ``HASHVIEW_*`` env
+    vars the live tests (and the hate_crack CLI) read so they target the local
+    instance instead of whatever ``config.json`` points at.
+
+    This runs in ``pytest_configure`` — *before* collection — on purpose: the
+    live subprocess tests gate on ``HASHVIEW_TEST_REAL`` via ``@skipif``, which
+    is evaluated at collection time. A session fixture would set the env too
+    late and every live test would skip. On failure we deliberately leave
+    ``HASHVIEW_TEST_REAL`` unset so the live tests skip with their normal
+    reason rather than erroring. See ``tests/_hashview_local.py`` for config.
+    """
+    from tests import _hashview_local as hv
+
+    if not hv.enabled():
+        return
+    reason = hv.setup()
+    if reason is not None:
+        config.issue_config_time_warning(
+            pytest.PytestWarning(
+                f"HASHVIEW_TEST_LOCAL set but local stack unavailable "
+                f"({reason}); live Hashview tests will skip."
+            ),
+            stacklevel=2,
+        )
+
+
+def pytest_unconfigure(config):
+    from tests import _hashview_local as hv
+
+    if hv.enabled():
+        hv.teardown()

tests/hashview_local_seed.py

@@ -0,0 +1,152 @@
+"""Seed a local Hashview docker stack to the state hate_crack's live tests need.
+
+Runs INSIDE the Hashview ``app`` container (``PYTHONPATH=/``), where the
+``hashview`` package is importable. Idempotent.
+
+Brings the database to:
+- admin user (id=1) with a known ``api_key`` (so hate_crack can authenticate)
+- a Settings row (so the /setup wizard does not intercept requests)
+- a Customer at ``HASHVIEW_CUSTOMER_ID``
+- a Hashfile at ``HASHVIEW_HASHFILE_ID`` holding one (uncracked) hash, so the
+  ``download-hashes`` / left-list endpoints return data
+- at least one *cracked* Hashes row per hash type in
+  ``HASHVIEW_SEED_HASH_TYPES`` (comma separated, default ``0,1000``) attached to
+  a real Task. ``/v1/jobs/add`` derives its "top 10 effective tasks" from
+  cracked hashes of the uploaded file's hash type; without this it answers
+  "Not enough data to determine effective tasks" and job creation fails.
+
+Required env: HASHVIEW_API_KEY, HASHVIEW_CUSTOMER_ID, HASHVIEW_HASHFILE_ID.
+Optional env: HASHVIEW_SEED_HASH_TYPES, HASHVIEW_SEED_TASK_ID (default 1).
+"""
+import os
+import sys
+
+from flask import Flask
+from flask_bcrypt import Bcrypt
+
+from hashview.config import Config
+from hashview.models import (
+    Customers,
+    Hashes,
+    HashfileHashes,
+    Hashfiles,
+    Settings,
+    Tasks,
+    Users,
+    db,
+)
+
+REQUIRED_ENV = (
+    "HASHVIEW_API_KEY",
+    "HASHVIEW_CUSTOMER_ID",
+    "HASHVIEW_HASHFILE_ID",
+)
+
+
+def build_app() -> Flask:
+    app = Flask(__name__)
+    app.config.from_object(Config)
+    db.init_app(app)
+    Bcrypt(app)
+    return app
+
+
+def seed(app: Flask) -> None:
+    api_key = os.environ["HASHVIEW_API_KEY"]
+    customer_id = int(os.environ["HASHVIEW_CUSTOMER_ID"])
+    hashfile_id = int(os.environ["HASHVIEW_HASHFILE_ID"])
+    task_id = int(os.environ.get("HASHVIEW_SEED_TASK_ID", "1"))
+    hash_types = [
+        int(x) for x in os.environ.get("HASHVIEW_SEED_HASH_TYPES", "0,1000").split(",") if x.strip()
+    ]
+
+    bcrypt = Bcrypt(app)
+
+    with app.app_context():
+        admin = db.session.get(Users, 1)
+        if admin is None:
+            raise RuntimeError(
+                "Admin user (id=1) not found. App must finish booting before seeding."
+            )
+        admin.api_key = api_key
+        admin.admin = True
+        # Hashview's setup guard (do_gui_setup_if_needed) redirects every request
+        # to /setup/admin-pass while the admin password is still the default.
+        # Set a non-default password so the API is reachable without the wizard.
+        admin.password = bcrypt.generate_password_hash(
+            os.environ.get("HASHVIEW_ADMIN_PASSWORD", "hate-crack-local-pass")
+        ).decode("utf-8")
+        admin.email_address = admin.email_address or "admin@hate-crack.local"
+
+        if db.session.query(Settings).first() is None:
+            db.session.add(
+                Settings(retention_period=30, max_runtime_jobs=0, max_runtime_tasks=0)
+            )
+
+        if db.session.get(Customers, customer_id) is None:
+            db.session.add(Customers(id=customer_id, name="hate_crack Local Customer"))
+
+        if db.session.get(Tasks, task_id) is None:
+            raise RuntimeError(
+                f"Task id={task_id} not found. Default tasks should exist after app boot."
+            )
+
+        if db.session.get(Hashfiles, hashfile_id) is None:
+            db.session.add(
+                Hashfiles(
+                    id=hashfile_id,
+                    name="hate-crack-local-hashfile",
+                    customer_id=customer_id,
+                    owner_id=admin.id,
+                )
+            )
+            db.session.flush()
+            hash_row = Hashes(
+                sub_ciphertext="d41d8cd98f00b204e9800998ecf8427e",
+                ciphertext="d41d8cd98f00b204e9800998ecf8427e",
+                hash_type=0,
+                cracked=False,
+            )
+            db.session.add(hash_row)
+            db.session.flush()
+            db.session.add(
+                HashfileHashes(
+                    hash_id=hash_row.id, username="local-user", hashfile_id=hashfile_id
+                )
+            )
+
+        # Cracked "effective task" data, one row per hash type the tests upload.
+        for ht in hash_types:
+            sub = f"seed-cracked-{ht}"
+            exists = (
+                db.session.query(Hashes)
+                .filter_by(sub_ciphertext=sub, hash_type=ht, cracked=True)
+                .first()
+            )
+            if exists is None:
+                db.session.add(
+                    Hashes(
+                        sub_ciphertext=sub,
+                        ciphertext=sub,
+                        hash_type=ht,
+                        cracked=True,
+                        task_id=task_id,
+                        plaintext="password",
+                    )
+                )
+
+        db.session.commit()
+
+
+def main() -> int:
+    missing = [k for k in REQUIRED_ENV if not os.getenv(k)]
+    if missing:
+        print(f"hashview_local_seed: missing env vars: {missing}", file=sys.stderr)
+        return 2
+    seed(build_app())
+    print("hashview_local_seed: ok")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_hashview.py

@@ -296,10 +296,14 @@ class TestHashviewAPI:
             assert result["size"] == len(content)
 
             # Verify auth headers were passed in the left hashes download call.
-            # The left download is the first GET (to /v1/hashfiles/<id>); the
-            # second GET is the best-effort found lookup that 404s here.
+            # The uncracked ("left") hashes come from GET /v1/hashfiles/<id>
+            # (the trailing /found call is a separate lookup).
             call_args_list = api.session.get.call_args_list
-            left_call = [c for c in call_args_list if "/v1/hashfiles/2" in str(c)][0]
+            left_call = [
+                c
+                for c in call_args_list
+                if "/v1/hashfiles/2" in str(c) and "found" not in str(c)
+            ][0]
             assert left_call.kwargs.get("headers") is not None
             auth_headers = left_call.kwargs.get("headers")
             assert "Cookie" in auth_headers or "uuid" in str(auth_headers)

tests/test_hashview_cli_subcommands_subprocess.py

@@ -105,6 +105,16 @@ def test_hashview_subcommands_require_api_key(tmp_path, args):
             path.write_text("dummy\n")
             args[idx + 1] = str(path)
 
+    # Strip any ambient Hashview credentials (e.g. exported by the local-stack
+    # fixture) so this exercises the genuine no-key-configured path. The CLI
+    # honours HASHVIEW_URL / HASHVIEW_API_KEY as overrides, so leaving them set
+    # would supply a key and defeat the check.
+    sub_env = {
+        k: v
+        for k, v in os.environ.items()
+        if k not in ("HASHVIEW_URL", "HASHVIEW_API_KEY")
+    }
+    sub_env["PYTHONUNBUFFERED"] = "1"
     cli_cmd = [sys.executable, HATE_CRACK_SCRIPT] + args
     result = subprocess.run(
         cli_cmd,
@@ -112,7 +122,7 @@ def test_hashview_subcommands_require_api_key(tmp_path, args):
         stderr=subprocess.PIPE,
         text=True,
         cwd=REPO_ROOT,
-        env={**os.environ, "PYTHONUNBUFFERED": "1"},
+        env=sub_env,
     )
     output = result.stdout + result.stderr
     assert "Hashview API key not configured" in output
@@ -215,7 +225,10 @@ def test_hashview_subcommands_live_upload_hashfile_job(tmp_path):
         )
     assert run.returncode == 0, output
     assert ("Hashfile uploaded" in output) or ("Hashfile added" in output)
-    assert ("Job created" in output) or ("Failed to add job" in output)
+    # Success surfaces the server's job id ("Job ID: N"); a graceful failure
+    # surfaces an "Error:" line. The CLI echoes the server's own message ("Job
+    # added"), so don't assert on the literal "Job created".
+    assert ("Job ID:" in output) or ("Error:" in output)
     if "Job ID:" in output:
         job_id = None
         for line in output.splitlines():
@@ -305,7 +318,10 @@ def test_hashview_subcommands_live_upload_hashfile_job_pwdump(tmp_path):
         )
     assert run.returncode == 0, output
     assert ("Hashfile uploaded" in output) or ("Hashfile added" in output)
-    assert ("Job created" in output) or ("Failed to add job" in output)
+    # Success surfaces the server's job id ("Job ID: N"); a graceful failure
+    # surfaces an "Error:" line. The CLI echoes the server's own message ("Job
+    # added"), so don't assert on the literal "Job created".
+    assert ("Job ID:" in output) or ("Error:" in output)
     if "Job ID:" in output:
         job_id = None
         for line in output.splitlines():
@@ -389,7 +405,10 @@ def test_hashview_subcommands_live_upload_hashfile_job_hashonly(tmp_path):
     output = run.stdout + run.stderr
     assert run.returncode == 0, output
     assert ("Hashfile uploaded" in output) or ("Hashfile added" in output)
-    assert ("Job created" in output) or ("Failed to add job" in output)
+    # Success surfaces the server's job id ("Job ID: N"); a graceful failure
+    # surfaces an "Error:" line. The CLI echoes the server's own message ("Job
+    # added"), so don't assert on the literal "Job created".
+    assert ("Job ID:" in output) or ("Error:" in output)
     if "Job ID:" in output:
         job_id = None
         for line in output.splitlines():