diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index 39b512bbe..ad1e31ccd 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -153,9 +153,20 @@ jobs: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: us-east-1 - # Sync the build to S3 + # Sync the build to S3. mdBook refreshes mtimes on every build, so first + # age byte-identical files based on S3 ETags; `s3 sync` then uploads only + # genuinely changed/new output while retaining normal --delete behavior. - name: Sync to S3 - run: aws s3 sync ./book s3://hacktricks-cloud/en --delete + run: | + aws s3api list-objects-v2 \ + --bucket hacktricks-cloud \ + --prefix en/ \ + --output json > /tmp/s3-en-manifest.json + python3 scripts/mark_unchanged_s3_files.py \ + --source ./book \ + --manifest /tmp/s3-en-manifest.json \ + --remote-prefix en/ + aws s3 sync ./book s3://hacktricks-cloud/en --delete - name: Upload root sitemap index run: | diff --git a/.github/workflows/translate_all.yml b/.github/workflows/translate_all.yml index b5867a5f3..1a03693ba 100644 --- a/.github/workflows/translate_all.yml +++ b/.github/workflows/translate_all.yml @@ -84,6 +84,7 @@ jobs: wget -O /tmp/compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py wget -O /tmp/translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py wget -O /tmp/seo_postprocess.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/seo_postprocess.py + cp scripts/mark_unchanged_s3_files.py /tmp/mark_unchanged_s3_files.py - name: Run get_and_save_refs.py run: | @@ -272,6 +273,16 @@ jobs: echo "Current branch:" git rev-parse --abbrev-ref HEAD echo "Syncing $BRANCH to S3" + # mdBook refreshes mtimes on every build. Age byte-identical files + # based on single-part S3 ETags so `s3 sync` skips redundant PUTs. + aws s3api list-objects-v2 \ + --bucket hacktricks-cloud \ + --prefix "$BRANCH/" \ + --output json > /tmp/s3-branch-manifest.json + python3 /tmp/mark_unchanged_s3_files.py \ + --source ./book \ + --manifest /tmp/s3-branch-manifest.json \ + --remote-prefix "$BRANCH/" aws s3 sync ./book s3://hacktricks-cloud/$BRANCH --delete echo "Sync completed" echo "Cat 3 files from the book" diff --git a/scripts/mark_unchanged_s3_files.py b/scripts/mark_unchanged_s3_files.py new file mode 100644 index 000000000..c9b85df5f --- /dev/null +++ b/scripts/mark_unchanged_s3_files.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Avoid redundant S3 uploads by aging byte-identical local files. + +`aws s3 sync` uploads a local file when its size differs from the remote object or +when the local modification time is newer. mdBook rebuilds refresh mtimes even +when output bytes are unchanged, which causes unnecessary PutObject requests. + +Given a ListObjectsV2 manifest, this script compares local MD5 digests with +single-part S3 ETags. Byte-identical files are assigned an old mtime so the +following `aws s3 sync --delete` skips them. New, changed, multipart, and +otherwise unverifiable objects are left untouched and therefore upload normally. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +OLD_MTIME_SECONDS = 1 + + +def file_md5(path: Path, chunk_size: int = 1024 * 1024) -> str: + digest = hashlib.md5(usedforsecurity=False) + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_remote_objects(manifest_path: Path, remote_prefix: str) -> dict[str, dict[str, Any]]: + if remote_prefix.startswith("/"): + raise ValueError("remote prefix must not start with '/'") + if remote_prefix and not remote_prefix.endswith("/"): + remote_prefix += "/" + + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + contents = payload.get("Contents", []) + if contents is None: + contents = [] + if not isinstance(contents, list): + raise ValueError("manifest Contents must be a list") + + objects: dict[str, dict[str, Any]] = {} + for item in contents: + if not isinstance(item, dict): + continue + key = str(item.get("Key") or "") + if not key.startswith(remote_prefix): + continue + relative_key = key[len(remote_prefix):] + if not relative_key or relative_key.endswith("/"): + continue + objects[relative_key] = item + return objects + + +def mark_unchanged_files(source: Path, remote: dict[str, dict[str, Any]]) -> dict[str, int]: + if not source.is_dir(): + raise ValueError(f"source is not a directory: {source}") + + stats = {"local_files": 0, "unchanged": 0, "upload_candidates": 0} + for path in source.rglob("*"): + if not path.is_file(): + continue + stats["local_files"] += 1 + relative_key = path.relative_to(source).as_posix() + item = remote.get(relative_key) + if not item: + stats["upload_candidates"] += 1 + continue + + etag = str(item.get("ETag") or "").strip('"').lower() + raw_size = item.get("Size") + try: + remote_size = int(raw_size) if raw_size is not None else -1 + except (TypeError, ValueError): + remote_size = -1 + + # A dashed ETag is normally multipart and is not the object's MD5. + verifiable = len(etag) == 32 and "-" not in etag and all(c in "0123456789abcdef" for c in etag) + if verifiable and remote_size == path.stat().st_size and file_md5(path) == etag: + os.utime(path, (OLD_MTIME_SECONDS, OLD_MTIME_SECONDS), follow_symlinks=True) + stats["unchanged"] += 1 + else: + stats["upload_candidates"] += 1 + return stats + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--remote-prefix", default="") + args = parser.parse_args() + + remote = load_remote_objects(args.manifest, args.remote_prefix) + stats = mark_unchanged_files(args.source, remote) + stats["remote_objects"] = len(remote) + print(json.dumps(stats, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_mark_unchanged_s3_files.py b/scripts/test_mark_unchanged_s3_files.py new file mode 100644 index 000000000..a41fbe923 --- /dev/null +++ b/scripts/test_mark_unchanged_s3_files.py @@ -0,0 +1,73 @@ +import hashlib +import json +import os +import tempfile +import unittest +from pathlib import Path + +from scripts.mark_unchanged_s3_files import OLD_MTIME_SECONDS, load_remote_objects, mark_unchanged_files + + +class MarkUnchangedS3FilesTest(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.root = Path(self.temp_dir.name) + self.source = self.root / "book" + self.source.mkdir() + + def tearDown(self) -> None: + self.temp_dir.cleanup() + + @staticmethod + def _etag(data: bytes) -> str: + return hashlib.md5(data, usedforsecurity=False).hexdigest() + + def _write_manifest(self, contents: list[dict]) -> Path: + path = self.root / "manifest.json" + path.write_text(json.dumps({"Contents": contents}), encoding="utf-8") + return path + + def test_only_byte_identical_single_part_objects_are_aged(self) -> None: + unchanged = self.source / "nested" / "same.html" + changed = self.source / "changed.html" + multipart = self.source / "large.bin" + new = self.source / "new.txt" + unchanged.parent.mkdir() + unchanged.write_bytes(b"same") + changed.write_bytes(b"local") + multipart.write_bytes(b"large") + new.write_bytes(b"new") + for path in (unchanged, changed, multipart, new): + os.utime(path, (1000, 1000)) + + manifest = self._write_manifest([ + {"Key": "en/nested/same.html", "Size": 4, "ETag": f'"{self._etag(b"same")}"'}, + {"Key": "en/changed.html", "Size": 5, "ETag": f'"{self._etag(b"other")}"'}, + {"Key": "en/large.bin", "Size": 5, "ETag": '"0123456789abcdef0123456789abcdef-2"'}, + {"Key": "other/ignored.txt", "Size": 1, "ETag": '"0cc175b9c0f1b6a831c399e269772661"'}, + ]) + + remote = load_remote_objects(manifest, "en") + stats = mark_unchanged_files(self.source, remote) + + self.assertEqual(int(unchanged.stat().st_mtime), OLD_MTIME_SECONDS) + self.assertEqual(int(changed.stat().st_mtime), 1000) + self.assertEqual(int(multipart.stat().st_mtime), 1000) + self.assertEqual(int(new.stat().st_mtime), 1000) + self.assertEqual(stats, {"local_files": 4, "unchanged": 1, "upload_candidates": 3}) + self.assertNotIn("ignored.txt", remote) + + def test_rejects_absolute_remote_prefix(self) -> None: + manifest = self._write_manifest([]) + with self.assertRaises(ValueError): + load_remote_objects(manifest, "/en/") + + def test_rejects_non_list_contents(self) -> None: + path = self.root / "manifest.json" + path.write_text('{"Contents": {}}', encoding="utf-8") + with self.assertRaises(ValueError): + load_remote_objects(path, "en/") + + +if __name__ == "__main__": + unittest.main()