ci: avoid redundant S3 uploads (#326)

Co-authored-by: Hermes Agent <hermes-agent@users.noreply.github.com>

.github/workflows/build_master.yml

@@ -153,9 +153,20 @@ jobs:
           role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
           aws-region: us-east-1
       
-      # Sync the build to S3
+      # Sync the build to S3. mdBook refreshes mtimes on every build, so first
+      # age byte-identical files based on S3 ETags; `s3 sync` then uploads only
+      # genuinely changed/new output while retaining normal --delete behavior.
       - name: Sync to S3
-        run: aws s3 sync ./book s3://hacktricks-cloud/en --delete
+        run: |
+          aws s3api list-objects-v2 \
+            --bucket hacktricks-cloud \
+            --prefix en/ \
+            --output json > /tmp/s3-en-manifest.json
+          python3 scripts/mark_unchanged_s3_files.py \
+            --source ./book \
+            --manifest /tmp/s3-en-manifest.json \
+            --remote-prefix en/
+          aws s3 sync ./book s3://hacktricks-cloud/en --delete
 
       - name: Upload root sitemap index
         run: |

.github/workflows/translate_all.yml

@@ -84,6 +84,7 @@ jobs:
           wget -O /tmp/compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py
           wget -O /tmp/translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py
           wget -O /tmp/seo_postprocess.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/seo_postprocess.py
+          cp scripts/mark_unchanged_s3_files.py /tmp/mark_unchanged_s3_files.py
 
       - name: Run get_and_save_refs.py
         run: |
@@ -272,6 +273,16 @@ jobs:
           echo "Current branch:"
           git rev-parse --abbrev-ref HEAD
           echo "Syncing $BRANCH to S3"
+          # mdBook refreshes mtimes on every build. Age byte-identical files
+          # based on single-part S3 ETags so `s3 sync` skips redundant PUTs.
+          aws s3api list-objects-v2 \
+            --bucket hacktricks-cloud \
+            --prefix "$BRANCH/" \
+            --output json > /tmp/s3-branch-manifest.json
+          python3 /tmp/mark_unchanged_s3_files.py \
+            --source ./book \
+            --manifest /tmp/s3-branch-manifest.json \
+            --remote-prefix "$BRANCH/"
           aws s3 sync ./book s3://hacktricks-cloud/$BRANCH --delete
           echo "Sync completed"
           echo "Cat 3 files from the book"

scripts/mark_unchanged_s3_files.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Avoid redundant S3 uploads by aging byte-identical local files.
+
+`aws s3 sync` uploads a local file when its size differs from the remote object or
+when the local modification time is newer. mdBook rebuilds refresh mtimes even
+when output bytes are unchanged, which causes unnecessary PutObject requests.
+
+Given a ListObjectsV2 manifest, this script compares local MD5 digests with
+single-part S3 ETags. Byte-identical files are assigned an old mtime so the
+following `aws s3 sync --delete` skips them. New, changed, multipart, and
+otherwise unverifiable objects are left untouched and therefore upload normally.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+OLD_MTIME_SECONDS = 1
+
+
+def file_md5(path: Path, chunk_size: int = 1024 * 1024) -> str:
+    digest = hashlib.md5(usedforsecurity=False)
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(chunk_size), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def load_remote_objects(manifest_path: Path, remote_prefix: str) -> dict[str, dict[str, Any]]:
+    if remote_prefix.startswith("/"):
+        raise ValueError("remote prefix must not start with '/'")
+    if remote_prefix and not remote_prefix.endswith("/"):
+        remote_prefix += "/"
+
+    payload = json.loads(manifest_path.read_text(encoding="utf-8"))
+    contents = payload.get("Contents", [])
+    if contents is None:
+        contents = []
+    if not isinstance(contents, list):
+        raise ValueError("manifest Contents must be a list")
+
+    objects: dict[str, dict[str, Any]] = {}
+    for item in contents:
+        if not isinstance(item, dict):
+            continue
+        key = str(item.get("Key") or "")
+        if not key.startswith(remote_prefix):
+            continue
+        relative_key = key[len(remote_prefix):]
+        if not relative_key or relative_key.endswith("/"):
+            continue
+        objects[relative_key] = item
+    return objects
+
+
+def mark_unchanged_files(source: Path, remote: dict[str, dict[str, Any]]) -> dict[str, int]:
+    if not source.is_dir():
+        raise ValueError(f"source is not a directory: {source}")
+
+    stats = {"local_files": 0, "unchanged": 0, "upload_candidates": 0}
+    for path in source.rglob("*"):
+        if not path.is_file():
+            continue
+        stats["local_files"] += 1
+        relative_key = path.relative_to(source).as_posix()
+        item = remote.get(relative_key)
+        if not item:
+            stats["upload_candidates"] += 1
+            continue
+
+        etag = str(item.get("ETag") or "").strip('"').lower()
+        raw_size = item.get("Size")
+        try:
+            remote_size = int(raw_size) if raw_size is not None else -1
+        except (TypeError, ValueError):
+            remote_size = -1
+
+        # A dashed ETag is normally multipart and is not the object's MD5.
+        verifiable = len(etag) == 32 and "-" not in etag and all(c in "0123456789abcdef" for c in etag)
+        if verifiable and remote_size == path.stat().st_size and file_md5(path) == etag:
+            os.utime(path, (OLD_MTIME_SECONDS, OLD_MTIME_SECONDS), follow_symlinks=True)
+            stats["unchanged"] += 1
+        else:
+            stats["upload_candidates"] += 1
+    return stats
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--source", type=Path, required=True)
+    parser.add_argument("--manifest", type=Path, required=True)
+    parser.add_argument("--remote-prefix", default="")
+    args = parser.parse_args()
+
+    remote = load_remote_objects(args.manifest, args.remote_prefix)
+    stats = mark_unchanged_files(args.source, remote)
+    stats["remote_objects"] = len(remote)
+    print(json.dumps(stats, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/test_mark_unchanged_s3_files.py

@@ -0,0 +1,73 @@
+import hashlib
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from scripts.mark_unchanged_s3_files import OLD_MTIME_SECONDS, load_remote_objects, mark_unchanged_files
+
+
+class MarkUnchangedS3FilesTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp_dir.name)
+        self.source = self.root / "book"
+        self.source.mkdir()
+
+    def tearDown(self) -> None:
+        self.temp_dir.cleanup()
+
+    @staticmethod
+    def _etag(data: bytes) -> str:
+        return hashlib.md5(data, usedforsecurity=False).hexdigest()
+
+    def _write_manifest(self, contents: list[dict]) -> Path:
+        path = self.root / "manifest.json"
+        path.write_text(json.dumps({"Contents": contents}), encoding="utf-8")
+        return path
+
+    def test_only_byte_identical_single_part_objects_are_aged(self) -> None:
+        unchanged = self.source / "nested" / "same.html"
+        changed = self.source / "changed.html"
+        multipart = self.source / "large.bin"
+        new = self.source / "new.txt"
+        unchanged.parent.mkdir()
+        unchanged.write_bytes(b"same")
+        changed.write_bytes(b"local")
+        multipart.write_bytes(b"large")
+        new.write_bytes(b"new")
+        for path in (unchanged, changed, multipart, new):
+            os.utime(path, (1000, 1000))
+
+        manifest = self._write_manifest([
+            {"Key": "en/nested/same.html", "Size": 4, "ETag": f'"{self._etag(b"same")}"'},
+            {"Key": "en/changed.html", "Size": 5, "ETag": f'"{self._etag(b"other")}"'},
+            {"Key": "en/large.bin", "Size": 5, "ETag": '"0123456789abcdef0123456789abcdef-2"'},
+            {"Key": "other/ignored.txt", "Size": 1, "ETag": '"0cc175b9c0f1b6a831c399e269772661"'},
+        ])
+
+        remote = load_remote_objects(manifest, "en")
+        stats = mark_unchanged_files(self.source, remote)
+
+        self.assertEqual(int(unchanged.stat().st_mtime), OLD_MTIME_SECONDS)
+        self.assertEqual(int(changed.stat().st_mtime), 1000)
+        self.assertEqual(int(multipart.stat().st_mtime), 1000)
+        self.assertEqual(int(new.stat().st_mtime), 1000)
+        self.assertEqual(stats, {"local_files": 4, "unchanged": 1, "upload_candidates": 3})
+        self.assertNotIn("ignored.txt", remote)
+
+    def test_rejects_absolute_remote_prefix(self) -> None:
+        manifest = self._write_manifest([])
+        with self.assertRaises(ValueError):
+            load_remote_objects(manifest, "/en/")
+
+    def test_rejects_non_list_contents(self) -> None:
+        path = self.root / "manifest.json"
+        path.write_text('{"Contents": {}}', encoding="utf-8")
+        with self.assertRaises(ValueError):
+            load_remote_objects(path, "en/")
+
+
+if __name__ == "__main__":
+    unittest.main()