import argparse import html import re from datetime import datetime, timezone from pathlib import Path import xml.etree.ElementTree as ET DEFAULT_LANGUAGES = [ "af", "zh", "es", "en", "fr", "de", "el", "hi", "it", "ja", "ko", "pl", "pt", "sr", "sw", "tr", "uk", ] SKIP_HTML = {"404.html", "print.html", "toc.html"} SEO_START = "" SEO_END = "" def parse_args(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command", required=True) pages = subparsers.add_parser("pages") pages.add_argument("--book-dir", required=True) pages.add_argument("--site-url", required=True) pages.add_argument("--lang", required=True) pages.add_argument("--default-lang", default="en") pages.add_argument("--languages", default=",".join(DEFAULT_LANGUAGES)) pages.add_argument("--site-name", default="HackTricks Cloud") index_cmd = subparsers.add_parser("index") index_cmd.add_argument("--site-url", required=True) index_cmd.add_argument("--languages", required=True) index_cmd.add_argument("--output", required=True) return parser.parse_args() def parse_languages(raw): langs = [] for item in raw.split(","): code = item.strip() if re.fullmatch(r"[a-z]{2}", code): langs.append(code) return sorted(set(langs)) def iter_html_files(book_dir): for html_file in sorted(Path(book_dir).rglob("*.html")): if html_file.name in SKIP_HTML: continue yield html_file def canonical_url(site_url, lang, rel_path): return f"{site_url.rstrip('/')}/{lang}/{rel_path.as_posix()}" def clean_text(fragment): fragment = re.sub(r"]*>.*?", " ", fragment, flags=re.I | re.S) fragment = re.sub(r"]*>.*?", " ", fragment, flags=re.I | re.S) fragment = re.sub(r"<[^>]+>", " ", fragment) fragment = html.unescape(fragment) fragment = re.sub(r"\s+", " ", fragment).strip() return fragment def trim_description(text, fallback): text = text or fallback text = re.sub(r"\s+", " ", text).strip() if len(text) <= 160: return text cut = text[:157] if " " in cut: cut = cut.rsplit(" ", 1)[0] return cut + "..." def extract_description(document, fallback): main_match = re.search(r"]*>(.*?)", document, flags=re.I | re.S) scope = main_match.group(1) if main_match else document for pattern in (r"]*>(.*?)

", r"]*>(.*?)", r"]*>(.*?)"): for match in re.finditer(pattern, scope, flags=re.I | re.S): text = clean_text(match.group(1)) if len(text) >= 40: return trim_description(text, fallback) return trim_description(clean_text(scope), fallback) def build_seo_block(site_url, lang, rel_path, languages, default_lang): current_url = canonical_url(site_url, lang, rel_path) lines = [SEO_START, f''] for alt_lang in languages: alt_url = canonical_url(site_url, alt_lang, rel_path) lines.append( f'' ) default_url = canonical_url(site_url, default_lang, rel_path) lines.append(f'') lines.append(SEO_END) return "\n ".join(lines) def update_document(document, site_url, lang, rel_path, languages, default_lang, site_name): title_match = re.search(r"(.*?)", document, flags=re.I | re.S) page_title = clean_text(title_match.group(1)) if title_match else site_name fallback_description = f"{site_name}: {page_title}" description = extract_description(document, fallback_description) seo_block = build_seo_block(site_url, lang, rel_path, languages, default_lang) document = re.sub( r"\s*.*?\s*", "\n", document, flags=re.S, ) if re.search(r'', document, flags=re.I): document = re.sub( r'()', r"\1" + html.escape(description, quote=True) + r"\2", document, count=1, flags=re.I, ) elif title_match: document = document.replace( title_match.group(0), title_match.group(0) + f'\n ', 1, ) document = re.sub(r"", f" {seo_block}\n ", document, count=1, flags=re.I) return document def generate_language_sitemap(book_dir, site_url, lang, languages, default_lang): ET.register_namespace("", "http://www.sitemaps.org/schemas/sitemap/0.9") ET.register_namespace("xhtml", "http://www.w3.org/1999/xhtml") urlset = ET.Element("{http://www.sitemaps.org/schemas/sitemap/0.9}urlset") for html_file in iter_html_files(book_dir): rel_path = html_file.relative_to(book_dir) url = ET.SubElement(urlset, "{http://www.sitemaps.org/schemas/sitemap/0.9}url") ET.SubElement(url, "{http://www.sitemaps.org/schemas/sitemap/0.9}loc").text = canonical_url( site_url, lang, rel_path ) lastmod = datetime.fromtimestamp(html_file.stat().st_mtime, tz=timezone.utc).date().isoformat() ET.SubElement(url, "{http://www.sitemaps.org/schemas/sitemap/0.9}lastmod").text = lastmod for alt_lang in languages: ET.SubElement( url, "{http://www.w3.org/1999/xhtml}link", { "rel": "alternate", "hreflang": alt_lang, "href": canonical_url(site_url, alt_lang, rel_path), }, ) ET.SubElement( url, "{http://www.w3.org/1999/xhtml}link", { "rel": "alternate", "hreflang": "x-default", "href": canonical_url(site_url, default_lang, rel_path), }, ) tree = ET.ElementTree(urlset) output = Path(book_dir) / "sitemap.xml" tree.write(output, encoding="utf-8", xml_declaration=True) def process_pages(args): book_dir = Path(args.book_dir) languages = parse_languages(args.languages) for html_file in iter_html_files(book_dir): rel_path = html_file.relative_to(book_dir) content = html_file.read_text(encoding="utf-8") updated = update_document( content, args.site_url, args.lang, rel_path, languages, args.default_lang, args.site_name, ) html_file.write_text(updated, encoding="utf-8") generate_language_sitemap(book_dir, args.site_url, args.lang, languages, args.default_lang) def generate_sitemap_index(args): ET.register_namespace("", "http://www.sitemaps.org/schemas/sitemap/0.9") sitemapindex = ET.Element("{http://www.sitemaps.org/schemas/sitemap/0.9}sitemapindex") now = datetime.now(timezone.utc).date().isoformat() for lang in parse_languages(args.languages): sitemap = ET.SubElement(sitemapindex, "{http://www.sitemaps.org/schemas/sitemap/0.9}sitemap") ET.SubElement(sitemap, "{http://www.sitemaps.org/schemas/sitemap/0.9}loc").text = ( f"{args.site_url.rstrip('/')}/{lang}/sitemap.xml" ) ET.SubElement(sitemap, "{http://www.sitemaps.org/schemas/sitemap/0.9}lastmod").text = now ET.ElementTree(sitemapindex).write(args.output, encoding="utf-8", xml_declaration=True) def main(): args = parse_args() if args.command == "pages": process_pages(args) elif args.command == "index": generate_sitemap_index(args) if __name__ == "__main__": main()