Fix translator directive corruption and add Docker Compose (#325)

Co-authored-by: Hermes <hermes@users.noreply.github.com>

.github/workflows/translate_all.yml

@@ -66,6 +66,9 @@ jobs:
         uses: actions/checkout@v6
         with:
           fetch-depth: 0
+
+      - name: Validate translator directive safeguards
+        run: python scripts/test_translator_directives.py -v
       
       - name: Update and download scripts
         run: |

docker-compose.yml

@@ -0,0 +1,19 @@
+# Run locally: docker compose up -> http://localhost:3377
+services:
+  hacktricks-cloud:
+    image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest
+    platform: linux/amd64  # published image is amd64-only
+    container_name: hacktricks_cloud
+    ports:
+      - "3377:3000"
+    volumes:
+      - .:/app
+    working_dir: /app
+    environment:
+      MDBOOK_PREPROCESSOR__HACKTRICKS__ENV: dev
+    command:
+      - bash
+      - -c
+      - >
+        git config --global --add safe.directory /app &&
+        mdbook serve --hostname 0.0.0.0 --port 3000

scripts/test_translator_directives.py

@@ -0,0 +1,51 @@
+import unittest
+
+from translator import preserve_mdbook_directives
+
+
+class PreserveMdbookDirectivesTests(unittest.TestCase):
+    def test_restores_translated_directive_and_attribute_names(self):
+        source = '''{{#tabs }}
+{{#tab name="API" }}
+Use the API.
+{{#endtab }}
+{{#endtabs }}'''
+        translated = '''{{#oortjies }}
+{{#oortjie naam="API" }}
+Gebruik die API.
+{{#eindoortjie }}
+{{#eindoortjies }}'''
+
+        self.assertEqual(
+            preserve_mdbook_directives(source, translated),
+            '''{{#tabs }}
+{{#tab name="API" }}
+Gebruik die API.
+{{#endtab }}
+{{#endtabs }}''',
+        )
+
+    def test_restores_attribute_when_directive_name_was_preserved(self):
+        source = '{{#tab name="API" }}\nUse the API.\n{{#endtab }}'
+        translated = '{{#tab naam="API" }}\nGebruik die API.\n{{#endtab }}'
+
+        self.assertEqual(
+            preserve_mdbook_directives(source, translated),
+            '{{#tab name="API" }}\nGebruik die API.\n{{#endtab }}',
+        )
+
+    def test_falls_back_to_source_when_directive_count_changes(self):
+        source = '{{#tabs }}\n{{#endtabs }}'
+        translated = '{{#tabs }}\nVertaalde teks'
+
+        self.assertEqual(preserve_mdbook_directives(source, translated), source)
+
+    def test_leaves_plain_translation_unchanged(self):
+        self.assertEqual(
+            preserve_mdbook_directives('Original prose', 'Translated prose'),
+            'Translated prose',
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/translator.py

@@ -22,6 +22,12 @@ MAX_TOKENS = 50000 #gpt-4-1106-preview
 DISALLOWED_SPECIAL = "<|endoftext|>"
 REPLACEMENT_TOKEN  = "<END_OF_TEXT>"
 
+# mdBook directives are executable document structure, not prose. Models can
+# occasionally translate directive names or attributes (for example
+# `name` -> `naam`), which makes preprocessors abort a whole language build.
+MDBOOK_DIRECTIVE_RE = re.compile(r"\{\{#[^{}]*\}\}")
+MDBOOK_TAB_OPEN_RE = re.compile(r"\{\{#tab\b([^{}]*)\}\}")
+
 TOKENIZER_FALLBACKS = [
     ("gpt-5", "o200k_base"),
     ("gpt-4o", "o200k_base"),
@@ -124,9 +130,37 @@ def _get_encoding_for_model(model: str):
         print(f"Tokenizer for model {model} not found. Falling back to {FINAL_TOKENIZER_FALLBACK}.")
         return tiktoken.get_encoding(FINAL_TOKENIZER_FALLBACK)
 
-def _fix_translated_shortcodes(text: str) -> str:
-    """Keep mdBook shortcode attribute names valid after translation."""
-    return re.sub(r'(\{\{#tab\s+)(?!name=)[^=\s}]+=', r'\1name=', text)
+def preserve_mdbook_directives(source: str, translated: str) -> str:
+    """Restore mdBook directives exactly as they appeared in the source.
+
+    Prompt instructions are not an integrity boundary: a translated directive
+    can still look plausible while being invalid to a preprocessor. When the
+    model preserves the number/order of directives, restore each one
+    byte-for-byte. If it adds or removes one, keep the source chunk instead of
+    committing structurally broken Markdown to a language branch.
+    """
+    source_directives = MDBOOK_DIRECTIVE_RE.findall(source)
+    if not source_directives:
+        return translated
+
+    translated_directives = MDBOOK_DIRECTIVE_RE.findall(translated)
+    if len(source_directives) != len(translated_directives):
+        print(
+            "Directive count changed during translation "
+            f"({len(source_directives)} -> {len(translated_directives)}); "
+            "returning the source chunk unchanged."
+        )
+        return source
+
+    directives = iter(source_directives)
+    restored = MDBOOK_DIRECTIVE_RE.sub(lambda _match: next(directives), translated)
+
+    for match in MDBOOK_TAB_OPEN_RE.finditer(restored):
+        if not re.search(r'\bname\s*=\s*(["\']).*?\1', match.group(1)):
+            print("Invalid mdBook tab directive after translation; returning the source chunk unchanged.")
+            return source
+
+    return restored
 
 def reportTokens(prompt, model):
     encoding = _get_encoding_for_model(model)
@@ -306,7 +340,10 @@ Also don't add any extra stuff in your response that is not part of the translat
     response_message = response_message.replace("bypassy", "bypasses") # PL translations translates that from time to time
     response_message = response_message.replace("Bypassy", "Bypasses")
     response_message = response_message.replace("-privec.md", "-privesc.md") # PL translations translates that from time to time
-    response_message = _fix_translated_shortcodes(response_message)
+
+    # Restore source directives after all model-provided text normalization.
+    # This preserves exact mdBook syntax while retaining translated prose.
+    response_message = preserve_mdbook_directives(text, response_message)
 
     # Sometimes chatgpt modified the number of "#" at the beginning of the text, so we need to fix that. This is specially important for the first line of the MD that mucst have only 1 "#"
     cont2 = 0

src/README.md

@@ -13,7 +13,7 @@ _Hacktricks logos & motion designed by_ [_@ppieranacho_](https://www.instagram.c
 git clone https://github.com/HackTricks-wiki/hacktricks-cloud
 
 # Select the language you want to use
-export LANG="master" # Leave master for English
+export HT_LANG="master" # Leave master for English
 # "af" for Afrikaans
 # "de" for German
 # "el" for Greek
@@ -32,11 +32,19 @@ export LANG="master" # Leave master for English
 # "zh" for Chinese
 
 # Run the docker container indicating the path to the hacktricks-cloud folder
-docker run -d --rm --platform linux/amd64 -p 3377:3000 --name hacktricks_cloud -v $(pwd)/hacktricks-cloud:/app ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image bash -c "mkdir -p ~/.ssh && ssh-keyscan -H github.com >> ~/.ssh/known_hosts && cd /app && git checkout $LANG && git pull && MDBOOK_PREPROCESSOR__HACKTRICKS__ENV=dev mdbook serve --hostname 0.0.0.0"
+docker run -d --rm --platform linux/amd64 -p 3377:3000 --name hacktricks_cloud -v $(pwd)/hacktricks-cloud:/app ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image bash -c "mkdir -p ~/.ssh && ssh-keyscan -H github.com >> ~/.ssh/known_hosts && cd /app && git checkout $HT_LANG && git pull && MDBOOK_PREPROCESSOR__HACKTRICKS__ENV=dev mdbook serve --hostname 0.0.0.0"
 ```
 
 Your local copy of HackTricks Cloud will be **available at [http://localhost:3377](http://localhost:3377)** after a minute.
 
+Alternatively, if you have Docker Compose, run this from the repository root:
+
+```bash
+docker compose up
+```
+
+The bundled `docker-compose.yml` serves your currently checked-out branch at [http://localhost:3377](http://localhost:3377) with live reload.
+
 ### **Pentesting CI/CD Methodology**
 
 **In the HackTricks CI/CD Methodology you will find how to pentest infrastructure related to CI/CD activities.** Read the following page for an **introduction:**