diff --git a/.github/workflows/translate_all.yml b/.github/workflows/translate_all.yml index 000b0b4f3..b5867a5f3 100644 --- a/.github/workflows/translate_all.yml +++ b/.github/workflows/translate_all.yml @@ -66,6 +66,9 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 + + - name: Validate translator directive safeguards + run: python scripts/test_translator_directives.py -v - name: Update and download scripts run: | diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..b36c616af --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +# Run locally: docker compose up -> http://localhost:3377 +services: + hacktricks-cloud: + image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest + platform: linux/amd64 # published image is amd64-only + container_name: hacktricks_cloud + ports: + - "3377:3000" + volumes: + - .:/app + working_dir: /app + environment: + MDBOOK_PREPROCESSOR__HACKTRICKS__ENV: dev + command: + - bash + - -c + - > + git config --global --add safe.directory /app && + mdbook serve --hostname 0.0.0.0 --port 3000 diff --git a/scripts/test_translator_directives.py b/scripts/test_translator_directives.py new file mode 100644 index 000000000..7b85df664 --- /dev/null +++ b/scripts/test_translator_directives.py @@ -0,0 +1,51 @@ +import unittest + +from translator import preserve_mdbook_directives + + +class PreserveMdbookDirectivesTests(unittest.TestCase): + def test_restores_translated_directive_and_attribute_names(self): + source = '''{{#tabs }} +{{#tab name="API" }} +Use the API. +{{#endtab }} +{{#endtabs }}''' + translated = '''{{#oortjies }} +{{#oortjie naam="API" }} +Gebruik die API. +{{#eindoortjie }} +{{#eindoortjies }}''' + + self.assertEqual( + preserve_mdbook_directives(source, translated), + '''{{#tabs }} +{{#tab name="API" }} +Gebruik die API. +{{#endtab }} +{{#endtabs }}''', + ) + + def test_restores_attribute_when_directive_name_was_preserved(self): + source = '{{#tab name="API" }}\nUse the API.\n{{#endtab }}' + translated = '{{#tab naam="API" }}\nGebruik die API.\n{{#endtab }}' + + self.assertEqual( + preserve_mdbook_directives(source, translated), + '{{#tab name="API" }}\nGebruik die API.\n{{#endtab }}', + ) + + def test_falls_back_to_source_when_directive_count_changes(self): + source = '{{#tabs }}\n{{#endtabs }}' + translated = '{{#tabs }}\nVertaalde teks' + + self.assertEqual(preserve_mdbook_directives(source, translated), source) + + def test_leaves_plain_translation_unchanged(self): + self.assertEqual( + preserve_mdbook_directives('Original prose', 'Translated prose'), + 'Translated prose', + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/translator.py b/scripts/translator.py index 65154c5dd..e5d679464 100644 --- a/scripts/translator.py +++ b/scripts/translator.py @@ -22,6 +22,12 @@ MAX_TOKENS = 50000 #gpt-4-1106-preview DISALLOWED_SPECIAL = "<|endoftext|>" REPLACEMENT_TOKEN = "" +# mdBook directives are executable document structure, not prose. Models can +# occasionally translate directive names or attributes (for example +# `name` -> `naam`), which makes preprocessors abort a whole language build. +MDBOOK_DIRECTIVE_RE = re.compile(r"\{\{#[^{}]*\}\}") +MDBOOK_TAB_OPEN_RE = re.compile(r"\{\{#tab\b([^{}]*)\}\}") + TOKENIZER_FALLBACKS = [ ("gpt-5", "o200k_base"), ("gpt-4o", "o200k_base"), @@ -124,9 +130,37 @@ def _get_encoding_for_model(model: str): print(f"Tokenizer for model {model} not found. Falling back to {FINAL_TOKENIZER_FALLBACK}.") return tiktoken.get_encoding(FINAL_TOKENIZER_FALLBACK) -def _fix_translated_shortcodes(text: str) -> str: - """Keep mdBook shortcode attribute names valid after translation.""" - return re.sub(r'(\{\{#tab\s+)(?!name=)[^=\s}]+=', r'\1name=', text) +def preserve_mdbook_directives(source: str, translated: str) -> str: + """Restore mdBook directives exactly as they appeared in the source. + + Prompt instructions are not an integrity boundary: a translated directive + can still look plausible while being invalid to a preprocessor. When the + model preserves the number/order of directives, restore each one + byte-for-byte. If it adds or removes one, keep the source chunk instead of + committing structurally broken Markdown to a language branch. + """ + source_directives = MDBOOK_DIRECTIVE_RE.findall(source) + if not source_directives: + return translated + + translated_directives = MDBOOK_DIRECTIVE_RE.findall(translated) + if len(source_directives) != len(translated_directives): + print( + "Directive count changed during translation " + f"({len(source_directives)} -> {len(translated_directives)}); " + "returning the source chunk unchanged." + ) + return source + + directives = iter(source_directives) + restored = MDBOOK_DIRECTIVE_RE.sub(lambda _match: next(directives), translated) + + for match in MDBOOK_TAB_OPEN_RE.finditer(restored): + if not re.search(r'\bname\s*=\s*(["\']).*?\1', match.group(1)): + print("Invalid mdBook tab directive after translation; returning the source chunk unchanged.") + return source + + return restored def reportTokens(prompt, model): encoding = _get_encoding_for_model(model) @@ -306,7 +340,10 @@ Also don't add any extra stuff in your response that is not part of the translat response_message = response_message.replace("bypassy", "bypasses") # PL translations translates that from time to time response_message = response_message.replace("Bypassy", "Bypasses") response_message = response_message.replace("-privec.md", "-privesc.md") # PL translations translates that from time to time - response_message = _fix_translated_shortcodes(response_message) + + # Restore source directives after all model-provided text normalization. + # This preserves exact mdBook syntax while retaining translated prose. + response_message = preserve_mdbook_directives(text, response_message) # Sometimes chatgpt modified the number of "#" at the beginning of the text, so we need to fix that. This is specially important for the first line of the MD that mucst have only 1 "#" cont2 = 0 diff --git a/src/README.md b/src/README.md index 5e384aa69..61cd02239 100644 --- a/src/README.md +++ b/src/README.md @@ -13,7 +13,7 @@ _Hacktricks logos & motion designed by_ [_@ppieranacho_](https://www.instagram.c git clone https://github.com/HackTricks-wiki/hacktricks-cloud # Select the language you want to use -export LANG="master" # Leave master for English +export HT_LANG="master" # Leave master for English # "af" for Afrikaans # "de" for German # "el" for Greek @@ -32,11 +32,19 @@ export LANG="master" # Leave master for English # "zh" for Chinese # Run the docker container indicating the path to the hacktricks-cloud folder -docker run -d --rm --platform linux/amd64 -p 3377:3000 --name hacktricks_cloud -v $(pwd)/hacktricks-cloud:/app ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image bash -c "mkdir -p ~/.ssh && ssh-keyscan -H github.com >> ~/.ssh/known_hosts && cd /app && git checkout $LANG && git pull && MDBOOK_PREPROCESSOR__HACKTRICKS__ENV=dev mdbook serve --hostname 0.0.0.0" +docker run -d --rm --platform linux/amd64 -p 3377:3000 --name hacktricks_cloud -v $(pwd)/hacktricks-cloud:/app ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image bash -c "mkdir -p ~/.ssh && ssh-keyscan -H github.com >> ~/.ssh/known_hosts && cd /app && git checkout $HT_LANG && git pull && MDBOOK_PREPROCESSOR__HACKTRICKS__ENV=dev mdbook serve --hostname 0.0.0.0" ``` Your local copy of HackTricks Cloud will be **available at [http://localhost:3377](http://localhost:3377)** after a minute. +Alternatively, if you have Docker Compose, run this from the repository root: + +```bash +docker compose up +``` + +The bundled `docker-compose.yml` serves your currently checked-out branch at [http://localhost:3377](http://localhost:3377) with live reload. + ### **Pentesting CI/CD Methodology** **In the HackTricks CI/CD Methodology you will find how to pentest infrastructure related to CI/CD activities.** Read the following page for an **introduction:**