From c0972aba1edef104e4ba1b7ff4490282821cb37a Mon Sep 17 00:00:00 2001 From: Carlos Polop Date: Sun, 19 Jul 2026 12:17:54 +0200 Subject: [PATCH] Recognize tokenizers for known model prefixes --- scripts/translator.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/translator.py b/scripts/translator.py index e5d679464..77eb7b9bd 100644 --- a/scripts/translator.py +++ b/scripts/translator.py @@ -28,7 +28,7 @@ REPLACEMENT_TOKEN = "" MDBOOK_DIRECTIVE_RE = re.compile(r"\{\{#[^{}]*\}\}") MDBOOK_TAB_OPEN_RE = re.compile(r"\{\{#tab\b([^{}]*)\}\}") -TOKENIZER_FALLBACKS = [ +MODEL_PREFIX_ENCODINGS = [ ("gpt-5", "o200k_base"), ("gpt-4o", "o200k_base"), ("gpt-4.1", "o200k_base"), @@ -116,16 +116,19 @@ def _sanitize(text: str) -> str: def _get_encoding_for_model(model: str): """ - Return a tokenizer for the requested model, with fallbacks for newer - model names that tiktoken may not recognize yet. + Return a tokenizer for the requested model. + + Tiktoken does not necessarily know custom or newly released model names, + so resolve known model families by prefix before treating a model as + unknown. For example, ``gpt-5.6-luna`` uses the GPT-5 tokenizer even if + that exact name is absent from tiktoken's model registry. """ try: return tiktoken.encoding_for_model(model) except KeyError: lowered_model = model.lower() - for prefix, encoding_name in TOKENIZER_FALLBACKS: + for prefix, encoding_name in MODEL_PREFIX_ENCODINGS: if lowered_model.startswith(prefix): - print(f"Tokenizer for model {model} not found. Falling back to {encoding_name}.") return tiktoken.get_encoding(encoding_name) print(f"Tokenizer for model {model} not found. Falling back to {FINAL_TOKENIZER_FALLBACK}.") return tiktoken.get_encoding(FINAL_TOKENIZER_FALLBACK)