gitea-mirror/hacktricks-cloud
1
0
Fork 0
mirror of https://github.com/HackTricks-wiki/hacktricks-cloud.git synced 2026-07-28 14:47:17 -07:00
Code Issues Packages Projects Releases Wiki Activity

Recognize tokenizers for known model prefixes

Browse Source
This commit is contained in:
Carlos Polop
2026-07-19 12:17:54 +02:00
parent 38e26664a3
commit c0972aba1e
1 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/translator.py
+8 -5
View File
@@ -28,7 +28,7 @@ REPLACEMENT_TOKEN = "<END_OF_TEXT>"
MDBOOK_DIRECTIVE_RE = re.compile(r"\{\{#[^{}]*\}\}") MDBOOK_DIRECTIVE_RE = re.compile(r"\{\{#[^{}]*\}\}")
MDBOOK_TAB_OPEN_RE = re.compile(r"\{\{#tab\b([^{}]*)\}\}") MDBOOK_TAB_OPEN_RE = re.compile(r"\{\{#tab\b([^{}]*)\}\}")
TOKENIZER_FALLBACKS = [ MODEL_PREFIX_ENCODINGS = [
("gpt-5", "o200k_base"), ("gpt-5", "o200k_base"),
("gpt-4o", "o200k_base"), ("gpt-4o", "o200k_base"),
("gpt-4.1", "o200k_base"), ("gpt-4.1", "o200k_base"),
@@ -116,16 +116,19 @@ def _sanitize(text: str) -> str:
def _get_encoding_for_model(model: str): def _get_encoding_for_model(model: str):
""" """
Return a tokenizer for the requested model, with fallbacks for newer Return a tokenizer for the requested model.
model names that tiktoken may not recognize yet.
Tiktoken does not necessarily know custom or newly released model names,
so resolve known model families by prefix before treating a model as
unknown. For example, ``gpt-5.6-luna`` uses the GPT-5 tokenizer even if
that exact name is absent from tiktoken's model registry.
""" """
try: try:
return tiktoken.encoding_for_model(model) return tiktoken.encoding_for_model(model)
except KeyError: except KeyError:
lowered_model = model.lower() lowered_model = model.lower()
for prefix, encoding_name in TOKENIZER_FALLBACKS: for prefix, encoding_name in MODEL_PREFIX_ENCODINGS:
if lowered_model.startswith(prefix): if lowered_model.startswith(prefix):
print(f"Tokenizer for model {model} not found. Falling back to {encoding_name}.")
return tiktoken.get_encoding(encoding_name) return tiktoken.get_encoding(encoding_name)
print(f"Tokenizer for model {model} not found. Falling back to {FINAL_TOKENIZER_FALLBACK}.") print(f"Tokenizer for model {model} not found. Falling back to {FINAL_TOKENIZER_FALLBACK}.")
return tiktoken.get_encoding(FINAL_TOKENIZER_FALLBACK) return tiktoken.get_encoding(FINAL_TOKENIZER_FALLBACK)