gitea-mirror/hacktricks-cloud
1
0
Fork 0
mirror of https://github.com/HackTricks-wiki/hacktricks-cloud.git synced 2025-12-05 20:40:18 -08:00
Code Issues Packages Projects Releases Wiki Activity

fix searchindex

Browse Source
This commit is contained in:
carlospolop
2025-10-01 11:37:13 +02:00
parent d0ebc37eb3
commit 570c0f46af
5 changed files with 131 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
.github/workflows/build_master.yml vendored
View File

@@ -35,64 +35,60 @@ jobs:
- name: Build mdBook - name: Build mdBook
run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1) run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
- name: Update searchindex in repo (purge history, keep current on HEAD) - name: Install GitHub CLI
run: |
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
&& sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
&& sudo apt update \
&& sudo apt install gh -y
- name: Publish search index release asset
shell: bash shell: bash
env:
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
run: | run: |
set -euo pipefail set -euo pipefail
pwd ASSET="book/searchindex.js"
ls -la TAG="searchindex-en"
ls -la book TITLE="Search Index (en)"
git config --global --add safe.directory /__w/hacktricks-cloud/hacktricks-cloud if [ ! -f "$ASSET" ]; then
git config --global user.email "build@example.com" echo "Expected $ASSET to exist after build" >&2
git config --global user.name "Build master" exit 1
git config pull.rebase false
# Ensure we're on the target branch and up to date
# Handle force-pushed branches by resetting to origin/master
git fetch origin
git reset --hard origin/master
# Choose the file to keep at HEAD:
# 1) Prefer freshly built version from book/
# 2) Fallback to the file currently at HEAD (if it exists)
HAS_FILE=0
if [ -f "book/searchindex.js" ]; then
cp "book/searchindex.js" /tmp/sidx.js
HAS_FILE=1
elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
git show "HEAD:searchindex.js" > /tmp/sidx.js
HAS_FILE=1
fi fi
# Skip if there's nothing to purge AND nothing to keep TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 HEAD -- "searchindex.js" >/dev/null 2>&1; then if [ -z "$TOKEN" ]; then
# Fail early if working tree is dirty (avoid confusing rewrites) echo "No token available for GitHub CLI" >&2
git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; } exit 1
fi
export GH_TOKEN="$TOKEN"
# Install git-filter-repo and ensure it's on PATH # Delete the release if it exists
python -m pip install --quiet --user git-filter-repo echo "Checking if release $TAG exists..."
export PATH="$HOME/.local/bin:$PATH" if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
echo "Release $TAG already exists, deleting it..."
# Rewrite ONLY the current branch, dropping all historical blobs of searchindex.js gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag || {
git filter-repo --force --path "searchindex.js" --invert-paths --refs "$(git symbolic-ref -q HEAD)" echo "Failed to delete release, trying without cleanup-tag..."
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" || {
# Re-add the current version on top of rewritten history (keep it in HEAD) echo "Warning: Could not delete existing release, will try to recreate..."
if [ "$HAS_FILE" = "1" ]; then }
mv /tmp/sidx.js "searchindex.js" }
git add "searchindex.js" sleep 2 # Give GitHub API a moment to process the deletion
git commit -m "Update searchindex (purged history; keep current)"
else
echo "No current searchindex.js to re-add after purge."
fi
# Safer force push (only updates if remote hasn't advanced)
git push --force-with-lease
else else
echo "Nothing to purge; skipping." echo "Release $TAG does not exist, proceeding with creation..."
fi fi
# Create new release (with force flag to overwrite if deletion failed)
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY" || {
echo "Failed to create release, trying with force flag..."
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag >/dev/null 2>&1 || true
sleep 2
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY"
}
# Login in AWs # Login in AWs
- name: Configure AWS credentials using OIDC - name: Configure AWS credentials using OIDC
uses: aws-actions/configure-aws-credentials@v3 uses: aws-actions/configure-aws-credentials@v3

73
.github/workflows/translate_all.yml vendored
View File

@@ -66,7 +66,13 @@ jobs:
- name: Update and download scripts - name: Update and download scripts
run: | run: |
sudo apt-get update sudo apt-get update
sudo apt-get install wget -y # Install GitHub CLI properly
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
&& sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
&& sudo apt update \
&& sudo apt install gh -y \
&& sudo apt-get install -y wget
wget -O /tmp/get_and_save_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/get_and_save_refs.py wget -O /tmp/get_and_save_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/get_and_save_refs.py
wget -O /tmp/compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py wget -O /tmp/compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py
wget -O /tmp/translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py wget -O /tmp/translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py
@@ -138,56 +144,37 @@ jobs:
git pull git pull
MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1) MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
- name: Update searchindex.js in repo (purge history, keep current on HEAD) - name: Publish search index release asset
shell: bash shell: bash
env:
PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
run: | run: |
set -euo pipefail set -euo pipefail
# Be explicit about workspace trust (avoids "dubious ownership") ASSET="book/searchindex.js"
git config --global --add safe.directory "$GITHUB_WORKSPACE" TAG="searchindex-${BRANCH}"
TITLE="Search Index (${BRANCH})"
git checkout "$BRANCH" if [ ! -f "$ASSET" ]; then
git fetch origin "$BRANCH" --quiet echo "Expected $ASSET to exist after build" >&2
git pull --ff-only exit 1
# Choose the file to keep at HEAD:
# 1) Prefer freshly built version from book/
# 2) Fallback to the file currently at HEAD (if it exists)
HAS_FILE=0
if [ -f "book/searchindex.js" ]; then
cp "book/searchindex.js" /tmp/sidx.js
HAS_FILE=1
elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
git show "HEAD:searchindex.js" > /tmp/sidx.js
HAS_FILE=1
fi fi
# Skip if there's nothing to purge AND nothing to keep TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 "$BRANCH" -- "searchindex.js" >/dev/null 2>&1; then if [ -z "$TOKEN" ]; then
# **Fail early if working tree is dirty** (prevents confusing filter results) echo "No token available for GitHub CLI" >&2
git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; } exit 1
# Make sure git-filter-repo is callable via `git filter-repo`
python -m pip install --quiet --user git-filter-repo
export PATH="$HOME/.local/bin:$PATH"
# Rewrite ONLY this branch, dropping all historical blobs of searchindex.js
git filter-repo --force --path "searchindex.js" --invert-paths --refs "refs/heads/$BRANCH"
# Re-add the current version on top of rewritten history (keep it in HEAD)
if [ "$HAS_FILE" = "1" ]; then
mv /tmp/sidx.js "searchindex.js"
git add "searchindex.js"
git commit -m "Update searchindex (purged history; keep current)"
else
echo "No current searchindex.js to re-add after purge."
fi
# **Safer force push** (prevents clobbering unexpected remote updates)
git push --force-with-lease origin "$BRANCH"
else
echo "Nothing to purge; skipping."
fi fi
export GH_TOKEN="$TOKEN"
# Delete the release if it exists
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
echo "Release $TAG already exists, deleting it..."
gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY"
fi
# Create new release
gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY"
# Login in AWs # Login in AWs
- name: Configure AWS credentials using OIDC - name: Configure AWS credentials using OIDC

1
.gitignore vendored
View File

@@ -35,3 +35,4 @@ book
book/* book/*
hacktricks-preprocessor.log hacktricks-preprocessor.log
hacktricks-preprocessor-error.log hacktricks-preprocessor-error.log
searchindex.js

1
searchindex.js
View File

File diff suppressed because one or more lines are too long

71
theme/ht_searcher.js
View File

@@ -21,33 +21,60 @@
try { importScripts('https://cdn.jsdelivr.net/npm/elasticlunr@0.9.5/elasticlunr.min.js'); } try { importScripts('https://cdn.jsdelivr.net/npm/elasticlunr@0.9.5/elasticlunr.min.js'); }
catch { importScripts(abs('/elasticlunr.min.js')); } catch { importScripts(abs('/elasticlunr.min.js')); }
/* 2 — load a single index (remote → local) */ /* 2 — load a single index (remote → local) */
async function loadIndex(remote, local, isCloud=false){ async function loadIndex(remote, local, isCloud=false){
let rawLoaded = false; let rawLoaded = false;
if(remote){
try { try {
const r = await fetch(remote,{mode:'cors'}); const r = await fetch(remote,{mode:'cors'});
if (!r.ok) throw new Error('HTTP '+r.status); if (!r.ok) throw new Error('HTTP '+r.status);
importScripts(URL.createObjectURL(new Blob([await r.text()],{type:'application/javascript'}))); importScripts(URL.createObjectURL(new Blob([await r.text()],{type:'application/javascript'})));
rawLoaded = true; rawLoaded = true;
} catch(e){ console.warn('remote',remote,'failed →',e); } } catch(e){ console.warn('remote',remote,'failed →',e); }
if(!rawLoaded){ }
try { importScripts(abs(local)); rawLoaded = true; } if(!rawLoaded && local){
catch(e){ console.error('local',local,'failed →',e); } try { importScripts(abs(local)); rawLoaded = true; }
catch(e){ console.error('local',local,'failed →',e); }
}
if(!rawLoaded) return null; /* give up on this index */
const data = { json:self.search.index, urls:self.search.doc_urls, cloud:isCloud };
delete self.search.index; delete self.search.doc_urls;
return data;
}
async function loadWithFallback(remotes, local, isCloud=false){
if(remotes.length){
const [primary, ...secondary] = remotes;
const primaryData = await loadIndex(primary, null, isCloud);
if(primaryData) return primaryData;
if(local){
const localData = await loadIndex(null, local, isCloud);
if(localData) return localData;
}
for (const remote of secondary){
const data = await loadIndex(remote, null, isCloud);
if(data) return data;
} }
if(!rawLoaded) return null; /* give up on this index */
const data = { json:self.search.index, urls:self.search.doc_urls, cloud:isCloud };
delete self.search.index; delete self.search.doc_urls;
return data;
} }
(async () => { return local ? loadIndex(null, local, isCloud) : null;
const MAIN_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks/refs/heads/master/searchindex.js'; } (async () => {
const CLOUD_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/refs/heads/master/searchindex.js'; const htmlLang = (document.documentElement.lang || 'en').toLowerCase();
const lang = htmlLang.split('-')[0];
const mainReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks/releases/download';
const cloudReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks-cloud/releases/download';
const indices = []; const mainTags = Array.from(new Set(['searchindex-' + lang, 'searchindex-en', 'searchindex-master']));
const main = await loadIndex(MAIN_RAW , '/searchindex-book.js', false); if(main) indices.push(main); const cloudTags = Array.from(new Set(['searchindex-' + lang, 'searchindex-en', 'searchindex-master']));
const cloud= await loadIndex(CLOUD_RAW, '/searchindex.js', true ); if(cloud) indices.push(cloud);
const MAIN_REMOTE_SOURCES = mainTags.map(function(tag) { return mainReleaseBase + '/' + tag + '/searchindex.js'; });
const CLOUD_REMOTE_SOURCES = cloudTags.map(function(tag) { return cloudReleaseBase + '/' + tag + '/searchindex.js'; });
const indices = [];
const main = await loadWithFallback(MAIN_REMOTE_SOURCES , '/searchindex-book.js', false); if(main) indices.push(main);
const cloud= await loadWithFallback(CLOUD_REMOTE_SOURCES, '/searchindex.js', true ); if(cloud) indices.push(cloud);
if(!indices.length){ postMessage({ready:false, error:'no-index'}); return; } if(!indices.length){ postMessage({ready:false, error:'no-index'}); return; }
/* build index objects */ /* build index objects */
@@ -160,8 +187,16 @@
/* ───────────── worker messages ───────────── */ /* ───────────── worker messages ───────────── */
worker.onmessage = ({data}) => { worker.onmessage = ({data}) => {
if(data && data.ready!==undefined){ if(data && data.ready!==undefined){
if(data.ready){ icon.innerHTML=READY_ICON; icon.setAttribute('aria-label','Open search (S)'); } if(data.ready){
else { icon.textContent='❌'; icon.setAttribute('aria-label','Search unavailable'); } icon.innerHTML=READY_ICON;
icon.setAttribute('aria-label','Open search (S)');
icon.removeAttribute('title');
}
else {
icon.textContent='❌';
icon.setAttribute('aria-label','Search unavailable');
icon.setAttribute('title','Search is unavailable');
}
return; return;
} }
const docs=data, q=bar.value.trim(), terms=q.split(/\s+/).filter(Boolean); const docs=data, q=bar.value.trim(), terms=q.split(/\s+/).filter(Boolean);