fix searchindex

.github/workflows/build_master.yml

@@ -35,63 +35,59 @@ jobs:
       - name: Build mdBook
         run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
       
-      - name: Update searchindex in repo (purge history, keep current on HEAD)
+      - name: Install GitHub CLI
+        run: |
+          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
+          && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
+          && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+          && sudo apt update \
+          && sudo apt install gh -y
+      
+      - name: Publish search index release asset
         shell: bash
+        env:
+          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
         run: |
           set -euo pipefail
 
-          pwd
-          ls -la
-          ls -la book
+          ASSET="book/searchindex.js"
+          TAG="searchindex-en"
+          TITLE="Search Index (en)"
 
-          git config --global --add safe.directory /__w/hacktricks-cloud/hacktricks-cloud
-          git config --global user.email "build@example.com"
-          git config --global user.name "Build master"
-          git config pull.rebase false
-
-          # Ensure we're on the target branch and up to date
-          # Handle force-pushed branches by resetting to origin/master
-          git fetch origin
-          git reset --hard origin/master
-
-          # Choose the file to keep at HEAD:
-          # 1) Prefer freshly built version from book/
-          # 2) Fallback to the file currently at HEAD (if it exists)
-          HAS_FILE=0
-          if [ -f "book/searchindex.js" ]; then
-            cp "book/searchindex.js" /tmp/sidx.js
-            HAS_FILE=1
-          elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
-            git show "HEAD:searchindex.js" > /tmp/sidx.js
-            HAS_FILE=1
+          if [ ! -f "$ASSET" ]; then
+            echo "Expected $ASSET to exist after build" >&2
+            exit 1
           fi
 
-          # Skip if there's nothing to purge AND nothing to keep
-          if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 HEAD -- "searchindex.js" >/dev/null 2>&1; then
-            # Fail early if working tree is dirty (avoid confusing rewrites)
-            git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
+          TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
+          if [ -z "$TOKEN" ]; then
+            echo "No token available for GitHub CLI" >&2
+            exit 1
+          fi
+          export GH_TOKEN="$TOKEN"
 
-            # Install git-filter-repo and ensure it's on PATH
-            python -m pip install --quiet --user git-filter-repo
-            export PATH="$HOME/.local/bin:$PATH"
-
-            # Rewrite ONLY the current branch, dropping all historical blobs of searchindex.js
-            git filter-repo --force --path "searchindex.js" --invert-paths --refs "$(git symbolic-ref -q HEAD)"
-
-            # Re-add the current version on top of rewritten history (keep it in HEAD)
-            if [ "$HAS_FILE" = "1" ]; then
-              mv /tmp/sidx.js "searchindex.js"
-              git add "searchindex.js"
-              git commit -m "Update searchindex (purged history; keep current)"
-            else
-              echo "No current searchindex.js to re-add after purge."
-            fi
-
-            # Safer force push (only updates if remote hasn't advanced)
-            git push --force-with-lease
+          # Delete the release if it exists
+          echo "Checking if release $TAG exists..."
+          if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+            echo "Release $TAG already exists, deleting it..."
+            gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag || {
+              echo "Failed to delete release, trying without cleanup-tag..."
+              gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" || {
+                echo "Warning: Could not delete existing release, will try to recreate..."
+              }
+            }
+            sleep 2  # Give GitHub API a moment to process the deletion
           else
-            echo "Nothing to purge; skipping."
+            echo "Release $TAG does not exist, proceeding with creation..."
           fi
+          
+          # Create new release (with force flag to overwrite if deletion failed)
+          gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY" || {
+            echo "Failed to create release, trying with force flag..."
+            gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY" --cleanup-tag >/dev/null 2>&1 || true
+            sleep 2
+            gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for master" --repo "$GITHUB_REPOSITORY"
+          }
 
       # Login in AWs
       - name: Configure AWS credentials using OIDC

.github/workflows/translate_all.yml

@@ -66,7 +66,13 @@ jobs:
       - name: Update and download scripts
         run: |
           sudo apt-get update
-          sudo apt-get install wget -y
+          # Install GitHub CLI properly
+          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
+          && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
+          && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+          && sudo apt update \
+          && sudo apt install gh -y \
+          && sudo apt-get install -y wget
           wget -O /tmp/get_and_save_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/get_and_save_refs.py
           wget -O /tmp/compare_and_fix_refs.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/compare_and_fix_refs.py
           wget -O /tmp/translator.py https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/master/scripts/translator.py
@@ -138,56 +144,37 @@ jobs:
           git pull
           MDBOOK_BOOK__LANGUAGE=$BRANCH mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
 
-      - name: Update searchindex.js in repo (purge history, keep current on HEAD)
+      - name: Publish search index release asset
         shell: bash
+        env:
+          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
         run: |
           set -euo pipefail
 
-          # Be explicit about workspace trust (avoids "dubious ownership")
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          ASSET="book/searchindex.js"
+          TAG="searchindex-${BRANCH}"
+          TITLE="Search Index (${BRANCH})"
 
-          git checkout "$BRANCH"
-          git fetch origin "$BRANCH" --quiet
-          git pull --ff-only
-
-          # Choose the file to keep at HEAD:
-          # 1) Prefer freshly built version from book/
-          # 2) Fallback to the file currently at HEAD (if it exists)
-          HAS_FILE=0
-          if [ -f "book/searchindex.js" ]; then
-            cp "book/searchindex.js" /tmp/sidx.js
-            HAS_FILE=1
-          elif git cat-file -e "HEAD:searchindex.js" 2>/dev/null; then
-            git show "HEAD:searchindex.js" > /tmp/sidx.js
-            HAS_FILE=1
+          if [ ! -f "$ASSET" ]; then
+            echo "Expected $ASSET to exist after build" >&2
+            exit 1
           fi
 
-          # Skip if there's nothing to purge AND nothing to keep
-          if [ "$HAS_FILE" = "1" ] || git rev-list -n 1 "$BRANCH" -- "searchindex.js" >/dev/null 2>&1; then
-            # **Fail early if working tree is dirty** (prevents confusing filter results)
-            git diff --quiet || { echo "Working tree has uncommitted changes; aborting purge." >&2; exit 1; }
-
-            # Make sure git-filter-repo is callable via `git filter-repo`
-            python -m pip install --quiet --user git-filter-repo
-            export PATH="$HOME/.local/bin:$PATH"
-
-            # Rewrite ONLY this branch, dropping all historical blobs of searchindex.js
-            git filter-repo --force --path "searchindex.js" --invert-paths --refs "refs/heads/$BRANCH"
-
-            # Re-add the current version on top of rewritten history (keep it in HEAD)
-            if [ "$HAS_FILE" = "1" ]; then
-              mv /tmp/sidx.js "searchindex.js"
-              git add "searchindex.js"
-              git commit -m "Update searchindex (purged history; keep current)"
-            else
-              echo "No current searchindex.js to re-add after purge."
-            fi
-
-            # **Safer force push** (prevents clobbering unexpected remote updates)
-            git push --force-with-lease origin "$BRANCH"
-          else
-            echo "Nothing to purge; skipping."
+          TOKEN="${PAT_TOKEN:-${GITHUB_TOKEN:-}}"
+          if [ -z "$TOKEN" ]; then
+            echo "No token available for GitHub CLI" >&2
+            exit 1
           fi
+          export GH_TOKEN="$TOKEN"
+
+          # Delete the release if it exists
+          if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+            echo "Release $TAG already exists, deleting it..."
+            gh release delete "$TAG" --yes --repo "$GITHUB_REPOSITORY"
+          fi
+          
+          # Create new release
+          gh release create "$TAG" "$ASSET" --title "$TITLE" --notes "Automated search index build for $BRANCH" --repo "$GITHUB_REPOSITORY"
       
       # Login in AWs
       - name: Configure AWS credentials using OIDC

.gitignore

@@ -35,3 +35,4 @@ book
 book/*
 hacktricks-preprocessor.log
 hacktricks-preprocessor-error.log
+searchindex.js

theme/ht_searcher.js

@@ -21,33 +21,60 @@
       try { importScripts('https://cdn.jsdelivr.net/npm/elasticlunr@0.9.5/elasticlunr.min.js'); }
       catch { importScripts(abs('/elasticlunr.min.js')); }
   
-      /* 2 — load a single index (remote → local) */
-      async function loadIndex(remote, local, isCloud=false){
-        let rawLoaded = false;
+    /* 2 — load a single index (remote → local) */
+    async function loadIndex(remote, local, isCloud=false){
+      let rawLoaded = false;
+      if(remote){
         try {
           const r = await fetch(remote,{mode:'cors'});
           if (!r.ok) throw new Error('HTTP '+r.status);
           importScripts(URL.createObjectURL(new Blob([await r.text()],{type:'application/javascript'})));
           rawLoaded = true;
         } catch(e){ console.warn('remote',remote,'failed →',e); }
-        if(!rawLoaded){
-          try { importScripts(abs(local)); rawLoaded = true; }
-          catch(e){ console.error('local',local,'failed →',e); }
-        }
-        if(!rawLoaded) return null;                 /* give up on this index */
-        const data = { json:self.search.index, urls:self.search.doc_urls, cloud:isCloud };
-        delete self.search.index; delete self.search.doc_urls;
-        return data;
       }
-  
-      (async () => {
-        const MAIN_RAW  = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks/refs/heads/master/searchindex.js';
-        const CLOUD_RAW = 'https://raw.githubusercontent.com/HackTricks-wiki/hacktricks-cloud/refs/heads/master/searchindex.js';
-  
-        const indices = [];
-        const main = await loadIndex(MAIN_RAW , '/searchindex-book.js',        false); if(main)  indices.push(main);
-        const cloud= await loadIndex(CLOUD_RAW, '/searchindex.js',  true ); if(cloud) indices.push(cloud);
-  
+      if(!rawLoaded && local){
+        try { importScripts(abs(local)); rawLoaded = true; }
+        catch(e){ console.error('local',local,'failed →',e); }
+      }
+      if(!rawLoaded) return null;                 /* give up on this index */
+      const data = { json:self.search.index, urls:self.search.doc_urls, cloud:isCloud };
+      delete self.search.index; delete self.search.doc_urls;
+      return data;
+    }
+
+    async function loadWithFallback(remotes, local, isCloud=false){
+      if(remotes.length){
+        const [primary, ...secondary] = remotes;
+        const primaryData = await loadIndex(primary, null, isCloud);
+        if(primaryData) return primaryData;
+
+        if(local){
+          const localData = await loadIndex(null, local, isCloud);
+          if(localData) return localData;
+        }
+
+        for (const remote of secondary){
+          const data = await loadIndex(remote, null, isCloud);
+          if(data) return data;
+        }
+      }
+
+      return local ? loadIndex(null, local, isCloud) : null;
+    }          (async () => {
+      const htmlLang = (document.documentElement.lang || 'en').toLowerCase();
+      const lang = htmlLang.split('-')[0];
+      const mainReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks/releases/download';
+      const cloudReleaseBase = 'https://github.com/HackTricks-wiki/hacktricks-cloud/releases/download';
+
+      const mainTags = Array.from(new Set(['searchindex-' + lang, 'searchindex-en', 'searchindex-master']));
+      const cloudTags = Array.from(new Set(['searchindex-' + lang, 'searchindex-en', 'searchindex-master']));
+
+      const MAIN_REMOTE_SOURCES  = mainTags.map(function(tag) { return mainReleaseBase + '/' + tag + '/searchindex.js'; });
+      const CLOUD_REMOTE_SOURCES = cloudTags.map(function(tag) { return cloudReleaseBase + '/' + tag + '/searchindex.js'; });
+
+      const indices = [];
+      const main = await loadWithFallback(MAIN_REMOTE_SOURCES , '/searchindex-book.js',        false); if(main)  indices.push(main);
+      const cloud= await loadWithFallback(CLOUD_REMOTE_SOURCES, '/searchindex.js',  true ); if(cloud) indices.push(cloud);  
         if(!indices.length){ postMessage({ready:false, error:'no-index'}); return; }
   
         /* build index objects */
@@ -160,8 +187,16 @@
     /* ───────────── worker messages ───────────── */
     worker.onmessage = ({data}) => {
       if(data && data.ready!==undefined){
-        if(data.ready){ icon.innerHTML=READY_ICON; icon.setAttribute('aria-label','Open search (S)'); }
-        else { icon.textContent='❌'; icon.setAttribute('aria-label','Search unavailable'); }
+        if(data.ready){ 
+          icon.innerHTML=READY_ICON; 
+          icon.setAttribute('aria-label','Open search (S)'); 
+          icon.removeAttribute('title');
+        }
+        else { 
+          icon.textContent='❌'; 
+          icon.setAttribute('aria-label','Search unavailable'); 
+          icon.setAttribute('title','Search is unavailable');
+        }
         return;
       }
       const docs=data, q=bar.value.trim(), terms=q.split(/\s+/).filter(Boolean);