From 9e28d1f000eab0f4d876ab96eafaf773b6c82501 Mon Sep 17 00:00:00 2001 From: Carlos Polop Date: Wed, 4 Feb 2026 11:27:07 +0100 Subject: [PATCH] f --- .github/workflows/build_master.yml | 87 +++++++++++++++++------------ .github/workflows/translate_all.yml | 85 ++++++++++++---------------- 2 files changed, 86 insertions(+), 86 deletions(-) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index b1209bb83..ecf04fcac 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -57,27 +57,6 @@ jobs: exit 1 fi - # Clone the searchindex repo - git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo - - cd /tmp/searchindex-repo - git config user.name "GitHub Actions" - git config user.email "github-actions@github.com" - - # Save all current files from main branch to temp directory - mkdir -p /tmp/searchindex-backup - cp -r * /tmp/searchindex-backup/ 2>/dev/null || true - - # Create a fresh orphan branch (no history) - git checkout --orphan new-main - - # Remove all files from git index (but keep working directory) - git rm -rf . 2>/dev/null || true - - # Restore all the files from backup (keeps all language files) - cp -r /tmp/searchindex-backup/* . 2>/dev/null || true - - # Now update/add our English searchindex file # First, compress the original file (in the build directory) cd "${GITHUB_WORKSPACE}" gzip -9 -k -f "$ASSET" @@ -107,21 +86,57 @@ jobs: EOF python3 /tmp/xor_encrypt.py "$KEY" "${ASSET}.gz" "${ASSET}.gz.enc" - # Copy ONLY the encrypted .gz version to the searchindex repo (no uncompressed .js) - cd /tmp/searchindex-repo - cp "${GITHUB_WORKSPACE}/${ASSET}.gz.enc" "${FILENAME}.gz" - - # Stage all files - git add -A - - # Commit with timestamp - TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") - git commit -m "Update searchindex files - ${TIMESTAMP}" --allow-empty - - # Force push to replace master branch (deletes history, keeps all files) - git push -f origin new-main:master - - echo "Successfully reset repository history and pushed all searchindex files" + # Rebuild and force-push with retries to handle concurrent updates. + MAX_RETRIES=20 + RETRY_COUNT=0 + while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + RETRY_COUNT=$((RETRY_COUNT + 1)) + echo "Push attempt ${RETRY_COUNT}/${MAX_RETRIES}" + + rm -rf /tmp/searchindex-repo /tmp/searchindex-backup + git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo + + cd /tmp/searchindex-repo + git config user.name "GitHub Actions" + git config user.email "github-actions@github.com" + + # Save all current files from master branch to temp directory. + mkdir -p /tmp/searchindex-backup + cp -r * /tmp/searchindex-backup/ 2>/dev/null || true + + # Create a fresh orphan branch (no history). + git checkout --orphan new-main + + # Remove all files from git index (but keep working directory). + git rm -rf . 2>/dev/null || true + + # Restore all files from backup (keeps all language files). + cp -r /tmp/searchindex-backup/* . 2>/dev/null || true + + # Update English searchindex artifact. + cp "${GITHUB_WORKSPACE}/${ASSET}.gz.enc" "${FILENAME}.gz" + + git add -A + TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") + git commit -m "Update searchindex files - ${TIMESTAMP}" --allow-empty + + if git push -f origin new-main:master 2>&1 | tee /tmp/push_output.txt; then + echo "Successfully reset repository history and pushed all searchindex files" + break + fi + + if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then + if grep -q "cannot lock ref 'refs/heads/master'" /tmp/push_output.txt; then + echo "Concurrent update detected on remote master. Retrying..." + else + echo "Force push failed. Retrying..." + fi + sleep 1 + else + echo "Failed to push after ${MAX_RETRIES} attempts" + exit 1 + fi + done # Login in AWs - name: Configure AWS credentials using OIDC diff --git a/.github/workflows/translate_all.yml b/.github/workflows/translate_all.yml index 68eb0b588..cc6c178ed 100644 --- a/.github/workflows/translate_all.yml +++ b/.github/workflows/translate_all.yml @@ -206,62 +206,47 @@ jobs: # Copy ONLY the encrypted .gz version to the searchindex repo (no uncompressed .js) cp "${ASSET}.gz.enc" "/tmp/searchindex-repo/${FILENAME}.gz" - # Commit and push with retry logic + # Commit and push with retry logic resilient to concurrent writers/history rewrites cd /tmp/searchindex-repo git config user.name "GitHub Actions" git config user.email "github-actions@github.com" - git add "${FILENAME}.gz" - - if git diff --staged --quiet; then - echo "No changes to commit" - else + + MAX_RETRIES=20 + RETRY_COUNT=0 + while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do + RETRY_COUNT=$((RETRY_COUNT + 1)) + echo "Push attempt ${RETRY_COUNT}/${MAX_RETRIES}" + + # Rebuild commit from latest remote state each attempt. + git fetch origin master + git checkout -B master origin/master + cp "${GITHUB_WORKSPACE}/${ASSET}.gz.enc" "${FILENAME}.gz" + git add "${FILENAME}.gz" + + if git diff --staged --quiet; then + echo "No changes to commit" + break + fi + git commit -m "Update ${FILENAME} from hacktricks-cloud build" - - # Retry push up to 20 times with pull --rebase between attempts - MAX_RETRIES=20 - RETRY_COUNT=0 - while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do - if git push origin master; then - echo "Successfully pushed on attempt $((RETRY_COUNT + 1))" - break + + if git push origin master 2>&1 | tee /tmp/push_output.txt; then + echo "Successfully pushed on attempt ${RETRY_COUNT}" + break + fi + + if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then + if grep -q "cannot lock ref 'refs/heads/master'" /tmp/push_output.txt; then + echo "Concurrent update detected on remote master. Retrying..." else - RETRY_COUNT=$((RETRY_COUNT + 1)) - if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then - echo "Push failed, attempt $RETRY_COUNT/$MAX_RETRIES. Pulling and retrying..." - - # Try normal rebase first - if git pull --rebase origin master 2>&1 | tee /tmp/pull_output.txt; then - echo "Rebase successful, retrying push..." - else - # If rebase fails due to divergent histories (orphan branch reset), re-clone - if grep -q "unrelated histories\|refusing to merge\|fatal: invalid upstream\|couldn't find remote ref" /tmp/pull_output.txt; then - echo "Detected history rewrite, re-cloning repository..." - cd /tmp - rm -rf searchindex-repo - git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git searchindex-repo - cd searchindex-repo - git config user.name "GitHub Actions" - git config user.email "github-actions@github.com" - - # Re-copy ONLY the encrypted .gz version (no uncompressed .js) - cp "${ASSET}.gz.enc" "${FILENAME}.gz" - - git add "${FILENAME}.gz" - git commit -m "Update ${FILENAME}.gz from hacktricks-cloud build" - echo "Re-cloned and re-committed, will retry push..." - else - echo "Rebase failed for unknown reason, retrying anyway..." - fi - fi - - sleep 1 - else - echo "Failed to push after $MAX_RETRIES attempts" - exit 1 - fi + echo "Push failed. Retrying with fresh remote state..." fi - done - fi + sleep 1 + else + echo "Failed to push after ${MAX_RETRIES} attempts" + exit 1 + fi + done # Login in AWs - name: Configure AWS credentials using OIDC