mirror of
https://github.com/HackTricks-wiki/hacktricks-cloud.git
synced 2026-03-12 21:22:57 -07:00
f
This commit is contained in:
Carlos Polop
79
.github/workflows/build_master.yml
vendored
79
.github/workflows/build_master.yml
vendored
@@ -57,27 +57,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Clone the searchindex repo
|
||||
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo
|
||||
|
||||
cd /tmp/searchindex-repo
|
||||
git config user.name "GitHub Actions"
|
||||
git config user.email "github-actions@github.com"
|
||||
|
||||
# Save all current files from main branch to temp directory
|
||||
mkdir -p /tmp/searchindex-backup
|
||||
cp -r * /tmp/searchindex-backup/ 2>/dev/null || true
|
||||
|
||||
# Create a fresh orphan branch (no history)
|
||||
git checkout --orphan new-main
|
||||
|
||||
# Remove all files from git index (but keep working directory)
|
||||
git rm -rf . 2>/dev/null || true
|
||||
|
||||
# Restore all the files from backup (keeps all language files)
|
||||
cp -r /tmp/searchindex-backup/* . 2>/dev/null || true
|
||||
|
||||
# Now update/add our English searchindex file
|
||||
# First, compress the original file (in the build directory)
|
||||
cd "${GITHUB_WORKSPACE}"
|
||||
gzip -9 -k -f "$ASSET"
|
||||
@@ -107,21 +86,57 @@ jobs:
|
||||
EOF
|
||||
python3 /tmp/xor_encrypt.py "$KEY" "${ASSET}.gz" "${ASSET}.gz.enc"
|
||||
|
||||
# Copy ONLY the encrypted .gz version to the searchindex repo (no uncompressed .js)
|
||||
cd /tmp/searchindex-repo
|
||||
cp "${GITHUB_WORKSPACE}/${ASSET}.gz.enc" "${FILENAME}.gz"
|
||||
# Rebuild and force-push with retries to handle concurrent updates.
|
||||
MAX_RETRIES=20
|
||||
RETRY_COUNT=0
|
||||
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
|
||||
RETRY_COUNT=$((RETRY_COUNT + 1))
|
||||
echo "Push attempt ${RETRY_COUNT}/${MAX_RETRIES}"
|
||||
|
||||
# Stage all files
|
||||
git add -A
|
||||
rm -rf /tmp/searchindex-repo /tmp/searchindex-backup
|
||||
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo
|
||||
|
||||
# Commit with timestamp
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
|
||||
git commit -m "Update searchindex files - ${TIMESTAMP}" --allow-empty
|
||||
cd /tmp/searchindex-repo
|
||||
git config user.name "GitHub Actions"
|
||||
git config user.email "github-actions@github.com"
|
||||
|
||||
# Force push to replace master branch (deletes history, keeps all files)
|
||||
git push -f origin new-main:master
|
||||
# Save all current files from master branch to temp directory.
|
||||
mkdir -p /tmp/searchindex-backup
|
||||
cp -r * /tmp/searchindex-backup/ 2>/dev/null || true
|
||||
|
||||
echo "Successfully reset repository history and pushed all searchindex files"
|
||||
# Create a fresh orphan branch (no history).
|
||||
git checkout --orphan new-main
|
||||
|
||||
# Remove all files from git index (but keep working directory).
|
||||
git rm -rf . 2>/dev/null || true
|
||||
|
||||
# Restore all files from backup (keeps all language files).
|
||||
cp -r /tmp/searchindex-backup/* . 2>/dev/null || true
|
||||
|
||||
# Update English searchindex artifact.
|
||||
cp "${GITHUB_WORKSPACE}/${ASSET}.gz.enc" "${FILENAME}.gz"
|
||||
|
||||
git add -A
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
|
||||
git commit -m "Update searchindex files - ${TIMESTAMP}" --allow-empty
|
||||
|
||||
if git push -f origin new-main:master 2>&1 | tee /tmp/push_output.txt; then
|
||||
echo "Successfully reset repository history and pushed all searchindex files"
|
||||
break
|
||||
fi
|
||||
|
||||
if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
|
||||
if grep -q "cannot lock ref 'refs/heads/master'" /tmp/push_output.txt; then
|
||||
echo "Concurrent update detected on remote master. Retrying..."
|
||||
else
|
||||
echo "Force push failed. Retrying..."
|
||||
fi
|
||||
sleep 1
|
||||
else
|
||||
echo "Failed to push after ${MAX_RETRIES} attempts"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Login in AWs
|
||||
- name: Configure AWS credentials using OIDC
|
||||
|
||||
81
.github/workflows/translate_all.yml
vendored
81
.github/workflows/translate_all.yml
vendored
@@ -206,62 +206,47 @@ jobs:
|
||||
# Copy ONLY the encrypted .gz version to the searchindex repo (no uncompressed .js)
|
||||
cp "${ASSET}.gz.enc" "/tmp/searchindex-repo/${FILENAME}.gz"
|
||||
|
||||
# Commit and push with retry logic
|
||||
# Commit and push with retry logic resilient to concurrent writers/history rewrites
|
||||
cd /tmp/searchindex-repo
|
||||
git config user.name "GitHub Actions"
|
||||
git config user.email "github-actions@github.com"
|
||||
git add "${FILENAME}.gz"
|
||||
|
||||
if git diff --staged --quiet; then
|
||||
echo "No changes to commit"
|
||||
else
|
||||
MAX_RETRIES=20
|
||||
RETRY_COUNT=0
|
||||
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
|
||||
RETRY_COUNT=$((RETRY_COUNT + 1))
|
||||
echo "Push attempt ${RETRY_COUNT}/${MAX_RETRIES}"
|
||||
|
||||
# Rebuild commit from latest remote state each attempt.
|
||||
git fetch origin master
|
||||
git checkout -B master origin/master
|
||||
cp "${GITHUB_WORKSPACE}/${ASSET}.gz.enc" "${FILENAME}.gz"
|
||||
git add "${FILENAME}.gz"
|
||||
|
||||
if git diff --staged --quiet; then
|
||||
echo "No changes to commit"
|
||||
break
|
||||
fi
|
||||
|
||||
git commit -m "Update ${FILENAME} from hacktricks-cloud build"
|
||||
|
||||
# Retry push up to 20 times with pull --rebase between attempts
|
||||
MAX_RETRIES=20
|
||||
RETRY_COUNT=0
|
||||
while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
|
||||
if git push origin master; then
|
||||
echo "Successfully pushed on attempt $((RETRY_COUNT + 1))"
|
||||
break
|
||||
if git push origin master 2>&1 | tee /tmp/push_output.txt; then
|
||||
echo "Successfully pushed on attempt ${RETRY_COUNT}"
|
||||
break
|
||||
fi
|
||||
|
||||
if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
|
||||
if grep -q "cannot lock ref 'refs/heads/master'" /tmp/push_output.txt; then
|
||||
echo "Concurrent update detected on remote master. Retrying..."
|
||||
else
|
||||
RETRY_COUNT=$((RETRY_COUNT + 1))
|
||||
if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
|
||||
echo "Push failed, attempt $RETRY_COUNT/$MAX_RETRIES. Pulling and retrying..."
|
||||
|
||||
# Try normal rebase first
|
||||
if git pull --rebase origin master 2>&1 | tee /tmp/pull_output.txt; then
|
||||
echo "Rebase successful, retrying push..."
|
||||
else
|
||||
# If rebase fails due to divergent histories (orphan branch reset), re-clone
|
||||
if grep -q "unrelated histories\|refusing to merge\|fatal: invalid upstream\|couldn't find remote ref" /tmp/pull_output.txt; then
|
||||
echo "Detected history rewrite, re-cloning repository..."
|
||||
cd /tmp
|
||||
rm -rf searchindex-repo
|
||||
git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git searchindex-repo
|
||||
cd searchindex-repo
|
||||
git config user.name "GitHub Actions"
|
||||
git config user.email "github-actions@github.com"
|
||||
|
||||
# Re-copy ONLY the encrypted .gz version (no uncompressed .js)
|
||||
cp "${ASSET}.gz.enc" "${FILENAME}.gz"
|
||||
|
||||
git add "${FILENAME}.gz"
|
||||
git commit -m "Update ${FILENAME}.gz from hacktricks-cloud build"
|
||||
echo "Re-cloned and re-committed, will retry push..."
|
||||
else
|
||||
echo "Rebase failed for unknown reason, retrying anyway..."
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
else
|
||||
echo "Failed to push after $MAX_RETRIES attempts"
|
||||
exit 1
|
||||
fi
|
||||
echo "Push failed. Retrying with fresh remote state..."
|
||||
fi
|
||||
done
|
||||
fi
|
||||
sleep 1
|
||||
else
|
||||
echo "Failed to push after ${MAX_RETRIES} attempts"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Login in AWs
|
||||
- name: Configure AWS credentials using OIDC
|
||||
|
||||
Reference in New Issue
Block a user