fix(ci): purge corrupted buildx cache on crc32/unpigz failures
Buildx failed with unpigz crc32 mismatch; retry now does full prune, drops stale builder volumes, and uses --no-cache on the last attempt.
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Retry docker/buildx commands on transient network/registry failures.
|
# Retry docker/buildx commands on transient network/registry/cache failures.
|
||||||
# Usage: retry-docker-build.sh [max_attempts] [wait_seconds] -- <command...>
|
# Usage: retry-docker-build.sh [max_attempts] [wait_seconds] -- <command...>
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
@@ -13,22 +13,59 @@ if [[ $# -eq 0 ]]; then
|
|||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
is_corrupt_cache() {
|
||||||
|
grep -qiE 'crc32 mismatch|unpigz|corrupted|failed to compute cache key' <<<"$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
purge_buildx_cache() {
|
||||||
|
echo "== purge buildx cache (all + stale builders)"
|
||||||
|
docker buildx prune -af >/dev/null 2>&1 || true
|
||||||
|
docker builder prune -af >/dev/null 2>&1 || true
|
||||||
|
docker ps -aq --filter "name=buildx_buildkit" --filter "status=exited" |
|
||||||
|
xargs -r docker rm -f >/dev/null 2>&1 || true
|
||||||
|
docker volume ls -q | grep -E '^buildx_buildkit_.*_state$' | while read -r vol; do
|
||||||
|
if docker ps --filter "volume=${vol}" --format '{{.ID}}' | grep -q .; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "== remove volume ${vol}"
|
||||||
|
docker volume rm -f "${vol}" >/dev/null 2>&1 || true
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
attempt=1
|
attempt=1
|
||||||
|
corrupt_seen=0
|
||||||
while true; do
|
while true; do
|
||||||
echo "== docker build attempt ${attempt}/${MAX}: $*"
|
cmd=("$@")
|
||||||
|
# After a corrupt-cache failure, force a clean rebuild on the last attempt.
|
||||||
|
if (( corrupt_seen == 1 && attempt == MAX )); then
|
||||||
|
if [[ "${cmd[*]}" == *buildx\ build* ]] && [[ "${cmd[*]}" != *--no-cache* ]]; then
|
||||||
|
echo "== last attempt: injecting --no-cache"
|
||||||
|
cmd+=(--no-cache)
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
echo "== docker build attempt ${attempt}/${MAX}: ${cmd[*]}"
|
||||||
|
logf="$(mktemp)"
|
||||||
set +e
|
set +e
|
||||||
"$@"
|
"${cmd[@]}" 2>&1 | tee "${logf}"
|
||||||
code=$?
|
code=${PIPESTATUS[0]}
|
||||||
set -e
|
set -e
|
||||||
if (( code == 0 )); then
|
if (( code == 0 )); then
|
||||||
|
rm -f "${logf}"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
log="$(cat "${logf}" 2>/dev/null || true)"
|
||||||
|
rm -f "${logf}"
|
||||||
if (( attempt >= MAX )); then
|
if (( attempt >= MAX )); then
|
||||||
echo "== all ${MAX} attempts failed (last exit ${code})"
|
echo "== all ${MAX} attempts failed (last exit ${code})"
|
||||||
exit "${code}"
|
exit "${code}"
|
||||||
fi
|
fi
|
||||||
echo "== attempt ${attempt} failed (exit ${code}), retry in ${WAIT}s..."
|
echo "== attempt ${attempt} failed (exit ${code}), retry in ${WAIT}s..."
|
||||||
|
if is_corrupt_cache "${log}"; then
|
||||||
|
corrupt_seen=1
|
||||||
|
purge_buildx_cache
|
||||||
|
else
|
||||||
docker buildx prune -f >/dev/null 2>&1 || true
|
docker buildx prune -f >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
sleep "${WAIT}"
|
sleep "${WAIT}"
|
||||||
attempt=$((attempt + 1))
|
attempt=$((attempt + 1))
|
||||||
done
|
done
|
||||||
|
|||||||
Reference in New Issue
Block a user