aleksey
ccca8d0a1f
Buildx failed with unpigz crc32 mismatch; retry now does full prune, drops stale builder volumes, and uses --no-cache on the last attempt.
72 lines
2.0 KiB
Bash
72 lines
2.0 KiB
Bash
#!/usr/bin/env bash
|
|
# Retry docker/buildx commands on transient network/registry/cache failures.
|
|
# Usage: retry-docker-build.sh [max_attempts] [wait_seconds] -- <command...>
|
|
set -euo pipefail
|
|
|
|
MAX="${1:-3}"
|
|
WAIT="${2:-30}"
|
|
shift 2
|
|
[[ "${1:-}" == "--" ]] && shift
|
|
|
|
if [[ $# -eq 0 ]]; then
|
|
echo "Usage: $0 [max_attempts] [wait_seconds] -- <command...>" >&2
|
|
exit 2
|
|
fi
|
|
|
|
is_corrupt_cache() {
|
|
grep -qiE 'crc32 mismatch|unpigz|corrupted|failed to compute cache key' <<<"$1"
|
|
}
|
|
|
|
purge_buildx_cache() {
|
|
echo "== purge buildx cache (all + stale builders)"
|
|
docker buildx prune -af >/dev/null 2>&1 || true
|
|
docker builder prune -af >/dev/null 2>&1 || true
|
|
docker ps -aq --filter "name=buildx_buildkit" --filter "status=exited" |
|
|
xargs -r docker rm -f >/dev/null 2>&1 || true
|
|
docker volume ls -q | grep -E '^buildx_buildkit_.*_state$' | while read -r vol; do
|
|
if docker ps --filter "volume=${vol}" --format '{{.ID}}' | grep -q .; then
|
|
continue
|
|
fi
|
|
echo "== remove volume ${vol}"
|
|
docker volume rm -f "${vol}" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
attempt=1
|
|
corrupt_seen=0
|
|
while true; do
|
|
cmd=("$@")
|
|
# After a corrupt-cache failure, force a clean rebuild on the last attempt.
|
|
if (( corrupt_seen == 1 && attempt == MAX )); then
|
|
if [[ "${cmd[*]}" == *buildx\ build* ]] && [[ "${cmd[*]}" != *--no-cache* ]]; then
|
|
echo "== last attempt: injecting --no-cache"
|
|
cmd+=(--no-cache)
|
|
fi
|
|
fi
|
|
echo "== docker build attempt ${attempt}/${MAX}: ${cmd[*]}"
|
|
logf="$(mktemp)"
|
|
set +e
|
|
"${cmd[@]}" 2>&1 | tee "${logf}"
|
|
code=${PIPESTATUS[0]}
|
|
set -e
|
|
if (( code == 0 )); then
|
|
rm -f "${logf}"
|
|
exit 0
|
|
fi
|
|
log="$(cat "${logf}" 2>/dev/null || true)"
|
|
rm -f "${logf}"
|
|
if (( attempt >= MAX )); then
|
|
echo "== all ${MAX} attempts failed (last exit ${code})"
|
|
exit "${code}"
|
|
fi
|
|
echo "== attempt ${attempt} failed (exit ${code}), retry in ${WAIT}s..."
|
|
if is_corrupt_cache "${log}"; then
|
|
corrupt_seen=1
|
|
purge_buildx_cache
|
|
else
|
|
docker buildx prune -f >/dev/null 2>&1 || true
|
|
fi
|
|
sleep "${WAIT}"
|
|
attempt=$((attempt + 1))
|
|
done
|