fix(ci): purge corrupted buildx cache on crc32/unpigz failures
CI / test (push) Failing after 8m8s
CI / deploy-ift (push) Has been skipped
CI / e2e-ift (push) Has been skipped
CI / deploy-stage (push) Has been skipped
CI / e2e-stage (push) Has been skipped

Buildx failed with unpigz crc32 mismatch; retry now does full prune, drops stale builder volumes, and uses --no-cache on the last attempt.
This commit is contained in:
2026-07-19 11:21:23 +03:00
parent e8addae3b7
commit ccca8d0a1f
+42 -5
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Retry docker/buildx commands on transient network/registry failures. # Retry docker/buildx commands on transient network/registry/cache failures.
# Usage: retry-docker-build.sh [max_attempts] [wait_seconds] -- <command...> # Usage: retry-docker-build.sh [max_attempts] [wait_seconds] -- <command...>
set -euo pipefail set -euo pipefail
@@ -13,22 +13,59 @@ if [[ $# -eq 0 ]]; then
exit 2 exit 2
fi fi
is_corrupt_cache() {
grep -qiE 'crc32 mismatch|unpigz|corrupted|failed to compute cache key' <<<"$1"
}
purge_buildx_cache() {
echo "== purge buildx cache (all + stale builders)"
docker buildx prune -af >/dev/null 2>&1 || true
docker builder prune -af >/dev/null 2>&1 || true
docker ps -aq --filter "name=buildx_buildkit" --filter "status=exited" |
xargs -r docker rm -f >/dev/null 2>&1 || true
docker volume ls -q | grep -E '^buildx_buildkit_.*_state$' | while read -r vol; do
if docker ps --filter "volume=${vol}" --format '{{.ID}}' | grep -q .; then
continue
fi
echo "== remove volume ${vol}"
docker volume rm -f "${vol}" >/dev/null 2>&1 || true
done
}
attempt=1 attempt=1
corrupt_seen=0
while true; do while true; do
echo "== docker build attempt ${attempt}/${MAX}: $*" cmd=("$@")
# After a corrupt-cache failure, force a clean rebuild on the last attempt.
if (( corrupt_seen == 1 && attempt == MAX )); then
if [[ "${cmd[*]}" == *buildx\ build* ]] && [[ "${cmd[*]}" != *--no-cache* ]]; then
echo "== last attempt: injecting --no-cache"
cmd+=(--no-cache)
fi
fi
echo "== docker build attempt ${attempt}/${MAX}: ${cmd[*]}"
logf="$(mktemp)"
set +e set +e
"$@" "${cmd[@]}" 2>&1 | tee "${logf}"
code=$? code=${PIPESTATUS[0]}
set -e set -e
if (( code == 0 )); then if (( code == 0 )); then
rm -f "${logf}"
exit 0 exit 0
fi fi
log="$(cat "${logf}" 2>/dev/null || true)"
rm -f "${logf}"
if (( attempt >= MAX )); then if (( attempt >= MAX )); then
echo "== all ${MAX} attempts failed (last exit ${code})" echo "== all ${MAX} attempts failed (last exit ${code})"
exit "${code}" exit "${code}"
fi fi
echo "== attempt ${attempt} failed (exit ${code}), retry in ${WAIT}s..." echo "== attempt ${attempt} failed (exit ${code}), retry in ${WAIT}s..."
docker buildx prune -f >/dev/null 2>&1 || true if is_corrupt_cache "${log}"; then
corrupt_seen=1
purge_buildx_cache
else
docker buildx prune -f >/dev/null 2>&1 || true
fi
sleep "${WAIT}" sleep "${WAIT}"
attempt=$((attempt + 1)) attempt=$((attempt + 1))
done done