From ccca8d0a1f20831becf6b2bf555c8d2f50514f32 Mon Sep 17 00:00:00 2001 From: Aleksey Sabilin Date: Sun, 19 Jul 2026 11:21:23 +0300 Subject: [PATCH] fix(ci): purge corrupted buildx cache on crc32/unpigz failures Buildx failed with unpigz crc32 mismatch; retry now does full prune, drops stale builder volumes, and uses --no-cache on the last attempt. --- scripts/retry-docker-build.sh | 47 +++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/scripts/retry-docker-build.sh b/scripts/retry-docker-build.sh index 0b0190e..9c6ef59 100644 --- a/scripts/retry-docker-build.sh +++ b/scripts/retry-docker-build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Retry docker/buildx commands on transient network/registry failures. +# Retry docker/buildx commands on transient network/registry/cache failures. # Usage: retry-docker-build.sh [max_attempts] [wait_seconds] -- set -euo pipefail @@ -13,22 +13,59 @@ if [[ $# -eq 0 ]]; then exit 2 fi +is_corrupt_cache() { + grep -qiE 'crc32 mismatch|unpigz|corrupted|failed to compute cache key' <<<"$1" +} + +purge_buildx_cache() { + echo "== purge buildx cache (all + stale builders)" + docker buildx prune -af >/dev/null 2>&1 || true + docker builder prune -af >/dev/null 2>&1 || true + docker ps -aq --filter "name=buildx_buildkit" --filter "status=exited" | + xargs -r docker rm -f >/dev/null 2>&1 || true + docker volume ls -q | grep -E '^buildx_buildkit_.*_state$' | while read -r vol; do + if docker ps --filter "volume=${vol}" --format '{{.ID}}' | grep -q .; then + continue + fi + echo "== remove volume ${vol}" + docker volume rm -f "${vol}" >/dev/null 2>&1 || true + done +} + attempt=1 +corrupt_seen=0 while true; do - echo "== docker build attempt ${attempt}/${MAX}: $*" + cmd=("$@") + # After a corrupt-cache failure, force a clean rebuild on the last attempt. + if (( corrupt_seen == 1 && attempt == MAX )); then + if [[ "${cmd[*]}" == *buildx\ build* ]] && [[ "${cmd[*]}" != *--no-cache* ]]; then + echo "== last attempt: injecting --no-cache" + cmd+=(--no-cache) + fi + fi + echo "== docker build attempt ${attempt}/${MAX}: ${cmd[*]}" + logf="$(mktemp)" set +e - "$@" - code=$? + "${cmd[@]}" 2>&1 | tee "${logf}" + code=${PIPESTATUS[0]} set -e if (( code == 0 )); then + rm -f "${logf}" exit 0 fi + log="$(cat "${logf}" 2>/dev/null || true)" + rm -f "${logf}" if (( attempt >= MAX )); then echo "== all ${MAX} attempts failed (last exit ${code})" exit "${code}" fi echo "== attempt ${attempt} failed (exit ${code}), retry in ${WAIT}s..." - docker buildx prune -f >/dev/null 2>&1 || true + if is_corrupt_cache "${log}"; then + corrupt_seen=1 + purge_buildx_cache + else + docker buildx prune -f >/dev/null 2>&1 || true + fi sleep "${WAIT}" attempt=$((attempt + 1)) done