diff --git a/scripts/ci-full-mnesia-wipe.sh b/scripts/ci-full-mnesia-wipe.sh index d87711b..826fd19 100644 --- a/scripts/ci-full-mnesia-wipe.sh +++ b/scripts/ci-full-mnesia-wipe.sh @@ -49,7 +49,66 @@ for ((i = 1; i <= 40; i++)); do done [[ "${n}" == "0" ]] || { echo "scale-down timeout"; exit 1; } -ssh_cmd "set -euo pipefail; MAX=${MAX_REPLICAS}; for n in \$(seq 1 \$MAX); do vol=eventhub-data-\$n; docker volume rm -f \$vol; docker volume create \$vol; echo wiped \$vol; done" +# After Swarm scale=0, shutdown/orphan task containers may still hold the volume. +# Find holders via Mounts / --filter volume=, docker rm -f, then retry volume rm. +echo "### wipe volumes (remove holders + retry volume rm)" +ssh_cmd "bash -s" </dev/null || true)" + if [[ -z "\${cids// }" ]]; then + for cid in \$(docker ps -aq 2>/dev/null || true); do + if docker inspect "\$cid" --format '{{range .Mounts}}{{println .Name}}{{end}}' 2>/dev/null | grep -qx "\$vol"; then + cids="\${cids} \$cid" + fi + done + fi + for cid in \$cids; do + [[ -z "\$cid" ]] && continue + echo "removing container holding \$vol:" + docker inspect "\$cid" --format ' id={{.Id}} name={{.Name}} status={{.State.Status}}' 2>/dev/null || echo " id=\$cid" + docker rm -f "\$cid" || true + done +} + +for n in \$(seq 1 \$MAX); do + vol=eventhub-data-\$n + echo "### wipe \$vol" + if ! docker volume inspect "\$vol" >/dev/null 2>&1; then + echo "volume \$vol absent, create fresh" + docker volume create "\$vol" + echo "wiped \$vol" + continue + fi + rm_volume_holders "\$vol" + removed=0 + for attempt in \$(seq 1 12); do + if docker volume rm -f "\$vol" >/tmp/volrm.out 2>/tmp/volrm.err; then + echo "volume rm ok \$vol attempt=\$attempt" + removed=1 + break + fi + err="\$(tr -d '\\r' /dev/null || true)" + echo "volume rm failed \$vol attempt=\$attempt: \$err" + rm_volume_holders "\$vol" + sleep 3 + done + if [[ "\$removed" != "1" ]]; then + if docker volume inspect "\$vol" >/dev/null 2>&1; then + echo "ERROR: could not remove \$vol after retries" >&2 + exit 1 + fi + echo "volume \$vol already absent after retries" + fi + docker volume create "\$vol" + echo "wiped \$vol" +done +EOF ssh_cmd "docker service scale ${SVC}=1" for ((i = 1; i <= 60; i++)); do