aleksey
104892bdde
After Swarm scale=0, shutdown containers can still hold eventhub-data-N; force-rm holders and retry volume rm so wipe-mnesia-ift does not fail.
130 lines
4.1 KiB
Bash
130 lines
4.1 KiB
Bash
#!/usr/bin/env bash
|
|
# Full Mnesia wipe for IFT/stage from CI runner (NOT prod).
|
|
# Order: scale 0 -> wipe eventhub-data-1..N -> scale 1 -> mnesia-safe-redeploy up -> ensure-smoke-user
|
|
# Env: SSH_HOST SSH_USER SSH_KEY; arg: ift|stage
|
|
set -euo pipefail
|
|
|
|
STAND="${1:?usage: ci-full-mnesia-wipe.sh ift|stage}"
|
|
case "${STAND}" in
|
|
ift)
|
|
MAX_REPLICAS=2
|
|
SVC="eventhub-ift-core_eventhub"
|
|
SCRIPTS="/opt/eventhub-ift/devops/scripts"
|
|
;;
|
|
stage)
|
|
MAX_REPLICAS=1
|
|
SVC="eventhub-stage-core_eventhub"
|
|
SCRIPTS="/opt/eventhub-stage/devops/scripts"
|
|
;;
|
|
*)
|
|
echo "Unknown stand: ${STAND} (ift|stage only; not prod)" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
SSH_HOST="${SSH_HOST:?SSH_HOST required}"
|
|
SSH_USER="${SSH_USER:?SSH_USER required}"
|
|
SSH_KEY="${SSH_KEY:?SSH_KEY required}"
|
|
|
|
KEY="$(mktemp)"
|
|
printf '%s\n' "${SSH_KEY}" > "${KEY}"
|
|
chmod 600 "${KEY}"
|
|
cleanup() { rm -f "${KEY}"; }
|
|
trap cleanup EXIT
|
|
|
|
ssh_cmd() {
|
|
ssh -i "${KEY}" -o BatchMode=yes -o ConnectTimeout=45 -o StrictHostKeyChecking=accept-new \
|
|
"${SSH_USER}@${SSH_HOST}" "$@"
|
|
}
|
|
|
|
echo "### FULL MNESIA WIPE ${STAND} slots=1..${MAX_REPLICAS}"
|
|
ssh_cmd "docker volume ls --format '{{.Name}}' | grep eventhub-data || true"
|
|
ssh_cmd "docker service scale ${SVC}=0"
|
|
|
|
for ((i = 1; i <= 40; i++)); do
|
|
n="$(ssh_cmd "docker service ps ${SVC} --filter desired-state=running -q 2>/dev/null | wc -l | tr -d ' '")"
|
|
echo "running=${n} try=${i}"
|
|
[[ "${n}" == "0" ]] && break
|
|
sleep 3
|
|
done
|
|
[[ "${n}" == "0" ]] || { echo "scale-down timeout"; exit 1; }
|
|
|
|
# After Swarm scale=0, shutdown/orphan task containers may still hold the volume.
|
|
# Find holders via Mounts / --filter volume=, docker rm -f, then retry volume rm.
|
|
echo "### wipe volumes (remove holders + retry volume rm)"
|
|
ssh_cmd "bash -s" <<EOF
|
|
set -euo pipefail
|
|
MAX=${MAX_REPLICAS}
|
|
|
|
rm_volume_holders() {
|
|
local vol="\$1"
|
|
local cids=""
|
|
local cid
|
|
cids="\$(docker ps -aq --filter "volume=\${vol}" 2>/dev/null || true)"
|
|
if [[ -z "\${cids// }" ]]; then
|
|
for cid in \$(docker ps -aq 2>/dev/null || true); do
|
|
if docker inspect "\$cid" --format '{{range .Mounts}}{{println .Name}}{{end}}' 2>/dev/null | grep -qx "\$vol"; then
|
|
cids="\${cids} \$cid"
|
|
fi
|
|
done
|
|
fi
|
|
for cid in \$cids; do
|
|
[[ -z "\$cid" ]] && continue
|
|
echo "removing container holding \$vol:"
|
|
docker inspect "\$cid" --format ' id={{.Id}} name={{.Name}} status={{.State.Status}}' 2>/dev/null || echo " id=\$cid"
|
|
docker rm -f "\$cid" || true
|
|
done
|
|
}
|
|
|
|
for n in \$(seq 1 \$MAX); do
|
|
vol=eventhub-data-\$n
|
|
echo "### wipe \$vol"
|
|
if ! docker volume inspect "\$vol" >/dev/null 2>&1; then
|
|
echo "volume \$vol absent, create fresh"
|
|
docker volume create "\$vol"
|
|
echo "wiped \$vol"
|
|
continue
|
|
fi
|
|
rm_volume_holders "\$vol"
|
|
removed=0
|
|
for attempt in \$(seq 1 12); do
|
|
if docker volume rm -f "\$vol" >/tmp/volrm.out 2>/tmp/volrm.err; then
|
|
echo "volume rm ok \$vol attempt=\$attempt"
|
|
removed=1
|
|
break
|
|
fi
|
|
err="\$(tr -d '\\r' </tmp/volrm.err 2>/dev/null || true)"
|
|
echo "volume rm failed \$vol attempt=\$attempt: \$err"
|
|
rm_volume_holders "\$vol"
|
|
sleep 3
|
|
done
|
|
if [[ "\$removed" != "1" ]]; then
|
|
if docker volume inspect "\$vol" >/dev/null 2>&1; then
|
|
echo "ERROR: could not remove \$vol after retries" >&2
|
|
exit 1
|
|
fi
|
|
echo "volume \$vol already absent after retries"
|
|
fi
|
|
docker volume create "\$vol"
|
|
echo "wiped \$vol"
|
|
done
|
|
EOF
|
|
|
|
ssh_cmd "docker service scale ${SVC}=1"
|
|
for ((i = 1; i <= 60; i++)); do
|
|
n="$(ssh_cmd "docker service ps ${SVC} --filter desired-state=running --format '{{.CurrentState}}' 2>/dev/null | grep -c '^Running' || true")"
|
|
echo "Running=${n}/1 try=${i}"
|
|
[[ "${n}" -eq 1 ]] && break
|
|
sleep 3
|
|
done
|
|
[[ "${n}" -eq 1 ]] || { echo "scale-up timeout"; exit 1; }
|
|
|
|
ssh_cmd "bash ${SCRIPTS}/mnesia-safe-redeploy.sh up ${STAND} ${MAX_REPLICAS}"
|
|
ssh_cmd "bash ${SCRIPTS}/ensure-smoke-user.sh ${STAND}"
|
|
if ssh_cmd "test -x ${SCRIPTS}/ensure-smoke-admin.sh"; then
|
|
ssh_cmd "bash ${SCRIPTS}/ensure-smoke-admin.sh ${STAND}" || true
|
|
fi
|
|
|
|
ssh_cmd "docker volume ls --format '{{.Name}}' | grep eventhub-data || true"
|
|
echo "OK full mnesia wipe ${STAND}"
|