#!/usr/bin/env bash # Full Mnesia wipe for IFT/stage from CI runner (NOT prod). # Order: scale 0 -> wipe eventhub-data-1..N -> scale 1 -> mnesia-safe-redeploy up -> ensure-smoke-user # Env: SSH_HOST SSH_USER SSH_KEY; arg: ift|stage set -euo pipefail STAND="${1:?usage: ci-full-mnesia-wipe.sh ift|stage}" case "${STAND}" in ift) MAX_REPLICAS=2 SVC="eventhub-ift-core_eventhub" SCRIPTS="/opt/eventhub-ift/devops/scripts" ;; stage) MAX_REPLICAS=1 SVC="eventhub-stage-core_eventhub" SCRIPTS="/opt/eventhub-stage/devops/scripts" ;; *) echo "Unknown stand: ${STAND} (ift|stage only; not prod)" >&2 exit 1 ;; esac SSH_HOST="${SSH_HOST:?SSH_HOST required}" SSH_USER="${SSH_USER:?SSH_USER required}" SSH_KEY="${SSH_KEY:?SSH_KEY required}" KEY="$(mktemp)" printf '%s\n' "${SSH_KEY}" > "${KEY}" chmod 600 "${KEY}" cleanup() { rm -f "${KEY}"; } trap cleanup EXIT ssh_cmd() { ssh -i "${KEY}" -o BatchMode=yes -o ConnectTimeout=45 -o StrictHostKeyChecking=accept-new \ "${SSH_USER}@${SSH_HOST}" "$@" } echo "### FULL MNESIA WIPE ${STAND} slots=1..${MAX_REPLICAS}" ssh_cmd "docker volume ls --format '{{.Name}}' | grep eventhub-data || true" ssh_cmd "docker service scale ${SVC}=0" for ((i = 1; i <= 40; i++)); do n="$(ssh_cmd "docker service ps ${SVC} --filter desired-state=running -q 2>/dev/null | wc -l | tr -d ' '")" echo "running=${n} try=${i}" [[ "${n}" == "0" ]] && break sleep 3 done [[ "${n}" == "0" ]] || { echo "scale-down timeout"; exit 1; } # After Swarm scale=0, shutdown/orphan task containers may still hold the volume. # Find holders via Mounts / --filter volume=, docker rm -f, then retry volume rm. echo "### wipe volumes (remove holders + retry volume rm)" ssh_cmd "bash -s" </dev/null || true)" if [[ -z "\${cids// }" ]]; then for cid in \$(docker ps -aq 2>/dev/null || true); do if docker inspect "\$cid" --format '{{range .Mounts}}{{println .Name}}{{end}}' 2>/dev/null | grep -qx "\$vol"; then cids="\${cids} \$cid" fi done fi for cid in \$cids; do [[ -z "\$cid" ]] && continue echo "removing container holding \$vol:" docker inspect "\$cid" --format ' id={{.Id}} name={{.Name}} status={{.State.Status}}' 2>/dev/null || echo " id=\$cid" docker rm -f "\$cid" || true done } for n in \$(seq 1 \$MAX); do vol=eventhub-data-\$n echo "### wipe \$vol" if ! docker volume inspect "\$vol" >/dev/null 2>&1; then echo "volume \$vol absent, create fresh" docker volume create "\$vol" echo "wiped \$vol" continue fi rm_volume_holders "\$vol" removed=0 for attempt in \$(seq 1 12); do if docker volume rm -f "\$vol" >/tmp/volrm.out 2>/tmp/volrm.err; then echo "volume rm ok \$vol attempt=\$attempt" removed=1 break fi err="\$(tr -d '\\r' /dev/null || true)" echo "volume rm failed \$vol attempt=\$attempt: \$err" rm_volume_holders "\$vol" sleep 3 done if [[ "\$removed" != "1" ]]; then if docker volume inspect "\$vol" >/dev/null 2>&1; then echo "ERROR: could not remove \$vol after retries" >&2 exit 1 fi echo "volume \$vol already absent after retries" fi docker volume create "\$vol" echo "wiped \$vol" done EOF ssh_cmd "docker service scale ${SVC}=1" for ((i = 1; i <= 60; i++)); do n="$(ssh_cmd "docker service ps ${SVC} --filter desired-state=running --format '{{.CurrentState}}' 2>/dev/null | grep -c '^Running' || true")" echo "Running=${n}/1 try=${i}" [[ "${n}" -eq 1 ]] && break sleep 3 done [[ "${n}" -eq 1 ]] || { echo "scale-up timeout"; exit 1; } ssh_cmd "bash ${SCRIPTS}/mnesia-safe-redeploy.sh up ${STAND} ${MAX_REPLICAS}" ssh_cmd "bash ${SCRIPTS}/ensure-smoke-user.sh ${STAND}" if ssh_cmd "test -x ${SCRIPTS}/ensure-smoke-admin.sh"; then ssh_cmd "bash ${SCRIPTS}/ensure-smoke-admin.sh ${STAND}" || true fi ssh_cmd "docker volume ls --format '{{.Name}}' | grep eventhub-data || true" echo "OK full mnesia wipe ${STAND}"