diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 1256aaf..0ccc441 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -21,6 +21,8 @@ on: - e2e-ift - deploy-stage - e2e-stage + - wipe-mnesia-ift + - wipe-mnesia-stage concurrency: group: eventhub-back-${{ github.ref }} @@ -472,3 +474,30 @@ jobs: REGISTRY_USER: ${{ secrets.REGISTRY_USER }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} run: bash scripts/run-stand-api-tests.sh stage + + # Manual only: full Mnesia wipe (NOT prod). No deploy/e2e needs. + wipe-mnesia-ift: + if: github.event_name == 'workflow_dispatch' && github.event.inputs.pipeline == 'wipe-mnesia-ift' + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - name: Full Mnesia wipe IFT + env: + SSH_HOST: ${{ secrets.IFT_SSH_HOST }} + SSH_USER: ${{ secrets.IFT_SSH_USER }} + SSH_KEY: ${{ secrets.IFT_SSH_PRIVATE_KEY }} + run: bash scripts/ci-full-mnesia-wipe.sh ift + + wipe-mnesia-stage: + if: github.event_name == 'workflow_dispatch' && github.event.inputs.pipeline == 'wipe-mnesia-stage' + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - name: Full Mnesia wipe stage + env: + SSH_HOST: ${{ secrets.STAGE_SSH_HOST }} + SSH_USER: ${{ secrets.STAGE_SSH_USER }} + SSH_KEY: ${{ secrets.STAGE_SSH_PRIVATE_KEY }} + run: bash scripts/ci-full-mnesia-wipe.sh stage diff --git a/scripts/ci-full-mnesia-wipe.sh b/scripts/ci-full-mnesia-wipe.sh new file mode 100644 index 0000000..d87711b --- /dev/null +++ b/scripts/ci-full-mnesia-wipe.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Full Mnesia wipe for IFT/stage from CI runner (NOT prod). +# Order: scale 0 -> wipe eventhub-data-1..N -> scale 1 -> mnesia-safe-redeploy up -> ensure-smoke-user +# Env: SSH_HOST SSH_USER SSH_KEY; arg: ift|stage +set -euo pipefail + +STAND="${1:?usage: ci-full-mnesia-wipe.sh ift|stage}" +case "${STAND}" in + ift) + MAX_REPLICAS=2 + SVC="eventhub-ift-core_eventhub" + SCRIPTS="/opt/eventhub-ift/devops/scripts" + ;; + stage) + MAX_REPLICAS=1 + SVC="eventhub-stage-core_eventhub" + SCRIPTS="/opt/eventhub-stage/devops/scripts" + ;; + *) + echo "Unknown stand: ${STAND} (ift|stage only; not prod)" >&2 + exit 1 + ;; +esac + +SSH_HOST="${SSH_HOST:?SSH_HOST required}" +SSH_USER="${SSH_USER:?SSH_USER required}" +SSH_KEY="${SSH_KEY:?SSH_KEY required}" + +KEY="$(mktemp)" +printf '%s\n' "${SSH_KEY}" > "${KEY}" +chmod 600 "${KEY}" +cleanup() { rm -f "${KEY}"; } +trap cleanup EXIT + +ssh_cmd() { + ssh -i "${KEY}" -o BatchMode=yes -o ConnectTimeout=45 -o StrictHostKeyChecking=accept-new \ + "${SSH_USER}@${SSH_HOST}" "$@" +} + +echo "### FULL MNESIA WIPE ${STAND} slots=1..${MAX_REPLICAS}" +ssh_cmd "docker volume ls --format '{{.Name}}' | grep eventhub-data || true" +ssh_cmd "docker service scale ${SVC}=0" + +for ((i = 1; i <= 40; i++)); do + n="$(ssh_cmd "docker service ps ${SVC} --filter desired-state=running -q 2>/dev/null | wc -l | tr -d ' '")" + echo "running=${n} try=${i}" + [[ "${n}" == "0" ]] && break + sleep 3 +done +[[ "${n}" == "0" ]] || { echo "scale-down timeout"; exit 1; } + +ssh_cmd "set -euo pipefail; MAX=${MAX_REPLICAS}; for n in \$(seq 1 \$MAX); do vol=eventhub-data-\$n; docker volume rm -f \$vol; docker volume create \$vol; echo wiped \$vol; done" + +ssh_cmd "docker service scale ${SVC}=1" +for ((i = 1; i <= 60; i++)); do + n="$(ssh_cmd "docker service ps ${SVC} --filter desired-state=running --format '{{.CurrentState}}' 2>/dev/null | grep -c '^Running' || true")" + echo "Running=${n}/1 try=${i}" + [[ "${n}" -eq 1 ]] && break + sleep 3 +done +[[ "${n}" -eq 1 ]] || { echo "scale-up timeout"; exit 1; } + +ssh_cmd "bash ${SCRIPTS}/mnesia-safe-redeploy.sh up ${STAND} ${MAX_REPLICAS}" +ssh_cmd "bash ${SCRIPTS}/ensure-smoke-user.sh ${STAND}" +if ssh_cmd "test -x ${SCRIPTS}/ensure-smoke-admin.sh"; then + ssh_cmd "bash ${SCRIPTS}/ensure-smoke-admin.sh ${STAND}" || true +fi + +ssh_cmd "docker volume ls --format '{{.Name}}' | grep eventhub-data || true" +echo "OK full mnesia wipe ${STAND}"