feat(ci): add manual wipe-mnesia jobs for IFT/stage [skip ci]

workflow_dispatch only: wipe-mnesia-ift / wipe-mnesia-stage and scripts/ci-full-mnesia-wipe.sh. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-01 18:30:11 +03:00
parent 0a5e86c4a0
commit 4eb1f326ec
2 changed files with 99 additions and 0 deletions
@@ -21,6 +21,8 @@ on:
          - e2e-ift
          - deploy-stage
          - e2e-stage
+          - wipe-mnesia-ift
+          - wipe-mnesia-stage

 concurrency:
  group: eventhub-back-${{ github.ref }}
@@ -472,3 +474,30 @@ jobs:
          REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
        run: bash scripts/run-stand-api-tests.sh stage
+
+  # Manual only: full Mnesia wipe (NOT prod). No deploy/e2e needs.
+  wipe-mnesia-ift:
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.pipeline == 'wipe-mnesia-ift'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - name: Full Mnesia wipe IFT
+        env:
+          SSH_HOST: ${{ secrets.IFT_SSH_HOST }}
+          SSH_USER: ${{ secrets.IFT_SSH_USER }}
+          SSH_KEY: ${{ secrets.IFT_SSH_PRIVATE_KEY }}
+        run: bash scripts/ci-full-mnesia-wipe.sh ift
+
+  wipe-mnesia-stage:
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.pipeline == 'wipe-mnesia-stage'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - name: Full Mnesia wipe stage
+        env:
+          SSH_HOST: ${{ secrets.STAGE_SSH_HOST }}
+          SSH_USER: ${{ secrets.STAGE_SSH_USER }}
+          SSH_KEY: ${{ secrets.STAGE_SSH_PRIVATE_KEY }}
+        run: bash scripts/ci-full-mnesia-wipe.sh stage
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Full Mnesia wipe for IFT/stage from CI runner (NOT prod).
+# Order: scale 0 -> wipe eventhub-data-1..N -> scale 1 -> mnesia-safe-redeploy up -> ensure-smoke-user
+# Env: SSH_HOST SSH_USER SSH_KEY; arg: ift|stage
+set -euo pipefail
+
+STAND="${1:?usage: ci-full-mnesia-wipe.sh ift|stage}"
+case "${STAND}" in
+  ift)
+    MAX_REPLICAS=2
+    SVC="eventhub-ift-core_eventhub"
+    SCRIPTS="/opt/eventhub-ift/devops/scripts"
+    ;;
+  stage)
+    MAX_REPLICAS=1
+    SVC="eventhub-stage-core_eventhub"
+    SCRIPTS="/opt/eventhub-stage/devops/scripts"
+    ;;
+  *)
+    echo "Unknown stand: ${STAND} (ift|stage only; not prod)" >&2
+    exit 1
+    ;;
+esac
+
+SSH_HOST="${SSH_HOST:?SSH_HOST required}"
+SSH_USER="${SSH_USER:?SSH_USER required}"
+SSH_KEY="${SSH_KEY:?SSH_KEY required}"
+
+KEY="$(mktemp)"
+printf '%s\n' "${SSH_KEY}" > "${KEY}"
+chmod 600 "${KEY}"
+cleanup() { rm -f "${KEY}"; }
+trap cleanup EXIT
+
+ssh_cmd() {
+  ssh -i "${KEY}" -o BatchMode=yes -o ConnectTimeout=45 -o StrictHostKeyChecking=accept-new \
+    "${SSH_USER}@${SSH_HOST}" "$@"
+}
+
+echo "### FULL MNESIA WIPE ${STAND} slots=1..${MAX_REPLICAS}"
+ssh_cmd "docker volume ls --format '{{.Name}}' | grep eventhub-data || true"
+ssh_cmd "docker service scale ${SVC}=0"
+
+for ((i = 1; i <= 40; i++)); do
+  n="$(ssh_cmd "docker service ps ${SVC} --filter desired-state=running -q 2>/dev/null | wc -l | tr -d ' '")"
+  echo "running=${n} try=${i}"
+  [[ "${n}" == "0" ]] && break
+  sleep 3
+done
+[[ "${n}" == "0" ]] || { echo "scale-down timeout"; exit 1; }
+
+ssh_cmd "set -euo pipefail; MAX=${MAX_REPLICAS}; for n in \$(seq 1 \$MAX); do vol=eventhub-data-\$n; docker volume rm -f \$vol; docker volume create \$vol; echo wiped \$vol; done"
+
+ssh_cmd "docker service scale ${SVC}=1"
+for ((i = 1; i <= 60; i++)); do
+  n="$(ssh_cmd "docker service ps ${SVC} --filter desired-state=running --format '{{.CurrentState}}' 2>/dev/null | grep -c '^Running' || true")"
+  echo "Running=${n}/1 try=${i}"
+  [[ "${n}" -eq 1 ]] && break
+  sleep 3
+done
+[[ "${n}" -eq 1 ]] || { echo "scale-up timeout"; exit 1; }
+
+ssh_cmd "bash ${SCRIPTS}/mnesia-safe-redeploy.sh up ${STAND} ${MAX_REPLICAS}"
+ssh_cmd "bash ${SCRIPTS}/ensure-smoke-user.sh ${STAND}"
+if ssh_cmd "test -x ${SCRIPTS}/ensure-smoke-admin.sh"; then
+  ssh_cmd "bash ${SCRIPTS}/ensure-smoke-admin.sh ${STAND}" || true
+fi
+
+ssh_cmd "docker volume ls --format '{{.Name}}' | grep eventhub-data || true"
+echo "OK full mnesia wipe ${STAND}"