From 823274caab12bc2b664cd6b1fc8bd2f2e566cbd4 Mon Sep 17 00:00:00 2001 From: Aleksey Sabilin Date: Wed, 15 Jul 2026 16:00:33 +0300 Subject: [PATCH] =?UTF-8?q?docs(loadtest):=20hold/stress=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D1=84=D0=B8=D0=BB=D0=B8=20=D0=B8=20=D0=BF=D1=80=D0=B0?= =?UTF-8?q?=D0=B2=D0=B8=D0=BB=D0=B0=20=D0=B1=D0=B5=D0=B7=D0=BE=D0=BF=D0=B0?= =?UTF-8?q?=D1=81=D0=BD=D0=BE=D1=81=D1=82=D0=B8=20IFT.=20Refs=20EventHub/E?= =?UTF-8?q?ventHubBack#31=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + test/tsung/README.md | 80 ++++++++++-- test/tsung/eventhub_ift.xml | 76 ++++++------ test/tsung/eventhub_ift_smoke.xml | 38 +++--- test/tsung/eventhub_ift_stress.xml | 146 ++++++++++++++++++++++ test/tsung/eventhub_ift_stress_bump.xml | 141 +++++++++++++++++++++ test/tsung/eventhub_ift_stress_hold.xml | 141 +++++++++++++++++++++ test/tsung/ift-users.csv.example | 1 + test/tsung/seed-ift-users.sh | 88 ++++++++++++++ test/tsung/summarize-tsung-stress.py | 155 ++++++++++++++++++++++++ 10 files changed, 807 insertions(+), 61 deletions(-) create mode 100644 test/tsung/eventhub_ift_stress.xml create mode 100644 test/tsung/eventhub_ift_stress_bump.xml create mode 100644 test/tsung/eventhub_ift_stress_hold.xml create mode 100644 test/tsung/ift-users.csv.example create mode 100644 test/tsung/seed-ift-users.sh create mode 100644 test/tsung/summarize-tsung-stress.py diff --git a/.gitignore b/.gitignore index 9d939b1..3690937 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,8 @@ docker/.env.stage # Локальные учётки admin для Tsung IFT (см. test/tsung/ift-admin.csv.example) /test/tsung/ift-admin.csv +# Пул пользователей для stress hot-path (см. test/tsung/ift-users.csv.example) +/test/tsung/ift-users.csv # Traefik TLS private keys (generated locally for dev/staging). # Никогда не коммитим приватный ключ в репозиторий. diff --git a/test/tsung/README.md b/test/tsung/README.md index a5e8e41..42b2c0f 100644 --- a/test/tsung/README.md +++ b/test/tsung/README.md @@ -6,12 +6,34 @@ |------|------------| | `eventhub_ift_smoke.xml` | Короткий smoke на IFT после prepare | | `eventhub_ift.xml` | Mix: `auth_browse` 60% + `create_flow` 40%, фазы warmup/ramp/peak | +| `eventhub_ift_stress.xml` | **Hot-path stress**: browse 70% / write 30%, ladder arrival, поиск верхней границы | +| `eventhub_ift_stress_hold.xml` | Controlled hold: maxusers=300, thinktime=1s, ladder 0.2→0.1→0.05→0.035 | +| `eventhub_ift_stress_bump.xml` | **Запрещён по умолчанию** (uncapped arrival; клинит WSL IFT) | | `eventhub_tsung.xml` | Legacy localhost (не для IFT) | | `eventhub_http.xml` | Legacy короткий localhost | -Цель: **HTTPS** `api.ift.eventhub.local` (+ `admin-api.ift.eventhub.local` для verify). +Цель: **HTTPS** `api.ift.eventhub.local` (+ `admin-api.ift.eventhub.local` для seed/verify). -### Auth path +### Правила безопасности IFT (обязательно) + +IFT = Docker Swarm **внутри WSL2 на Windows**. Aggressive open-loop без лимитов клинит `vmmem` и рвёт SSH. + +| Правило | Значение | +|---------|----------| +| Рекомендуемый stress | `eventhub_ift_stress_hold.xml` (`stress-hold`) | +| `maxusers` | ≤ **300** | +| `thinktime` | ≥ **1** s на hot-path hold | +| Плановый interarrival | ≥ **0.1** s (~10 VU/s); ниже — только с наблюдением | +| Uncapped / bump / maxusers≫300 | **запрещено** на IFT без явного согласования | +| Рабочий потолок (2026-07-15) | ~**10 VU/s** hot-path на текущем железе | + +**Recover после клина (на Windows-хосте IFT):** + +1. Сбор логов: `\\192.168.1.116\eventhub-cursor\scripts\restart-wsl-and-collect-logs.ps1` (или `collect-ift-loadtest-logs.ps1`) +2. `wsl --shutdown` при необходимости +3. Restore: `restore-ift.ps1` (Windows OpenSSH) → `ift-loadtest-restore.sh` (retry/`--force` при `update out of sequence`) + +### Auth path (smoke / full) 1. `POST /v1/admin/login` (admin-api) — учётка из `ift-admin.csv` 2. `POST /v1/register` → `user.id` @@ -19,6 +41,35 @@ 4. `POST /v1/verify` 5. `POST /v1/login` → user JWT +### Stress hot-path + +Пользователи сидятся заранее (`seed-ift-users.sh` → `ift-users.csv`). Сессия Tsung: **login → browse/write**. Без admin/register/verify на VU. + +**Критерий верхней границы (2A):** + +- error rate > 1% (4xx/5xx + nomatch + match_stop относительно числа запросов), **или** +- cumulative / late-phase mean request > 500 ms (hold ≥2 мин в фазах ladder). + +Разбор: `python3 test/tsung/summarize-tsung-stress.py [tsung.log]` → `verdict=BOUNDARY_HIT|UNDER_THRESHOLD`. + +В error rate входят и transport-ошибки Tsung (`error_connect_closed`, `error_timeout`, `error_abort_max_conn_retries`). + +**Результат 2026-07-15 aggressive** (`20260715-1027/`): `BOUNDARY_HIT` — mean ~735 ms, error ~88%, maxusers=5000, SSH wedge. + +**Controlled hold ×1** (`20260715-1200/`): фазы 0.2/0.1 GREEN; 0.05 OVER (timeout 290, ~17%); overall BOUNDARY_HIT (err ~2.9%). + +**Controlled hold ×2** (`20260715-1336/`, `EVENTHUB_REPLICAS=2`): фазы 0.2/0.1 GREEN (mean ≈37 ms); 0.05 OVER мягче (timeout 106, ~3%); overall UNDER_THRESHOLD (err ~0.94%, mean ~64 ms). + +**Controlled hold ×3** (`20260715-1411/`, `EVENTHUB_REPLICAS=3`): connect по фазам 0.2→0.035 GREEN (0.05 mean≈40 ms); overall BOUNDARY_HIT из‑за **3772× HTTP 429** — узкое место сместилось на Traefik `api-ratelimit` (per source IP, было 5000/1m≈83 rps). После поднятия IFT-лимитов до 2000/1s (EventHubBack#32) перепроверить hold ×3. + +Restore fallback: оркестратор → WSL SSH, при fail Windows `restore-ift.ps1`. + +Если после stress SSH «висит на banner» — **reboot IFT**, затем: + +```bash +bash /opt/eventhub-ift/devops/scripts/ift-loadtest-restore.sh +``` + ## Подготовка учётки admin ```bash @@ -30,6 +81,14 @@ printf '%s;%s\n' "$ADMIN_SUPER_EMAIL" "$ADMIN_SUPER_PASSWORD" > test/tsung/ift-a `ift-admin.csv` в `.gitignore`. +## Пул пользователей для stress + +```bash +# из корня EventHubBack; нужны DNS/TLS до IFT и docker/.env.ift +COUNT=400 bash test/tsung/seed-ift-users.sh +# → test/tsung/ift-users.csv (email;password), gitignore +``` + ## Перед прогоном (хост IFT) ```bash @@ -45,14 +104,20 @@ EVENTHUB_REPLICAS=1 bash /opt/eventhub-ift/devops/scripts/ift-loadtest-prepare.s 2. `sudo apt install tsung` 3. TLS: доверить CA стенда (`EventHubDevOps/ift/traefik/certs`). Проверка: `curl -sf https://api.ift.eventhub.local/health` -4. Из корня `EventHubBack`: +4. Из корня `EventHubBack` или оркестратор: ```bash tsung -f test/tsung/eventhub_ift_smoke.xml start tsung -f test/tsung/eventhub_ift.xml start +tsung -f test/tsung/eventhub_ift_stress.xml start +# оркестратор (prepare + seed при необходимости + restore): +bash ~/.cursor/eventhub/run-ift-loadtest.sh smoke|full|stress|stress-bump|stress-hold +# replicas: EVENTHUB_REPLICAS=2|3 bash ~/.cursor/eventhub/run-ift-loadtest.sh stress-hold +# restore fallback: если WSL SSH не отвечает — Windows OpenSSH +# powershell -File $env:USERPROFILE\.cursor\eventhub\restore-ift.ps1 ``` -Отчёт: `~/.tsung/log//`. +Отчёт: `~/.tsung/log//`. Для stress — также `~/.tsung/ift-docker-stats-*.log`. ## После прогона @@ -62,10 +127,11 @@ bash /opt/eventhub-ift/devops/scripts/ift-loadtest-restore.sh ## Настройка интенсивности -В `eventhub_ift.xml` блок ``: `interarrival` и `duration` фаз. -`thinktime` — **секунды**. +В `eventhub_ift.xml` / `eventhub_ift_stress*.xml` блок ``: `interarrival` и `duration` фаз. +`thinktime` в stress XML **отключён** (Tsung отклоняет `min="0"`; дробные `max` тоже) — максимальная плотность для поиска потолка. ## Замечания -- Каждый VU делает admin login — это нагрузка и на admin-api; при необходимости позже вынести общий admin JWT в csv. +- Все URL в IFT-сценариях **абсолютные** (`https://api…` / `https://admin-api…`): после запроса на admin-api относительный путь у Tsung остаётся на том же Host → 404. +- В smoke/full каждый VU делает admin login — нагрузка и на admin-api; stress этого не делает. - Из hot path убраны: dummy refresh, reports/tickets, завышенные thinktime старого `eventhub_tsung.xml`. diff --git a/test/tsung/eventhub_ift.xml b/test/tsung/eventhub_ift.xml index 3e7e31d..8458c26 100644 --- a/test/tsung/eventhub_ift.xml +++ b/test/tsung/eventhub_ift.xml @@ -43,74 +43,74 @@ + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 + version="1.1" method="GET"> - HTTP/1.[0-1] 20 - HTTP/[0-9.]+ 20 + + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 + - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 + - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 + - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 + @@ -132,51 +132,51 @@ + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 + version="1.1" method="GET"> - HTTP/1.[0-1] 20 - HTTP/[0-9.]+ 20 + + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - @@ -184,17 +184,17 @@ - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 + + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - @@ -202,16 +202,16 @@ - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 + - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 + diff --git a/test/tsung/eventhub_ift_smoke.xml b/test/tsung/eventhub_ift_smoke.xml index 9290021..b7acc37 100644 --- a/test/tsung/eventhub_ift_smoke.xml +++ b/test/tsung/eventhub_ift_smoke.xml @@ -2,6 +2,7 @@ @@ -40,52 +41,56 @@ + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 + method="GET" version="1.1"> - HTTP/1.[0-1] 20 - HTTP/[0-9.]+ 20 + + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - @@ -93,9 +98,10 @@ + HTTP/[0-9.]+ 20 - HTTP/1.[0-1] 20 - @@ -103,8 +109,8 @@ - HTTP/1.[0-1] 20 - + HTTP/[0-9.]+ 20 + diff --git a/test/tsung/eventhub_ift_stress.xml b/test/tsung/eventhub_ift_stress.xml new file mode 100644 index 0000000..ef83025 --- /dev/null +++ b/test/tsung/eventhub_ift_stress.xml @@ -0,0 +1,146 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + + + + + + + + + + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + HTTP/[0-9.]+ 20 + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + diff --git a/test/tsung/eventhub_ift_stress_bump.xml b/test/tsung/eventhub_ift_stress_bump.xml new file mode 100644 index 0000000..7ef8d6e --- /dev/null +++ b/test/tsung/eventhub_ift_stress_bump.xml @@ -0,0 +1,141 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + + + + + + + + + + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + HTTP/[0-9.]+ 20 + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + diff --git a/test/tsung/eventhub_ift_stress_hold.xml b/test/tsung/eventhub_ift_stress_hold.xml new file mode 100644 index 0000000..4dd4bac --- /dev/null +++ b/test/tsung/eventhub_ift_stress_hold.xml @@ -0,0 +1,141 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + + + + + + + + + + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + HTTP/[0-9.]+ 20 + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + HTTP/[0-9.]+ 20 + + + + + + + diff --git a/test/tsung/ift-users.csv.example b/test/tsung/ift-users.csv.example new file mode 100644 index 0000000..9e8c045 --- /dev/null +++ b/test/tsung/ift-users.csv.example @@ -0,0 +1 @@ +user@example.com;testpassword123 diff --git a/test/tsung/seed-ift-users.sh b/test/tsung/seed-ift-users.sh new file mode 100644 index 0000000..bd38646 --- /dev/null +++ b/test/tsung/seed-ift-users.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Seed verified users for IFT stress hot-path (ift-users.csv). +# Usage (from EventHubBack root or anywhere): +# COUNT=400 bash test/tsung/seed-ift-users.sh +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +ENV_FILE="${ENV_FILE:-$ROOT/docker/.env.ift}" +OUT="${OUT:-$ROOT/test/tsung/ift-users.csv}" +COUNT="${COUNT:-400}" +PASSWORD="${SEED_PASSWORD:-testpassword123}" +API="${API_BASE:-https://api.ift.eventhub.local}" +ADMIN_API="${ADMIN_API_BASE:-https://admin-api.ift.eventhub.local}" +PREFIX="${SEED_PREFIX:-lt_stress_$(date +%Y%m%d%H%M%S)}" + +if [[ ! -f "$ENV_FILE" ]]; then + echo "Missing $ENV_FILE" >&2 + exit 1 +fi + +set -a +# shellcheck disable=SC1091 +source <(sed $'s/\xEF\xBB\xBF//' "$ENV_FILE" | tr -d '\r' | grep -E '^(ADMIN_SUPER_|ADMIN_EMAIL|ADMIN_PASSWORD|SMOKE_ADMIN_)') +set +a +ADMIN_EMAIL_V="${SMOKE_ADMIN_EMAIL:-${ADMIN_SUPER_EMAIL:-$ADMIN_EMAIL}}" +ADMIN_PASS_V="${SMOKE_ADMIN_PASSWORD:-${ADMIN_SUPER_PASSWORD:-$ADMIN_PASSWORD}}" +if [[ -z "${ADMIN_EMAIL_V:-}" || -z "${ADMIN_PASS_V:-}" ]]; then + echo "No admin credentials in $ENV_FILE" >&2 + exit 1 +fi + +echo "=== admin login ===" +ADMIN_JSON=$(curl -skf --connect-timeout 15 -X POST "$ADMIN_API/v1/admin/login" \ + -H 'Content-Type: application/json' \ + -d "{\"email\":\"${ADMIN_EMAIL_V}\",\"password\":\"${ADMIN_PASS_V}\"}") +ADMIN_TOKEN=$(printf '%s' "$ADMIN_JSON" | python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])') + +: > "$OUT" +OK=0 +FAIL=0 +for i in $(seq 1 "$COUNT"); do + EMAIL="${PREFIX}_${i}@loadtest.local" + REG=$(curl -sk --connect-timeout 15 -w '\n%{http_code}' -X POST "$API/v1/register" \ + -H 'Content-Type: application/json' \ + -d "{\"email\":\"${EMAIL}\",\"password\":\"${PASSWORD}\"}" || true) + BODY=$(printf '%s' "$REG" | head -n -1) + CODE=$(printf '%s' "$REG" | tail -n 1) + if [[ "$CODE" != "201" && "$CODE" != "200" ]]; then + echo "register fail i=$i code=$CODE body=$BODY" >&2 + FAIL=$((FAIL + 1)) + continue + fi + USER_ID=$(printf '%s' "$BODY" | python3 -c 'import json,sys; print(json.load(sys.stdin)["user"]["id"])') + + VT=$(curl -sk --connect-timeout 15 -w '\n%{http_code}' -X GET \ + "$ADMIN_API/v1/admin/users/${USER_ID}/verification-token" \ + -H "Authorization: Bearer ${ADMIN_TOKEN}" || true) + VT_BODY=$(printf '%s' "$VT" | head -n -1) + VT_CODE=$(printf '%s' "$VT" | tail -n 1) + if [[ "$VT_CODE" != "200" ]]; then + echo "vtoken fail i=$i code=$VT_CODE" >&2 + FAIL=$((FAIL + 1)) + continue + fi + VTOKEN=$(printf '%s' "$VT_BODY" | python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])') + + VER_CODE=$(curl -sk --connect-timeout 15 -o /dev/null -w '%{http_code}' -X POST "$API/v1/verify" \ + -H 'Content-Type: application/json' \ + -d "{\"token\":\"${VTOKEN}\"}" || true) + if [[ "$VER_CODE" != "200" && "$VER_CODE" != "201" && "$VER_CODE" != "204" ]]; then + echo "verify fail i=$i code=$VER_CODE" >&2 + FAIL=$((FAIL + 1)) + continue + fi + + printf '%s;%s\n' "$EMAIL" "$PASSWORD" >> "$OUT" + OK=$((OK + 1)) + if (( i % 25 == 0 )); then + echo " seeded $i/$COUNT (ok=$OK fail=$FAIL)" + fi +done + +chmod 600 "$OUT" +echo "=== done ok=$OK fail=$FAIL out=$OUT prefix=$PREFIX ===" +if (( OK < 10 )); then + echo "Too few users seeded" >&2 + exit 1 +fi diff --git a/test/tsung/summarize-tsung-stress.py b/test/tsung/summarize-tsung-stress.py new file mode 100644 index 0000000..94789b7 --- /dev/null +++ b/test/tsung/summarize-tsung-stress.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Summarize Tsung log against stress boundary 2A. + +Boundary hit if: + - error rate > 1% (4xx+5xx+nomatch+match_stop relative to requests), OR + - cumulative mean request > 500 ms + +Usage: summarize-tsung-stress.py [path/to/tsung.log] +Default: newest ~/.tsung/log/*/tsung.log +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + + +def latest_log() -> Path: + root = Path.home() / ".tsung" / "log" + dirs = sorted([p for p in root.iterdir() if p.is_dir()], reverse=True) + for d in dirs: + cand = d / "tsung.log" + if cand.is_file(): + return cand + raise SystemExit("No tsung.log under ~/.tsung/log") + + +def last_stats(text: str, key: str) -> list[str] | None: + ms = list(re.finditer(rf"^stats: {re.escape(key)} (.+)$", text, re.M)) + if not ms: + return None + return ms[-1].group(1).split() + + +def cumul_count(fields: list[str] | None) -> int: + """Tsung dump: for counters usually last field is cumulative total.""" + if not fields: + return 0 + return int(float(fields[-1])) + + +def mean_request_ms(fields: list[str] | None) -> float | None: + """request line: count mean stddev max min mean_cumul count_cumul (approx).""" + if not fields or len(fields) < 6: + return None + # Prefer overall mean (field index 5 in dump = running mean of means in later dumps) + try: + return float(fields[5]) + except (IndexError, ValueError): + return float(fields[1]) + + +def phase_means(text: str) -> list[tuple[int, float, int]]: + """Per-dump mean from request lines: (dump_idx, mean_interval, cumul_count).""" + out = [] + dumps = text.split("# stats: dump") + for i, block in enumerate(dumps[1:], start=1): + m = re.search(r"^stats: request (.+)$", block, re.M) + if not m: + continue + f = m.group(1).split() + if len(f) >= 2: + try: + out.append((i, float(f[1]), cumul_count(f))) + except ValueError: + pass + return out + + +def main() -> int: + log = Path(sys.argv[1]) if len(sys.argv) > 1 else latest_log() + text = log.read_text(encoding="utf-8", errors="replace") + print(f"log={log}") + + req = last_stats(text, "request") + users = last_stats(text, "users_count") + finish = last_stats(text, "finish_users_count") + match = last_stats(text, "match") + nomatch = last_stats(text, "nomatch") + match_stop = last_stats(text, "match_stop") + + codes = {} + for code in ("200", "201", "400", "401", "403", "404", "429", "500", "502", "503"): + st = last_stats(text, code) + if st: + codes[code] = cumul_count(st) + + transport_errs = {} + for key in ( + "error_connect_closed", + "error_timeout", + "error_abort_max_conn_retries", + "error_connect_econnrefused", + "error_connect_etimedout", + "error_unknown", + ): + st = last_stats(text, key) + if st: + transport_errs[key] = cumul_count(st) + + n_req = cumul_count(req) + n_nomatch = cumul_count(nomatch) + n_stop = cumul_count(match_stop) + n_4xx = sum(v for k, v in codes.items() if k.startswith("4")) + n_5xx = sum(v for k, v in codes.items() if k.startswith("5")) + n_2xx = sum(v for k, v in codes.items() if k.startswith("2")) + n_transport = sum(transport_errs.values()) + # Denominator: completed HTTP + transport failures (attempts that never got status) + attempts = max(n_req + n_transport, 1) + err = n_4xx + n_5xx + n_nomatch + n_stop + n_transport + err_rate = err / attempts * 100.0 + mean_ms = mean_request_ms(req) + + print(f"users_started={cumul_count(users)} finished={cumul_count(finish)}") + print(f"requests={n_req} match={cumul_count(match)} nomatch={n_nomatch} match_stop={n_stop}") + print(f"http_2xx={n_2xx} http_4xx={n_4xx} http_5xx={n_5xx} codes={codes}") + print(f"transport_errors={transport_errs} transport_total={n_transport}") + print(f"attempts≈{n_req + n_transport} error_units={err} error_rate_pct={err_rate:.3f}") + print(f"mean_request_ms_cumul={mean_ms}") + + # Hold-ish: last third of dumps (late phases) + means = phase_means(text) + if means: + tail = means[max(0, len(means) * 2 // 3) :] + hold_means = [m for _, m, _ in tail if m > 0] + hold_mean = sum(hold_means) / len(hold_means) if hold_means else None + print(f"dumps={len(means)} late_interval_mean_ms={hold_mean}") + peak_interval = max(means, key=lambda t: t[1]) + print(f"peak_interval_mean_ms={peak_interval[1]} at_dump={peak_interval[0]}") + else: + hold_mean = None + + boundary = False + reasons = [] + if err_rate > 1.0: + boundary = True + reasons.append(f"error_rate {err_rate:.3f}% > 1%") + check_mean = mean_ms if mean_ms is not None else hold_mean + if check_mean is not None and check_mean > 500: + boundary = True + reasons.append(f"mean_request {check_mean:.1f} ms > 500 ms") + if hold_mean is not None and hold_mean > 500: + boundary = True + if f"mean_request {hold_mean:.1f} ms > 500 ms" not in reasons: + reasons.append(f"late_phase_mean {hold_mean:.1f} ms > 500 ms") + + verdict = "BOUNDARY_HIT" if boundary else "UNDER_THRESHOLD" + print(f"verdict={verdict}") + if reasons: + print("reasons=" + "; ".join(reasons)) + return 0 if not boundary else 2 + + +if __name__ == "__main__": + raise SystemExit(main())