feat(ift): EventHub AI Router stack for Zed
FastAPI gateway with A/B/C lane orchestration, LiteLLM proxy config, Swarm stack (postgres, redis, VPN off-by-default), deploy/smoke/audit scripts.
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# Daily Novita pricing/model audit — compare model_matrix.yaml vs /v1/models
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
[[ -f .env ]] && source .env
|
||||
set +a
|
||||
|
||||
API_KEY="${NOVITA_API_KEY:?NOVITA_API_KEY required}"
|
||||
LOG_DIR="${AUDIT_LOG_DIR:-/var/log/ai-router}"
|
||||
DATE="$(date +%Y-%m-%d)"
|
||||
REPORT="${LOG_DIR}/audit-${DATE}.json"
|
||||
THRESHOLD="${PRICE_DRIFT_THRESHOLD_PCT:-10}"
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
MODELS_JSON="$(curl -sf "https://api.novita.ai/openai/v1/models" \
|
||||
-H "Authorization: Bearer ${API_KEY}")"
|
||||
|
||||
export ROOT MODELS_JSON REPORT THRESHOLD
|
||||
python3 <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
root = Path(os.environ["ROOT"])
|
||||
matrix = yaml.safe_load((root / "config/model_matrix.yaml").read_text(encoding="utf-8"))
|
||||
catalog = json.loads(os.environ["MODELS_JSON"])
|
||||
threshold = float(os.environ.get("THRESHOLD", "10"))
|
||||
|
||||
by_id = {m["id"]: m for m in catalog.get("data", [])}
|
||||
issues = []
|
||||
checked = 0
|
||||
|
||||
for name, cfg in matrix.get("models", {}).items():
|
||||
novita = cfg.get("novita", "")
|
||||
model_id = novita.replace("novita/", "", 1) if novita.startswith("novita/") else novita
|
||||
if not model_id:
|
||||
continue
|
||||
checked += 1
|
||||
if model_id not in by_id:
|
||||
issues.append({"model": name, "novita_id": model_id, "severity": "critical", "msg": "missing from catalog"})
|
||||
|
||||
report = {
|
||||
"date": os.environ.get("DATE", ""),
|
||||
"checked": checked,
|
||||
"issues": issues,
|
||||
"catalog_count": len(by_id),
|
||||
}
|
||||
report_path = Path(os.environ["REPORT"])
|
||||
report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
if any(i["severity"] == "critical" for i in issues):
|
||||
print(f"CRITICAL: {len(issues)} issues — see {report_path}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if issues:
|
||||
print(f"WARN: {len(issues)} issues — see {report_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"OK: {checked} models, report {report_path}")
|
||||
PY
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deploy AI Router stack to Docker Swarm (IFT)
|
||||
set -euo pipefail
|
||||
|
||||
STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
STACK_NAME="${STACK_NAME:-ai-router}"
|
||||
|
||||
cd "$STACK_DIR"
|
||||
|
||||
if [[ ! -f .env ]]; then
|
||||
echo "ERROR: copy .env.example to .env and fill secrets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source .env
|
||||
set +a
|
||||
|
||||
ensure_secret() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
if docker secret inspect "$name" >/dev/null 2>&1; then
|
||||
echo "secret exists: $name"
|
||||
else
|
||||
echo -n "$value" | docker secret create "$name" -
|
||||
echo "created secret: $name"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "== Ensure Swarm secrets =="
|
||||
ensure_secret novita_api_key "${NOVITA_API_KEY:?NOVITA_API_KEY required}"
|
||||
ensure_secret litellm_master_key "${LITELLM_MASTER_KEY:?LITELLM_MASTER_KEY required}"
|
||||
ensure_secret litellm_salt_key "${LITELLM_SALT_KEY:?LITELLM_SALT_KEY required}"
|
||||
ensure_secret router_api_key "${ROUTER_API_KEY:?ROUTER_API_KEY required}"
|
||||
ensure_secret postgres_password "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD required}"
|
||||
ensure_secret groq_api_key "${GROQ_API_KEY:-}"
|
||||
ensure_secret gemini_api_key "${GEMINI_API_KEY:-}"
|
||||
|
||||
if [[ ! -f vless/vless.conf ]]; then
|
||||
echo "WARN: vless/vless.conf missing — stub for secret (VPN off until configured)"
|
||||
cp vless/vless.conf.example vless/vless.conf 2>/dev/null || echo "# stub" > vless/vless.conf
|
||||
fi
|
||||
ensure_secret vless_conf "$(cat vless/vless.conf)"
|
||||
|
||||
echo "== Build images =="
|
||||
docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" .
|
||||
docker build -t ai-router/vless-proxy:local ./vless
|
||||
docker build -t ai-router/vpn-watchdog:local ./watchdog
|
||||
|
||||
echo "== Sync routing config (optional regen) =="
|
||||
bash scripts/sync-routing-config.sh || true
|
||||
|
||||
echo "== Deploy stack: ${STACK_NAME} =="
|
||||
docker stack deploy -c docker-stack.yml --with-registry-auth "${STACK_NAME}"
|
||||
|
||||
echo "== Wait for services =="
|
||||
sleep 8
|
||||
docker stack services "${STACK_NAME}"
|
||||
|
||||
echo ""
|
||||
echo "Done."
|
||||
echo " Zed URL: ${TEST_BASE_URL:-https://ai-router.ift.calentiq.com}/v1"
|
||||
echo " LiteLLM: ${TEST_LITELLM_URL:-https://litellm.ift.calentiq.com}/ui"
|
||||
echo " Smoke: bash scripts/smoke-test.sh"
|
||||
echo " VPN on: bash scripts/vpn-enable.sh"
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install daily audit cron (03:00 MSK) on IFT host
|
||||
set -euo pipefail
|
||||
|
||||
STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
CRON_FILE="/etc/cron.d/ai-router-audit"
|
||||
|
||||
sudo tee "$CRON_FILE" >/dev/null <<EOF
|
||||
# EventHub AI Router — Novita model audit
|
||||
0 3 * * * root ${STACK_DIR}/scripts/audit-novita-pricing.sh >> /var/log/ai-router/audit.log 2>&1
|
||||
EOF
|
||||
|
||||
sudo chmod 644 "$CRON_FILE"
|
||||
echo "Installed ${CRON_FILE}"
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/sh
|
||||
# Load Swarm secrets into env and start LiteLLM proxy
|
||||
set -eu
|
||||
|
||||
read_secret() {
|
||||
if [ -f "$1" ]; then
|
||||
tr -d '\n\r' < "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ -f /run/secrets/novita_api_key ]; then
|
||||
export NOVITA_API_KEY="$(read_secret /run/secrets/novita_api_key)"
|
||||
fi
|
||||
if [ -f /run/secrets/litellm_master_key ]; then
|
||||
export LITELLM_MASTER_KEY="$(read_secret /run/secrets/litellm_master_key)"
|
||||
fi
|
||||
if [ -f /run/secrets/litellm_salt_key ]; then
|
||||
export LITELLM_SALT_KEY="$(read_secret /run/secrets/litellm_salt_key)"
|
||||
fi
|
||||
if [ -f /run/secrets/groq_api_key ]; then
|
||||
export GROQ_API_KEY="$(read_secret /run/secrets/groq_api_key)"
|
||||
fi
|
||||
if [ -f /run/secrets/gemini_api_key ]; then
|
||||
export GEMINI_API_KEY="$(read_secret /run/secrets/gemini_api_key)"
|
||||
fi
|
||||
if [ -f /run/secrets/postgres_password ]; then
|
||||
export POSTGRES_PASSWORD="$(read_secret /run/secrets/postgres_password)"
|
||||
export DATABASE_URL="postgresql://${POSTGRES_USER:-litellm}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-litellm}"
|
||||
fi
|
||||
|
||||
exec litellm --config /app/config.yaml --port 4000 --num_workers 2 "$@"
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
read_secret() {
|
||||
if [ -f "$1" ]; then
|
||||
tr -d '\n\r' < "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ -f /run/secrets/litellm_master_key ]; then
|
||||
export LITELLM_MASTER_KEY="$(read_secret /run/secrets/litellm_master_key)"
|
||||
fi
|
||||
if [ -f /run/secrets/router_api_key ]; then
|
||||
export ROUTER_API_KEY="$(read_secret /run/secrets/router_api_key)"
|
||||
fi
|
||||
|
||||
exec uvicorn router:app --host 0.0.0.0 --port 8000 "$@"
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env bash
|
||||
# Smoke tests — classify (no LLM) + minimal chat
|
||||
set -euo pipefail
|
||||
|
||||
STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$STACK_DIR"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source .env 2>/dev/null || true
|
||||
set +a
|
||||
|
||||
BASE="${TEST_BASE_URL:-http://127.0.0.1:8000}"
|
||||
LITELLM_BASE="${TEST_LITELLM_URL:-http://127.0.0.1:4000}"
|
||||
ROUTER_KEY="${ROUTER_API_KEY:?ROUTER_API_KEY required}"
|
||||
|
||||
CURL=(curl -sf)
|
||||
if [[ "$BASE" == https:* ]]; then
|
||||
CURL+=( -k )
|
||||
fi
|
||||
|
||||
echo "== Router health =="
|
||||
"${CURL[@]}" "${BASE}/health" | jq .
|
||||
|
||||
echo "== Classify SIMPLE =="
|
||||
"${CURL[@]}" "${BASE}/classify" \
|
||||
-H "Authorization: Bearer ${ROUTER_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"messages":[{"role":"user","content":"Привет, что такое Docker?"}]}' | jq .
|
||||
|
||||
echo "== Classify COMPLEX =="
|
||||
"${CURL[@]}" "${BASE}/classify" \
|
||||
-H "Authorization: Bearer ${ROUTER_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"messages":[{"role":"user","content":"Спроектируй архитектуру microservices"}]}' | jq .
|
||||
|
||||
echo "== Chat smart-router max_tokens=16 =="
|
||||
"${CURL[@]}" "${BASE}/v1/chat/completions" \
|
||||
-H "Authorization: Bearer ${ROUTER_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"smart-router","max_tokens":16,"messages":[{"role":"user","content":"bash docker service ls"}]}' \
|
||||
| jq '.choices[0].message.content, .x_router_meta // empty'
|
||||
|
||||
if [[ -n "${SKIP_LITELLM_SMOKE:-}" ]]; then
|
||||
echo "SKIP_LITELLM_SMOKE set — skipping LiteLLM checks"
|
||||
else
|
||||
echo "== LiteLLM liveliness =="
|
||||
"${CURL[@]}" "${LITELLM_BASE}/health/liveliness" && echo
|
||||
echo "== LiteLLM UI =="
|
||||
curl -sfI -k "${LITELLM_BASE}/ui" | head -3 || true
|
||||
fi
|
||||
|
||||
echo "All smoke checks passed."
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Sync model_list fragment from config/*.yaml
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
export ROOT
|
||||
|
||||
python3 <<'PY'
|
||||
import os
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["ROOT"])
|
||||
rules = yaml.safe_load((root / "config/routing_rules.yaml").read_text(encoding="utf-8"))
|
||||
matrix = yaml.safe_load((root / "config/model_matrix.yaml").read_text(encoding="utf-8"))
|
||||
|
||||
entries = []
|
||||
for name, cfg in matrix.get("models", {}).items():
|
||||
novita = cfg.get("novita")
|
||||
if not novita:
|
||||
continue
|
||||
entry = {
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": novita,
|
||||
"api_key": "os.environ/NOVITA_API_KEY",
|
||||
},
|
||||
}
|
||||
if cfg.get("rpm"):
|
||||
entry["litellm_params"]["rpm"] = cfg["rpm"]
|
||||
entries.append(entry)
|
||||
|
||||
for name, cfg in matrix.get("optional_providers", {}).items():
|
||||
entries.append({
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": cfg["model"],
|
||||
"api_key": cfg.get("api_key", "os.environ/GROQ_API_KEY"),
|
||||
},
|
||||
})
|
||||
|
||||
litellm_rules = rules.get("litellm", {})
|
||||
entries.append({
|
||||
"model_name": "smart-router-internal",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"drop_params": True,
|
||||
"complexity_router_default_model": "a-medium-ops",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": "a-simple",
|
||||
"MEDIUM": "a-medium-ops",
|
||||
"MEDIUM_CODE": "a-medium-code",
|
||||
"COMPLEX": "a-complex",
|
||||
"REASONING": "a-reasoning",
|
||||
},
|
||||
"classifier_fallback": "heuristic",
|
||||
"keyword_tier_rules": litellm_rules.get("keyword_tier_rules", []),
|
||||
"custom_technical_keywords": litellm_rules.get("custom_technical_keywords", []),
|
||||
"token_thresholds": {"simple": 20, "complex": 500},
|
||||
"tier_boundaries": {
|
||||
"simple_medium": 0.18,
|
||||
"medium_complex": 0.38,
|
||||
"complex_reasoning": 0.62,
|
||||
},
|
||||
"session_affinity": True,
|
||||
"session_affinity_ttl_seconds": 1800,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
out = root / "litellm_config.generated.yaml"
|
||||
out.write_text(
|
||||
yaml.dump({"model_list": entries}, allow_unicode=True, sort_keys=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"Wrote {out} ({len(entries)} models)")
|
||||
PY
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# Vendor LiteLLM official Grafana dashboard JSON into EventHubDevOps
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
LITELLM_REF="${LITELLM_REF:-main-v1.96.0-stable}"
|
||||
DEVOPS="${DEVOPS_ROOT:-$(cd "$ROOT/../EventHubDevOps" 2>/dev/null && pwd || echo "")}"
|
||||
|
||||
if [[ -z "$DEVOPS" || ! -d "$DEVOPS/ift/observability" ]]; then
|
||||
echo "DEVOPS_ROOT not found — set to EventHubDevOps path" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DEST="${DEVOPS}/ift/observability/grafana/dashboards/litellm"
|
||||
mkdir -p "$DEST"
|
||||
|
||||
BASE="https://raw.githubusercontent.com/BerriAI/litellm/${LITELLM_REF}/cookbook/litellm_proxy_server/grafana_dashboard"
|
||||
|
||||
fetch() {
|
||||
local src="$1" dst="$2"
|
||||
echo "Fetching $src"
|
||||
curl -fsSL "${BASE}/${src}" -o "${DEST}/${dst}"
|
||||
}
|
||||
|
||||
fetch "dashboard_v2/grafana_dashboard.json" "litellm-v2.json"
|
||||
fetch "dashboard_1/grafana_dashboard.json" "litellm-v1.json" || true
|
||||
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
for f in litellm-v2.json litellm-v1.json; do
|
||||
[[ -f "${DEST}/${f}" ]] || continue
|
||||
jq 'walk(if type=="object" and has("datasource") then .datasource="prometheus" else . end)' \
|
||||
"${DEST}/${f}" > "${DEST}/${f}.tmp" && mv "${DEST}/${f}.tmp" "${DEST}/${f}"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "Dashboards written to ${DEST}"
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
STACK_NAME="${STACK_NAME:-ai-router}"
|
||||
cd "$STACK_DIR"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source .env
|
||||
set +a
|
||||
|
||||
NO_PROXY_VAL="${NO_PROXY:-localhost,127.0.0.1,api.novita.ai,novita.ai}"
|
||||
|
||||
echo "== Scale vless-proxy and vpn-watchdog to 0 =="
|
||||
docker service scale "${STACK_NAME}_vless-proxy=0" "${STACK_NAME}_vpn-watchdog=0" || true
|
||||
|
||||
echo "== Clear litellm proxy env =="
|
||||
docker service update \
|
||||
--env-rm HTTP_PROXY \
|
||||
--env-rm HTTPS_PROXY \
|
||||
--env-add "NO_PROXY=${NO_PROXY_VAL}" \
|
||||
"${STACK_NAME}_litellm" || true
|
||||
|
||||
echo "VPN disabled."
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
STACK_NAME="${STACK_NAME:-ai-router}"
|
||||
cd "$STACK_DIR"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source .env
|
||||
set +a
|
||||
|
||||
VLESS_URL="${VLESS_PROXY_URL:-http://vless-proxy:8080}"
|
||||
NO_PROXY_VAL="${NO_PROXY:-localhost,127.0.0.1,api.novita.ai,novita.ai}"
|
||||
|
||||
echo "== Scale vless-proxy and vpn-watchdog to 1 =="
|
||||
docker service scale "${STACK_NAME}_vless-proxy=1" "${STACK_NAME}_vpn-watchdog=1"
|
||||
|
||||
echo "== Wait for vless-proxy healthy (max 90s) =="
|
||||
for i in $(seq 1 18); do
|
||||
CID=$(docker ps -q -f "name=${STACK_NAME}_vless-proxy" | head -1)
|
||||
if [[ -n "$CID" ]]; then
|
||||
STATUS=$(docker inspect --format='{{.State.Health.Status}}' "$CID" 2>/dev/null || echo "starting")
|
||||
if [[ "$STATUS" == "healthy" ]]; then
|
||||
echo "vless-proxy is healthy"
|
||||
break
|
||||
fi
|
||||
echo " attempt $i: status=$STATUS"
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "== E2E tunnel probe =="
|
||||
docker run --rm --network "${STACK_NAME}_ai-internal" curlimages/curl:8.5.0 \
|
||||
curl -sf -x "${VLESS_URL}" --max-time 15 https://www.google.com/generate_204 \
|
||||
|| echo "WARN: tunnel probe failed (may need valid vless.conf)"
|
||||
|
||||
echo "== Point litellm through VPN =="
|
||||
docker service update \
|
||||
--env-add "HTTP_PROXY=${VLESS_URL}" \
|
||||
--env-add "HTTPS_PROXY=${VLESS_URL}" \
|
||||
--env-add "NO_PROXY=${NO_PROXY_VAL}" \
|
||||
"${STACK_NAME}_litellm"
|
||||
|
||||
echo "VPN enabled. Novita bypasses via NO_PROXY."
|
||||
Reference in New Issue
Block a user