From 83d516afa141f758c33d808771ef1178a6881124 Mon Sep 17 00:00:00 2001 From: Aleksey Sabilin Date: Fri, 7 Aug 2026 22:06:41 +0300 Subject: [PATCH] feat(ift): EventHub AI Router stack for Zed FastAPI gateway with A/B/C lane orchestration, LiteLLM proxy config, Swarm stack (postgres, redis, VPN off-by-default), deploy/smoke/audit scripts. --- .env.example | 50 +++++ .gitea/workflows/ci.yml | 25 +++ .gitignore | 8 + README.md | 113 ++++++++++ config/model_matrix.yaml | 105 ++++++++++ config/orchestration.yaml | 32 +++ config/routing_rules.yaml | 77 +++++++ docker-stack.yml | 243 ++++++++++++++++++++++ litellm_config.yaml | 200 ++++++++++++++++++ router/Dockerfile | 22 ++ router/metrics.py | 31 +++ router/orchestrator.py | 300 +++++++++++++++++++++++++++ router/requirements.txt | 6 + router/router.py | 288 +++++++++++++++++++++++++ router/rules_loader.py | 39 ++++ scripts/audit-novita-pricing.sh | 67 ++++++ scripts/deploy.sh | 66 ++++++ scripts/install-audit-cron.sh | 14 ++ scripts/litellm-entrypoint.sh | 31 +++ scripts/router-entrypoint.sh | 17 ++ scripts/smoke-test.sh | 53 +++++ scripts/sync-routing-config.sh | 78 +++++++ scripts/vendor-litellm-dashboards.sh | 36 ++++ scripts/vpn-disable.sh | 25 +++ scripts/vpn-enable.sh | 45 ++++ vless/Dockerfile | 3 + vless/vless.conf.example | 4 + watchdog/Dockerfile | 8 + watchdog/vpn-watchdog.sh | 19 ++ 29 files changed, 2005 insertions(+) create mode 100644 .env.example create mode 100644 .gitea/workflows/ci.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 config/model_matrix.yaml create mode 100644 config/orchestration.yaml create mode 100644 config/routing_rules.yaml create mode 100644 docker-stack.yml create mode 100644 litellm_config.yaml create mode 100644 router/Dockerfile create mode 100644 router/metrics.py create mode 100644 router/orchestrator.py create mode 100644 router/requirements.txt create mode 100644 router/router.py create mode 100644 router/rules_loader.py create mode 100644 scripts/audit-novita-pricing.sh create mode 100644 scripts/deploy.sh create mode 100644 scripts/install-audit-cron.sh create mode 100644 scripts/litellm-entrypoint.sh create mode 100644 scripts/router-entrypoint.sh create mode 100644 scripts/smoke-test.sh create mode 100644 scripts/sync-routing-config.sh create mode 100644 scripts/vendor-litellm-dashboards.sh create mode 100644 scripts/vpn-disable.sh create mode 100644 scripts/vpn-enable.sh create mode 100644 vless/Dockerfile create mode 100644 vless/vless.conf.example create mode 100644 watchdog/Dockerfile create mode 100644 watchdog/vpn-watchdog.sh diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..c14d4b6 --- /dev/null +++ b/.env.example @@ -0,0 +1,50 @@ +# ============================================================================= +# EventHubAiRouter — IFT Docker Swarm (.env not committed; use Swarm secrets) +# ============================================================================= + +STACK_NAME=ai-router +TZ=Europe/Moscow +DEPLOY_HOST=https://ai-router.ift.calentiq.com + +# --- LiteLLM --- +LITELLM_MASTER_KEY=sk-litellm-change-me +LITELLM_SALT_KEY=sk-salt-generate-once-never-change +LITELLM_IMAGE=ghcr.io/berriai/litellm:main-v1.96.0-stable +PROXY_BASE_URL=https://litellm.ift.calentiq.com + +# Postgres (LiteLLM Admin UI + spend logs) +POSTGRES_USER=litellm +POSTGRES_DB=litellm +POSTGRES_PASSWORD=change-me-postgres + +# Novita AI (direct from IFT, no VPN) +NOVITA_API_KEY= + +# Optional fallbacks +GROQ_API_KEY= +GEMINI_API_KEY= + +# Budget (USD/month, also in litellm_config.yaml) +LITELLM_MAX_BUDGET=50 + +# --- VPN (off by default) --- +VPN_ENABLED=false +VLESS_PROXY_URL=http://vless-proxy:8080 +NO_PROXY=localhost,127.0.0.1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,api.novita.ai,novita.ai,git.sabilin.com,*.ift.calentiq.com,ift.calentiq.com,*.ift.eventhub.local,*.eventhub.local + +DISABLE_AIOHTTP_TRANSPORT=True +USE_AIOHTTP_TRANSPORT=False + +# --- Gateway (Zed entrypoint) --- +ROUTER_API_KEY=sk-router-change-me +ROUTER_IMAGE=git.sabilin.com/eventhub/ai-router-gateway:ift +LITELLM_INTERNAL_URL=http://litellm:4000 +DEFAULT_QUALITY_MODE=auto +REDIS_URL=redis://redis:6379/0 + +# --- Observability --- +UPTIME_KUMA_PORT=3001 + +# Smoke test target (HTTPS after Traefik deploy) +TEST_BASE_URL=https://ai-router.ift.calentiq.com +TEST_LITELLM_URL=https://litellm.ift.calentiq.com diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..3e23b1d --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [master, main] + pull_request: + +jobs: + build-gateway: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build gateway image + run: docker build -f router/Dockerfile -t ai-router-gateway:ci . + + - name: Lint Python syntax + run: python3 -m py_compile router/*.py + + sync-config: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Sync routing config + run: bash scripts/sync-routing-config.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..225268a --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.env +vless/vless.conf +*.pyc +__pycache__/ +.venv/ +litellm_config.generated.yaml +/tmp/ +*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..b5e666e --- /dev/null +++ b/README.md @@ -0,0 +1,113 @@ +# EventHub AI Router — Zed gateway + LiteLLM on IFT + +OpenAI-compatible endpoint for [Zed](https://zed.dev) with automatic tier/lane routing (Novita AI), optional VLESS VPN, Grafana metrics, and LiteLLM Admin UI. + +## URLs (IFT, Calentiq TLS) + +| Service | URL | +|---------|-----| +| **Zed API** | `https://ai-router.ift.calentiq.com/v1` | +| **LiteLLM Admin UI** | `https://litellm.ift.calentiq.com/ui` (login: `admin` / `LITELLM_MASTER_KEY`) | +| **Grafana** | `https://grafana.ift.calentiq.com` | +| Legacy (self-signed) | `https://ai-router.ift.eventhub.local/v1` | + +## Zed settings.json + +```json +{ + "openai": { + "api_url": "https://ai-router.ift.calentiq.com/v1", + "api_key": "" + }, + "assistant": { + "default_model": { "provider": "openai", "model": "smart-router" } + } +} +``` + +## Architecture + +- **Gateway** (`router/`) — vision/OCR split, tier classification, lanes A/B/C orchestration, Redis session +- **LiteLLM** — named models `a-*` / `b-*` / `c-*`, Auto Router v2 fallback (`smart-router-internal`) +- **PostgreSQL** — LiteLLM Admin UI, spend logs +- **Redis** — response cache + gateway session context +- **VPN** — `vless-proxy` replicas=0 by default; `scripts/vpn-enable.sh` + +Config sources: + +- `config/routing_rules.yaml` — tier keywords +- `config/model_matrix.yaml` — tier × lane → Novita models +- `config/orchestration.yaml` — start lanes, escalation, budget caps + +## Deploy on IFT + +```bash +git clone git.sabilin.com/eventhub/EventHubAiRouter /opt/ai-router-stack +cd /opt/ai-router-stack +cp .env.example .env # fill NOVITA_API_KEY, keys, POSTGRES_PASSWORD, LITELLM_SALT_KEY +bash scripts/deploy.sh +bash scripts/smoke-test.sh +sudo bash scripts/install-audit-cron.sh # daily Novita audit 03:00 MSK +``` + +Prerequisites: Docker Swarm, external network `eventhub-ift-net`, Traefik routes in EventHubDevOps. + +### Swarm secrets + +Created automatically by `deploy.sh` from `.env`: `novita_api_key`, `litellm_master_key`, `litellm_salt_key`, `router_api_key`, `postgres_password`, optional `groq_api_key`, `gemini_api_key`, `vless_conf`. + +**`LITELLM_SALT_KEY`** — generate once, never change after first deploy. + +## VPN + +```bash +bash scripts/vpn-enable.sh # scale vless + watchdog, set HTTP_PROXY on litellm +bash scripts/vpn-disable.sh +``` + +Novita always direct (`NO_PROXY=api.novita.ai`). + +## Observability + +- Prometheus scrape: `litellm:4000/metrics`, `ai-router:8000/metrics` (via `eventhub-ift-net`) +- Grafana: vendored LiteLLM v2 dashboard + custom `gateway-tier-lane.json` (EventHubDevOps) +- Uptime Kuma monitors — see table in plan / configure manually + +## Scripts + +| Script | Purpose | +|--------|---------| +| `deploy.sh` | secrets + build + stack deploy | +| `smoke-test.sh` | health, classify, chat max_tokens=16 | +| `sync-routing-config.sh` | regen `litellm_config.generated.yaml` | +| `audit-novita-pricing.sh` | daily model catalog check | +| `vendor-litellm-dashboards.sh` | fetch Grafana JSON → EventHubDevOps | +| `vpn-enable.sh` / `vpn-disable.sh` | VPN toggle | + +## Quality modes + +Header `X-AI-Quality: auto|economy|balanced|max` or `metadata.quality_mode` in request body. Default: `auto` (start lane A/B by tier, escalate on retry/5xx). + +## Response metadata + +Each chat response includes `x_router_meta`: + +```json +{ + "tier": "MEDIUM_OPS", + "lane": "A", + "model": "a-medium-ops", + "escalation_level": 0, + "quality_mode": "auto" +} +``` + +## Local build + +```bash +docker build -f router/Dockerfile -t ai-router-gateway:local . +``` + +## CI + +Gitea Actions: `.gitea/workflows/ci.yml` — build gateway image, sync config check. diff --git a/config/model_matrix.yaml b/config/model_matrix.yaml new file mode 100644 index 0000000..cd57087 --- /dev/null +++ b/config/model_matrix.yaml @@ -0,0 +1,105 @@ +# tier × lane (A/B/C) → Novita model id + LiteLLM fallbacks + +lanes: + A: economy + B: balanced + C: max + +models: + a-simple: + novita: novita/qwen/qwen3-4b-fp8 + fallbacks: [b-simple, groq-llama-8b] + rpm: 60 + b-simple: + novita: novita/meta-llama/llama-3.1-8b-instruct + fallbacks: [a-simple, groq-llama-8b] + rpm: 60 + c-simple: + novita: novita/qwen/qwen3-8b-fp8 + fallbacks: [b-simple] + rpm: 60 + + a-medium-ops: + novita: novita/qwen/qwen3-8b-fp8 + fallbacks: [a-medium-code, b-medium-ops] + rpm: 40 + b-medium-ops: + novita: novita/deepseek/deepseek-v3.2 + fallbacks: [a-medium-code, c-medium-ops] + rpm: 40 + c-medium-ops: + novita: novita/deepseek/deepseek-v3.2 + fallbacks: [b-medium-ops] + rpm: 40 + + a-medium-code: + novita: novita/qwen/qwen3-coder-30b-a3b-instruct + fallbacks: [a-medium-ops, groq-qwen-coder] + rpm: 40 + b-medium-code: + novita: novita/qwen/qwen3-coder-30b-a3b-instruct + fallbacks: [b-medium-ops, c-medium-code] + rpm: 40 + c-medium-code: + novita: novita/qwen/qwen3-coder-30b-a3b-instruct + fallbacks: [b-medium-code, c-medium-ops] + rpm: 30 + + a-complex: + novita: novita/deepseek/deepseek-v3.2 + fallbacks: [b-complex, a-reasoning] + rpm: 30 + b-complex: + novita: novita/deepseek/deepseek-r1-0528 + fallbacks: [a-complex, c-complex] + rpm: 20 + c-complex: + novita: novita/deepseek/deepseek-r1-turbo + fallbacks: [b-complex] + rpm: 15 + + a-reasoning: + novita: novita/deepseek/deepseek-r1-0528-qwen3-8b + fallbacks: [b-reasoning, gemini-flash] + rpm: 30 + b-reasoning: + novita: novita/deepseek/deepseek-r1-0528 + fallbacks: [a-reasoning, c-reasoning] + rpm: 20 + c-reasoning: + novita: novita/deepseek/deepseek-r1-turbo + fallbacks: [b-reasoning, gemini-flash] + rpm: 15 + + a-vision-ocr: + novita: novita/paddlepaddle/paddleocr-vl + fallbacks: [a-vision] + rpm: 30 + no_escalation: true + a-vision: + novita: novita/qwen/qwen3-vl-30b-a3b-instruct + fallbacks: [a-vision-ocr] + rpm: 20 + b-vision: + novita: novita/qwen/qwen2.5-vl-72b-instruct + fallbacks: [a-vision, c-vision] + rpm: 15 + c-vision: + novita: novita/qwen/qwen3-vl-235b-a22b-instruct + fallbacks: [b-vision] + rpm: 10 + +optional_providers: + groq-llama-8b: + model: groq/llama-3.1-8b-instant + api_key: os.environ/GROQ_API_KEY + groq-qwen-coder: + model: groq/qwen-qwen-2.5-coder-32b + api_key: os.environ/GROQ_API_KEY + gemini-flash: + model: gemini/gemini-2.0-flash + api_key: os.environ/GEMINI_API_KEY + +audit: + price_drift_threshold_pct: 10 + cheaper_alternative_pct: 15 diff --git a/config/orchestration.yaml b/config/orchestration.yaml new file mode 100644 index 0000000..e32c6a9 --- /dev/null +++ b/config/orchestration.yaml @@ -0,0 +1,32 @@ +default_quality_mode: auto + +start_lanes: + SIMPLE: A + MEDIUM_OPS: A + MEDIUM_CODE: A + COMPLEX: B + REASONING: B + VISION_OCR: A + VISION_UI: A + +quality_mode_map: + economy: A + balanced: B + max: C + +lane_order: [A, B, C] + +budget_caps: + warn_pct: 80 + hard_pct: 95 + warn_max_lane: B + hard_max_lane: A + +escalation: + repeat_prompt_window_sec: 300 + context_tokens_min_lane_b: 32000 + session_ttl_sec: 1800 + +redis: + key_prefix: "ai-router:session:" + ttl_sec: 1800 diff --git a/config/routing_rules.yaml b/config/routing_rules.yaml new file mode 100644 index 0000000..db60567 --- /dev/null +++ b/config/routing_rules.yaml @@ -0,0 +1,77 @@ +# Tier classification — single source for gateway + LiteLLM keyword rules + +gateway: + simple_patterns: + - "^(привет|hello|hi|hey|спасибо|thanks|что такое|what is|define)\\b" + complex_keywords: + - рефакторинг + - refactor + - архитектур + - architecture + - спроектируй + - design system + - microservice + - distributed + - migration plan + - deep refactor + reasoning_keywords: + - step by step + - prove + - analyze deeply + - think through + - reasoning + medium_ops_keywords: + - bash + - docker + - swarm + - ci/cd + - gitea + - rebar + - erlang + - devops + - kubectl + - terraform + - playwright + - npm + - wsl + - litellm + - novita + - traefik + - mnesia + ocr_keywords: + - ocr + - прочитай текст + - extract text + - распознай текст + - read text from + escalation_keywords: + - не подходит + - лучше + - качественнее + - retry + - попробуй ещё + word_thresholds: + simple_max_words: 12 + simple_question_max_words: 25 + complex_min_words: 400 + medium_code_min_lines: 200 + confidence: + low_threshold: 0.6 + +litellm: + keyword_tier_rules: + - keywords: ["привет", "hello", "hi", "thanks", "спасибо", "что такое", "what is", "define"] + tier: SIMPLE + - keywords: ["bash", "docker", "swarm", "ci/cd", "gitea", "rebar", "erlang", "devops", "kubectl", "terraform", "playwright", "npm", "wsl"] + tier: MEDIUM + - keywords: ["рефакторинг", "refactor", "архитектура", "architecture", "спроектируй", "design system", "microservice", "distributed", "migration plan"] + tier: COMPLEX + - keywords: ["step by step", "reasoning", "prove", "analyze deeply", "think through"] + tier: REASONING + custom_technical_keywords: + - erlang + - rebar3 + - traefik + - mnesia + - litellm + - novita diff --git a/docker-stack.yml b/docker-stack.yml new file mode 100644 index 0000000..1bd60e5 --- /dev/null +++ b/docker-stack.yml @@ -0,0 +1,243 @@ +version: "3.8" + +# AI Router stack — IFT Docker Swarm +# VPN (vless-proxy, vpn-watchdog) scaled to 0 by default + +networks: + ai-internal: + driver: overlay + internal: true + eventhub-ift: + external: true + name: eventhub-ift-net + +volumes: + litellm-postgres-data: + uptime-kuma-data: + +configs: + litellm_config: + file: ./litellm_config.yaml + litellm_entrypoint: + file: ./scripts/litellm-entrypoint.sh + router_entrypoint: + file: ./scripts/router-entrypoint.sh + routing_config: + file: ./config/routing_rules.yaml + model_matrix: + file: ./config/model_matrix.yaml + orchestration_config: + file: ./config/orchestration.yaml + +secrets: + novita_api_key: + external: true + litellm_master_key: + external: true + litellm_salt_key: + external: true + router_api_key: + external: true + postgres_password: + external: true + groq_api_key: + external: true + gemini_api_key: + external: true + vless_conf: + external: true + +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB:-litellm} + POSTGRES_USER: ${POSTGRES_USER:-litellm} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - litellm-postgres-data:/var/lib/postgresql/data + networks: + - ai-internal + deploy: + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - node.role == manager + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-litellm} -d ${POSTGRES_DB:-litellm}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + + redis: + image: redis:7-alpine + command: ["redis-server", "--save", "", "--appendonly", "no"] + networks: + - ai-internal + deploy: + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - node.role == manager + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 3 + + litellm: + image: ${LITELLM_IMAGE:-ghcr.io/berriai/litellm:main-v1.96.0-stable} + entrypoint: ["/bin/sh", "/entrypoint/litellm-entrypoint.sh"] + configs: + - source: litellm_config + target: /app/config.yaml + - source: litellm_entrypoint + target: /entrypoint/litellm-entrypoint.sh + mode: 0555 + environment: + STORE_MODEL_IN_DB: "False" + PROXY_BASE_URL: ${PROXY_BASE_URL:-https://litellm.ift.calentiq.com} + FORWARDED_ALLOW_IPS: "*" + DISABLE_AIOHTTP_TRANSPORT: ${DISABLE_AIOHTTP_TRANSPORT:-True} + USE_AIOHTTP_TRANSPORT: ${USE_AIOHTTP_TRANSPORT:-False} + HTTP_PROXY: ${HTTP_PROXY:-} + HTTPS_PROXY: ${HTTPS_PROXY:-} + NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1,api.novita.ai,novita.ai} + POSTGRES_USER: ${POSTGRES_USER:-litellm} + POSTGRES_DB: ${POSTGRES_DB:-litellm} + secrets: + - novita_api_key + - litellm_master_key + - litellm_salt_key + - postgres_password + - groq_api_key + - gemini_api_key + networks: + ai-internal: + aliases: + - litellm + eventhub-ift: + aliases: + - litellm + deploy: + replicas: 1 + restart_policy: + condition: on-failure + delay: 5s + update_config: + parallelism: 1 + failure_action: rollback + placement: + constraints: + - node.role == manager + healthcheck: + test: ["CMD-SHELL", "wget -q -O- http://127.0.0.1:4000/health/liveliness || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + router: + image: ${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift} + entrypoint: ["/bin/sh", "/entrypoint/router-entrypoint.sh"] + configs: + - source: router_entrypoint + target: /entrypoint/router-entrypoint.sh + mode: 0555 + - source: routing_config + target: /app/config/routing_rules.yaml + - source: model_matrix + target: /app/config/model_matrix.yaml + - source: orchestration_config + target: /app/config/orchestration.yaml + environment: + LITELLM_INTERNAL_URL: http://litellm:4000 + REDIS_URL: redis://redis:6379/0 + CONFIG_DIR: /app/config + DEFAULT_QUALITY_MODE: ${DEFAULT_QUALITY_MODE:-auto} + secrets: + - litellm_master_key + - router_api_key + networks: + ai-internal: + eventhub-ift: + aliases: + - ai-router + deploy: + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - node.role == manager + healthcheck: + test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8000/health || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + + vless-proxy: + image: ${VLESS_IMAGE:-ai-router/vless-proxy:local} + secrets: + - source: vless_conf + target: /app/vless.conf + networks: + - ai-internal + deploy: + replicas: 0 + restart_policy: + condition: any + delay: 5s + max_attempts: 0 + placement: + constraints: + - node.role == manager + healthcheck: + test: ["CMD-SHELL", "wget -q -O- http://127.0.0.1:8080 || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + + vpn-watchdog: + image: ${WATCHDOG_IMAGE:-ai-router/vpn-watchdog:local} + environment: + STACK_NAME: ${STACK_NAME:-ai-router} + VLESS_PROXY_URL: ${VLESS_PROXY_URL:-http://vless-proxy:8080} + CHECK_INTERVAL: "60" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: + - ai-internal + deploy: + replicas: 0 + restart_policy: + condition: any + placement: + constraints: + - node.role == manager + + uptime-kuma: + image: louislam/uptime-kuma:1 + volumes: + - uptime-kuma-data:/app/data + networks: + ai-internal: + eventhub-ift: + aliases: + - uptime-kuma + deploy: + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - node.role == manager diff --git a/litellm_config.yaml b/litellm_config.yaml new file mode 100644 index 0000000..fd50523 --- /dev/null +++ b/litellm_config.yaml @@ -0,0 +1,200 @@ +# LiteLLM Proxy — Novita lanes A/B/C + Auto Router v2 fallback +# Regenerate model_list tail: bash scripts/sync-routing-config.sh + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + store_model_in_db: false + max_budget: 50 + budget_duration: 30d + +litellm_settings: + drop_params: true + set_verbose: false + request_timeout: 120 + num_retries: 2 + cache: true + cache_params: + type: redis + host: redis + port: 6379 + ttl: 3600 + callbacks: ["prometheus"] + require_auth_for_metrics_endpoint: false + +environment_variables: + NOVITA_API_KEY: os.environ/NOVITA_API_KEY + GROQ_API_KEY: os.environ/GROQ_API_KEY + GEMINI_API_KEY: os.environ/GEMINI_API_KEY + +router_settings: + routing_strategy: simple-shuffle + num_retries: 2 + timeout: 120 + allowed_fails: 2 + cooldown_time: 30 + fallbacks: + - a-simple: ["b-simple", "groq-llama-8b"] + - a-medium-code: ["a-medium-ops", "groq-qwen-coder"] + - a-complex: ["b-complex", "a-reasoning"] + - b-complex: ["a-complex", "c-complex"] + - a-reasoning: ["b-reasoning", "gemini-flash"] + - a-vision: ["a-vision-ocr"] + - smart-router-internal: ["a-medium-ops", "a-medium-code", "a-complex"] + default_fallbacks: ["a-medium-code", "a-complex", "groq-qwen-coder"] + +model_list: + # --- Lane models (generated from config/model_matrix.yaml) --- + - model_name: a-simple + litellm_params: + model: novita/qwen/qwen3-4b-fp8 + api_key: os.environ/NOVITA_API_KEY + rpm: 60 + - model_name: b-simple + litellm_params: + model: novita/meta-llama/llama-3.1-8b-instruct + api_key: os.environ/NOVITA_API_KEY + rpm: 60 + - model_name: c-simple + litellm_params: + model: novita/qwen/qwen3-8b-fp8 + api_key: os.environ/NOVITA_API_KEY + rpm: 60 + + - model_name: a-medium-ops + litellm_params: + model: novita/qwen/qwen3-8b-fp8 + api_key: os.environ/NOVITA_API_KEY + rpm: 40 + - model_name: b-medium-ops + litellm_params: + model: novita/deepseek/deepseek-v3.2 + api_key: os.environ/NOVITA_API_KEY + rpm: 40 + - model_name: c-medium-ops + litellm_params: + model: novita/deepseek/deepseek-v3.2 + api_key: os.environ/NOVITA_API_KEY + rpm: 40 + + - model_name: a-medium-code + litellm_params: + model: novita/qwen/qwen3-coder-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + rpm: 40 + - model_name: b-medium-code + litellm_params: + model: novita/qwen/qwen3-coder-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + rpm: 40 + - model_name: c-medium-code + litellm_params: + model: novita/qwen/qwen3-coder-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + rpm: 30 + + - model_name: a-complex + litellm_params: + model: novita/deepseek/deepseek-v3.2 + api_key: os.environ/NOVITA_API_KEY + rpm: 30 + - model_name: b-complex + litellm_params: + model: novita/deepseek/deepseek-r1-0528 + api_key: os.environ/NOVITA_API_KEY + rpm: 20 + - model_name: c-complex + litellm_params: + model: novita/deepseek/deepseek-r1-turbo + api_key: os.environ/NOVITA_API_KEY + rpm: 15 + + - model_name: a-reasoning + litellm_params: + model: novita/deepseek/deepseek-r1-0528-qwen3-8b + api_key: os.environ/NOVITA_API_KEY + rpm: 30 + - model_name: b-reasoning + litellm_params: + model: novita/deepseek/deepseek-r1-0528 + api_key: os.environ/NOVITA_API_KEY + rpm: 20 + - model_name: c-reasoning + litellm_params: + model: novita/deepseek/deepseek-r1-turbo + api_key: os.environ/NOVITA_API_KEY + rpm: 15 + + - model_name: a-vision-ocr + litellm_params: + model: novita/paddlepaddle/paddleocr-vl + api_key: os.environ/NOVITA_API_KEY + rpm: 30 + - model_name: a-vision + litellm_params: + model: novita/qwen/qwen3-vl-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + rpm: 20 + - model_name: b-vision + litellm_params: + model: novita/qwen/qwen2.5-vl-72b-instruct + api_key: os.environ/NOVITA_API_KEY + rpm: 15 + - model_name: c-vision + litellm_params: + model: novita/qwen/qwen3-vl-235b-a22b-instruct + api_key: os.environ/NOVITA_API_KEY + rpm: 10 + + # --- Optional fallbacks --- + - model_name: groq-llama-8b + litellm_params: + model: groq/llama-3.1-8b-instant + api_key: os.environ/GROQ_API_KEY + - model_name: groq-qwen-coder + litellm_params: + model: groq/qwen-qwen-2.5-coder-32b + api_key: os.environ/GROQ_API_KEY + - model_name: gemini-flash + litellm_params: + model: gemini/gemini-2.0-flash + api_key: os.environ/GEMINI_API_KEY + + # --- Auto Router v2 fallback when gateway confidence low --- + - model_name: smart-router-internal + litellm_params: + model: auto_router/complexity_router + drop_params: true + complexity_router_default_model: a-medium-ops + complexity_router_config: + tiers: + SIMPLE: a-simple + MEDIUM: a-medium-ops + MEDIUM_CODE: a-medium-code + COMPLEX: a-complex + REASONING: a-reasoning + classifier_fallback: heuristic + keyword_tier_rules: + - keywords: ["привет", "hello", "hi", "thanks", "спасибо", "что такое", "what is", "define"] + tier: SIMPLE + - keywords: ["bash", "docker", "swarm", "ci/cd", "gitea", "rebar", "erlang", "devops", "kubectl", "terraform"] + tier: MEDIUM + - keywords: ["рефакторинг", "refactor", "архитектура", "architecture", "спроектируй", "migration plan"] + tier: COMPLEX + - keywords: ["step by step", "prove", "analyze deeply", "think through"] + tier: REASONING + custom_technical_keywords: [erlang, rebar3, traefik, mnesia, litellm, novita] + token_thresholds: + simple: 20 + complex: 500 + tier_boundaries: + simple_medium: 0.18 + medium_complex: 0.38 + complex_reasoning: 0.62 + session_affinity: true + session_affinity_ttl_seconds: 1800 + + # Alias for Zed default model name (gateway resolves before LiteLLM) + - model_name: smart-router + litellm_params: + model: novita/qwen/qwen3-8b-fp8 + api_key: os.environ/NOVITA_API_KEY diff --git a/router/Dockerfile b/router/Dockerfile new file mode 100644 index 0000000..e195694 --- /dev/null +++ b/router/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY router/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY router/*.py ./ +COPY config /app/config + +ENV CONFIG_DIR=/app/config +ENV PYTHONUNBUFFERED=1 + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD curl -sf http://127.0.0.1:8000/health || exit 1 + +CMD ["uvicorn", "router:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/router/metrics.py b/router/metrics.py new file mode 100644 index 0000000..fc31ea5 --- /dev/null +++ b/router/metrics.py @@ -0,0 +1,31 @@ +"""Prometheus metrics for gateway tier/lane routing.""" + +from __future__ import annotations + +from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest + +REQUESTS = Counter( + "ai_router_requests_total", + "Total routed requests", + ["tier", "lane", "model", "status"], +) +ESCALATIONS = Counter( + "ai_router_escalations_total", + "Lane escalations", + ["from_lane", "to_lane", "reason"], +) +CLASSIFY = Counter( + "ai_router_classify_total", + "Classification results", + ["tier"], +) +DURATION = Histogram( + "ai_router_request_duration_seconds", + "Request duration", + ["tier", "model"], + buckets=(0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120, 180), +) + + +def metrics_payload() -> tuple[bytes, str]: + return generate_latest(), CONTENT_TYPE_LATEST diff --git a/router/orchestrator.py b/router/orchestrator.py new file mode 100644 index 0000000..c5a58e6 --- /dev/null +++ b/router/orchestrator.py @@ -0,0 +1,300 @@ +"""Lane orchestration A/B/C with Redis session context.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import re +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +from rules_loader import load_model_matrix, load_orchestration, load_routing_rules + +log = logging.getLogger("orchestrator") + +LANE_PREFIX = {"A": "a", "B": "b", "C": "c"} +TIER_SUFFIX = { + "SIMPLE": "simple", + "MEDIUM_OPS": "medium-ops", + "MEDIUM_CODE": "medium-code", + "COMPLEX": "complex", + "REASONING": "reasoning", + "VISION_OCR": "vision-ocr", + "VISION_UI": "vision", +} + + +class Tier(str, Enum): + SIMPLE = "SIMPLE" + MEDIUM_OPS = "MEDIUM_OPS" + MEDIUM_CODE = "MEDIUM_CODE" + COMPLEX = "COMPLEX" + REASONING = "REASONING" + VISION_OCR = "VISION_OCR" + VISION_UI = "VISION_UI" + + +@dataclass +class SessionCtx: + escalation_level: int = 0 + last_prompt_hash: str = "" + turn_count: int = 0 + budget_pct: float = 0.0 + + +@dataclass +class RouteDecision: + tier: Tier + lane: str + model: str + quality_mode: str + escalation_level: int + confidence: float + delegated_internal: bool = False + + +class SessionStore: + def __init__(self, redis_url: str | None) -> None: + self._redis = None + if redis_url: + try: + import redis + + self._redis = redis.from_url(redis_url, decode_responses=True) + self._redis.ping() + except Exception as exc: # noqa: BLE001 + log.warning("Redis unavailable (%s), using in-memory sessions", exc) + self._memory: dict[str, SessionCtx] = {} + orch = load_orchestration() + self._prefix = orch.get("redis", {}).get("key_prefix", "ai-router:session:") + self._ttl = int(orch.get("redis", {}).get("ttl_sec", 1800)) + + def get(self, session_id: str) -> SessionCtx: + if not session_id: + return SessionCtx() + key = f"{self._prefix}{session_id}" + if self._redis: + raw = self._redis.get(key) + if raw: + data = json.loads(raw) + return SessionCtx(**data) + return SessionCtx() + return self._memory.get(session_id, SessionCtx()) + + def save(self, session_id: str, ctx: SessionCtx) -> None: + if not session_id: + return + key = f"{self._prefix}{session_id}" + payload = json.dumps( + { + "escalation_level": ctx.escalation_level, + "last_prompt_hash": ctx.last_prompt_hash, + "turn_count": ctx.turn_count, + "budget_pct": ctx.budget_pct, + } + ) + if self._redis: + self._redis.setex(key, self._ttl, payload) + else: + self._memory[session_id] = ctx + + +class Classifier: + def __init__(self) -> None: + rules = load_routing_rules() + gw = rules.get("gateway", {}) + self._simple_re = re.compile( + gw.get("simple_patterns", [r"^(hello)\\b"])[0], + re.IGNORECASE, + ) + self._complex = self._kw_re(gw.get("complex_keywords", [])) + self._reasoning = self._kw_re(gw.get("reasoning_keywords", [])) + self._medium_ops = self._kw_re(gw.get("medium_ops_keywords", [])) + self._ocr = self._kw_re(gw.get("ocr_keywords", [])) + self._escalation = self._kw_re(gw.get("escalation_keywords", [])) + wt = gw.get("word_thresholds", {}) + self._simple_max = int(wt.get("simple_max_words", 12)) + self._simple_q_max = int(wt.get("simple_question_max_words", 25)) + self._complex_min = int(wt.get("complex_min_words", 400)) + self._low_conf = float(rules.get("gateway", {}).get("confidence", {}).get("low_threshold", 0.6)) + self._code_block = re.compile(r"```[\s\S]*?```|`[^`]+`") + + @staticmethod + def _kw_re(keywords: list[str]) -> re.Pattern[str]: + if not keywords: + return re.compile(r"(?!x)x") + escaped = [re.escape(k) for k in keywords] + return re.compile("|".join(escaped), re.IGNORECASE) + + def wants_escalation(self, text: str, prompt_hash: str, prev_hash: str) -> bool: + if self._escalation.search(text): + return True + return bool(prompt_hash and prompt_hash == prev_hash) + + def classify( + self, + messages: list[dict[str, Any]], + *, + has_image: bool, + text: str, + ) -> tuple[Tier, float]: + if has_image: + if self._ocr.search(text) or len(text.split()) < 30: + return Tier.VISION_OCR, 0.95 + return Tier.VISION_UI, 0.9 + + text = text.strip() + if not text: + return Tier.SIMPLE, 0.9 + + words = len(text.split()) + if words <= self._simple_max and self._simple_re.search(text): + return Tier.SIMPLE, 0.92 + if self._reasoning.search(text): + return Tier.REASONING, 0.88 + if self._complex.search(text): + return Tier.COMPLEX, 0.88 + if self._code_block.search(text): + return Tier.MEDIUM_CODE, 0.85 + if self._medium_ops.search(text): + return Tier.MEDIUM_OPS, 0.82 + if words <= self._simple_q_max and "?" in text and not self._code_block.search(text): + return Tier.SIMPLE, 0.75 + if words >= self._complex_min: + return Tier.COMPLEX, 0.7 + return Tier.MEDIUM_OPS, 0.55 + + +class Orchestrator: + def __init__(self, session_store: SessionStore) -> None: + self.sessions = session_store + self.classifier = Classifier() + self._orch = load_orchestration() + self._matrix = load_model_matrix() + self._models: dict[str, dict] = self._matrix.get("models", {}) + self._no_esc = { + name for name, cfg in self._models.items() if cfg.get("no_escalation") + } + self.default_quality = os.environ.get( + "DEFAULT_QUALITY_MODE", + self._orch.get("default_quality_mode", "auto"), + ) + + def _lane_from_mode(self, quality_mode: str) -> str | None: + if quality_mode == "auto": + return None + return self._orch.get("quality_mode_map", {}).get(quality_mode) + + def _start_lane(self, tier: Tier) -> str: + return self._orch.get("start_lanes", {}).get(tier.value, "A") + + def _bump_lane(self, lane: str, levels: int) -> str: + order: list[str] = self._orch.get("lane_order", ["A", "B", "C"]) + try: + idx = order.index(lane) + except ValueError: + return lane + return order[min(idx + levels, len(order) - 1)] + + def _cap_lane(self, lane: str, budget_pct: float) -> str: + caps = self._orch.get("budget_caps", {}) + hard = float(caps.get("hard_pct", 95)) + warn = float(caps.get("warn_pct", 80)) + if budget_pct >= hard: + return caps.get("hard_max_lane", "A") + if budget_pct >= warn: + max_lane = caps.get("warn_max_lane", "B") + order = self._orch.get("lane_order", ["A", "B", "C"]) + if order.index(lane) > order.index(max_lane): + return max_lane + return lane + + def _model_name(self, lane: str, tier: Tier) -> str: + prefix = LANE_PREFIX.get(lane, "a") + suffix = TIER_SUFFIX[tier] + return f"{prefix}-{suffix}" + + def resolve( + self, + messages: list[dict[str, Any]], + *, + quality_mode: str | None = None, + session_id: str = "", + text: str = "", + has_image: bool = False, + token_estimate: int = 0, + ) -> RouteDecision: + mode = quality_mode or self.default_quality + tier, confidence = self.classifier.classify(messages, has_image=has_image, text=text) + ctx = self.sessions.get(session_id) + prompt_hash = hashlib.sha256(text.encode()).hexdigest()[:16] + + if mode != "auto": + lane = self._lane_from_mode(mode) or "A" + escalation = 0 + else: + lane = self._start_lane(tier) + escalation = ctx.escalation_level + if tier != Tier.VISION_OCR and self.classifier.wants_escalation( + text, prompt_hash, ctx.last_prompt_hash + ): + escalation += 1 + min_tokens = int( + self._orch.get("escalation", {}).get("context_tokens_min_lane_b", 32000) + ) + if token_estimate > min_tokens: + order = self._orch.get("lane_order", ["A", "B", "C"]) + if order.index(lane) < order.index("B"): + lane = "B" + lane = self._bump_lane(lane, escalation) + lane = self._cap_lane(lane, ctx.budget_pct) + + model = self._model_name(lane, tier) + if model not in self._models: + log.warning("model %s missing from matrix, fallback smart-router-internal", model) + return RouteDecision( + tier=tier, + lane=lane, + model="smart-router-internal", + quality_mode=mode, + escalation_level=escalation, + confidence=confidence, + delegated_internal=True, + ) + + delegated = confidence < self.classifier._low_conf and tier in ( + Tier.MEDIUM_OPS, + Tier.SIMPLE, + ) + + return RouteDecision( + tier=tier, + lane=lane, + model="smart-router-internal" if delegated else model, + quality_mode=mode, + escalation_level=escalation, + confidence=confidence, + delegated_internal=delegated, + ) + + def after_request( + self, + session_id: str, + *, + prompt_hash: str, + success: bool, + escalate: bool, + ) -> None: + if not session_id: + return + ctx = self.sessions.get(session_id) + ctx.last_prompt_hash = prompt_hash + ctx.turn_count += 1 + if success and not escalate: + ctx.escalation_level = 0 + elif escalate: + ctx.escalation_level = min(ctx.escalation_level + 1, 2) + self.sessions.save(session_id, ctx) diff --git a/router/requirements.txt b/router/requirements.txt new file mode 100644 index 0000000..a04e60b --- /dev/null +++ b/router/requirements.txt @@ -0,0 +1,6 @@ +fastapi>=0.115.0 +uvicorn[standard]>=0.32.0 +httpx>=0.27.0 +pyyaml>=6.0.2 +prometheus-client>=0.21.0 +redis>=5.2.0 diff --git a/router/router.py b/router/router.py new file mode 100644 index 0000000..ad58d6f --- /dev/null +++ b/router/router.py @@ -0,0 +1,288 @@ +""" +FastAPI AI router gateway — Zed entrypoint with tier/lane orchestration. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import time +from typing import Any + +import httpx +from fastapi import FastAPI, Header, HTTPException, Request, Response +from fastapi.responses import JSONResponse, StreamingResponse + +from metrics import CLASSIFY, DURATION, ESCALATIONS, REQUESTS, metrics_payload +from orchestrator import Orchestrator, SessionStore, Tier +from rules_loader import reload_configs + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +log = logging.getLogger("ai-router") + +LITELLM_URL = os.environ.get("LITELLM_INTERNAL_URL", "http://litellm:4000").rstrip("/") +LITELLM_KEY = os.environ.get("LITELLM_MASTER_KEY", "") +ROUTER_API_KEY = os.environ.get("ROUTER_API_KEY", "") +DEFAULT_MODEL = os.environ.get("DEFAULT_LITELLM_MODEL", "smart-router") +REDIS_URL = os.environ.get("REDIS_URL", "redis://redis:6379/0") + +session_store = SessionStore(REDIS_URL or None) +orchestrator = Orchestrator(session_store) + +app = FastAPI(title="EventHub AI Router", version="2.0.0") + + +def _extract_text(messages: list[dict[str, Any]]) -> str: + parts: list[str] = [] + for msg in messages: + content = msg.get("content", "") + if isinstance(content, str): + parts.append(content) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(str(block.get("text", ""))) + return "\n".join(parts) + + +def _has_image(messages: list[dict[str, Any]]) -> bool: + for msg in messages: + content = msg.get("content", "") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") in ("image_url", "image"): + return True + return False + + +def _estimate_tokens(text: str) -> int: + return max(1, len(text) // 4) + + +def _auth_or_403(authorization: str | None) -> None: + if not ROUTER_API_KEY: + return + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Missing Bearer token") + token = authorization.removeprefix("Bearer ").strip() + if token != ROUTER_API_KEY: + raise HTTPException(status_code=403, detail="Invalid API key") + + +def _quality_mode(header: str | None, body: dict[str, Any]) -> str | None: + if header: + mode = header.strip().lower() + if mode in ("auto", "economy", "balanced", "max"): + return mode + meta = body.get("metadata") or {} + if isinstance(meta, dict): + mode = str(meta.get("quality_mode", "")).lower() + if mode in ("auto", "economy", "balanced", "max"): + return mode + return None + + +def _session_id(body: dict[str, Any]) -> str: + meta = body.get("metadata") or {} + if isinstance(meta, dict) and meta.get("session_id"): + return str(meta["session_id"]) + if body.get("user"): + return str(body["user"]) + return "" + + +def _router_meta(decision, *, requested: str) -> dict[str, Any]: + return { + "tier": decision.tier.value, + "lane": decision.lane, + "model": decision.model, + "escalation_level": decision.escalation_level, + "quality_mode": decision.quality_mode, + "confidence": round(decision.confidence, 3), + "requested_model": requested, + "delegated_internal": decision.delegated_internal, + } + + +@app.get("/health") +async def health() -> dict[str, str]: + return {"status": "ok"} + + +@app.get("/metrics") +async def metrics() -> Response: + body, content_type = metrics_payload() + return Response(content=body, media_type=content_type) + + +@app.post("/admin/reload-config") +async def admin_reload(authorization: str | None = Header(default=None)) -> dict[str, str]: + _auth_or_403(authorization) + reload_configs() + global orchestrator # noqa: PLW0603 + orchestrator = Orchestrator(session_store) + return {"status": "reloaded"} + + +@app.get("/v1/models") +async def list_models(authorization: str | None = Header(default=None)) -> dict[str, Any]: + _auth_or_403(authorization) + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.get( + f"{LITELLM_URL}/v1/models", + headers={"Authorization": f"Bearer {LITELLM_KEY}"}, + ) + if resp.status_code >= 400: + raise HTTPException(status_code=resp.status_code, detail=resp.text) + return resp.json() + + +@app.post("/classify") +async def classify_debug( + request: Request, + authorization: str | None = Header(default=None), + x_ai_quality: str | None = Header(default=None, alias="X-AI-Quality"), +) -> dict[str, Any]: + _auth_or_403(authorization) + body = await request.json() + messages = body.get("messages") or [] + text = _extract_text(messages) + decision = orchestrator.resolve( + messages, + quality_mode=_quality_mode(x_ai_quality, body), + session_id=_session_id(body), + text=text, + has_image=_has_image(messages), + token_estimate=_estimate_tokens(text), + ) + CLASSIFY.labels(tier=decision.tier.value).inc() + return { + "tier": decision.tier.value, + "lane": decision.lane, + "model": decision.model, + "x_router_meta": _router_meta(decision, requested=body.get("model", DEFAULT_MODEL)), + } + + +async def _forward_litellm( + forward: dict[str, Any], + *, + stream: bool, + decision_meta: dict[str, Any], +) -> Any: + headers = { + "Authorization": f"Bearer {LITELLM_KEY}", + "Content-Type": "application/json", + } + start = time.perf_counter() + tier = decision_meta.get("tier", "UNKNOWN") + model = forward.get("model", DEFAULT_MODEL) + + async with httpx.AsyncClient(timeout=180.0) as client: + if stream: + req = client.build_request( + "POST", + f"{LITELLM_URL}/v1/chat/completions", + headers=headers, + content=json.dumps(forward), + ) + resp = await client.send(req, stream=True) + + async def event_stream(): + async for chunk in resp.aiter_bytes(): + yield chunk + + REQUESTS.labels(tier=tier, lane=decision_meta["lane"], model=model, status=str(resp.status_code)).inc() + DURATION.labels(tier=tier, model=model).observe(time.perf_counter() - start) + return StreamingResponse( + event_stream(), + status_code=resp.status_code, + media_type=resp.headers.get("content-type", "text/event-stream"), + headers={"X-Router-Meta": json.dumps(decision_meta)}, + ) + + resp = await client.post( + f"{LITELLM_URL}/v1/chat/completions", + headers=headers, + json=forward, + ) + + status = str(resp.status_code) + REQUESTS.labels(tier=tier, lane=decision_meta["lane"], model=model, status=status).inc() + DURATION.labels(tier=tier, model=model).observe(time.perf_counter() - start) + + if resp.status_code >= 400: + return JSONResponse(status_code=resp.status_code, content=resp.json()) + + data = resp.json() + if isinstance(data, dict): + data["x_router_meta"] = decision_meta + return data + + +@app.post("/v1/chat/completions") +async def chat_completions( + request: Request, + authorization: str | None = Header(default=None), + x_ai_quality: str | None = Header(default=None, alias="X-AI-Quality"), +) -> Any: + _auth_or_403(authorization) + + body: dict[str, Any] = await request.json() + messages = body.get("messages") or [] + if not isinstance(messages, list): + raise HTTPException(status_code=400, detail="messages must be a list") + + text = _extract_text(messages) + has_image = _has_image(messages) + session_id = _session_id(body) + prompt_hash = hashlib.sha256(text.encode()).hexdigest()[:16] + + requested_model = body.get("model", DEFAULT_MODEL) + if requested_model in ("smart-router", "auto", ""): + decision = orchestrator.resolve( + messages, + quality_mode=_quality_mode(x_ai_quality, body), + session_id=session_id, + text=text, + has_image=has_image, + token_estimate=_estimate_tokens(text), + ) + target_model = decision.model + meta = _router_meta(decision, requested=requested_model) + else: + target_model = requested_model + meta = {"selected_model": target_model, "requested_model": requested_model} + + forward = dict(body) + forward["model"] = target_model + forward.setdefault("metadata", {}) + if isinstance(forward["metadata"], dict): + forward["metadata"]["semantic_tier"] = meta.get("tier") + forward["metadata"]["session_id"] = session_id or forward["metadata"].get("session_id") + + log.info( + "route tier=%s lane=%s model=%s requested=%s", + meta.get("tier"), + meta.get("lane"), + target_model, + requested_model, + ) + + stream = bool(body.get("stream", False)) + result = await _forward_litellm(forward, stream=stream, decision_meta=meta) + + escalate = isinstance(result, JSONResponse) and result.status_code >= 429 + orchestrator.after_request( + session_id, + prompt_hash=prompt_hash, + success=not escalate, + escalate=escalate, + ) + if escalate and meta.get("lane") != "C": + ESCALATIONS.labels(from_lane=meta.get("lane", "?"), to_lane="next", reason="http_error").inc() + + return result diff --git a/router/rules_loader.py b/router/rules_loader.py new file mode 100644 index 0000000..4e95478 --- /dev/null +++ b/router/rules_loader.py @@ -0,0 +1,39 @@ +"""Load routing YAML configs from CONFIG_DIR.""" + +from __future__ import annotations + +import os +from functools import lru_cache +from pathlib import Path +from typing import Any + +import yaml + +CONFIG_DIR = Path(os.environ.get("CONFIG_DIR", "/app/config")) + + +@lru_cache(maxsize=1) +def load_routing_rules() -> dict[str, Any]: + path = CONFIG_DIR / "routing_rules.yaml" + with path.open(encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +@lru_cache(maxsize=1) +def load_model_matrix() -> dict[str, Any]: + path = CONFIG_DIR / "model_matrix.yaml" + with path.open(encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +@lru_cache(maxsize=1) +def load_orchestration() -> dict[str, Any]: + path = CONFIG_DIR / "orchestration.yaml" + with path.open(encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +def reload_configs() -> None: + load_routing_rules.cache_clear() + load_model_matrix.cache_clear() + load_orchestration.cache_clear() diff --git a/scripts/audit-novita-pricing.sh b/scripts/audit-novita-pricing.sh new file mode 100644 index 0000000..406a27a --- /dev/null +++ b/scripts/audit-novita-pricing.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Daily Novita pricing/model audit — compare model_matrix.yaml vs /v1/models +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +set -a +# shellcheck disable=SC1091 +[[ -f .env ]] && source .env +set +a + +API_KEY="${NOVITA_API_KEY:?NOVITA_API_KEY required}" +LOG_DIR="${AUDIT_LOG_DIR:-/var/log/ai-router}" +DATE="$(date +%Y-%m-%d)" +REPORT="${LOG_DIR}/audit-${DATE}.json" +THRESHOLD="${PRICE_DRIFT_THRESHOLD_PCT:-10}" + +mkdir -p "$LOG_DIR" + +MODELS_JSON="$(curl -sf "https://api.novita.ai/openai/v1/models" \ + -H "Authorization: Bearer ${API_KEY}")" + +export ROOT MODELS_JSON REPORT THRESHOLD +python3 <<'PY' +import json +import os +import sys +from pathlib import Path + +import yaml + +root = Path(os.environ["ROOT"]) +matrix = yaml.safe_load((root / "config/model_matrix.yaml").read_text(encoding="utf-8")) +catalog = json.loads(os.environ["MODELS_JSON"]) +threshold = float(os.environ.get("THRESHOLD", "10")) + +by_id = {m["id"]: m for m in catalog.get("data", [])} +issues = [] +checked = 0 + +for name, cfg in matrix.get("models", {}).items(): + novita = cfg.get("novita", "") + model_id = novita.replace("novita/", "", 1) if novita.startswith("novita/") else novita + if not model_id: + continue + checked += 1 + if model_id not in by_id: + issues.append({"model": name, "novita_id": model_id, "severity": "critical", "msg": "missing from catalog"}) + +report = { + "date": os.environ.get("DATE", ""), + "checked": checked, + "issues": issues, + "catalog_count": len(by_id), +} +report_path = Path(os.environ["REPORT"]) +report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + +if any(i["severity"] == "critical" for i in issues): + print(f"CRITICAL: {len(issues)} issues — see {report_path}", file=sys.stderr) + sys.exit(2) +if issues: + print(f"WARN: {len(issues)} issues — see {report_path}", file=sys.stderr) + sys.exit(1) +print(f"OK: {checked} models, report {report_path}") +PY diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100644 index 0000000..82dc7cb --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Deploy AI Router stack to Docker Swarm (IFT) +set -euo pipefail + +STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACK_NAME="${STACK_NAME:-ai-router}" + +cd "$STACK_DIR" + +if [[ ! -f .env ]]; then + echo "ERROR: copy .env.example to .env and fill secrets" >&2 + exit 1 +fi + +set -a +# shellcheck disable=SC1091 +source .env +set +a + +ensure_secret() { + local name="$1" + local value="$2" + if docker secret inspect "$name" >/dev/null 2>&1; then + echo "secret exists: $name" + else + echo -n "$value" | docker secret create "$name" - + echo "created secret: $name" + fi +} + +echo "== Ensure Swarm secrets ==" +ensure_secret novita_api_key "${NOVITA_API_KEY:?NOVITA_API_KEY required}" +ensure_secret litellm_master_key "${LITELLM_MASTER_KEY:?LITELLM_MASTER_KEY required}" +ensure_secret litellm_salt_key "${LITELLM_SALT_KEY:?LITELLM_SALT_KEY required}" +ensure_secret router_api_key "${ROUTER_API_KEY:?ROUTER_API_KEY required}" +ensure_secret postgres_password "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD required}" +ensure_secret groq_api_key "${GROQ_API_KEY:-}" +ensure_secret gemini_api_key "${GEMINI_API_KEY:-}" + +if [[ ! -f vless/vless.conf ]]; then + echo "WARN: vless/vless.conf missing — stub for secret (VPN off until configured)" + cp vless/vless.conf.example vless/vless.conf 2>/dev/null || echo "# stub" > vless/vless.conf +fi +ensure_secret vless_conf "$(cat vless/vless.conf)" + +echo "== Build images ==" +docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" . +docker build -t ai-router/vless-proxy:local ./vless +docker build -t ai-router/vpn-watchdog:local ./watchdog + +echo "== Sync routing config (optional regen) ==" +bash scripts/sync-routing-config.sh || true + +echo "== Deploy stack: ${STACK_NAME} ==" +docker stack deploy -c docker-stack.yml --with-registry-auth "${STACK_NAME}" + +echo "== Wait for services ==" +sleep 8 +docker stack services "${STACK_NAME}" + +echo "" +echo "Done." +echo " Zed URL: ${TEST_BASE_URL:-https://ai-router.ift.calentiq.com}/v1" +echo " LiteLLM: ${TEST_LITELLM_URL:-https://litellm.ift.calentiq.com}/ui" +echo " Smoke: bash scripts/smoke-test.sh" +echo " VPN on: bash scripts/vpn-enable.sh" diff --git a/scripts/install-audit-cron.sh b/scripts/install-audit-cron.sh new file mode 100644 index 0000000..e24bed6 --- /dev/null +++ b/scripts/install-audit-cron.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Install daily audit cron (03:00 MSK) on IFT host +set -euo pipefail + +STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CRON_FILE="/etc/cron.d/ai-router-audit" + +sudo tee "$CRON_FILE" >/dev/null <> /var/log/ai-router/audit.log 2>&1 +EOF + +sudo chmod 644 "$CRON_FILE" +echo "Installed ${CRON_FILE}" diff --git a/scripts/litellm-entrypoint.sh b/scripts/litellm-entrypoint.sh new file mode 100644 index 0000000..b610158 --- /dev/null +++ b/scripts/litellm-entrypoint.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Load Swarm secrets into env and start LiteLLM proxy +set -eu + +read_secret() { + if [ -f "$1" ]; then + tr -d '\n\r' < "$1" + fi +} + +if [ -f /run/secrets/novita_api_key ]; then + export NOVITA_API_KEY="$(read_secret /run/secrets/novita_api_key)" +fi +if [ -f /run/secrets/litellm_master_key ]; then + export LITELLM_MASTER_KEY="$(read_secret /run/secrets/litellm_master_key)" +fi +if [ -f /run/secrets/litellm_salt_key ]; then + export LITELLM_SALT_KEY="$(read_secret /run/secrets/litellm_salt_key)" +fi +if [ -f /run/secrets/groq_api_key ]; then + export GROQ_API_KEY="$(read_secret /run/secrets/groq_api_key)" +fi +if [ -f /run/secrets/gemini_api_key ]; then + export GEMINI_API_KEY="$(read_secret /run/secrets/gemini_api_key)" +fi +if [ -f /run/secrets/postgres_password ]; then + export POSTGRES_PASSWORD="$(read_secret /run/secrets/postgres_password)" + export DATABASE_URL="postgresql://${POSTGRES_USER:-litellm}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-litellm}" +fi + +exec litellm --config /app/config.yaml --port 4000 --num_workers 2 "$@" diff --git a/scripts/router-entrypoint.sh b/scripts/router-entrypoint.sh new file mode 100644 index 0000000..e6db9c3 --- /dev/null +++ b/scripts/router-entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +read_secret() { + if [ -f "$1" ]; then + tr -d '\n\r' < "$1" + fi +} + +if [ -f /run/secrets/litellm_master_key ]; then + export LITELLM_MASTER_KEY="$(read_secret /run/secrets/litellm_master_key)" +fi +if [ -f /run/secrets/router_api_key ]; then + export ROUTER_API_KEY="$(read_secret /run/secrets/router_api_key)" +fi + +exec uvicorn router:app --host 0.0.0.0 --port 8000 "$@" diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh new file mode 100644 index 0000000..e8b6af6 --- /dev/null +++ b/scripts/smoke-test.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Smoke tests — classify (no LLM) + minimal chat +set -euo pipefail + +STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$STACK_DIR" + +set -a +# shellcheck disable=SC1091 +source .env 2>/dev/null || true +set +a + +BASE="${TEST_BASE_URL:-http://127.0.0.1:8000}" +LITELLM_BASE="${TEST_LITELLM_URL:-http://127.0.0.1:4000}" +ROUTER_KEY="${ROUTER_API_KEY:?ROUTER_API_KEY required}" + +CURL=(curl -sf) +if [[ "$BASE" == https:* ]]; then + CURL+=( -k ) +fi + +echo "== Router health ==" +"${CURL[@]}" "${BASE}/health" | jq . + +echo "== Classify SIMPLE ==" +"${CURL[@]}" "${BASE}/classify" \ + -H "Authorization: Bearer ${ROUTER_KEY}" \ + -H "Content-Type: application/json" \ + -d '{"messages":[{"role":"user","content":"Привет, что такое Docker?"}]}' | jq . + +echo "== Classify COMPLEX ==" +"${CURL[@]}" "${BASE}/classify" \ + -H "Authorization: Bearer ${ROUTER_KEY}" \ + -H "Content-Type: application/json" \ + -d '{"messages":[{"role":"user","content":"Спроектируй архитектуру microservices"}]}' | jq . + +echo "== Chat smart-router max_tokens=16 ==" +"${CURL[@]}" "${BASE}/v1/chat/completions" \ + -H "Authorization: Bearer ${ROUTER_KEY}" \ + -H "Content-Type: application/json" \ + -d '{"model":"smart-router","max_tokens":16,"messages":[{"role":"user","content":"bash docker service ls"}]}' \ + | jq '.choices[0].message.content, .x_router_meta // empty' + +if [[ -n "${SKIP_LITELLM_SMOKE:-}" ]]; then + echo "SKIP_LITELLM_SMOKE set — skipping LiteLLM checks" +else + echo "== LiteLLM liveliness ==" + "${CURL[@]}" "${LITELLM_BASE}/health/liveliness" && echo + echo "== LiteLLM UI ==" + curl -sfI -k "${LITELLM_BASE}/ui" | head -3 || true +fi + +echo "All smoke checks passed." diff --git a/scripts/sync-routing-config.sh b/scripts/sync-routing-config.sh new file mode 100644 index 0000000..f4009ba --- /dev/null +++ b/scripts/sync-routing-config.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Sync model_list fragment from config/*.yaml +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +export ROOT + +python3 <<'PY' +import os +import yaml +from pathlib import Path + +root = Path(os.environ["ROOT"]) +rules = yaml.safe_load((root / "config/routing_rules.yaml").read_text(encoding="utf-8")) +matrix = yaml.safe_load((root / "config/model_matrix.yaml").read_text(encoding="utf-8")) + +entries = [] +for name, cfg in matrix.get("models", {}).items(): + novita = cfg.get("novita") + if not novita: + continue + entry = { + "model_name": name, + "litellm_params": { + "model": novita, + "api_key": "os.environ/NOVITA_API_KEY", + }, + } + if cfg.get("rpm"): + entry["litellm_params"]["rpm"] = cfg["rpm"] + entries.append(entry) + +for name, cfg in matrix.get("optional_providers", {}).items(): + entries.append({ + "model_name": name, + "litellm_params": { + "model": cfg["model"], + "api_key": cfg.get("api_key", "os.environ/GROQ_API_KEY"), + }, + }) + +litellm_rules = rules.get("litellm", {}) +entries.append({ + "model_name": "smart-router-internal", + "litellm_params": { + "model": "auto_router/complexity_router", + "drop_params": True, + "complexity_router_default_model": "a-medium-ops", + "complexity_router_config": { + "tiers": { + "SIMPLE": "a-simple", + "MEDIUM": "a-medium-ops", + "MEDIUM_CODE": "a-medium-code", + "COMPLEX": "a-complex", + "REASONING": "a-reasoning", + }, + "classifier_fallback": "heuristic", + "keyword_tier_rules": litellm_rules.get("keyword_tier_rules", []), + "custom_technical_keywords": litellm_rules.get("custom_technical_keywords", []), + "token_thresholds": {"simple": 20, "complex": 500}, + "tier_boundaries": { + "simple_medium": 0.18, + "medium_complex": 0.38, + "complex_reasoning": 0.62, + }, + "session_affinity": True, + "session_affinity_ttl_seconds": 1800, + }, + }, +}) + +out = root / "litellm_config.generated.yaml" +out.write_text( + yaml.dump({"model_list": entries}, allow_unicode=True, sort_keys=False), + encoding="utf-8", +) +print(f"Wrote {out} ({len(entries)} models)") +PY diff --git a/scripts/vendor-litellm-dashboards.sh b/scripts/vendor-litellm-dashboards.sh new file mode 100644 index 0000000..6542a8a --- /dev/null +++ b/scripts/vendor-litellm-dashboards.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Vendor LiteLLM official Grafana dashboard JSON into EventHubDevOps +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LITELLM_REF="${LITELLM_REF:-main-v1.96.0-stable}" +DEVOPS="${DEVOPS_ROOT:-$(cd "$ROOT/../EventHubDevOps" 2>/dev/null && pwd || echo "")}" + +if [[ -z "$DEVOPS" || ! -d "$DEVOPS/ift/observability" ]]; then + echo "DEVOPS_ROOT not found — set to EventHubDevOps path" >&2 + exit 1 +fi + +DEST="${DEVOPS}/ift/observability/grafana/dashboards/litellm" +mkdir -p "$DEST" + +BASE="https://raw.githubusercontent.com/BerriAI/litellm/${LITELLM_REF}/cookbook/litellm_proxy_server/grafana_dashboard" + +fetch() { + local src="$1" dst="$2" + echo "Fetching $src" + curl -fsSL "${BASE}/${src}" -o "${DEST}/${dst}" +} + +fetch "dashboard_v2/grafana_dashboard.json" "litellm-v2.json" +fetch "dashboard_1/grafana_dashboard.json" "litellm-v1.json" || true + +if command -v jq >/dev/null 2>&1; then + for f in litellm-v2.json litellm-v1.json; do + [[ -f "${DEST}/${f}" ]] || continue + jq 'walk(if type=="object" and has("datasource") then .datasource="prometheus" else . end)' \ + "${DEST}/${f}" > "${DEST}/${f}.tmp" && mv "${DEST}/${f}.tmp" "${DEST}/${f}" + done +fi + +echo "Dashboards written to ${DEST}" diff --git a/scripts/vpn-disable.sh b/scripts/vpn-disable.sh new file mode 100644 index 0000000..465f1c4 --- /dev/null +++ b/scripts/vpn-disable.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACK_NAME="${STACK_NAME:-ai-router}" +cd "$STACK_DIR" + +set -a +# shellcheck disable=SC1091 +source .env +set +a + +NO_PROXY_VAL="${NO_PROXY:-localhost,127.0.0.1,api.novita.ai,novita.ai}" + +echo "== Scale vless-proxy and vpn-watchdog to 0 ==" +docker service scale "${STACK_NAME}_vless-proxy=0" "${STACK_NAME}_vpn-watchdog=0" || true + +echo "== Clear litellm proxy env ==" +docker service update \ + --env-rm HTTP_PROXY \ + --env-rm HTTPS_PROXY \ + --env-add "NO_PROXY=${NO_PROXY_VAL}" \ + "${STACK_NAME}_litellm" || true + +echo "VPN disabled." diff --git a/scripts/vpn-enable.sh b/scripts/vpn-enable.sh new file mode 100644 index 0000000..1ecfbc7 --- /dev/null +++ b/scripts/vpn-enable.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACK_NAME="${STACK_NAME:-ai-router}" +cd "$STACK_DIR" + +set -a +# shellcheck disable=SC1091 +source .env +set +a + +VLESS_URL="${VLESS_PROXY_URL:-http://vless-proxy:8080}" +NO_PROXY_VAL="${NO_PROXY:-localhost,127.0.0.1,api.novita.ai,novita.ai}" + +echo "== Scale vless-proxy and vpn-watchdog to 1 ==" +docker service scale "${STACK_NAME}_vless-proxy=1" "${STACK_NAME}_vpn-watchdog=1" + +echo "== Wait for vless-proxy healthy (max 90s) ==" +for i in $(seq 1 18); do + CID=$(docker ps -q -f "name=${STACK_NAME}_vless-proxy" | head -1) + if [[ -n "$CID" ]]; then + STATUS=$(docker inspect --format='{{.State.Health.Status}}' "$CID" 2>/dev/null || echo "starting") + if [[ "$STATUS" == "healthy" ]]; then + echo "vless-proxy is healthy" + break + fi + echo " attempt $i: status=$STATUS" + fi + sleep 5 +done + +echo "== E2E tunnel probe ==" +docker run --rm --network "${STACK_NAME}_ai-internal" curlimages/curl:8.5.0 \ + curl -sf -x "${VLESS_URL}" --max-time 15 https://www.google.com/generate_204 \ + || echo "WARN: tunnel probe failed (may need valid vless.conf)" + +echo "== Point litellm through VPN ==" +docker service update \ + --env-add "HTTP_PROXY=${VLESS_URL}" \ + --env-add "HTTPS_PROXY=${VLESS_URL}" \ + --env-add "NO_PROXY=${NO_PROXY_VAL}" \ + "${STACK_NAME}_litellm" + +echo "VPN enabled. Novita bypasses via NO_PROXY." diff --git a/vless/Dockerfile b/vless/Dockerfile new file mode 100644 index 0000000..78fa8aa --- /dev/null +++ b/vless/Dockerfile @@ -0,0 +1,3 @@ +FROM thejohnd0e/vless-to-http:latest + +# vless.conf mounted via Swarm secret at /app/vless.conf diff --git a/vless/vless.conf.example b/vless/vless.conf.example new file mode 100644 index 0000000..d0ced5c --- /dev/null +++ b/vless/vless.conf.example @@ -0,0 +1,4 @@ +# Example VLESS config — copy to vless.conf and fill, or use Swarm secret vless_conf +# Format: see https://github.com/thejohnd0e/VLESS-to-HTTP + +vless://UUID@host:port?encryption=none&security=tls&sni=example.com#label diff --git a/watchdog/Dockerfile b/watchdog/Dockerfile new file mode 100644 index 0000000..c307975 --- /dev/null +++ b/watchdog/Dockerfile @@ -0,0 +1,8 @@ +FROM docker:27-cli + +RUN apk add --no-cache curl bash + +COPY vpn-watchdog.sh /usr/local/bin/vpn-watchdog.sh +RUN chmod +x /usr/local/bin/vpn-watchdog.sh + +CMD ["/usr/local/bin/vpn-watchdog.sh"] diff --git a/watchdog/vpn-watchdog.sh b/watchdog/vpn-watchdog.sh new file mode 100644 index 0000000..374cfed --- /dev/null +++ b/watchdog/vpn-watchdog.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# End-to-end VPN tunnel probe; restart vless-proxy on failure +set -euo pipefail + +STACK_NAME="${STACK_NAME:-ai-router}" +VLESS_PROXY_URL="${VLESS_PROXY_URL:-http://vless-proxy:8080}" +CHECK_INTERVAL="${CHECK_INTERVAL:-60}" + +log() { echo "[vpn-watchdog] $(date -Iseconds) $*"; } + +while true; do + if curl -sf -x "${VLESS_PROXY_URL}" --max-time 15 https://www.google.com/generate_204 >/dev/null 2>&1; then + log "tunnel OK" + else + log "tunnel FAIL — force restart ${STACK_NAME}_vless-proxy" + docker service update --force "${STACK_NAME}_vless-proxy" || log "service update failed" + fi + sleep "${CHECK_INTERVAL}" +done