diff --git a/.env.example b/.env.example index 33d0810..3efb492 100644 --- a/.env.example +++ b/.env.example @@ -9,7 +9,7 @@ DEPLOY_HOST=https://ai-router.ift.calentiq.com # --- LiteLLM --- LITELLM_MASTER_KEY=sk-litellm-change-me LITELLM_SALT_KEY=sk-salt-generate-once-never-change -LITELLM_IMAGE=ghcr.io/berriai/litellm:main-v1.96.0-stable +LITELLM_IMAGE=ghcr.io/berriai/litellm:v1.95.0 PROXY_BASE_URL=https://litellm.ift.calentiq.com # Postgres (LiteLLM Admin UI + spend logs) @@ -17,15 +17,27 @@ POSTGRES_USER=litellm POSTGRES_DB=litellm POSTGRES_PASSWORD=change-me-postgres -# Novita AI (direct from IFT, no VPN) +# --- Provider switch (lane backends) --- +# hybrid (default) = Novita workers + Novita plan/verify | openrouter = VPN reserve +PRIMARY_PROVIDER=hybrid + +# Novita AI (direct from IFT, no VPN) — required for hybrid NOVITA_API_KEY= -# Optional fallbacks +# Anthropic — optional (card/region often blocked); hierarchical uses Novita plan/verify +ANTHROPIC_API_KEY= + +# Optional fixed fallbacks GROQ_API_KEY= GEMINI_API_KEY= +TOGETHER_API_KEY= +# OpenRouter reserve (VPN). Note: Anthropic/OpenAI/Google may be region-blocked on billing address. +OPENROUTER_API_KEY= -# GigaChat — optional LLM classifier (hybrid mode, freemium for физлица) -# Authorization key from https://developers.sber.ru/studio/ → GigaChat API +# xAI Grok — VPN only +XAI_API_KEY= + +# GigaChat — classifier (hybrid mode) GIGACHAT_CREDENTIALS= CLASSIFIER_MODE=hybrid CLASSIFIER_LLM_MODEL=gigachat-classifier @@ -33,10 +45,18 @@ CLASSIFIER_LLM_MODEL=gigachat-classifier # Budget (USD/month, also in litellm_config.yaml) LITELLM_MAX_BUDGET=50 -# --- VPN (off by default) --- +# --- VPN (OpenRouter + Grok + Anthropic → VPN; Novita/Groq/Gemini/GigaChat → NO_PROXY) --- +# Hierarchical plan/verify на Novita — VPN НЕ обязателен +# 1) vless/vless.conf или VLESS_SUB_URL +# 2) VPN_ENABLED=true → deploy + vpn-enable.sh (для OR/Grok/Anthropic) +# 3) api.anthropic.com, openrouter.ai, api.x.ai НЕ в NO_PROXY VPN_ENABLED=false +VLESS_SUB_URL= +VLESS_HWID= +VLESS_UA=Happ/3.3.6/Windows/2607171516600 VLESS_PROXY_URL=http://vless-proxy:8080 -NO_PROXY=localhost,127.0.0.1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,api.novita.ai,novita.ai,git.sabilin.com,*.ift.calentiq.com,ift.calentiq.com,*.ift.eventhub.local,*.eventhub.local +# См. scripts/no-proxy-default.sh +NO_PROXY=localhost,127.0.0.1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,api.novita.ai,novita.ai,api.groq.com,groq.com,generativelanguage.googleapis.com,ngw.devices.sberbank.ru,gigachat.devices.sberbank.ru,git.sabilin.com,*.ift.calentiq.com,ift.calentiq.com,*.ift.eventhub.local,*.eventhub.local DISABLE_AIOHTTP_TRANSPORT=True USE_AIOHTTP_TRANSPORT=False diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 3e23b1d..97297bf 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -17,9 +17,22 @@ jobs: - name: Lint Python syntax run: python3 -m py_compile router/*.py + - name: Unit tests (hierarchical) + run: | + pip install httpx pyyaml + PYTHONPATH=router python3 -m unittest discover -s test/unit -v + sync-config: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Sync routing config + - name: Install PyYAML + run: pip install pyyaml + - name: Sync routing config (hybrid) + env: + PRIMARY_PROVIDER: hybrid run: bash scripts/sync-routing-config.sh + - name: Sync routing config (openrouter reserve) + env: + PRIMARY_PROVIDER: openrouter + run: bash scripts/gen-litellm-config.py diff --git a/.gitignore b/.gitignore index 225268a..d8faf44 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ __pycache__/ litellm_config.generated.yaml /tmp/ *.log +.tmp-* +.tmp/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..20fd9d8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,38 @@ +# AGENTS.md — EventHubAiRouter + +Zed OpenAI-compatible gateway на IFT. Карта продукта: `../EventHubSpec/ZED-ARCHITECTURE.md`. + +## Ключевые файлы + +| Файл | Роль | +|------|------| +| `router/router.py` | `/v1/chat/completions`; ветка `tools` → agent hierarchical | +| `router/hierarchical.py` | Plan (Max) / workers / verify (DeepSeek) / `plan_confirm` | +| `router/agent_hier.py` | При tools: executor + tool_calls (Zed Write); без synthetic по умолчанию | +| `router/orchestrator.py` | Tiers / lanes, Redis session | +| `config/orchestration.yaml` | Роли моделей, таймауты, `plan_confirm`, synthetic=never | +| `config/providers.yaml` | Novita / VPN (`novita-planner` = Qwen3.8-Max) | +| `scripts/gen-litellm-config.py` | LiteLLM config | + +## Модели (бюджет) + +| Роль | Модель | +|------|--------| +| Planner | `novita-planner` → **qwen/qwen3.8-max** | +| Routine tools / medium_code | **qwen3-coder-30b** (`a-medium-code`) | +| Hard | DeepSeek V3.2 (`b-complex`); `quality=max` → Max (`c-complex`) | +| Verify | DeepSeek V3.2 | +| Text SIMPLE (без tools) | Llama 8B — **не** в Write/tools | + +## Поведение для Zed + +- **Primary клиент:** Zed Agent → `smart-router` (не Claude Code/Codex ACP). +- **Write** (`tools`): Max-план → утверждение → Coder-30B `tool_calls` (файлы правит Zed); escalate на DeepSeek после edit fails. +- **Без tools:** text hierarchical (plan → workers → synth). +- Endpoint: `https://ai-router.ift.calentiq.com/v1`. + +## Не делать + +- Не коммитить секреты (`.env`, ключи) и локальные `.tmp-*`. +- Не сажать Llama / `a-simple` на tool-loop. +- Deploy на IFT — только по просьбе; shell через WSL + `run-wsl-sh.sh`. diff --git a/README.md b/README.md index 06b3e13..822e59c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # EventHub AI Router — Zed gateway + LiteLLM on IFT -OpenAI-compatible endpoint for [Zed](https://zed.dev) with automatic tier/lane routing (Novita AI), optional VLESS VPN, Grafana metrics, and LiteLLM Admin UI. +OpenAI-compatible endpoint for [Zed](https://zed.dev) with tier/lane routing, optional **hierarchical** orchestration (**Qwen3.8-Max** plan → cheap **Qwen3-Coder-30B** workers/executor → conditional **DeepSeek V3.2** verify), GigaChat classifier, VLESS VPN, Grafana, LiteLLM Admin UI. + +Цель: качество правок близко к Cursor при экономии бюджета — дорогие модели только на plan/hard/verify, рутина на дешёвом coder. ## URLs (IFT, Calentiq TLS) @@ -11,123 +13,146 @@ OpenAI-compatible endpoint for [Zed](https://zed.dev) with automatic tier/lane r | **Grafana** | `https://grafana.ift.calentiq.com` | | Legacy (self-signed) | `https://ai-router.ift.eventhub.local/v1` | -## Zed settings.json +## Клиент (канон): Zed Agent → AiRouter + +**Primary:** встроенный Zed Agent (OpenAI-compatible), не Claude Code / Codex ACP — те обходят Novita-бюджет. + +Пример `settings.json` (Zed 2026 openai-compatible provider): ```json { - "openai": { - "api_url": "https://ai-router.ift.calentiq.com/v1", - "api_key": "" + "language_models": { + "openai_compatible": { + "AiRouter": { + "api_url": "https://ai-router.ift.calentiq.com/v1", + "available_models": [ + { + "name": "smart-router", + "display_name": "EventHub smart-router", + "max_tokens": 16384, + "max_output_tokens": 4096, + "capabilities": { + "tools": true, + "images": false, + "parallel_tool_calls": false + } + } + ] + } + } }, - "assistant": { - "default_model": { "provider": "openai", "model": "smart-router" } + "agent": { + "default_model": { + "provider": "AiRouter", + "model": "smart-router" + } } } ``` +API key: `ROUTER_API_KEY` в Agent Settings / provider UI. + +Checklist: + +- модель `smart-router`, `capabilities.tools: true` +- профиль **Write** (tools on) для правок файлов +- Ask без tools → text hierarchical / lane routing + +Optional headers / metadata: + +- `X-AI-Quality: auto|economy|balanced|max` — `max` поднимает hard на Qwen3.8-Max +- `X-AI-Orchestrate: off|auto|force` — hierarchical (`force` always; `auto` на COMPLEX/REASONING; `economy` quality пропускает hierarchical) + +### Запасной клиент (тот же AiRouter) + +Если native Zed Agent loop хрупкий: **OpenCode** или **Cline** с `base_url=https://ai-router.ift.calentiq.com/v1` и тем же ключом — иерархия бюджета сохраняется. Claude Code / Codex ACP в Zed — **не** primary (свой биллинг). + +### Zed Agent Write + hierarchical + +Если в запросе есть `tools` (профиль **Write**), gateway **не** подменяет agent-цикл текстовым синтезом: + +1. **Max**-планировщик строит план (`paths` / `edit_goal` / `constraints`) → при `plan_confirm: true` ждёт **утвердить / правка / отмена** +2. После `ok` — **Qwen3-Coder-30B** executor получает `tools` и отвечает **`tool_calls`** (Zed сам правит файлы) +3. Пока в истории есть `role: tool` / `tool_calls` — только executor; при повторных fail edit → escalate на DeepSeek (без synthetic tool_calls) + +Чат без tools (Ask / Minimal) по-прежнему может идти через полный text hierarchical. + +### Progress в Zed (hierarchical) + +При `stream: true` (дефолт Zed) gateway сразу стримит блок **«Ход выполнения»**: план → волны workers → verify → синтез, затем `---` и финальный ответ. Конфиг: `hierarchical.stream_progress` / `progress_in_content` в `config/orchestration.yaml`. + +## Provider profiles (`PRIMARY_PROVIDER`) + +| Profile | Keys | VPN | Role | +|---------|------|-----|------| +| **`hybrid`** (default) | `NOVITA_API_KEY` | workers direct | Novita workers + **Novita** plan/verify | +| **`openrouter`** | `OPENROUTER_API_KEY` | **VPN** | Reserve (`:free` / non-Anthropic); Claude/OpenAI/Google may be region-blocked | + +GigaChat (`GIGACHAT_CREDENTIALS`) — classifier. Optional: Groq / Gemini / Grok / Anthropic (если биллинг заработает). + +```bash +bash scripts/switch-provider.sh hybrid +bash scripts/deploy.sh +# VPN нужен только для OpenRouter / Grok / Anthropic — не для Novita hierarchical +``` + +## Hierarchical flow (COMPLEX / force) + +1. **`novita-planner`** (**Qwen3.8-Max**) — JSON subtasks (`simple` | `medium_code` | `hard`) + `paths` / `edit_goal` +2. **Workers / agent executor**: `simple` → Llama (text only); `medium_code` → **Qwen3-Coder-30B**; `hard` → **DeepSeek V3.2**; `quality=max` hard → **Qwen3.8-Max** +3. Checks; **`novita-verifier`** (DeepSeek V3.2) только на fail / hard / `quality=max` +4. **Synthesize** (text path) на `a-medium-code` + +> Anthropic / OpenRouter-Claude недоступны при RU billing — plan/verify идут через Novita. + +Response `x_router_meta.mode=hierarchical` / `hierarchical_agent` with `worker_calls`, `executor_model`, `role_cost`, `verify_skipped`, etc. + ## Architecture -- **Gateway** (`router/`) — vision/OCR split, tier classification, lanes A/B/C orchestration, Redis session -- **LiteLLM** — named models `a-*` / `b-*` / `c-*`, Auto Router v2 fallback (`smart-router-internal`) -- **PostgreSQL** — LiteLLM Admin UI, spend logs -- **Redis** — response cache + gateway session context -- **VPN** — `vless-proxy` replicas=0 by default; `scripts/vpn-enable.sh` +- **Gateway** (`router/`) — classify, lanes A/B/C, hierarchical runner, Redis session +- **LiteLLM** — named models + Auto Router (`smart-router-internal`) +- **PostgreSQL** — spend logs / Admin UI +- **Redis** — cache + session +- **VPN** — `vless-proxy` (Anthropic, OpenRouter, Grok) -Config sources: - -- `config/routing_rules.yaml` — tier keywords -- `config/model_matrix.yaml` — tier × lane → Novita models -- `config/orchestration.yaml` — start lanes, escalation, budget caps +Config: `config/providers.yaml`, `orchestration.yaml`, `routing_rules.yaml`, `model_matrix.yaml`. ## Deploy on IFT ```bash git clone git.sabilin.com/eventhub/EventHubAiRouter /opt/ai-router-stack cd /opt/ai-router-stack -cp .env.example .env # fill NOVITA_API_KEY, keys, POSTGRES_PASSWORD, LITELLM_SALT_KEY +cp .env.example .env # NOVITA_API_KEY; PRIMARY_PROVIDER=hybrid bash scripts/deploy.sh bash scripts/smoke-test.sh -sudo bash scripts/install-audit-cron.sh # daily Novita audit 03:00 MSK +SMOKE_HIERARCHICAL=1 bash scripts/smoke-test.sh ``` -Prerequisites: Docker Swarm, external network `eventhub-ift-net`, Traefik routes in EventHubDevOps. - -### Swarm secrets - -Created automatically by `deploy.sh` from `.env`: `novita_api_key`, `litellm_master_key`, `litellm_salt_key`, `router_api_key`, `postgres_password`, optional `groq_api_key`, `gemini_api_key`, `vless_conf`. +Swarm secrets: `novita_api_key`, optional `anthropic_api_key` / `openrouter_api_key`, `gigachat_credentials`, … **`LITELLM_SALT_KEY`** — generate once, never change after first deploy. ## VPN ```bash -bash scripts/vpn-enable.sh # scale vless + watchdog, set HTTP_PROXY on litellm +bash scripts/vpn-enable.sh bash scripts/vpn-disable.sh ``` -Novita always direct (`NO_PROXY=api.novita.ai`). +| Provider | Host | Route | +|----------|------|-------| +| Novita | `api.novita.ai` | **direct** (NO_PROXY) | +| Groq / Gemini / GigaChat | … | **direct** | +| **Anthropic** | `api.anthropic.com` | **VPN** (App unavailable without) | +| **OpenRouter** | `openrouter.ai` | **VPN** | +| **xAI Grok** | `api.x.ai` | **VPN** | -## Observability +Canon: `scripts/no-proxy-default.sh`. -- Prometheus scrape: `litellm:4000/metrics`, `ai-router:8000/metrics` (via `eventhub-ift-net`) -- Grafana: vendored LiteLLM v2 dashboard + custom `gateway-tier-lane.json` (EventHubDevOps) -- Uptime Kuma monitors — see table in plan / configure manually +## LLM classifier (GigaChat) + +`CLASSIFIER_MODE=hybrid|heuristic|llm` — see `config/orchestration.yaml`. Freemium GigaChat for personal test. ## Scripts -| Script | Purpose | -|--------|---------| -| `deploy.sh` | secrets + build + stack deploy | -| `smoke-test.sh` | health, classify, chat max_tokens=16 | -| `sync-routing-config.sh` | regen `litellm_config.generated.yaml` | -| `audit-novita-pricing.sh` | daily model catalog check | -| `vendor-litellm-dashboards.sh` | fetch Grafana JSON → EventHubDevOps | -| `vpn-enable.sh` / `vpn-disable.sh` | VPN toggle | - -## Quality modes - -Header `X-AI-Quality: auto|economy|balanced|max` or `metadata.quality_mode` in request body. Default: `auto` (start lane A/B by tier, escalate on retry/5xx). - -## LLM classifier (GigaChat, optional) - -When `CLASSIFIER_MODE=hybrid` (default) and `GIGACHAT_CREDENTIALS` is set: - -1. **Heuristic** classify first (0 cost) -2. If `confidence < 0.6` → one call to **GigaChat-2-Lite** via LiteLLM (`gigachat-classifier`) -3. Lane orchestration unchanged (A/B/C, Redis, budget) - -| `CLASSIFIER_MODE` | Behavior | -|-------------------|----------| -| `heuristic` | Keywords only (no GigaChat) | -| `hybrid` | GigaChat only on low confidence | -| `llm` | Always GigaChat for text (except vision) | - -Freemium GigaChat — для личного некомmercial теста; prod — юр. тариф Сбера. - -Setup: [developers.sber.ru](https://developers.sber.ru/docs/ru/gigachat/quickstart/ind-create-project) → Authorization key → `.env` `GIGACHAT_CREDENTIALS`. - -Response field: `x_router_meta.classifier_source` = `heuristic` | `gigachat` | `heuristic_fallback`. - -## Response metadata - -Each chat response includes `x_router_meta`: - -```json -{ - "tier": "MEDIUM_OPS", - "lane": "A", - "model": "a-medium-ops", - "escalation_level": 0, - "quality_mode": "auto" -} -``` - -## Local build - -```bash -docker build -f router/Dockerfile -t ai-router-gateway:local . -``` - -## CI - -Gitea Actions: `.gitea/workflows/ci.yml` — build gateway image, sync config check. +See `scripts/` — `deploy.sh`, `smoke-test.sh`, `gen-litellm-config.py`, `switch-provider.sh`, VPN helpers. diff --git a/config/litellm.base.yaml b/config/litellm.base.yaml new file mode 100644 index 0000000..a6f942e --- /dev/null +++ b/config/litellm.base.yaml @@ -0,0 +1,37 @@ +# Static LiteLLM settings — model_list generated by scripts/gen-litellm-config.py + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + store_model_in_db: false + max_budget: 50 + budget_duration: 30d + +litellm_settings: + drop_params: true + set_verbose: false + request_timeout: 300 + num_retries: 2 + cache: true + cache_params: + type: redis + host: redis + port: 6379 + ttl: 3600 + callbacks: ["prometheus"] + require_auth_for_metrics_endpoint: false + +environment_variables: + NOVITA_API_KEY: os.environ/NOVITA_API_KEY + GROQ_API_KEY: os.environ/GROQ_API_KEY + XAI_API_KEY: os.environ/XAI_API_KEY + GEMINI_API_KEY: os.environ/GEMINI_API_KEY + GIGACHAT_CREDENTIALS: os.environ/GIGACHAT_CREDENTIALS + TOGETHER_API_KEY: os.environ/TOGETHER_API_KEY + OPENROUTER_API_KEY: os.environ/OPENROUTER_API_KEY + +router_settings: + routing_strategy: simple-shuffle + num_retries: 2 + timeout: 300 + allowed_fails: 5 + cooldown_time: 15 diff --git a/config/model_matrix.yaml b/config/model_matrix.yaml index cd57087..1df54c7 100644 --- a/config/model_matrix.yaml +++ b/config/model_matrix.yaml @@ -1,104 +1,76 @@ -# tier × lane (A/B/C) → Novita model id + LiteLLM fallbacks +# Tier × lane orchestration metadata (model IDs → config/providers.yaml) lanes: A: economy B: balanced C: max +# LiteLLM per-lane RPM (proxy-side). Zed agent bursts tool calls — keep high for personal IFT. +# OpenRouter :free upstream may still 429; router fallbacks + cooldown_time apply separately. models: a-simple: - novita: novita/qwen/qwen3-4b-fp8 fallbacks: [b-simple, groq-llama-8b] - rpm: 60 + rpm: 300 b-simple: - novita: novita/meta-llama/llama-3.1-8b-instruct fallbacks: [a-simple, groq-llama-8b] - rpm: 60 + rpm: 300 c-simple: - novita: novita/qwen/qwen3-8b-fp8 fallbacks: [b-simple] - rpm: 60 + rpm: 300 a-medium-ops: - novita: novita/qwen/qwen3-8b-fp8 fallbacks: [a-medium-code, b-medium-ops] - rpm: 40 + rpm: 240 b-medium-ops: - novita: novita/deepseek/deepseek-v3.2 fallbacks: [a-medium-code, c-medium-ops] - rpm: 40 + rpm: 240 c-medium-ops: - novita: novita/deepseek/deepseek-v3.2 fallbacks: [b-medium-ops] - rpm: 40 + rpm: 240 a-medium-code: - novita: novita/qwen/qwen3-coder-30b-a3b-instruct fallbacks: [a-medium-ops, groq-qwen-coder] - rpm: 40 + rpm: 240 b-medium-code: - novita: novita/qwen/qwen3-coder-30b-a3b-instruct fallbacks: [b-medium-ops, c-medium-code] - rpm: 40 + rpm: 240 c-medium-code: - novita: novita/qwen/qwen3-coder-30b-a3b-instruct fallbacks: [b-medium-code, c-medium-ops] - rpm: 30 + rpm: 180 a-complex: - novita: novita/deepseek/deepseek-v3.2 fallbacks: [b-complex, a-reasoning] - rpm: 30 + rpm: 180 b-complex: - novita: novita/deepseek/deepseek-r1-0528 fallbacks: [a-complex, c-complex] - rpm: 20 + rpm: 120 c-complex: - novita: novita/deepseek/deepseek-r1-turbo fallbacks: [b-complex] - rpm: 15 + rpm: 120 a-reasoning: - novita: novita/deepseek/deepseek-r1-0528-qwen3-8b fallbacks: [b-reasoning, gemini-flash] - rpm: 30 + rpm: 120 b-reasoning: - novita: novita/deepseek/deepseek-r1-0528 fallbacks: [a-reasoning, c-reasoning] - rpm: 20 + rpm: 120 c-reasoning: - novita: novita/deepseek/deepseek-r1-turbo fallbacks: [b-reasoning, gemini-flash] - rpm: 15 + rpm: 120 a-vision-ocr: - novita: novita/paddlepaddle/paddleocr-vl fallbacks: [a-vision] - rpm: 30 + rpm: 120 no_escalation: true a-vision: - novita: novita/qwen/qwen3-vl-30b-a3b-instruct fallbacks: [a-vision-ocr] - rpm: 20 + rpm: 120 b-vision: - novita: novita/qwen/qwen2.5-vl-72b-instruct fallbacks: [a-vision, c-vision] - rpm: 15 + rpm: 90 c-vision: - novita: novita/qwen/qwen3-vl-235b-a22b-instruct fallbacks: [b-vision] - rpm: 10 - -optional_providers: - groq-llama-8b: - model: groq/llama-3.1-8b-instant - api_key: os.environ/GROQ_API_KEY - groq-qwen-coder: - model: groq/qwen-qwen-2.5-coder-32b - api_key: os.environ/GROQ_API_KEY - gemini-flash: - model: gemini/gemini-2.0-flash - api_key: os.environ/GEMINI_API_KEY + rpm: 90 audit: price_drift_threshold_pct: 10 diff --git a/config/orchestration.yaml b/config/orchestration.yaml index 6b23ecf..4e80cc3 100644 --- a/config/orchestration.yaml +++ b/config/orchestration.yaml @@ -32,9 +32,89 @@ redis: ttl_sec: 1800 classifier: - # heuristic | hybrid (default) | llm mode: hybrid llm_model: gigachat-classifier low_confidence_threshold: 0.6 timeout_sec: 15 max_tokens: 64 + +# Strong Max plan → cheap coder workers/executor → conditional DeepSeek verify +hierarchical: + enabled: true + trigger_tiers: [COMPLEX, REASONING] + header: X-AI-Orchestrate # off | auto | force + planner_model: novita-planner + verifier_model: novita-verifier + planner_fallback: b-complex + verifier_fallback: b-complex + verify_policy: on_fail_or_hard + verify_input: digests + synthesize_model: a-medium-code + plan_confirm: true + agent_executor_model: a-medium-code + agent_escalate_model: b-complex + agent_escalate_after_edit_failures: 2 + skip_synthesize_if_single: true + max_subtasks: 3 + max_verify_retries: 1 + plan_max_tokens: 1024 + verify_max_tokens: 384 + synthesize_max_tokens: 1024 + worker_max_tokens: 768 + wall_clock_sec: 600 + digest_chars: 300 + call_timeout_sec: 300 + # First turn (kickstart read) can stay longer; after read_file body Coder hangs — + # tool_loop uses short timeout + escalate_after_read → DeepSeek. + executor_timeout_sec: 90 + executor_num_retries: 0 + executor_input_chars: 16000 + executor_tool_loop_chars: 12000 + executor_tool_loop_timeout_sec: 45 + executor_newest_tool_chars: 3500 + executor_minimal_context: true + executor_max_tools: 6 + executor_tool_desc_chars: 120 + executor_tools_full_chars: 8000 + executor_max_tokens: 1024 + executor_use_stream: false + executor_synthetic_on_timeout: true + # First turn only: synthetic read_file on plan path if model returns prose. + # Mid-loop still stops (no fake edits). + executor_synthetic_mode: kickstart_only + executor_max_synthetic_continues: 1 + executor_max_edit_failures: 4 + executor_midloop_model_retry: true + executor_midloop_escalate: true + # After successful read_file skip DeepSeek (was 408→400 loop) → edit_file + executor_force_edit_after_read: true + # After a successful read_file, skip cheap coder — it stalls on fat tool results + executor_escalate_after_read: true + # Rewrite read_file on already-read paths → edit (breaks re-read loops) + executor_rewrite_reread: true + # When model returns 200 without tool_calls: one escalate retry before kickstart + executor_no_tools_escalate: true + # After plan approve: 1) deterministic path index 2) if gaps → cheap find_* model + path_resolve_enabled: true + path_resolve_model: a-simple + path_resolve_timeout_sec: 45 + path_resolve_max_tools: 3 + # DevOps (*.calentiq.com / Bad Gateway): probe live ports before edit + runtime_probe_enabled: true + runtime_probe_timeout_sec: 8 + # Optional: ssh host for docker inspect (empty = TCP probe from gateway only) + runtime_probe_ssh_host: "" + # Quiet Zed: milestones only (no ctx bars / LiteLLM waits / subtask dumps) + progress_verbose: false + show_context_fill: false + progress_in_content: true + stream_progress: true + context_window_tokens: 32768 + plan_input_chars: 48000 + llm_cache_ttl_sec: 3600 + worker_map: + simple: a-simple + medium_code: a-medium-code + hard: b-complex + worker_map_max: + hard: c-complex diff --git a/config/providers.yaml b/config/providers.yaml new file mode 100644 index 0000000..71159c3 --- /dev/null +++ b/config/providers.yaml @@ -0,0 +1,133 @@ +# Provider profiles — active: PRIMARY_PROVIDER env (default: hybrid) +# Regenerate: bash scripts/gen-litellm-config.py + +profiles: + hybrid: + label: "Hybrid: Novita workers + Novita plan/verify (no Claude geo-billing)" + welcome_credit: false + requires: [NOVITA_API_KEY] + smart_router: + model: novita/meta-llama/llama-3.1-8b-instruct + api_key: NOVITA_API_KEY + fallbacks: + # No groq/gemini here — VPN off on IFT; those hang until client timeout. + a-simple: [b-simple, a-medium-ops] + a-medium-code: [b-medium-code, a-complex] + a-complex: [b-complex, a-reasoning] + b-complex: [a-complex, c-complex] + a-reasoning: [b-reasoning, a-complex] + a-vision: [a-vision-ocr] + smart-router-internal: [a-medium-ops, a-medium-code, a-complex] + # Planner is Max — fallback to DeepSeek then coder (never Llama) + novita-planner: [b-complex, a-medium-code] + novita-verifier: [b-complex, a-medium-code] + default_fallbacks: [a-medium-code, a-complex, a-medium-ops] + lanes: + # Prefer Llama for fast lanes — Qwen3 puts answers in reasoning_content + a-simple: { model: novita/meta-llama/llama-3.1-8b-instruct, api_key: NOVITA_API_KEY } + b-simple: { model: novita/meta-llama/llama-3.1-8b-instruct, api_key: NOVITA_API_KEY } + c-simple: { model: novita/meta-llama/llama-3.1-8b-instruct, api_key: NOVITA_API_KEY } + a-medium-ops: { model: novita/meta-llama/llama-3.1-8b-instruct, api_key: NOVITA_API_KEY } + b-medium-ops: { model: novita/meta-llama/llama-3.1-8b-instruct, api_key: NOVITA_API_KEY } + c-medium-ops: { model: novita/meta-llama/llama-3.1-8b-instruct, api_key: NOVITA_API_KEY } + # medium_code → coder / Plus-class (disable thinking — else stream hangs) + a-medium-code: { model: novita/qwen/qwen3-coder-30b-a3b-instruct, api_key: NOVITA_API_KEY, disable_thinking: true } + b-medium-code: { model: novita/qwen/qwen3-coder-30b-a3b-instruct, api_key: NOVITA_API_KEY, disable_thinking: true } + c-medium-code: { model: novita/qwen/qwen3-coder-30b-a3b-instruct, api_key: NOVITA_API_KEY, disable_thinking: true } + # complex/reasoning A/B → coder/deepseek; C (hard) → Qwen3.8-Max only + a-complex: { model: novita/qwen/qwen3-coder-30b-a3b-instruct, api_key: NOVITA_API_KEY, disable_thinking: true } + b-complex: { model: novita/deepseek/deepseek-v3.2, api_key: NOVITA_API_KEY } + c-complex: { model: novita/qwen/qwen3.8-max, api_key: NOVITA_API_KEY, disable_thinking: true } + a-reasoning: { model: novita/qwen/qwen3-coder-30b-a3b-instruct, api_key: NOVITA_API_KEY, disable_thinking: true } + b-reasoning: { model: novita/deepseek/deepseek-v3.2, api_key: NOVITA_API_KEY } + c-reasoning: { model: novita/qwen/qwen3.8-max, api_key: NOVITA_API_KEY, disable_thinking: true } + a-vision-ocr: { model: novita/paddlepaddle/paddleocr-vl, api_key: NOVITA_API_KEY } + a-vision: { model: novita/qwen/qwen3-vl-30b-a3b-instruct, api_key: NOVITA_API_KEY } + b-vision: { model: novita/qwen/qwen2.5-vl-72b-instruct, api_key: NOVITA_API_KEY } + c-vision: { model: novita/qwen/qwen3-vl-235b-a22b-instruct, api_key: NOVITA_API_KEY } + + openrouter: + label: OpenRouter reserve (VPN; free :free + pay-as-you-go) + welcome_credit: true + requires: [OPENROUTER_API_KEY] + requires_vpn: true + smart_router: + model: openrouter/openrouter/free + api_key: OPENROUTER_API_KEY + fallbacks: + a-simple: [b-simple, c-simple] + a-medium-code: [b-medium-code, c-medium-code] + a-complex: [b-complex, c-complex] + smart-router-internal: [a-medium-ops, a-medium-code] + default_fallbacks: [a-medium-code, a-medium-ops] + lanes: + a-simple: { model: openrouter/openai/gpt-oss-20b:free, api_key: OPENROUTER_API_KEY } + b-simple: { model: openrouter/openai/gpt-oss-20b:free, api_key: OPENROUTER_API_KEY } + c-simple: { model: openrouter/openrouter/free, api_key: OPENROUTER_API_KEY } + a-medium-ops: { model: openrouter/openrouter/free, api_key: OPENROUTER_API_KEY } + b-medium-ops: { model: openrouter/openrouter/free, api_key: OPENROUTER_API_KEY } + c-medium-ops: { model: openrouter/openrouter/free, api_key: OPENROUTER_API_KEY } + a-medium-code: { model: openrouter/cohere/north-mini-code:free, api_key: OPENROUTER_API_KEY } + b-medium-code: { model: openrouter/cohere/north-mini-code:free, api_key: OPENROUTER_API_KEY } + c-medium-code: { model: openrouter/cohere/north-mini-code:free, api_key: OPENROUTER_API_KEY } + a-complex: { model: openrouter/google/gemma-4-26b-a4b-it:free, api_key: OPENROUTER_API_KEY } + b-complex: { model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free, api_key: OPENROUTER_API_KEY } + c-complex: { model: openrouter/nvidia/nemotron-3-nano-30b-a3b:free, api_key: OPENROUTER_API_KEY } + a-reasoning: { model: openrouter/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free, api_key: OPENROUTER_API_KEY } + b-reasoning: { model: openrouter/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free, api_key: OPENROUTER_API_KEY } + c-reasoning: { model: openrouter/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free, api_key: OPENROUTER_API_KEY } + a-vision-ocr: { model: openrouter/openrouter/free, api_key: OPENROUTER_API_KEY } + a-vision: { model: openrouter/openrouter/free, api_key: OPENROUTER_API_KEY } + b-vision: { model: openrouter/openrouter/free, api_key: OPENROUTER_API_KEY } + c-vision: { model: openrouter/openrouter/free, api_key: OPENROUTER_API_KEY } + +# Always registered when keys present (all profiles) +fixed_models: + # Hierarchical: strong Max plan (rare); DeepSeek verify (conditional) + novita-planner: + model: novita/qwen/qwen3.8-max + api_key: NOVITA_API_KEY + max_tokens: 1024 + temperature: 0 + disable_thinking: true + novita-verifier: + model: novita/deepseek/deepseek-v3.2 + api_key: NOVITA_API_KEY + max_tokens: 1024 + temperature: 0 + # Optional Claude — only if ANTHROPIC_API_KEY works later (VPN + billing) + claude-haiku-planner: + model: anthropic/claude-haiku-4-5-20251001 + api_key: ANTHROPIC_API_KEY + requires_vpn: true + max_tokens: 1024 + temperature: 0 + claude-sonnet-verifier: + model: anthropic/claude-sonnet-5 + api_key: ANTHROPIC_API_KEY + requires_vpn: true + max_tokens: 2048 + temperature: 0 + grok-3: + model: xai/grok-3 + api_key: XAI_API_KEY + requires_vpn: true + groq-llama-8b: + model: groq/llama-3.1-8b-instant + api_key: GROQ_API_KEY + groq-qwen-coder: + model: groq/qwen-qwen-2.5-coder-32b + api_key: GROQ_API_KEY + gemini-flash: + model: gemini/gemini-2.0-flash + api_key: GEMINI_API_KEY + gigachat-classifier: + model: gigachat/GigaChat-2-Lite + api_key: GIGACHAT_CREDENTIALS + ssl_verify: false + max_tokens: 64 + temperature: 0 + gigachat-pro: + model: gigachat/GigaChat-2-Pro + api_key: GIGACHAT_CREDENTIALS + ssl_verify: false diff --git a/config/routing_rules.yaml b/config/routing_rules.yaml index db60567..4fa705e 100644 --- a/config/routing_rules.yaml +++ b/config/routing_rules.yaml @@ -38,6 +38,18 @@ gateway: - novita - traefik - mnesia + medium_code_keywords: + - поправь + - исправь + - fix + - edit + - patch + - bugfix + - typo + - rename + - add test + - unit test + - lint ocr_keywords: - ocr - прочитай текст @@ -64,6 +76,8 @@ litellm: tier: SIMPLE - keywords: ["bash", "docker", "swarm", "ci/cd", "gitea", "rebar", "erlang", "devops", "kubectl", "terraform", "playwright", "npm", "wsl"] tier: MEDIUM + - keywords: ["поправь", "исправь", "fix", "edit", "patch", "bugfix", "typo", "rename", "add test", "lint"] + tier: MEDIUM - keywords: ["рефакторинг", "refactor", "архитектура", "architecture", "спроектируй", "design system", "microservice", "distributed", "migration plan"] tier: COMPLEX - keywords: ["step by step", "reasoning", "prove", "analyze deeply", "think through"] diff --git a/docker-stack.yml b/docker-stack.yml index 2288673..8fc29c5 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -42,8 +42,16 @@ secrets: external: true groq_api_key: external: true + xai_api_key: + external: true gemini_api_key: external: true + together_api_key: + external: true + openrouter_api_key: + external: true + anthropic_api_key: + external: true gigachat_credentials: external: true vless_conf: @@ -95,7 +103,7 @@ services: retries: 3 litellm: - image: ${LITELLM_IMAGE:-ghcr.io/berriai/litellm:main-v1.96.0-stable} + image: ${LITELLM_IMAGE:-ghcr.io/berriai/litellm:v1.95.0} entrypoint: ["/bin/sh", "/entrypoint/litellm-entrypoint.sh"] configs: - source: litellm_config @@ -111,7 +119,7 @@ services: USE_AIOHTTP_TRANSPORT: ${USE_AIOHTTP_TRANSPORT:-False} HTTP_PROXY: ${HTTP_PROXY:-} HTTPS_PROXY: ${HTTPS_PROXY:-} - NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1,api.novita.ai,novita.ai} + NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,api.novita.ai,novita.ai,api.groq.com,groq.com,generativelanguage.googleapis.com,ngw.devices.sberbank.ru,gigachat.devices.sberbank.ru,git.sabilin.com,*.ift.calentiq.com,ift.calentiq.com,*.ift.eventhub.local,*.eventhub.local} POSTGRES_USER: ${POSTGRES_USER:-litellm} POSTGRES_DB: ${POSTGRES_DB:-litellm} secrets: @@ -120,7 +128,11 @@ services: - litellm_salt_key - postgres_password - groq_api_key + - xai_api_key - gemini_api_key + - together_api_key + - openrouter_api_key + - anthropic_api_key - gigachat_credentials networks: ai-internal: @@ -136,16 +148,16 @@ services: delay: 5s update_config: parallelism: 1 - failure_action: rollback + failure_action: pause placement: constraints: - node.role == manager healthcheck: - test: ["CMD-SHELL", "wget -q -O- http://127.0.0.1:4000/health/liveliness || exit 1"] + test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=5)\" || exit 1"] interval: 30s - timeout: 10s - retries: 3 - start_period: 60s + timeout: 15s + retries: 5 + start_period: 120s router: image: ${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift} @@ -195,7 +207,12 @@ services: - source: vless_conf target: /app/vless.conf networks: - - ai-internal + ai-internal: + aliases: + - vless-proxy + eventhub-ift: + aliases: + - vless-proxy deploy: replicas: 0 restart_policy: @@ -206,7 +223,7 @@ services: constraints: - node.role == manager healthcheck: - test: ["CMD-SHELL", "wget -q -O- http://127.0.0.1:8080 || exit 1"] + test: ["CMD-SHELL", "pgrep Xray >/dev/null || exit 1"] interval: 15s timeout: 5s retries: 5 diff --git a/litellm_config.yaml b/litellm_config.yaml index 4ea73ac..6a437f3 100644 --- a/litellm_config.yaml +++ b/litellm_config.yaml @@ -1,16 +1,16 @@ -# LiteLLM Proxy — Novita lanes A/B/C + Auto Router v2 fallback -# Regenerate model_list tail: bash scripts/sync-routing-config.sh +# LiteLLM — generated for PRIMARY_PROVIDER=hybrid +# Profile: Hybrid: Novita workers + Novita plan/verify (no Claude geo-billing) +# Regenerate: PRIMARY_PROVIDER=hybrid bash scripts/gen-litellm-config.py general_settings: master_key: os.environ/LITELLM_MASTER_KEY store_model_in_db: false max_budget: 50 budget_duration: 30d - litellm_settings: drop_params: true set_verbose: false - request_timeout: 120 + request_timeout: 300 num_retries: 2 cache: true cache_params: @@ -18,193 +18,358 @@ litellm_settings: host: redis port: 6379 ttl: 3600 - callbacks: ["prometheus"] + callbacks: + - prometheus require_auth_for_metrics_endpoint: false - environment_variables: NOVITA_API_KEY: os.environ/NOVITA_API_KEY GROQ_API_KEY: os.environ/GROQ_API_KEY + XAI_API_KEY: os.environ/XAI_API_KEY GEMINI_API_KEY: os.environ/GEMINI_API_KEY GIGACHAT_CREDENTIALS: os.environ/GIGACHAT_CREDENTIALS - + TOGETHER_API_KEY: os.environ/TOGETHER_API_KEY + OPENROUTER_API_KEY: os.environ/OPENROUTER_API_KEY router_settings: routing_strategy: simple-shuffle num_retries: 2 - timeout: 120 - allowed_fails: 2 - cooldown_time: 30 + timeout: 300 + allowed_fails: 5 + cooldown_time: 15 fallbacks: - - a-simple: ["b-simple", "groq-llama-8b"] - - a-medium-code: ["a-medium-ops", "groq-qwen-coder"] - - a-complex: ["b-complex", "a-reasoning"] - - b-complex: ["a-complex", "c-complex"] - - a-reasoning: ["b-reasoning", "gemini-flash"] - - a-vision: ["a-vision-ocr"] - - smart-router-internal: ["a-medium-ops", "a-medium-code", "a-complex"] - default_fallbacks: ["a-medium-code", "a-complex", "groq-qwen-coder"] - + - a-simple: + - b-simple + - a-medium-ops + - a-medium-code: + - b-medium-code + - a-complex + - a-complex: + - b-complex + - a-reasoning + - b-complex: + - a-complex + - c-complex + - a-reasoning: + - b-reasoning + - a-complex + - a-vision: + - a-vision-ocr + - smart-router-internal: + - a-medium-ops + - a-medium-code + - a-complex + - novita-planner: + - b-complex + - a-medium-code + - novita-verifier: + - b-complex + - a-medium-code + default_fallbacks: + - a-medium-code + - a-complex + - a-medium-ops model_list: - # --- LLM tier classifier (GigaChat freemium, hybrid mode only) --- - - model_name: gigachat-classifier - litellm_params: - model: gigachat/GigaChat-2-Lite - api_key: os.environ/GIGACHAT_CREDENTIALS - ssl_verify: false - max_tokens: 64 - temperature: 0 - - # --- Lane models (generated from config/model_matrix.yaml) --- - - model_name: a-simple - litellm_params: - model: novita/qwen/qwen3-4b-fp8 - api_key: os.environ/NOVITA_API_KEY - rpm: 60 - - model_name: b-simple - litellm_params: - model: novita/meta-llama/llama-3.1-8b-instruct - api_key: os.environ/NOVITA_API_KEY - rpm: 60 - - model_name: c-simple - litellm_params: - model: novita/qwen/qwen3-8b-fp8 - api_key: os.environ/NOVITA_API_KEY - rpm: 60 - - - model_name: a-medium-ops - litellm_params: - model: novita/qwen/qwen3-8b-fp8 - api_key: os.environ/NOVITA_API_KEY - rpm: 40 - - model_name: b-medium-ops - litellm_params: - model: novita/deepseek/deepseek-v3.2 - api_key: os.environ/NOVITA_API_KEY - rpm: 40 - - model_name: c-medium-ops - litellm_params: - model: novita/deepseek/deepseek-v3.2 - api_key: os.environ/NOVITA_API_KEY - rpm: 40 - - - model_name: a-medium-code - litellm_params: - model: novita/qwen/qwen3-coder-30b-a3b-instruct - api_key: os.environ/NOVITA_API_KEY - rpm: 40 - - model_name: b-medium-code - litellm_params: - model: novita/qwen/qwen3-coder-30b-a3b-instruct - api_key: os.environ/NOVITA_API_KEY - rpm: 40 - - model_name: c-medium-code - litellm_params: - model: novita/qwen/qwen3-coder-30b-a3b-instruct - api_key: os.environ/NOVITA_API_KEY - rpm: 30 - - - model_name: a-complex - litellm_params: - model: novita/deepseek/deepseek-v3.2 - api_key: os.environ/NOVITA_API_KEY - rpm: 30 - - model_name: b-complex - litellm_params: - model: novita/deepseek/deepseek-r1-0528 - api_key: os.environ/NOVITA_API_KEY - rpm: 20 - - model_name: c-complex - litellm_params: - model: novita/deepseek/deepseek-r1-turbo - api_key: os.environ/NOVITA_API_KEY - rpm: 15 - - - model_name: a-reasoning - litellm_params: - model: novita/deepseek/deepseek-r1-0528-qwen3-8b - api_key: os.environ/NOVITA_API_KEY - rpm: 30 - - model_name: b-reasoning - litellm_params: - model: novita/deepseek/deepseek-r1-0528 - api_key: os.environ/NOVITA_API_KEY - rpm: 20 - - model_name: c-reasoning - litellm_params: - model: novita/deepseek/deepseek-r1-turbo - api_key: os.environ/NOVITA_API_KEY - rpm: 15 - - - model_name: a-vision-ocr - litellm_params: - model: novita/paddlepaddle/paddleocr-vl - api_key: os.environ/NOVITA_API_KEY - rpm: 30 - - model_name: a-vision - litellm_params: - model: novita/qwen/qwen3-vl-30b-a3b-instruct - api_key: os.environ/NOVITA_API_KEY - rpm: 20 - - model_name: b-vision - litellm_params: - model: novita/qwen/qwen2.5-vl-72b-instruct - api_key: os.environ/NOVITA_API_KEY - rpm: 15 - - model_name: c-vision - litellm_params: - model: novita/qwen/qwen3-vl-235b-a22b-instruct - api_key: os.environ/NOVITA_API_KEY - rpm: 10 - - # --- Optional fallbacks --- - - model_name: groq-llama-8b - litellm_params: - model: groq/llama-3.1-8b-instant - api_key: os.environ/GROQ_API_KEY - - model_name: groq-qwen-coder - litellm_params: - model: groq/qwen-qwen-2.5-coder-32b - api_key: os.environ/GROQ_API_KEY - - model_name: gemini-flash - litellm_params: - model: gemini/gemini-2.0-flash - api_key: os.environ/GEMINI_API_KEY - - # --- Auto Router v2 fallback when gateway confidence low --- - - model_name: smart-router-internal - litellm_params: - model: auto_router/complexity_router - drop_params: true - complexity_router_default_model: a-medium-ops - complexity_router_config: - tiers: - SIMPLE: a-simple - MEDIUM: a-medium-ops - MEDIUM_CODE: a-medium-code - COMPLEX: a-complex - REASONING: a-reasoning - classifier_fallback: heuristic - keyword_tier_rules: - - keywords: ["привет", "hello", "hi", "thanks", "спасибо", "что такое", "what is", "define"] - tier: SIMPLE - - keywords: ["bash", "docker", "swarm", "ci/cd", "gitea", "rebar", "erlang", "devops", "kubectl", "terraform"] - tier: MEDIUM - - keywords: ["рефакторинг", "refactor", "архитектура", "architecture", "спроектируй", "migration plan"] - tier: COMPLEX - - keywords: ["step by step", "prove", "analyze deeply", "think through"] - tier: REASONING - custom_technical_keywords: [erlang, rebar3, traefik, mnesia, litellm, novita] - token_thresholds: - simple: 20 - complex: 500 - tier_boundaries: - simple_medium: 0.18 - medium_complex: 0.38 - complex_reasoning: 0.62 - session_affinity: true - session_affinity_ttl_seconds: 1800 - - # Alias for Zed default model name (gateway resolves before LiteLLM) - - model_name: smart-router - litellm_params: - model: novita/qwen/qwen3-8b-fp8 - api_key: os.environ/NOVITA_API_KEY +- model_name: gigachat-classifier + litellm_params: + model: gigachat/GigaChat-2-Lite + api_key: os.environ/GIGACHAT_CREDENTIALS + timeout: 180 + ssl_verify: false + max_tokens: 64 + temperature: 0 +- model_name: a-simple + litellm_params: + model: novita/meta-llama/llama-3.1-8b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 300 +- model_name: b-simple + litellm_params: + model: novita/meta-llama/llama-3.1-8b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 300 +- model_name: c-simple + litellm_params: + model: novita/meta-llama/llama-3.1-8b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 300 +- model_name: a-medium-ops + litellm_params: + model: novita/meta-llama/llama-3.1-8b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 240 +- model_name: b-medium-ops + litellm_params: + model: novita/meta-llama/llama-3.1-8b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 240 +- model_name: c-medium-ops + litellm_params: + model: novita/meta-llama/llama-3.1-8b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 240 +- model_name: a-medium-code + litellm_params: + model: novita/qwen/qwen3-coder-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 240 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: b-medium-code + litellm_params: + model: novita/qwen/qwen3-coder-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 240 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: c-medium-code + litellm_params: + model: novita/qwen/qwen3-coder-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 180 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: a-complex + litellm_params: + model: novita/qwen/qwen3-coder-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 180 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: b-complex + litellm_params: + model: novita/deepseek/deepseek-v3.2 + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 120 +- model_name: c-complex + litellm_params: + model: novita/qwen/qwen3.8-max + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 120 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: a-reasoning + litellm_params: + model: novita/qwen/qwen3-coder-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 120 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: b-reasoning + litellm_params: + model: novita/deepseek/deepseek-v3.2 + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 120 +- model_name: c-reasoning + litellm_params: + model: novita/qwen/qwen3.8-max + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 120 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: a-vision-ocr + litellm_params: + model: novita/paddlepaddle/paddleocr-vl + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 120 +- model_name: a-vision + litellm_params: + model: novita/qwen/qwen3-vl-30b-a3b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 120 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: b-vision + litellm_params: + model: novita/qwen/qwen2.5-vl-72b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 90 +- model_name: c-vision + litellm_params: + model: novita/qwen/qwen3-vl-235b-a22b-instruct + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + rpm: 90 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: novita-planner + litellm_params: + model: novita/qwen/qwen3.8-max + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + max_tokens: 1024 + temperature: 0 + extra_body: + chat_template_kwargs: + enable_thinking: false + enable_thinking: false +- model_name: novita-verifier + litellm_params: + model: novita/deepseek/deepseek-v3.2 + api_key: os.environ/NOVITA_API_KEY + timeout: 180 + max_tokens: 1024 + temperature: 0 +- model_name: claude-haiku-planner + litellm_params: + model: anthropic/claude-haiku-4-5-20251001 + api_key: os.environ/ANTHROPIC_API_KEY + timeout: 180 + max_tokens: 1024 + temperature: 0 +- model_name: claude-sonnet-verifier + litellm_params: + model: anthropic/claude-sonnet-5 + api_key: os.environ/ANTHROPIC_API_KEY + timeout: 180 + max_tokens: 2048 + temperature: 0 +- model_name: groq-llama-8b + litellm_params: + model: groq/llama-3.1-8b-instant + api_key: os.environ/GROQ_API_KEY + timeout: 180 +- model_name: groq-qwen-coder + litellm_params: + model: groq/qwen-qwen-2.5-coder-32b + api_key: os.environ/GROQ_API_KEY + timeout: 180 +- model_name: gemini-flash + litellm_params: + model: gemini/gemini-2.0-flash + api_key: os.environ/GEMINI_API_KEY + timeout: 180 +- model_name: grok-3 + litellm_params: + model: xai/grok-3 + api_key: os.environ/XAI_API_KEY + timeout: 180 +- model_name: gigachat-pro + litellm_params: + model: gigachat/GigaChat-2-Pro + api_key: os.environ/GIGACHAT_CREDENTIALS + timeout: 180 + ssl_verify: false +- model_name: smart-router-internal + litellm_params: + model: auto_router/complexity_router + drop_params: true + complexity_router_default_model: a-medium-ops + complexity_router_config: + tiers: + SIMPLE: a-simple + MEDIUM: a-medium-ops + MEDIUM_CODE: a-medium-code + COMPLEX: a-complex + REASONING: a-reasoning + classifier_fallback: heuristic + keyword_tier_rules: + - keywords: + - привет + - hello + - hi + - thanks + - спасибо + - что такое + - what is + - define + tier: SIMPLE + - keywords: + - bash + - docker + - swarm + - ci/cd + - gitea + - rebar + - erlang + - devops + - kubectl + - terraform + - playwright + - npm + - wsl + tier: MEDIUM + - keywords: + - поправь + - исправь + - fix + - edit + - patch + - bugfix + - typo + - rename + - add test + - lint + tier: MEDIUM + - keywords: + - рефакторинг + - refactor + - архитектура + - architecture + - спроектируй + - design system + - microservice + - distributed + - migration plan + tier: COMPLEX + - keywords: + - step by step + - reasoning + - prove + - analyze deeply + - think through + tier: REASONING + custom_technical_keywords: + - erlang + - rebar3 + - traefik + - mnesia + - litellm + - novita + token_thresholds: + simple: 20 + complex: 500 + tier_boundaries: + simple_medium: 0.18 + medium_complex: 0.38 + complex_reasoning: 0.62 + session_affinity: true + session_affinity_ttl_seconds: 1800 +- model_name: smart-router + litellm_params: + model: novita/meta-llama/llama-3.1-8b-instruct + api_key: os.environ/NOVITA_API_KEY diff --git a/router/agent_hier.py b/router/agent_hier.py new file mode 100644 index 0000000..3568628 --- /dev/null +++ b/router/agent_hier.py @@ -0,0 +1,2546 @@ +"""Agent-native hierarchical: plan/verify on gateway, tools stay with Zed.""" + +from __future__ import annotations + +import json +import logging +import re +import time +import uuid +from typing import Any + +from hierarchical import ( + find_pending_plan, + worker_model_for, +) + +log = logging.getLogger("ai-router") + +# Prefer coding tools; Zed sends huge schemas for diagnostics/browser/etc. +_TOOL_PRIORITY = ( + "read_file", + "write_file", + "edit_file", + "create_file", + "delete_file", + "move_file", + "find_path", + "grep", + "search", + "list_directory", + "terminal", + "bash", + "shell", + "run_command", + "execute", + "open", + "save", + "replace", + "apply_patch", +) + + +def request_has_tools(body: dict[str, Any]) -> bool: + tools = body.get("tools") + return isinstance(tools, list) and len(tools) > 0 + + +def messages_have_tool_activity(messages: list[dict[str, Any]] | None) -> bool: + for msg in messages or []: + if not isinstance(msg, dict): + continue + role = str(msg.get("role") or "") + if role == "tool": + return True + if msg.get("tool_calls"): + return True + if msg.get("function_call"): + return True + return False + + +def _msg_chars(msg: dict[str, Any]) -> int: + content = msg.get("content") + n = 0 + if isinstance(content, str): + n += len(content) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict): + n += len(str(block.get("text") or "")) + n += len(str(block.get("content") or "")) + elif content is not None: + n += len(json.dumps(content, ensure_ascii=False, default=str)) + if msg.get("tool_calls"): + n += len(json.dumps(msg.get("tool_calls"), ensure_ascii=False, default=str)) + if msg.get("name"): + n += len(str(msg.get("name"))) + return n + + +def estimate_tokens_from_chars(n: int) -> int: + return max(0, (n + 3) // 4) + + +def measure_forward_size(forward: dict[str, Any]) -> dict[str, int]: + """Chars actually sent to LiteLLM (messages + slim tools).""" + msgs = forward.get("messages") or [] + tools = forward.get("tools") or [] + msg_chars = sum(_msg_chars(m) for m in msgs if isinstance(m, dict)) + tools_chars = len(json.dumps(tools, ensure_ascii=False, default=str)) if tools else 0 + total = msg_chars + tools_chars + return { + "msg_chars": msg_chars, + "tools_chars": tools_chars, + "total_chars": total, + "msgs": len(msgs) if isinstance(msgs, list) else 0, + "tools": len(tools) if isinstance(tools, list) else 0, + "est_tokens": estimate_tokens_from_chars(total), + } + + +def format_context_fill( + *, + used_chars: int, + budget_chars: int, + model_window_tokens: int = 32768, + label: str = "ctx", + tools_chars: int = 0, +) -> str: + """Human-readable context fill for Zed progress stream.""" + budget = max(1, int(budget_chars or 1)) + window = max(1, int(model_window_tokens or 1)) + pct_budget = min(999, int(round(100.0 * used_chars / budget))) + tok = estimate_tokens_from_chars(used_chars) + pct_model = min(999, int(round(100.0 * tok / window))) + filled = min(10, max(0, min(pct_budget, 100) // 10)) + bar = "█" * filled + "░" * (10 - filled) + over = " OVER" if used_chars > budget else "" + tools_bit = f" tools={tools_chars}" if tools_chars else "" + return ( + f"{label} {used_chars}/{budget} ({pct_budget}%{over}) [{bar}]" + f"{tools_bit} ~{tok}tok · model≈{pct_model}%/{window}" + ) + + +def context_fill_for_forward( + forward: dict[str, Any], + cfg: dict[str, Any], + *, + budget_chars: int | None = None, +) -> tuple[str, dict[str, Any]]: + """Return (progress line, meta fields) for an executor/plan forward body.""" + size = measure_forward_size(forward) + budget = int( + budget_chars + if budget_chars is not None + else cfg.get("executor_tool_loop_chars") + or cfg.get("executor_input_chars") + or 6000 + ) + window = int(cfg.get("context_window_tokens") or 32768) + line = format_context_fill( + used_chars=size["total_chars"], + budget_chars=budget, + model_window_tokens=window, + label="ctx", + tools_chars=size["tools_chars"], + ) + meta = { + "ctx_used_chars": size["total_chars"], + "ctx_budget_chars": budget, + "ctx_pct": min(999, int(round(100.0 * size["total_chars"] / max(1, budget)))), + "ctx_est_tokens": size["est_tokens"], + "ctx_tools_chars": size["tools_chars"], + "ctx_msgs": size["msgs"], + "ctx_window_tokens": window, + "ctx_fill": line, + } + return line, meta + + +def trim_keeping_system( + messages: list[dict[str, Any]], + *, + max_chars: int, +) -> list[dict[str, Any]]: + """Trim from the oldest, but always keep a leading system message.""" + if not messages: + return [] + if messages[0].get("role") == "system": + sys = messages[0] + sys_n = _msg_chars(sys) + rest = trim_messages_for_executor( + messages[1:], max_chars=max(0, max_chars - sys_n) + ) + return [sys, *rest] + return trim_messages_for_executor(messages, max_chars=max_chars) + + +def sanitize_outbound_messages( + messages: list[dict[str, Any]] | None, +) -> list[dict[str, Any]]: + """Novita DeepSeek returns 400 on content:null / orphan tools / bad args.""" + cleaned: list[dict[str, Any]] = [] + for msg in messages or []: + if not isinstance(msg, dict): + continue + m = dict(msg) + role = str(m.get("role") or "") + for junk in ( + "reasoning", + "reasoning_content", + "reasoning_details", + "provider_specific_fields", + ): + m.pop(junk, None) + if role == "assistant": + tcs = m.get("tool_calls") + if isinstance(tcs, list) and tcs: + if m.get("content") is None: + m["content"] = "" + clean_tcs: list[dict[str, Any]] = [] + for tc in tcs: + if not isinstance(tc, dict): + continue + tc2 = dict(tc) + fn = tc2.get("function") + if isinstance(fn, dict): + fn2 = dict(fn) + args = fn2.get("arguments") + if args is None: + fn2["arguments"] = "{}" + elif isinstance(args, dict): + fn2["arguments"] = json.dumps(args, ensure_ascii=False) + elif not isinstance(args, str): + fn2["arguments"] = str(args) + tc2["function"] = fn2 + if not tc2.get("type"): + tc2["type"] = "function" + clean_tcs.append(tc2) + m["tool_calls"] = clean_tcs + elif m.get("content") is None: + m["content"] = "" + elif role == "tool": + if m.get("content") is None: + m["content"] = "" + else: + if m.get("content") is None: + m["content"] = "" + cleaned.append(m) + + offered: set[str] = set() + for m in cleaned: + if m.get("role") != "assistant": + continue + for tc in m.get("tool_calls") or []: + if isinstance(tc, dict): + cid = str(tc.get("id") or "") + if cid: + offered.add(cid) + out: list[dict[str, Any]] = [] + for m in cleaned: + if m.get("role") == "tool": + cid = str(m.get("tool_call_id") or "") + if not cid or cid not in offered: + continue + out.append(m) + return out + + +def _count_edit_tool_calls(messages: list[dict[str, Any]] | None) -> int: + n = 0 + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + if "edit" in name or "write" in name: + n += 1 + return n + + +def _last_tool_result_ok(messages: list[dict[str, Any]] | None) -> bool: + for msg in reversed(messages or []): + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + low = str(msg.get("content") or "").lower() + if not low.strip(): + return False + if any( + x in low + for x in ( + "not found", + "could not find matching", + "error", + "failed", + "expected struct", + ) + ): + return False + return True + return False + + +def should_stop_after_edit( + messages: list[dict[str, Any]] | None, + cfg: dict[str, Any] | None = None, +) -> bool: + """After any edit_file was issued — do not force another edit loop.""" + cfg = cfg or {} + if _count_edit_tool_calls(messages) < 1: + return False + # Prefer stop when last edit looks done (success or already failed enough) + if _edit_failure_count(messages) >= int( + cfg.get("executor_max_edit_failures", 4) or 4 + ): + return True + # Last assistant turn was edit → stop after tool result (success or fail once) + for msg in reversed(messages or []): + if not isinstance(msg, dict): + continue + if msg.get("role") == "tool": + continue + if msg.get("role") == "assistant" and msg.get("tool_calls"): + names = [ + str(((tc.get("function") or {}).get("name")) or "").lower() + for tc in (msg.get("tool_calls") or []) + if isinstance(tc, dict) + ] + return any("edit" in n or "write" in n for n in names) + if msg.get("role") in ("assistant", "user"): + break + return _count_edit_tool_calls(messages) >= 1 + + +def should_force_edit_after_read( + messages: list[dict[str, Any]] | None, + cfg: dict[str, Any] | None = None, + plan: dict[str, Any] | None = None, +) -> bool: + """Only once: right after a successful read, before any edit was attempted. + + Bug before: returned True on ANY trailing tool result — including successful + edit_file — which made Zed apply the same marker edit in an infinite loop. + """ + cfg = cfg or {} + if not cfg.get("executor_force_edit_after_read", True): + return False + # DevOps without runtime facts — do not blind-edit + try: + from runtime_probe import devops_blocks_blind_force_edit + + if devops_blocks_blind_force_edit(plan): + return False + except Exception: # noqa: BLE001 + pass + if not _successful_read_keys(messages): + return False + # Already edited (or tried) → never force again + if _count_edit_tool_calls(messages) >= 1: + return False + if _edit_failure_count(messages) > 0: + return False + if _edit_mismatch_recently(messages): + return False + # Last assistant tool_calls must be read_* (skip tool results — do not + # treat edit success as "force another edit") + for msg in reversed(messages or []): + if not isinstance(msg, dict): + continue + if msg.get("role") == "tool": + continue + if msg.get("role") == "assistant" and msg.get("tool_calls"): + names = [ + str(((tc.get("function") or {}).get("name")) or "").lower() + for tc in (msg.get("tool_calls") or []) + if isinstance(tc, dict) + ] + if any("edit" in n or "write" in n for n in names): + return False + if any( + any(k in n for k in ("read", "open", "cat", "get_file")) for n in names + ): + return True + return False + if msg.get("role") in ("user", "system"): + break + return False + + +def force_edit_after_read_completion( + *, + plan: dict[str, Any] | None, + tools: list[Any], + model: str, + messages: list[dict[str, Any]] | None, +) -> dict[str, Any]: + # Prefer deterministic Traefik port fix from live probe + try: + from runtime_probe import force_runtime_fix_edit + + targeted = force_runtime_fix_edit(plan=plan, tools=tools, model=model) + if targeted: + return targeted + except Exception: # noqa: BLE001 + pass + path = _last_successful_read_path(messages) or _path_for_user_status( + plan, tools, messages + ) + if not path: + return _stop_synth(model, "Нет пути для edit после read_file.") + return _continue_edit_or_grep( + model=model, + tools=tools, + path=path, + plan=plan, + messages=messages, + ) + + +def stop_after_edit_completion( + *, + model: str, + messages: list[dict[str, Any]] | None, +) -> dict[str, Any]: + path = _last_successful_read_path(messages) or _last_edit_path(messages) + ok = _last_tool_result_ok(messages) and _edit_failure_count(messages) == 0 + if ok: + text = "Готово: правка применена." + if path: + text += f" Файл: `{path}`." + text += " Если нужно ещё — напиши одной фразой." + else: + fails = _edit_failure_count(messages) + text = ( + f"Правка не применилась ({fails} ошибка). " + "Нужен точный фрагмент из файла, либо правь вручную." + ) + if path: + text += f" Файл: `{path}`." + return _stop_synth(model, text) + + +def enforce_forward_budget( + forward: dict[str, Any], + *, + budget_chars: int, +) -> dict[str, Any]: + """Hard-cap messages+tools so measure_forward_size ≤ budget (stops 160%+ spills).""" + out = dict(forward) + budget = max(500, int(budget_chars)) + for round_i in range(10): + size = measure_forward_size(out) + if size["total_chars"] <= budget: + return out + tools = list(out.get("tools") or []) + msgs = list(out.get("messages") or []) + # 1) Drop lowest-priority tools while tools take >35% of budget + if tools and size["tools_chars"] > budget * 0.35: + out["tools"] = tools[:-1] + continue + # 2) Aggressively truncate tool role contents + max_tool = max(200, 900 - round_i * 80) + new_msgs: list[dict[str, Any]] = [] + for m in msgs: + if not isinstance(m, dict): + continue + mm = dict(m) + if mm.get("role") == "tool" and isinstance(mm.get("content"), str): + if len(mm["content"]) > max_tool: + mm["content"] = mm["content"][: max_tool - 1] + "…" + new_msgs.append(mm) + msgs = new_msgs + # 3) Shrink message window (keep system) + msg_budget = max(400, budget - len(json.dumps(out.get("tools") or [], ensure_ascii=False))) + msgs = trim_keeping_system(msgs, max_chars=msg_budget) + # 4) Drop oldest non-system pairs if still over + if len(msgs) > 2 and measure_forward_size({"messages": msgs, "tools": out.get("tools")})[ + "total_chars" + ] > budget: + if msgs[0].get("role") == "system": + msgs = [msgs[0], *msgs[2:]] + else: + msgs = msgs[1:] + out["messages"] = msgs + # Final hard cut on last tool payload if still over + size = measure_forward_size(out) + if size["total_chars"] > budget: + msgs = list(out.get("messages") or []) + for i in range(len(msgs) - 1, -1, -1): + m = msgs[i] + if isinstance(m, dict) and m.get("role") == "tool" and isinstance(m.get("content"), str): + room = max(80, budget - (size["total_chars"] - len(m["content"]))) + m = dict(m) + m["content"] = m["content"][:room] + "…" + msgs[i] = m + out["messages"] = msgs + break + return out + + +def trim_messages_for_executor( + messages: list[dict[str, Any]], + *, + max_chars: int = 24000, +) -> list[dict[str, Any]]: + """Keep newest messages under budget — huge Zed dumps make Novita hang past timeout.""" + if max_chars <= 0: + return list(messages) + kept: list[dict[str, Any]] = [] + total = 0 + for msg in reversed(messages or []): + if not isinstance(msg, dict): + continue + size = _msg_chars(msg) + if kept and total + size > max_chars: + break + kept.append(msg) + total += size + kept.reverse() + return kept + + +def compress_messages_for_tool_loop( + messages: list[dict[str, Any]], + *, + max_chars: int = 6000, + max_tool_result_chars: int = 1200, + newest_tool_result_chars: int = 3500, + max_messages: int = 8, +) -> list[dict[str, Any]]: + """Keep plan + newest tool turns; truncate huge tool payloads (Zed dumps).""" + compressed: list[dict[str, Any]] = [] + tool_indices: list[int] = [] + for msg in messages or []: + if not isinstance(msg, dict): + continue + m = dict(msg) + if m.get("role") == "tool": + tool_indices.append(len(compressed)) + content = m.get("content") + if isinstance(content, str) and len(content) > max_tool_result_chars: + m["content"] = ( + content[: max_tool_result_chars - 14].rstrip() + "\n<>" + ) + elif m.get("role") == "assistant" and m.get("tool_calls"): + # Novita DeepSeek 400 on content:null — use empty string + m["content"] = "" + compressed.append(m) + # Keep the newest tool result fuller (edit_file needs exact text, not "…") + if tool_indices: + newest_i = tool_indices[-1] + orig = None + # recover original from messages by counting tool roles + tool_n = 0 + for msg in messages or []: + if isinstance(msg, dict) and msg.get("role") == "tool": + tool_n += 1 + if tool_n == len(tool_indices): + orig = msg.get("content") + break + if isinstance(orig, str) and orig.strip(): + # Cap newest read — large dumps make Novita coder hang for minutes + keep_cap = max(800, int(newest_tool_result_chars or 3500)) + keep = min(len(orig), keep_cap) + restored = orig[:keep] + if len(orig) > keep: + restored = restored.rstrip() + "\n<>" + compressed[newest_i] = dict(compressed[newest_i]) + compressed[newest_i]["content"] = restored + # Prefer trailing window (tool loop is newest-first relevance) + if len(compressed) > max_messages: + # Keep leading system if present + head: list[dict[str, Any]] = [] + rest = compressed + if compressed and compressed[0].get("role") == "system": + head = [compressed[0]] + rest = compressed[1:] + rest = rest[-(max_messages - len(head)) :] + compressed = head + rest + return trim_messages_for_executor(compressed, max_chars=max_chars) + + +def inject_plan_context( + messages: list[dict[str, Any]], + plan_payload: dict[str, Any] | None, + *, + max_chars: int = 24000, + minimal: bool = False, + tool_loop: bool = False, + newest_tool_result_chars: int = 3500, + path_resolve: bool = False, +) -> list[dict[str, Any]]: + """Build executor messages. + + minimal=True (first turn after plan approve): drop Zed history/AGENTS + dumps — tools+history makes Novita hang past timeout. + tool_loop=True: compress tool results + keep only newest turns. + path_resolve=True: cheap find/list only — do not invent read paths. + """ + tasks = (plan_payload or {}).get("subtasks") or [] + plan_blob = json.dumps(plan_payload or {}, ensure_ascii=False) + if path_resolve: + qs = (plan_payload or {}).get("path_resolve_queries") or [] + q_line = ", ".join(str(q) for q in qs[:4]) or "dynamic_conf.yml" + root = _DEFAULT_DEVOPS_ROOT + sys_msg = { + "role": "system", + "content": "\n".join( + [ + "You locate files for an approved plan. Do NOT invent paths.", + "Call find_path or list_directory only — no read_file/edit yet.", + f"Search under: {root}", + f"Queries: {q_line}", + "EventHubDevOps Traefik file is ift\\traefik\\dynamic_conf.yml " + "(NOT traefik.yml). Compose: ift\\docker-compose.core.yml " + "(NO root docker-compose.yml).", + ] + ), + } + return [ + sys_msg, + { + "role": "user", + "content": ( + f"Find real files for: {q_line}. " + f"Start with find_path under {root}." + ), + }, + ] + + lines = [ + "You are a coding agent. Execute the approved plan via tools.", + "Do not paste full files as markdown — use edit/write tools.", + "Reply with a tool call only — no long prose.", + "edit_file MUST be: {\"path\":\"...\",\"edits\":[{\"old_text\":\"...\",\"new_text\":\"...\"}]}. " + "Never pass a string as edits (Zed: expected struct Edit).", + ] + if _looks_like_devops_plan(plan_blob): + root = _DEFAULT_DEVOPS_ROOT + yml = _join_under(root, _DEVOPS_KEY_RELS[0]) + lines.extend( + [ + f"EventHubDevOps root (absolute): {root}", + f"Target file: {yml}", + "There is NO src/ under EventHubDevOps.", + "There is NO root docker-compose.yml — use ift\\docker-compose.core.yml " + "(or stage\\…).", + "Traefik: ONLY ift\\traefik\\dynamic_conf.yml " + "(NOT traefik.yml / traefik.toml / static.yml).", + "Do NOT use relative paths (ift/... fails in Zed multi-root).", + "Read each file at most once; then edit_file — never re-read the same path.", + "Bad Gateway: fix Traefik backend URL/port to match the open container port " + "(do NOT invent traefik.yml or random markers).", + ] + ) + facts = (plan_payload or {}).get("runtime_facts") + if isinstance(facts, dict) and facts: + lines.append("RUNTIME FACTS (from live probe — trust these over guesses):") + lines.append( + f" host={facts.get('host')} http={facts.get('http_status')} " + f"service={facts.get('service')} open_ports={facts.get('open_ports')}" + ) + if facts.get("suggested_backend_url"): + lines.append( + f" REQUIRED FIX: set loadbalancer url to {facts.get('suggested_backend_url')} " + f"(replace http://{facts.get('service')}:80 if present)." + ) + if facts.get("hint"): + lines.append(f" hint: {facts.get('hint')}") + for i, item in enumerate(tasks): + if not isinstance(item, dict): + continue + tid = item.get("id") or str(i + 1) + prompt = item.get("prompt") or "" + tier = item.get("worker_tier") or "simple" + deps = item.get("depends_on") or item.get("deps") or [] + dep_s = f", deps={deps}" if deps else "" + lines.append(f"{tid}. [{tier}] {prompt}{dep_s}") + paths = item.get("paths") or [] + if isinstance(paths, list) and paths: + lines.append(" paths: " + ", ".join(str(p) for p in paths[:6])) + edit_goal = item.get("edit_goal") or item.get("goal") or "" + if edit_goal: + lines.append(f" edit_goal: {edit_goal}") + constraints = item.get("constraints") or [] + if isinstance(constraints, list) and constraints: + lines.append( + " constraints: " + "; ".join(str(c) for c in constraints[:6]) + ) + goal = (plan_payload or {}).get("user_goal") or "" + if goal: + lines.append("User goal: " + str(goal)[:800]) + acc = (plan_payload or {}).get("acceptance") or [] + if acc: + lines.append("Acceptance: " + "; ".join(str(x) for x in acc)) + sys_msg = {"role": "system", "content": "\n".join(lines)} + + if minimal: + start = "Plan approved. Start with step 1. Call tools now." + path_hints: list[str] = [] + for item in tasks: + if isinstance(item, dict): + for p in item.get("paths") or []: + if p and str(p) not in path_hints: + path_hints.append(str(p)) + if path_hints: + start = ( + "Plan approved. Prefer absolute paths. Start with read_file on: " + + ", ".join(path_hints[:3]) + ) + if _looks_like_devops_plan(plan_blob): + start = ( + "Plan approved. Call read_file now with absolute path " + f"{_join_under(_DEFAULT_DEVOPS_ROOT, _DEVOPS_KEY_RELS[0])} " + "(no src/, no relative ift/...)." + ) + return [ + sys_msg, + {"role": "user", "content": start}, + ] + + if tool_loop: + lines.append( + "Tool loop: do NOT call read_file on paths already present in tool results. " + "Next action must be edit/write/terminal." + ) + sys_msg = {"role": "system", "content": "\n".join(lines)} + # Leave room for sys_msg inside max_chars + sys_n = _msg_chars(sys_msg) + trimmed = compress_messages_for_tool_loop( + messages, + max_chars=max(400, max_chars - sys_n), + max_tool_result_chars=1200, + newest_tool_result_chars=newest_tool_result_chars, + max_messages=6, + ) + if trimmed and trimmed[0].get("role") == "system": + out = [sys_msg, *trimmed[1:]] + else: + out = [sys_msg, *trimmed] + return trim_keeping_system(out, max_chars=max_chars) + + trimmed = trim_messages_for_executor(messages, max_chars=max_chars) + if not plan_payload: + return trimmed + return [sys_msg, *trimmed] + + +def _tool_name(tool: dict[str, Any]) -> str: + fn = tool.get("function") if isinstance(tool.get("function"), dict) else {} + return str(fn.get("name") or tool.get("name") or "") + + +def _tool_priority(name: str) -> int: + low = name.lower() + for i, key in enumerate(_TOOL_PRIORITY): + if key in low: + return i + return 1000 + len(low) + + +def _skeleton_params(params: Any) -> dict[str, Any]: + """Drop nested descriptions / oneOf / $ref — they bloat Novita to death. + + Preserve array-of-object shapes for Zed `edits: Edit[]` — flattening to + `items: string` makes the model pass a prose string → + `expected struct Edit`. + """ + if not isinstance(params, dict): + return {"type": "object", "properties": {}} + props_in = params.get("properties") + props_out: dict[str, Any] = {} + if isinstance(props_in, dict): + for key, spec in list(props_in.items())[:16]: + t = "string" + if isinstance(spec, dict): + raw_t = spec.get("type") + if isinstance(raw_t, str): + t = raw_t + elif isinstance(raw_t, list) and raw_t: + t = str(raw_t[0]) + key_s = str(key) + if t == "array": + items = spec.get("items") if isinstance(spec, dict) else None + # Zed edit_file.edits → [{old_text, new_text}] + if key_s == "edits" or ( + isinstance(items, dict) + and ( + items.get("type") == "object" + or isinstance(items.get("properties"), dict) + ) + ): + item_props: dict[str, Any] = { + "old_text": {"type": "string"}, + "new_text": {"type": "string"}, + } + if isinstance(items, dict) and isinstance( + items.get("properties"), dict + ): + # Keep known Edit-like fields if present under other names + src = items["properties"] + if "old_string" in src and "old_text" not in src: + item_props = { + "old_string": {"type": "string"}, + "new_string": {"type": "string"}, + } + elif "old_text" in src or "new_text" in src: + item_props = { + "old_text": {"type": "string"}, + "new_text": {"type": "string"}, + } + props_out[key_s] = { + "type": "array", + "items": { + "type": "object", + "properties": item_props, + "required": list(item_props.keys()), + }, + } + else: + props_out[key_s] = { + "type": "array", + "items": {"type": "string"}, + } + elif t == "object": + props_out[key_s] = {"type": "object"} + elif t == "number" or t == "integer" or t == "boolean": + props_out[key_s] = {"type": t} + else: + props_out[key_s] = {"type": "string"} + required = params.get("required") + req_out = ( + [str(r) for r in required if str(r) in props_out][:12] + if isinstance(required, list) + else [] + ) + out: dict[str, Any] = {"type": "object", "properties": props_out} + if req_out: + out["required"] = req_out + return out + + +_ZED_EDIT_FILE_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "path": {"type": "string"}, + "edits": { + "type": "array", + "items": { + "type": "object", + "properties": { + "old_text": {"type": "string"}, + "new_text": {"type": "string"}, + }, + "required": ["old_text", "new_text"], + }, + }, + }, + "required": ["path", "edits"], +} + + +def slim_tools_for_executor( + tools: list[Any] | None, + *, + max_tools: int = 12, + max_desc_chars: int = 80, + full_schema_chars: int = 12000, +) -> list[Any]: + """Keep coding tools; use full schemas under threshold, else skeletonize. + + Full Zed JSON Schema (~20KB+) hangs weak models — threshold avoids blind + skeleton when payload is already small. + """ + if not isinstance(tools, list): + return [] + ranked = [t for t in tools if isinstance(t, dict) and _tool_name(t)] + ranked.sort(key=lambda t: _tool_priority(_tool_name(t))) + ranked = ranked[:max_tools] + raw_chars = len(json.dumps(ranked, ensure_ascii=False, default=str)) + use_full = raw_chars <= max(500, int(full_schema_chars)) + out: list[Any] = [] + for tool in ranked: + fn_in = tool.get("function") if isinstance(tool.get("function"), dict) else {} + name = str(fn_in.get("name") or tool.get("name") or "tool") + desc = fn_in.get("description") + if not isinstance(desc, str): + desc = name + if len(desc) > max_desc_chars: + desc = desc[: max_desc_chars - 1] + "…" + low = name.lower() + if use_full and "edit_file" not in low and low != "edit": + # Pass through trimmed description only + params = fn_in.get("parameters") + if not isinstance(params, dict): + params = {"type": "object", "properties": {}} + out.append( + { + "type": "function", + "function": { + "name": name, + "description": desc, + "parameters": params, + }, + } + ) + continue + if "edit_file" in low or low == "edit": + params = dict(_ZED_EDIT_FILE_PARAMS) + desc = ( + "Apply edits: {path, edits:[{old_text, new_text}]}. " + "edits MUST be objects, never a string." + )[:max_desc_chars] + else: + params = _skeleton_params(fn_in.get("parameters")) + out.append( + { + "type": "function", + "function": { + "name": name, + "description": desc, + "parameters": params, + }, + } + ) + return out + + +def pick_agent_executor_model(cfg: dict[str, Any]) -> str: + explicit = cfg.get("agent_executor_model") + if explicit: + model = str(explicit) + # Never hang tool-loop on Llama simple lanes + if model in ("a-simple", "b-simple", "c-simple", "a-medium-ops", "b-medium-ops", "c-medium-ops"): + return worker_model_for("medium_code") + return model + return worker_model_for("medium_code") + + +def pick_agent_escalate_model(cfg: dict[str, Any]) -> str: + """Stronger model after repeated edit failures (DeepSeek / Max).""" + explicit = cfg.get("agent_escalate_model") + if explicit: + return str(explicit) + return worker_model_for("hard") + + +def should_escalate_executor( + messages: list[dict[str, Any]] | None, + cfg: dict[str, Any], +) -> bool: + """Escalate mid tool_loop after edit failures or a successful read. + + Cheap coder routinely hangs 2–3 minutes on fat read_file payloads; once the + file is in context, jump to DeepSeek for the edit decision. + """ + max_ef = int(cfg.get("agent_escalate_after_edit_failures", 2) or 2) + if _edit_failure_count(messages) >= max(1, max_ef): + return True + if cfg.get("executor_escalate_after_read", True) and _successful_read_keys( + messages + ): + return True + return False + + +def prepare_agent_executor_forward( + body: dict[str, Any], + messages: list[dict[str, Any]], + plan: dict[str, Any] | None, + cfg: dict[str, Any], + *, + executor: str | None = None, + stream: bool | None = None, + minimal: bool | None = None, + path_resolve: bool = False, +) -> dict[str, Any]: + """Build LiteLLM body for Zed tool executor. + + Critical: num_retries=0 and fallbacks=[] — otherwise LiteLLM stacks + retries×fallback chain past gateway httpx ReadTimeout. + Never fall back to Llama on tools (except dedicated path_resolve). + """ + from path_resolve import discovery_tools_only, pick_path_resolve_model + + model = executor or pick_agent_executor_model(cfg) + if path_resolve: + model = pick_path_resolve_model(cfg) + elif should_escalate_executor(messages, cfg): + model = pick_agent_escalate_model(cfg) + plan = sanitize_plan_paths(plan) + max_ctx = int(cfg.get("executor_input_chars", 12000)) + timeout = float( + cfg.get("executor_timeout_sec") or cfg.get("call_timeout_sec") or 90 + ) + retries = int(cfg.get("executor_num_retries", 0)) + in_tool_loop = messages_have_tool_activity(messages) + use_minimal = ( + bool(cfg.get("executor_minimal_context", True)) + if minimal is None + else bool(minimal) + ) + if path_resolve: + use_minimal = True + timeout = float(cfg.get("path_resolve_timeout_sec") or 45) + elif in_tool_loop: + use_minimal = False + max_ctx = int(cfg.get("executor_tool_loop_chars", 6000)) + timeout = float(cfg.get("executor_tool_loop_timeout_sec") or timeout or 100) + + tools = slim_tools_for_executor( + body.get("tools") if isinstance(body.get("tools"), list) else [], + max_tools=int(cfg.get("executor_max_tools", 12)), + max_desc_chars=int(cfg.get("executor_tool_desc_chars", 80)), + full_schema_chars=int(cfg.get("executor_tools_full_chars", 12000)), + ) + if path_resolve: + tools = discovery_tools_only( + tools, max_tools=int(cfg.get("path_resolve_max_tools", 3) or 3) + ) + tools_chars = len(json.dumps(tools, ensure_ascii=False, default=str)) if tools else 0 + # Tools count toward the same budget — reserve space so msgs+tools ≤ max_ctx + while tools and tools_chars > max(800, max_ctx // 3): + tools = tools[:-1] + tools_chars = len(json.dumps(tools, ensure_ascii=False, default=str)) + msg_budget = max(600, max_ctx - tools_chars) + + newest_tool = int(cfg.get("executor_newest_tool_chars", 3500) or 3500) + msgs = inject_plan_context( + messages, + plan, + max_chars=msg_budget, + minimal=use_minimal, + tool_loop=in_tool_loop and not path_resolve, + newest_tool_result_chars=newest_tool, + path_resolve=path_resolve, + ) + msgs = sanitize_outbound_messages(msgs) + forward: dict[str, Any] = { + "model": model, + "messages": msgs, + "tools": tools, + # Force a tool call — prose-only replies waste the whole timeout budget + "tool_choice": "required", + "max_tokens": int(cfg.get("executor_max_tokens") or 768), + "temperature": 0.1, + "stream": bool(stream) if stream is not None else False, + "num_retries": retries, + "fallbacks": [], + "timeout": timeout, + } + return enforce_forward_budget(forward, budget_chars=max_ctx) + + +# Zed multi-root: relative paths fail ("not in the project"). Prefer absolute. +_DEFAULT_DEVOPS_ROOT = r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps" +_DEVOPS_KEY_RELS = ( + r"ift\traefik\dynamic_conf.yml", + r"ift\traefik\dynamic_conf.loadtest.yml", + r"ift\docker-compose.core.yml", + r"README.md", +) +_ABS_WIN_RE = re.compile(r"[A-Za-z]:\\[^\"'\n\r]+") + + +def _norm_path_key(path: str) -> str: + return path.replace("/", "\\").rstrip("\\").lower() + + +def _join_under(root: str, rel: str) -> str: + return root.rstrip("\\/") + "\\" + rel.replace("/", "\\").lstrip("\\/") + + +def _looks_like_devops_plan(text: str) -> bool: + low = (text or "").lower() + return any( + k in low + for k in ( + "traefik", + "балансир", + "eventhubdevops", + "ift.calentiq", + "ai-router.ift", + ) + ) + + +def _paths_mentioned(text: str) -> list[str]: + found = re.findall( + r"([A-Za-z0-9_./\\-]+\.(?:ya?ml|toml|md|erl|ts|tsx|js|go|py|json|sh))", + text or "", + ) + return found + + +def _extract_abs_win_paths(text: str) -> list[str]: + out: list[str] = [] + for m in _ABS_WIN_RE.finditer(text or ""): + p = m.group(0).rstrip("\\/.,;:)") + # json.dumps of Windows paths → doubled backslashes + while "\\\\" in p: + p = p.replace("\\\\", "\\") + if p: + out.append(p) + return out + + +def _tool_arg_paths(messages: list[dict[str, Any]] | None) -> list[str]: + """Paths from tool_call arguments (prefer parsed JSON over regex on dumps).""" + out: list[str] = [] + for msg in messages or []: + if not isinstance(msg, dict) or not msg.get("tool_calls"): + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + parsed = None + if isinstance(parsed, dict): + for key in ("path", "file", "target", "directory", "dir"): + val = parsed.get(key) + if isinstance(val, str) and val.strip(): + out.append(val.strip()) + else: + out.extend(_extract_abs_win_paths(str(args))) + return out + + +def _devops_root_from_messages(messages: list[dict[str, Any]] | None) -> str | None: + for p in _tool_arg_paths(messages): + norm = p.replace("/", "\\") + m = re.search(r"(?i)^(.+?\\EventHubDevOps)(?:\\|$)", norm) + if m: + raw = m.group(1) + idx = raw.lower().rfind("eventhubdevops") + return raw[:idx] + "EventHubDevOps" + for msg in messages or []: + if not isinstance(msg, dict): + continue + blob = json.dumps(msg, ensure_ascii=False, default=str) + for p in _extract_abs_win_paths(blob): + m = re.search(r"(?i)^(.+?\\EventHubDevOps)(?:\\|$)", p) + if m: + raw = m.group(1) + idx = raw.lower().rfind("eventhubdevops") + return raw[:idx] + "EventHubDevOps" + return None + + +def _failed_path_keys(messages: list[dict[str, Any]] | None) -> set[str]: + failed: set[str] = set() + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + content = str(msg.get("content") or "") + low = content.lower() + if not ( + "not found" in low + or "not in the project" in low + or "path not found" in low + ): + continue + for p in _extract_abs_win_paths(content): + failed.add(_norm_path_key(p)) + # Zed often returns JSON: {"Text":"C:\\...\\file.yml not found"} + for m in re.finditer( + r'([A-Za-z]:\\[^"\n\r]+?)\s+not found', content, re.I + ): + failed.add(_norm_path_key(m.group(1).strip())) + for p in _paths_mentioned(content): + failed.add(_norm_path_key(p)) + m = re.search(r"Path\s+(.+?)\s+is not", content, re.I) + if m: + failed.add(_norm_path_key(m.group(1).strip())) + m2 = re.search(r"Path not found:\s*(.+)", content, re.I) + if m2: + failed.add(_norm_path_key(m2.group(1).strip())) + return failed + + +def _seen_path_keys(messages: list[dict[str, Any]] | None) -> set[str]: + seen: set[str] = set() + for msg in messages or []: + if not isinstance(msg, dict): + continue + if msg.get("role") == "assistant" and msg.get("tool_calls"): + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + parsed = None + if isinstance(parsed, dict): + for key in ("path", "file", "target", "directory", "dir"): + val = parsed.get(key) + if isinstance(val, str) and val.strip(): + seen.add(_norm_path_key(val)) + for p in _extract_abs_win_paths(str(args)): + seen.add(_norm_path_key(p)) + for p in _paths_mentioned(str(args)): + seen.add(_norm_path_key(p)) + blob = json.dumps(msg, ensure_ascii=False, default=str) + for p in _paths_mentioned(blob): + if "\\" in p or "/" in p: + seen.add(_norm_path_key(p)) + return seen + + +def _is_bogus_devops_path(path: str) -> bool: + key = _norm_path_key(path) + if key.endswith("\\src") or key.endswith("/src"): + return True + # bare relative under multi-root Zed — reject for DevOps synthetics + if not re.match(r"^[a-z]:\\", key) and ( + key.startswith("ift\\") + or key.startswith("ift/") + or key.startswith("eventhubdevops\\") + or key.startswith("eventhubdevops/") + ): + return True + # EventHubDevOps has no root compose — only ift|stage/docker-compose.*.yml + base = key.rsplit("\\", 1)[-1].rsplit("/", 1)[-1] + if base in ("docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"): + if "\\ift\\" not in key and "/ift/" not in key and "\\stage\\" not in key and "/stage/" not in key: + return True + # Classic Traefik static names — this repo only has dynamic_conf*.yml under */traefik/ + if "\\traefik\\" in key or "/traefik/" in key: + allowed_traefik = { + "dynamic_conf.yml", + "dynamic_conf.yaml", + "dynamic_conf.loadtest.yml", + "dynamic_conf.loadtest.yaml", + ".gitkeep", + } + if base not in allowed_traefik and not base.endswith(".crt") and not base.endswith(".key"): + return True + # Other frequently invented static Traefik filenames anywhere under DevOps + if base in ( + "traefik.yml", + "traefik.yaml", + "traefik.toml", + "traefik-static.yml", + "static.yml", + "static_conf.yml", + ): + return True + return False + + +def sanitize_plan_paths(plan: dict[str, Any] | None) -> dict[str, Any] | None: + """Drop invented DevOps paths from planner JSON before kickstart/hints.""" + if not isinstance(plan, dict): + return plan + out = dict(plan) + tasks = out.get("subtasks") + if not isinstance(tasks, list): + return out + fixed_tasks: list[Any] = [] + for item in tasks: + if not isinstance(item, dict): + fixed_tasks.append(item) + continue + it = dict(item) + paths = it.get("paths") + if isinstance(paths, list): + clean = [ + p + for p in paths + if isinstance(p, str) and p.strip() and not _is_bogus_devops_path(p) + ] + if not clean and _looks_like_devops_plan(json.dumps(plan, ensure_ascii=False)): + clean = [ + _join_under(_DEFAULT_DEVOPS_ROOT, _DEVOPS_KEY_RELS[0]) + ] + it["paths"] = clean + fixed_tasks.append(it) + out["subtasks"] = fixed_tasks + return out + + +def _first_path_hint( + plan: dict[str, Any] | None, + tools: list[Any], + messages: list[dict[str, Any]] | None = None, +) -> str | None: + plan_text = json.dumps(plan or {}, ensure_ascii=False) + blob_all = plan_text + "\n" + json.dumps(messages or [], ensure_ascii=False, default=str) + devops = _looks_like_devops_plan(blob_all) + root = _devops_root_from_messages(messages) + if devops and not root: + root = _DEFAULT_DEVOPS_ROOT + + candidates: list[str] = [] + if root: + for rel in _DEVOPS_KEY_RELS: + candidates.append(_join_under(root, rel)) + for p in _paths_mentioned(plan_text): + if re.match(r"^[A-Za-z]:\\", p) or p.startswith("/"): + candidates.append(p) + elif not _is_bogus_devops_path(p): + if "src" not in p.replace("\\", "/").split("/"): + candidates.append(_join_under(root, p)) + + seen = _seen_path_keys(messages) + failed = _failed_path_keys(messages) + read_counts = _read_path_counts(messages) + success = _successful_read_keys(messages) + + def _ok(path: str) -> bool: + key = _norm_path_key(path) + if key in seen or key in failed or key in success: + return False + if read_counts.get(key, 0) >= 1: + return False + if devops and _is_bogus_devops_path(path): + return False + return True + + for path in candidates: + if _ok(path): + return path + if root: + fallback = _join_under(root, "README.md") + if _ok(fallback): + return fallback + # Do NOT re-return dynamic_conf.yml — that caused infinite read loops + return None + names = [_tool_name(t) for t in tools if isinstance(t, dict)] + if any("read" in n.lower() for n in names): + return "README.md" + return "AGENTS.md" + + +def _read_path_counts(messages: list[dict[str, Any]] | None) -> dict[str, int]: + counts: dict[str, int] = {} + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + if not any(k in name for k in ("read", "open", "cat", "get_file")): + continue + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + parsed = None + path = None + if isinstance(parsed, dict): + for key in ("path", "file", "target"): + val = parsed.get(key) + if isinstance(val, str) and val.strip(): + path = val.strip() + break + if path: + k = _norm_path_key(path) + counts[k] = counts.get(k, 0) + 1 + return counts + + +def _successful_read_keys(messages: list[dict[str, Any]] | None) -> set[str]: + """Paths whose read tool result looks like file content (not an error).""" + call_path: dict[str, str] = {} + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + if not any(k in name for k in ("read", "open", "cat", "get_file")): + continue + cid = str(tc.get("id") or "") + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + parsed = None + if isinstance(parsed, dict): + for key in ("path", "file", "target"): + val = parsed.get(key) + if isinstance(val, str) and val.strip() and cid: + call_path[cid] = val.strip() + break + ok: set[str] = set() + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + cid = str(msg.get("tool_call_id") or "") + path = call_path.get(cid) + if not path: + continue + content = str(msg.get("content") or "") + low = content.lower() + if ( + "not found" in low + or "not in the project" in low + or "path not found" in low + or "error" in low[:80] + ): + continue + if len(content.strip()) < 8: + continue + ok.add(_norm_path_key(path)) + return ok + + +def _pick_read_tool(tools: list[Any]) -> str: + for t in tools: + if not isinstance(t, dict): + continue + name = _tool_name(t) + low = name.lower() + if "read" in low and "file" in low: + return name + if low in ("read", "open", "cat"): + return name + for t in tools: + if isinstance(t, dict) and _tool_name(t): + return _tool_name(t) + return "read_file" + + +def _pick_edit_tool(tools: list[Any]) -> str | None: + prefer = ( + "edit_file", + "apply_patch", + "search_replace", + "replace", + "write_file", + "create_file", + "edit", + "write", + ) + names = {_tool_name(t): t for t in tools if isinstance(t, dict) and _tool_name(t)} + low_map = {n.lower(): n for n in names} + for key in prefer: + for low, orig in low_map.items(): + if key == low or key in low: + return orig + return None + + +def _synthetic_tool_completion( + *, + model: str, + tool_name: str, + arguments: dict[str, Any], +) -> dict[str, Any]: + call_id = f"call_synth_{uuid.uuid4().hex[:10]}" + return { + "id": f"chatcmpl-synth-{uuid.uuid4().hex[:12]}", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": { + "name": tool_name, + "arguments": json.dumps( + arguments, ensure_ascii=False + ), + }, + } + ], + }, + } + ], + } + + +def _strip_read_file_line_numbers(content: str) -> str: + """Zed read_file prefixes 'NNNNNN\\t' — must strip before edit old_text.""" + out: list[str] = [] + for line in content.replace("\r\n", "\n").split("\n"): + # right-aligned 6-digit field + tab (Zed agent read_file) + if len(line) >= 7 and line[6] == "\t" and line[:6].strip().isdigit(): + out.append(line[7:]) + else: + m = re.match(r"^\s*\d+\t(.*)$", line) + out.append(m.group(1) if m else line) + return "\n".join(out) + + +_TRUNC_MARKERS = ("\n<>", "<>", "\n…", "…", "\n...", "...") + + +def _strip_tool_truncation(content: str) -> str: + """Remove gateway/compress truncation suffixes; keep usable file head.""" + c = content.replace("\r\n", "\n") + changed = True + while changed: + changed = False + for mark in _TRUNC_MARKERS: + if c.endswith(mark): + c = c[: -len(mark)].rstrip() + changed = True + break + return c + + +def _is_edit_mismatch_text(content: str) -> bool: + low = content.lower() + return any( + x in low + for x in ( + "did not match", + "could not find matching", + "old_text did not match", + "read the file again", + "matching text for edit", + "no match for old_text", + ) + ) + + +def _looks_like_file_body(content: str) -> bool: + """True if tool content is file text (not an edit error / empty).""" + c = _strip_tool_truncation(content).strip() + if len(c) < 20: + return False + low = c.lower() + if _is_edit_mismatch_text(c): + return False + if ( + "not found" in low + or "not in the project" in low + or low.startswith("error") + or "invalid type" in low + or "expected struct" in low + or "failed to edit" in low + ): + return False + return True + + +def _tool_result_for_path( + messages: list[dict[str, Any]] | None, path: str +) -> str | None: + """Return successful read_file content for path, if present in history.""" + want = _norm_path_key(path) + call_path: dict[str, str] = {} + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + if not any(k in name for k in ("read", "open", "cat", "get_file")): + continue + cid = str(tc.get("id") or "") + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + parsed = None + if isinstance(parsed, dict): + p = parsed.get("path") or parsed.get("file") or parsed.get("target") + if isinstance(p, str) and cid: + call_path[cid] = p + # Prefer newest matching read (allow truncated head — strip marker) + found: str | None = None + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + cid = str(msg.get("tool_call_id") or "") + p = call_path.get(cid) + if not p or _norm_path_key(p) != want: + continue + content = str(msg.get("content") or "") + if not _looks_like_file_body(content): + continue + body = _strip_tool_truncation(content) + found = _strip_read_file_line_numbers(body) + return found + + +def _edit_mismatch_recently(messages: list[dict[str, Any]] | None) -> bool: + """True only if latest edit failed with mismatch AND no successful re-read after it. + + Scanning any of the last N tools caused infinite read_file after a good re-read. + """ + # Map tool_call_id → path for reads + read_call_path: dict[str, str] = {} + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + cid = str(tc.get("id") or "") + if not cid: + continue + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + parsed = None + path = None + if isinstance(parsed, dict): + p = parsed.get("path") or parsed.get("file") or parsed.get("target") + if isinstance(p, str) and p.strip(): + path = p.strip() + if path and any(k in name for k in ("read", "open", "cat", "get_file")): + read_call_path[cid] = path + + edit_path = _last_edit_path(messages) + want = _norm_path_key(edit_path) if edit_path else None + for msg in reversed(messages or []): + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + content = str(msg.get("content") or "") + cid = str(msg.get("tool_call_id") or "") + # Successful read of the edited path after mismatch → do NOT re-read again + if want and cid in read_call_path: + if _norm_path_key(read_call_path[cid]) == want and _looks_like_file_body( + content + ): + return False + if _is_edit_mismatch_text(content): + return True + return False + + +def _last_edit_path(messages: list[dict[str, Any]] | None) -> str | None: + for msg in reversed(messages or []): + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in reversed(msg.get("tool_calls") or []): + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + if "edit" not in name: + continue + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + continue + if isinstance(parsed, dict): + p = parsed.get("path") + if isinstance(p, str) and p.strip(): + return p.strip() + return None + + +def _edit_failure_count(messages: list[dict[str, Any]] | None) -> int: + n = 0 + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + low = str(msg.get("content") or "").lower() + if any( + x in low + for x in ( + "failed to edit", + "edit failed", + "not unique", + "ошибка редактир", + "could not find", + "multiple matches", + "expected struct edit", + "invalid type", + "fuzzy match", + "no match", + "did not match", + "matching text for edit", + "eof while parsing", + "invalid json", + "error parsing input json", + "parent directory doesn't exist", + ) + ): + n += 1 + return n + + +def _tried_old_texts(messages: list[dict[str, Any]] | None) -> set[str]: + tried: set[str] = set() + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + if "edit" not in name: + continue + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + continue + if not isinstance(parsed, dict): + continue + for edit in parsed.get("edits") or []: + if isinstance(edit, dict): + ot = edit.get("old_text") or edit.get("old_string") + if isinstance(ot, str) and ot.strip(): + tried.add(ot) + ot2 = parsed.get("old_text") or parsed.get("old_string") + if isinstance(ot2, str) and ot2.strip(): + tried.add(ot2) + return tried + + +def _unique_snippet_edit( + content: str, + *, + exclude: set[str] | None = None, +) -> dict[str, str] | None: + """Build a unique old→new replace from file body (avoid short ambiguous strings).""" + marker = "# calentiq-ift-services-links: pending" + if marker in content: + return None + exclude = exclude or set() + lines = content.replace("\r\n", "\n").split("\n") + # Prefer longer windows first after a failed edit (more unique) + for width in (8, 7, 6, 5, 4, 3): + limit = min(120, max(0, len(lines) - width + 1)) + for i in range(limit): + chunk_lines = lines[i : i + width] + if any(not ln.strip() for ln in chunk_lines): + continue + if any("…" in ln or ln.strip() == "..." for ln in chunk_lines): + continue + old = "\n".join(chunk_lines) + if len(old) < 40: + continue + if old in exclude: + continue + if content.count(old) != 1: + continue + new = old + "\n" + marker + return {"old_text": old, "new_text": new} + return None + + +def _edit_args_for_path( + path: str, + plan: dict[str, Any] | None, + messages: list[dict[str, Any]] | None = None, +) -> dict[str, Any] | None: + """Zed edit_file shape: {path, edits:[{old_text, new_text}]} — not flat old_string.""" + del plan # reserved for future plan-aware snippets + body = _tool_result_for_path(messages, path) + if not body: + return None + snippet = _unique_snippet_edit(body, exclude=_tried_old_texts(messages)) + if not snippet: + return None + return { + "path": path, + "edits": [ + { + "old_text": snippet["old_text"], + "new_text": snippet["new_text"], + } + ], + } + + +def count_synthetic_tool_calls(messages: list[dict[str, Any]] | None) -> int: + """How many gateway-injected tool_calls already ran (call_synth_* ids).""" + n = 0 + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + cid = str(tc.get("id") or "") + if cid.startswith("call_synth_"): + n += 1 + return n + + +def _assistant_tool_fingerprint(msg: dict[str, Any]) -> str | None: + tcs = msg.get("tool_calls") or [] + if not isinstance(tcs, list) or not tcs: + return None + parts: list[str] = [] + for tc in tcs: + if not isinstance(tc, dict): + continue + fn = tc.get("function") or {} + name = str(fn.get("name") or "") + args = str(fn.get("arguments") or "")[:240] + parts.append(f"{name}:{args}") + return "|".join(parts) if parts else None + + +def _sticky_tool_loop(messages: list[dict[str, Any]] | None, *, repeat: int = 3) -> bool: + """True if the last `repeat` assistant tool turns are identical (Zed spin).""" + fps: list[str] = [] + for msg in reversed(messages or []): + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + if not msg.get("tool_calls"): + continue + fp = _assistant_tool_fingerprint(msg) + if not fp: + continue + fps.append(fp) + if len(fps) >= repeat: + break + return len(fps) >= repeat and len(set(fps)) == 1 + + +def synthetic_abort_reason( + messages: list[dict[str, Any]] | None, + *, + max_synthetic: int = 3, + max_edit_failures: int = 4, +) -> str | None: + """If non-None, gateway must stop (finish=stop) — no more synthetic tool_calls.""" + synth_n = count_synthetic_tool_calls(messages) + if max_synthetic > 0 and synth_n >= max_synthetic: + return ( + f"Стоп: слишком много synthetic continue ({synth_n}≥{max_synthetic}). " + "Модель Novita/LiteLLM отвечает 408 или пустым tool_calls — цикл прерван. " + "Сократи задачу / попробуй снова; правку файла лучше сделать точечно вручную." + ) + fails = _edit_failure_count(messages) + if max_edit_failures > 0 and fails >= max_edit_failures: + return ( + f"Стоп: edit_file падал {fails} раз(а). " + "Нужен точный old_text из актуального read_file (без усечения) " + "в формате {\"path\":\"...\",\"edits\":[{\"old_text\":\"...\",\"new_text\":\"...\"}]}." + ) + if _sticky_tool_loop(messages, repeat=3): + return ( + "Стоп: одинаковые tool_calls повторяются — агент зациклился. " + "Прервано, чтобы не крутить litellm 408 → synthetic бесконечно." + ) + return None + + +def _stop_synth(model: str, text: str) -> dict[str, Any]: + return { + "id": f"chatcmpl-synth-{uuid.uuid4().hex[:12]}", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": text}, + } + ], + } + + +def _continue_edit_or_grep( + *, + model: str, + tools: list[Any], + path: str, + plan: dict[str, Any] | None, + messages: list[dict[str, Any]] | None, +) -> dict[str, Any]: + """Never finish_reason=stop after edit fail — that kills the Zed agent loop.""" + read_counts = _read_path_counts(messages) + path_key = _norm_path_key(path) + already_read = ( + read_counts.get(path_key, 0) >= 1 + or path_key in _successful_read_keys(messages) + or bool(_tool_result_for_path(messages, path)) + ) + # Mismatch only if it is newer than any successful re-read of that path + if _edit_mismatch_recently(messages): + rpath = _last_edit_path(messages) or path + return _synthetic_tool_completion( + model=model, + tool_name=_pick_read_tool(tools), + arguments={"path": rpath}, + ) + edit_name = _pick_edit_tool(tools) + if edit_name: + args = _edit_args_for_path(path, plan, messages) + if args: + return _synthetic_tool_completion( + model=model, tool_name=edit_name, arguments=args + ) + # No body yet → one read; never re-read the same path in a loop + if not already_read: + return _synthetic_tool_completion( + model=model, + tool_name=_pick_read_tool(tools), + arguments={"path": path}, + ) + # Fallback: grep for routers section to re-anchor without full re-read + grep_name = None + for t in tools: + if not isinstance(t, dict): + continue + n = _tool_name(t).lower() + if "grep" in n or n in ("search", "find"): + grep_name = _tool_name(t) + break + if grep_name: + return _synthetic_tool_completion( + model=model, + tool_name=grep_name, + arguments={ + "pattern": "routers:|services:|middlewares:", + "path": path.rsplit("\\", 1)[0] if "\\" in path else path, + }, + ) + # Last resort: still emit edit_file with a minimal unique line from body + body = _tool_result_for_path(messages, path) or "" + lines = [ln for ln in body.split("\n") if ln.strip()] + if edit_name and len(lines) >= 1: + # pick longest unique single line + for ln in sorted(lines, key=len, reverse=True): + if len(ln) >= 24 and body.count(ln) == 1 and ln not in _tried_old_texts(messages): + marker = "# calentiq-ift-services-links: pending" + return _synthetic_tool_completion( + model=model, + tool_name=edit_name, + arguments={ + "path": path, + "edits": [ + { + "old_text": ln, + "new_text": ln + "\n" + marker, + } + ], + }, + ) + # Absolute last: ask model via a no-op-looking but valid tool — list_directory parent + list_name = None + for t in tools: + if not isinstance(t, dict): + continue + n = _tool_name(t).lower() + if "list" in n and "dir" in n: + list_name = _tool_name(t) + break + if list_name: + parent = path.rsplit("\\", 1)[0] if "\\" in path else path.rsplit("/", 1)[0] + return _synthetic_tool_completion( + model=model, + tool_name=list_name, + arguments={"path": parent}, + ) + # Only if no tools at all + return _stop_synth( + model, + "edit_file нужен в формате " + '{"path":"...","edits":[{"old_text":"...","new_text":"..."}]}. ' + f"Файл уже в контексте: {path}", + ) + + +def _last_successful_read_path( + messages: list[dict[str, Any]] | None, +) -> str | None: + """Absolute path of the newest successful read_file (for status / edit).""" + success = _successful_read_keys(messages) + if not success: + return None + last: str | None = None + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + if not any(k in name for k in ("read", "open", "cat", "get_file")): + continue + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + parsed = None + if not isinstance(parsed, dict): + continue + for key in ("path", "file", "target"): + val = parsed.get(key) + if isinstance(val, str) and _norm_path_key(val) in success: + last = val.strip() + return last + + +def _path_for_user_status( + plan: dict[str, Any] | None, + tools: list[Any], + messages: list[dict[str, Any]] | None, +) -> str | None: + """Prefer already-read / plan path — never the next unread KEY file (loadtest).""" + got = _last_successful_read_path(messages) + if got: + return got + for item in (plan or {}).get("subtasks") or []: + if not isinstance(item, dict): + continue + for p in item.get("paths") or []: + if isinstance(p, str) and p.strip() and not _is_bogus_devops_path(p): + return p.strip() + return _first_path_hint(plan, tools, messages) + + +def timeout_stop_message(reason: str, *, path: str | None = None) -> str: + base = ( + "Executor (Novita/LiteLLM) не вернул tool_calls вовремя " + f"({reason}). Synthetic mid-loop отключён — иначе Zed крутит ctx/408. " + "Повтори короче или правь файл вручную." + ) + if path: + return base + f" Цель: `{path}`." + return base + + +def midloop_stop_message(reason: str, *, path: str | None = None) -> str: + """User-facing stop after tools already ran — never 'synthetic 1≥1' noise.""" + base = ( + f"Novita/LiteLLM timeout в tool_loop ({reason}). " + "Повторный synthetic edit не делается (это и давало бесконечный ctx). " + "Уточни шаг одной фразой / «ok» ещё раз, либо правь файл вручную." + ) + if path: + return base + f" Файл: `{path}`." + return base + + +def executor_fallback_completion( + *, + plan: dict[str, Any] | None, + tools: list[Any], + model: str, + messages: list[dict[str, Any]] | None, + cfg: dict[str, Any] | None, + reason: str = "timeout", +) -> dict[str, Any]: + """Decide gateway fallback when LiteLLM 408 / empty tool_calls. + + Modes (hierarchical.executor_synthetic_mode): + never: always stop with a clear message — no fake tool_calls. + kickstart_only (default): at most one synthetic *read_file* before any + tool activity; mid tool_loop → finish=stop (no fake edits). + always: legacy synthetic edit/read with hard abort limits. + """ + cfg = cfg or {} + mode = str(cfg.get("executor_synthetic_mode") or "kickstart_only").lower() + max_s = int(cfg.get("executor_max_synthetic_continues", 1) or 1) + max_ef = int(cfg.get("executor_max_edit_failures", 4) or 4) + in_loop = messages_have_tool_activity(messages) + synth_n = count_synthetic_tool_calls(messages) + + if mode == "never": + path = _path_for_user_status(plan, tools, messages) + return _stop_synth(model, timeout_stop_message(reason, path=path)) + if mode != "always": + # kickstart_only: never invent edit_file after tools already ran + if in_loop or synth_n >= max(1, max_s): + path = _path_for_user_status(plan, tools, messages) + fails = _edit_failure_count(messages) + if fails >= max_ef: + text = ( + f"Стоп: edit_file/write_file падал {fails} раз(а). " + "Нужен точный old_text из read_file " + '{"path":"...","edits":[{"old_text":"...","new_text":"..."}]}.' + ) + elif _sticky_tool_loop(messages, repeat=3): + text = ( + "Стоп: одинаковые tool_calls повторяются — цикл прерван." + ) + elif path and _last_successful_read_path(messages): + # Prefer one more real edit attempt over a dead-end stop after 408 + cont = _continue_edit_or_grep( + model=model, + tools=tools, + path=path, + plan=plan, + messages=messages, + ) + if completion_has_tool_calls(cont): + tc0 = (cont.get("choices") or [{}])[0].get("message", {}).get( + "tool_calls" + ) or [] + name = str( + ((tc0[0].get("function") or {}).get("name")) if tc0 else "" + ).lower() + if "edit" in name or "write" in name or "grep" in name: + return cont + text = midloop_stop_message(reason, path=path) + else: + # Do NOT surface synthetic_abort_reason "1≥1" — kickstart counts as 1 + text = midloop_stop_message(reason, path=path) + return _stop_synth(model, text) + # First kick only: read primary file — model continues after Zed returns body + plan_text = json.dumps(plan or {}, ensure_ascii=False) + devops = _looks_like_devops_plan( + plan_text + + "\n" + + json.dumps(messages or [], ensure_ascii=False, default=str) + ) + root = _devops_root_from_messages(messages) + if devops and not root: + root = _DEFAULT_DEVOPS_ROOT + path = _first_path_hint(plan, tools, messages) + if not path and root: + path = _join_under(root, _DEVOPS_KEY_RELS[0]) + if not path: + return _stop_synth(model, timeout_stop_message(reason)) + return _synthetic_tool_completion( + model=model, + tool_name=_pick_read_tool(tools), + arguments={"path": path}, + ) + + # Legacy always-mode + return synthetic_first_tool_completion( + plan=plan, + tools=tools, + model=model, + messages=messages, + max_synthetic=max_s, + max_edit_failures=max_ef, + ) + + +def synthetic_first_tool_completion( + *, + plan: dict[str, Any] | None, + tools: list[Any], + model: str, + messages: list[dict[str, Any]] | None = None, + max_synthetic: int = 3, + max_edit_failures: int = 4, +) -> dict[str, Any]: + """Unblock Zed on timeout / empty tool_calls — keep the tool loop alive. + + Hard-stops after too many gateway synthetics / edit failures / sticky repeats + so litellm 408 → synthetic cannot spin forever (ctx fill spam in Zed). + """ + abort = synthetic_abort_reason( + messages, + max_synthetic=max_synthetic, + max_edit_failures=max_edit_failures, + ) + if abort: + return _stop_synth(model, abort) + + plan_text = json.dumps(plan or {}, ensure_ascii=False) + devops = _looks_like_devops_plan( + plan_text + "\n" + json.dumps(messages or [], ensure_ascii=False, default=str) + ) + root = _devops_root_from_messages(messages) + if devops and not root: + root = _DEFAULT_DEVOPS_ROOT + primary = _join_under(root, _DEVOPS_KEY_RELS[0]) if root else None + + read_counts = _read_path_counts(messages) + success = _successful_read_keys(messages) + edit_name = _pick_edit_tool(tools) + + def _should_edit(path: str) -> bool: + key = _norm_path_key(path) + return key in success or read_counts.get(key, 0) >= 1 + + target = primary + path_hint = _first_path_hint(plan, tools, messages) + if path_hint and _should_edit(path_hint): + target = path_hint + elif primary and _should_edit(primary): + target = primary + + # After failed edit(s): retry with a different unique old_text — never stop the agent + if target and _edit_failure_count(messages) > 0 and edit_name: + return _continue_edit_or_grep( + model=model, + tools=tools, + path=target, + plan=plan, + messages=messages, + ) + + if primary and _should_edit(primary) and edit_name: + args = _edit_args_for_path(primary, plan, messages) + if args: + return _synthetic_tool_completion( + model=model, tool_name=edit_name, arguments=args + ) + return _continue_edit_or_grep( + model=model, + tools=tools, + path=primary, + plan=plan, + messages=messages, + ) + + path = path_hint + if path and not _should_edit(path): + return _synthetic_tool_completion( + model=model, + tool_name=_pick_read_tool(tools), + arguments={"path": path}, + ) + if path and _should_edit(path) and edit_name: + args = _edit_args_for_path(path, plan, messages) + if args: + return _synthetic_tool_completion( + model=model, tool_name=edit_name, arguments=args + ) + return _continue_edit_or_grep( + model=model, tools=tools, path=path, plan=plan, messages=messages + ) + if primary and edit_name: + args = _edit_args_for_path(primary, plan, messages) + if args: + return _synthetic_tool_completion( + model=model, tool_name=edit_name, arguments=args + ) + return _continue_edit_or_grep( + model=model, + tools=tools, + path=primary, + plan=plan, + messages=messages, + ) + if primary: + return _continue_edit_or_grep( + model=model, + tools=tools, + path=primary, + plan=plan, + messages=messages, + ) + return _synthetic_tool_completion( + model=model, + tool_name=_pick_read_tool(tools), + arguments={"path": "README.md"}, + ) + + +def plan_payload_from_meta(meta: dict[str, Any]) -> dict[str, Any] | None: + payload = meta.get("plan_payload") + if isinstance(payload, dict) and payload.get("subtasks"): + return sanitize_plan_paths(payload) + return None + + +def merge_agent_meta( + base: dict[str, Any], + *, + executor: str, + plan: dict[str, Any] | None, + phase: str, +) -> dict[str, Any]: + out = dict(base) + out["mode"] = "hierarchical_agent" + out["agent_phase"] = phase + out["executor_model"] = executor + out["selected_model"] = executor + if plan: + out["plan_payload"] = plan + out["subtask_count"] = len(plan.get("subtasks") or []) + return out + + +def meta_header(meta: dict[str, Any] | None) -> str: + """Serialize router meta for HTTP headers (latin-1 only). + + Cyrillic plan_payload in X-Router-Meta → UnicodeEncodeError → 500 in Zed. + """ + skip = {"plan_payload", "plan", "content", "messages", "body"} + safe: dict[str, Any] = {} + for key, val in (meta or {}).items(): + if key in skip: + if key == "plan_payload" and isinstance(val, dict): + safe["subtask_count"] = len(val.get("subtasks") or []) + continue + if isinstance(val, (str, int, float, bool)) or val is None: + safe[key] = val + elif isinstance(val, (list, dict)): + try: + encoded = json.dumps(val, ensure_ascii=True, default=str) + except Exception: # noqa: BLE001 + continue + if len(encoded) <= 600: + safe[key] = json.loads(encoded) + else: + safe[key] = str(val)[:80] + return json.dumps(safe, ensure_ascii=True, default=str) + + +def completion_has_tool_calls(data: dict[str, Any] | None) -> bool: + if not isinstance(data, dict): + return False + choice = (data.get("choices") or [{}])[0] + msg = choice.get("message") or {} + tcs = msg.get("tool_calls") + return isinstance(tcs, list) and len(tcs) > 0 + + +def _tool_calls_from_completion(data: dict[str, Any] | None) -> list[dict[str, Any]]: + if not isinstance(data, dict): + return [] + choice = (data.get("choices") or [{}])[0] + msg = choice.get("message") or {} + tcs = msg.get("tool_calls") if isinstance(msg, dict) else None + return [tc for tc in tcs if isinstance(tc, dict)] if isinstance(tcs, list) else [] + + +def _read_paths_from_tool_calls(tcs: list[dict[str, Any]]) -> list[str]: + paths: list[str] = [] + for tc in tcs: + name = str((tc.get("function") or {}).get("name") or "").lower() + if not any(k in name for k in ("read", "open", "cat", "get_file")): + return [] # mixed with non-read → leave alone + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + parsed = None + path = None + if isinstance(parsed, dict): + for key in ("path", "file", "target"): + val = parsed.get(key) + if isinstance(val, str) and val.strip(): + path = val.strip() + break + if path: + paths.append(path) + return paths + + +def rewrite_redundant_reread_completion( + data: dict[str, Any], + *, + messages: list[dict[str, Any]] | None, + plan: dict[str, Any] | None, + tools: list[Any], + model: str, + cfg: dict[str, Any] | None = None, +) -> dict[str, Any]: + """If model only re-reads already-loaded / bogus paths, force a better tool. + + After a 408→DeepSeek escalate we still saw read_file on the same yml — + that looks like 'долго читает файл' and burns another round-trip. + Also blocks invented EventHubDevOps\\docker-compose.yml (does not exist). + """ + cfg = cfg or {} + if not cfg.get("executor_rewrite_reread", True): + return data + tcs = _tool_calls_from_completion(data) + if not tcs: + return data + paths = _read_paths_from_tool_calls(tcs) + if not paths: + return data + success = _successful_read_keys(messages) + read_counts = _read_path_counts(messages) + failed = _failed_path_keys(messages) + already = all( + _norm_path_key(p) in success or read_counts.get(_norm_path_key(p), 0) >= 1 + for p in paths + ) + bogus_or_failed = any( + _is_bogus_devops_path(p) or _norm_path_key(p) in failed for p in paths + ) + if not already and not bogus_or_failed: + return data + + if bogus_or_failed and not already: + # Prefer editing a file already in context over inventing another read + if success: + # recover a real path string matching a successful key + path_for_edit = None + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + if not any(k in name for k in ("read", "open", "cat", "get_file")): + continue + args = (tc.get("function") or {}).get("arguments") or "" + try: + parsed = json.loads(args) if isinstance(args, str) else args + except Exception: # noqa: BLE001 + parsed = None + if not isinstance(parsed, dict): + continue + for key in ("path", "file", "target"): + val = parsed.get(key) + if isinstance(val, str) and _norm_path_key(val) in success: + path_for_edit = val.strip() + if path_for_edit: + log.info( + "rewrite bogus read → edit already-read path=%s", path_for_edit + ) + return _continue_edit_or_grep( + model=model, + tools=tools, + path=path_for_edit, + plan=plan, + messages=messages, + ) + # Invented / previously missing path → read a real DevOps file instead + good = _first_path_hint(plan, tools, messages) + if not good: + return data + log.info("rewrite bogus/failed read_file → %s", good) + return _synthetic_tool_completion( + model=model, + tool_name=_pick_read_tool(tools), + arguments={"path": good}, + ) + + # Prefer edit on the path that was re-requested + path = paths[0] + rewritten = _continue_edit_or_grep( + model=model, + tools=tools, + path=path, + plan=plan, + messages=messages, + ) + # If continue still wants read (mismatch case), keep original re-read + r_tcs = _tool_calls_from_completion(rewritten) + if r_tcs: + r_name = str((r_tcs[0].get("function") or {}).get("name") or "").lower() + if any(k in r_name for k in ("read", "open", "cat", "get_file")): + return data + log.info( + "rewrite redundant read_file → %s path=%s", + ( + ((r_tcs[0].get("function") or {}).get("name")) + if r_tcs + else "stop" + ), + path, + ) + return rewritten + + +def completion_preview(data: dict[str, Any] | None, *, n: int = 160) -> str: + """Short preview for logs when model returns prose instead of tools.""" + if not isinstance(data, dict): + return "" + choice = (data.get("choices") or [{}])[0] + msg = choice.get("message") or {} + fr = choice.get("finish_reason") + content = msg.get("content") if isinstance(msg, dict) else None + rc = msg.get("reasoning_content") if isinstance(msg, dict) else None + text = content if isinstance(content, str) and content.strip() else "" + if not text and isinstance(rc, str): + text = rc + return f"finish={fr} content={str(text)[:n]!r}" + + +def force_kickstart_read( + *, + plan: dict[str, Any] | None, + tools: list[Any], + model: str, + messages: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Unblock Zed: one read_file on plan path (first turn only).""" + path = _first_path_hint(plan, tools, messages) + if not path: + root = _devops_root_from_messages(messages) or _DEFAULT_DEVOPS_ROOT + path = _join_under(root, _DEVOPS_KEY_RELS[0]) + return _synthetic_tool_completion( + model=model, + tool_name=_pick_read_tool(tools), + arguments={"path": path}, + ) + + +def completion_to_sse_chunks( + data: dict[str, Any], + *, + cid: str, + model: str, + drop_content_if_tools: bool = True, +) -> list[bytes]: + """Turn a non-stream chat.completion into OpenAI SSE chunks for Zed. + + Emit tool_calls in a single delta (full arguments) — fragmented argument + streams confuse some Zed builds and look like a stuck/one-shot agent. + """ + choice = (data.get("choices") or [{}])[0] + msg = choice.get("message") or {} + finish = choice.get("finish_reason") or "stop" + chunks: list[bytes] = [] + + def pack(delta: dict[str, Any], fr: str | None = None) -> bytes: + payload = { + "id": cid, + "object": "chat.completion.chunk", + "created": int(data.get("created") or 0) or int(time.time()), + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": fr}], + } + return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n".encode("utf-8") + + tool_calls = msg.get("tool_calls") + has_tools = isinstance(tool_calls, list) and bool(tool_calls) + + chunks.append(pack({"role": "assistant"})) + + content = msg.get("content") + if ( + isinstance(content, str) + and content.strip() + and not (drop_content_if_tools and has_tools) + ): + text = content + step = 800 + for i in range(0, len(text), step): + chunks.append(pack({"content": text[i : i + step]})) + + if has_tools: + finish = "tool_calls" + streamed = [] + for i, tc in enumerate(tool_calls): + if not isinstance(tc, dict): + continue + fn = tc.get("function") or {} + args = fn.get("arguments") or "" + if not isinstance(args, str): + args = json.dumps(args, ensure_ascii=False) + streamed.append( + { + "index": i, + "id": tc.get("id") or f"call_{i}", + "type": tc.get("type") or "function", + "function": { + "name": fn.get("name") or "", + "arguments": args, + }, + } + ) + chunks.append(pack({"tool_calls": streamed}, finish)) + else: + chunks.append(pack({}, finish)) + chunks.append(b"data: [DONE]\n\n") + return chunks diff --git a/router/agent_stream.py b/router/agent_stream.py new file mode 100644 index 0000000..b3a29ce --- /dev/null +++ b/router/agent_stream.py @@ -0,0 +1,587 @@ +"""Streaming agent-hierarchical path — never block Zed on a silent await.""" + +from __future__ import annotations + +import asyncio +import json +import logging +import time +from typing import Any + +import httpx +from fastapi.responses import StreamingResponse + +log = logging.getLogger("ai-router.agent_stream") + + +def sse_chunk( + *, + cid: str, + model: str, + delta: dict[str, Any], + finish_reason: str | None = None, +) -> bytes: + payload = { + "id": cid, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": delta, + "finish_reason": finish_reason, + } + ], + } + return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n".encode("utf-8") + + +def format_progress_block(lines: list[str]) -> str: + from progress_ui import format_progress_block as _fmt + + return _fmt(lines) + + +async def stream_agent_plan_then_act( + *, + runner: Any, + last_user: str, + quality_mode: str | None, + session_id: str, + messages: list[dict[str, Any]], + body: dict[str, Any], + meta: dict[str, Any], + decision: Any, + prompt_hash: str, + litellm_url: str, + litellm_key: str, + cfg: dict[str, Any], + find_pending_plan: Any, + plan_payload_from_meta: Any, + inject_plan_context: Any, + pick_agent_executor_model: Any, + prepare_agent_executor_forward: Any, + completion_to_sse_chunks: Any, + synthetic_first_tool_completion: Any, + meta_header: Any, + orchestrator: Any, + requests_metric: Any, + executor_fallback_completion: Any = None, +) -> StreamingResponse: + """Immediately SSE-stream progress; plan in background; never silent-hang.""" + + cid = f"hier-agent-{int(time.time() * 1000)}" + queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + + async def on_progress(msg: str) -> None: + await queue.put(("progress", msg)) + + async def work() -> None: + try: + hier = await runner.run( + last_user, + quality_mode=quality_mode, + session_id=session_id, + messages=messages, + on_progress=on_progress, + agent_mode=True, + ) + await queue.put(("done", hier)) + except Exception as exc: # noqa: BLE001 + log.exception("agent plan failed: %s", exc) + await queue.put(("error", exc)) + + async def gen(): + # First bytes ASAP — Zed must leave spinner for content + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={ + "role": "assistant", + "content": "**Agent**\n· Планирую задачу\n", + }, + ) + task = asyncio.create_task(work()) + hier = None + failed: Exception | None = None + progress_lines: list[str] = ["Планирую задачу"] + + while True: + try: + kind, payload = await asyncio.wait_for(queue.get(), timeout=12.0) + except asyncio.TimeoutError: + # SSE comment keepalive (proxies / Zed idle timeout) + yield b": keepalive\n\n" + if task.done() and queue.empty(): + # drain outcome if any + if not task.cancelled(): + try: + _ = task.result() + except Exception as exc: # noqa: BLE001 + failed = exc + break + continue + + if kind == "progress": + from progress_ui import humanize_line, stream_step + + line = humanize_line(str(payload)) + if not line: + continue + progress_lines.append(line) + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={"content": stream_step(line)}, + ) + continue + if kind == "error": + failed = payload # type: ignore[assignment] + break + hier = payload + break + + if not task.done(): + await task + + if failed is not None: + err = (str(failed) or type(failed).__name__)[:200] + meta["hierarchical_error"] = err + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={ + "content": ( + f"\n---\nОшибка плана (роутер, не LiteLLM UI): `{err}`.\n" + "Повтори запрос или напиши проще. Спиннер без шагов = баг " + "роутера — мы должны стримить progress; если снова тишина, " + "проверь деплой.\n" + ) + }, + ) + yield sse_chunk( + cid=cid, model="hierarchical", delta={}, finish_reason="stop" + ) + yield b"data: [DONE]\n\n" + return + + if hier is None: + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={"content": "\n---\nПустой ответ плана.\n"}, + ) + yield sse_chunk( + cid=cid, model="hierarchical", delta={}, finish_reason="stop" + ) + yield b"data: [DONE]\n\n" + return + + hier_meta = {**meta, **hier.meta} + + if hier.meta.get("agent_execute"): + plan = plan_payload_from_meta(hier.meta) or find_pending_plan(messages) + orchestrator.after_request( + session_id, prompt_hash=prompt_hash, success=True, escalate=False + ) + from path_resolve import ( + apply_deterministic_path_index, + path_resolve_needed, + pick_path_resolve_model, + ) + + if cfg.get("path_resolve_enabled", True): + plan, unresolved = apply_deterministic_path_index(plan) + if unresolved: + log.info("path_index miss (stream) → %s", unresolved) + if cfg.get("runtime_probe_enabled", True) and isinstance(plan, dict): + try: + from runtime_probe import attach_runtime_probe, runtime_facts_line + + user_bits = [] + for m in reversed(messages or []): + if isinstance(m, dict) and m.get("role") == "user": + c = m.get("content") + user_bits.append(c if isinstance(c, str) else str(c or "")) + break + plan = attach_runtime_probe( + plan, user_text=user_bits[0] if user_bits else "", cfg=cfg + ) + line = runtime_facts_line(plan) + if line: + log.info("%s", line) + except Exception as exc: # noqa: BLE001 + log.warning("runtime_probe skipped (stream): %s", exc) + path_mode = bool(cfg.get("path_resolve_enabled", True)) and path_resolve_needed( + plan + ) + if path_mode: + executor = pick_path_resolve_model(cfg) + else: + executor = pick_agent_executor_model(cfg) + timeout = float( + cfg.get("path_resolve_timeout_sec") + if path_mode + else ( + cfg.get("executor_timeout_sec") + or cfg.get("call_timeout_sec") + or 90 + ) + ) + max_ctx = int(cfg.get("executor_input_chars", 12000)) + use_stream = bool(cfg.get("executor_use_stream", False)) + verbose = bool(cfg.get("progress_verbose", False)) + from progress_ui import execution_banner + + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={ + "content": execution_banner( + path_mode=path_mode, plan=plan, model=executor + ) + }, + ) + if verbose: + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={ + "content": f"· модель `{executor}` · таймаут {int(timeout)}с\n" + }, + ) + _ = inject_plan_context + forward = prepare_agent_executor_forward( + body, + messages, + plan, + cfg, + executor=executor, + stream=use_stream, + minimal=True, + path_resolve=path_mode, + ) + n_tools = len(forward.get("tools") or []) + approx = len(json.dumps(forward, ensure_ascii=False, default=str)) + from agent_hier import context_fill_for_forward + + ctx_line, ctx_meta = context_fill_for_forward( + forward, cfg, budget_chars=max_ctx + ) + log.info( + "executor forward model=%s tools=%s bytes≈%s stream=%s %s", + executor, + n_tools, + approx, + use_stream, + ctx_line, + ) + if verbose or cfg.get("show_context_fill", False): + payload_line = f"- tools={n_tools}\n" + if cfg.get("show_context_fill", False): + payload_line += f"- {ctx_line}\n" + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={"content": payload_line}, + ) + _ = ctx_meta + try: + body_timeout = float(forward.get("timeout") or timeout) + client_timeout = body_timeout + 30.0 + if use_stream: + async with httpx.AsyncClient( + timeout=httpx.Timeout(client_timeout, connect=15.0) + ) as client: + async with client.stream( + "POST", + f"{litellm_url.rstrip('/')}/v1/chat/completions", + headers={ + "Authorization": f"Bearer {litellm_key}", + "Content-Type": "application/json", + }, + json=forward, + ) as resp: + if resp.status_code >= 400: + err_body = (await resp.aread())[:300] + raise RuntimeError( + f"litellm HTTP {resp.status_code}: " + f"{err_body.decode('utf-8', 'replace')}" + ) + buf = b"" + aiter = resp.aiter_bytes().__aiter__() + deadline = time.perf_counter() + client_timeout + got_data = False + while time.perf_counter() < deadline: + try: + piece = await asyncio.wait_for( + aiter.__anext__(), timeout=12.0 + ) + except StopAsyncIteration: + break + except asyncio.TimeoutError: + yield b": keepalive\n\n" + if not got_data: + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={ + "content": "- executor: ждём LiteLLM…\n" + }, + ) + continue + got_data = True + buf += piece + while True: + nl = buf.find(b"\n") + if nl < 0: + break + line = buf[:nl].rstrip(b"\r") + buf = buf[nl + 1 :] + if not line: + yield b"\n" + continue + yield line + b"\n" + if buf.strip(): + yield buf.rstrip(b"\r") + b"\n" + yield b"data: [DONE]\n\n" + else: + # Non-stream: keepalive while awaiting Novita (headers otherwise block) + async def _post() -> httpx.Response: + async with httpx.AsyncClient( + timeout=httpx.Timeout(client_timeout, connect=15.0) + ) as client: + return await client.post( + f"{litellm_url.rstrip('/')}/v1/chat/completions", + headers={ + "Authorization": f"Bearer {litellm_key}", + "Content-Type": "application/json", + }, + json=forward, + ) + + task = asyncio.create_task(_post()) + while not task.done(): + try: + await asyncio.wait_for(asyncio.shield(task), timeout=8.0) + except asyncio.TimeoutError: + yield b": keepalive\n\n" + yield ( + b": executor waiting LiteLLM non-stream\n\n" + ) + resp = task.result() + if resp.status_code >= 400: + raise RuntimeError( + f"litellm HTTP {resp.status_code}: " + f"{resp.text[:300]}" + ) + data = resp.json() + msg = ((data.get("choices") or [{}])[0].get("message") or {}) + has_tools = bool(msg.get("tool_calls")) + if not has_tools: + from agent_hier import ( + completion_preview, + force_kickstart_read, + pick_agent_escalate_model, + completion_has_tool_calls as _has_tc, + ) + + log.warning( + "executor no tool_calls after plan: %s", + completion_preview(data), + ) + # 1) Escalate to DeepSeek with hard tool_choice + if cfg.get("executor_no_tools_escalate", True): + esc = pick_agent_escalate_model(cfg) + esc_fwd = dict(forward) + esc_fwd["model"] = esc + esc_fwd["tool_choice"] = "required" + path_hint = None + try: + from agent_hier import _first_path_hint + + path_hint = _first_path_hint( + plan, forward.get("tools") or [], messages + ) + except Exception: # noqa: BLE001 + path_hint = None + nudge = ( + "Call a tool now. Start with read_file" + + (f" on `{path_hint}`." if path_hint else ".") + + " No prose." + ) + esc_fwd["messages"] = list(esc_fwd.get("messages") or []) + [ + {"role": "user", "content": nudge} + ] + yield ( + f": no tool_calls → escalate `{esc}`\n\n".encode( + "utf-8" + ) + ) + async with httpx.AsyncClient( + timeout=httpx.Timeout(client_timeout, connect=15.0) + ) as client2: + resp2 = await client2.post( + f"{litellm_url.rstrip('/')}/v1/chat/completions", + headers={ + "Authorization": f"Bearer {litellm_key}", + "Content-Type": "application/json", + }, + json=esc_fwd, + ) + if resp2.status_code < 400: + data2 = resp2.json() + if isinstance(data2, dict) and _has_tc(data2): + data = data2 + has_tools = True + log.info("escalate no_tools → tool_calls OK") + # 2) Kickstart read_file on plan path (keeps Zed loop alive) + if not has_tools and cfg.get( + "executor_synthetic_on_timeout", True + ): + yield b": no tool_calls -> kickstart read_file\n\n" + if str(cfg.get("executor_synthetic_mode") or "").lower() in ( + "never", + ): + # still kickstart once on first turn — otherwise dead end + data = force_kickstart_read( + plan=plan, + tools=forward.get("tools") or [], + model=executor, + messages=messages, + ) + elif executor_fallback_completion: + data = executor_fallback_completion( + plan=plan, + tools=forward.get("tools") or [], + model=executor, + messages=messages, + cfg=cfg, + reason="no_tool_calls", + ) + else: + data = force_kickstart_read( + plan=plan, + tools=forward.get("tools") or [], + model=executor, + messages=messages, + ) + has_tools = bool( + ( + (data.get("choices") or [{}])[0].get("message") or {} + ).get("tool_calls") + ) + # Visible status for Zed (short), details stay in SSE comments + from progress_ui import plan_paths, tool_status + + tc_names: list[str] = [] + msg0 = ((data.get("choices") or [{}])[0].get("message") or {}) + for tc in msg0.get("tool_calls") or []: + if isinstance(tc, dict): + tc_names.append( + str(((tc.get("function") or {}).get("name")) or "") + ) + paths = plan_paths(plan) + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={ + "content": tool_status( + tc_names, path=paths[0] if paths else None + ) + }, + ) + yield ( + f": executor OK tool_calls={has_tools}\n\n".encode("utf-8") + ) + for chunk in completion_to_sse_chunks( + data, cid=cid, model=str(data.get("model") or executor) + ): + yield chunk + except Exception as exc: # noqa: BLE001 + err = (str(exc) or type(exc).__name__)[:220] + log.exception("executor after plan failed: %s", err) + if cfg.get("executor_synthetic_on_timeout", True): + yield ( + f": timeout → executor fallback ({err[:80]})\n\n".encode( + "utf-8" + ) + ) + if executor_fallback_completion: + data = executor_fallback_completion( + plan=plan, + tools=forward.get("tools") or [], + model=executor, + messages=messages, + cfg=cfg, + reason=f"exception:{err[:80]}", + ) + else: + data = synthetic_first_tool_completion( + plan=plan, + tools=forward.get("tools") or [], + model=executor, + messages=messages, + max_synthetic=int( + cfg.get("executor_max_synthetic_continues", 1) or 1 + ), + max_edit_failures=int( + cfg.get("executor_max_edit_failures", 4) or 4 + ), + ) + for chunk in completion_to_sse_chunks( + data, cid=cid, model=executor + ): + yield chunk + else: + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={ + "content": ( + f"\nExecutor не ответил: `{err}`. " + "Роутер жив — увеличь timeout / урежь контекст; " + "повтор после ok обычно быстрее (cache).\n" + ) + }, + ) + yield sse_chunk( + cid=cid, model="hierarchical", delta={}, finish_reason="stop" + ) + yield b"data: [DONE]\n\n" + return + + # Confirm / cancel / amend — text plan for user + content = hier.content or "" + yield sse_chunk( + cid=cid, + model="hierarchical", + delta={"content": f"\n---\n\n{content}"}, + ) + try: + requests_metric.labels( + tier=decision.tier.value, + lane=decision.lane, + model="hierarchical_agent", + status="200", + ).inc() + except Exception: # noqa: BLE001 + pass + orchestrator.after_request( + session_id, prompt_hash=prompt_hash, success=True, escalate=False + ) + yield sse_chunk( + cid=cid, model="hierarchical", delta={}, finish_reason="stop" + ) + yield b"data: [DONE]\n\n" + + return StreamingResponse( + gen(), + media_type="text/event-stream", + headers={ + "X-Router-Meta": meta_header({**meta, "mode": "hierarchical_agent"}), + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + }, + ) diff --git a/router/hierarchical.py b/router/hierarchical.py new file mode 100644 index 0000000..81d40ac --- /dev/null +++ b/router/hierarchical.py @@ -0,0 +1,1137 @@ +"""Hierarchical orchestration: strong Max plan → cheap workers → conditional DeepSeek verify → synthesize.""" + +from __future__ import annotations + +import asyncio +import json +import logging +import re +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import Any + +from rules_loader import load_orchestration + +log = logging.getLogger("hierarchical") + +ProgressCallback = Callable[[str], Awaitable[None] | None] + +PLAN_SYSTEM = """You are a task planner for a coding agent. Split the user request into at most {max_subtasks} subtasks. +Reply with ONLY valid compact JSON (no markdown, no commentary, no trailing commas): +{{"subtasks":[{{"id":"1","prompt":"...","worker_tier":"simple|medium_code|hard","depends_on":[],"paths":["C:\\\\abs\\\\or\\\\rel\\\\file"],"edit_goal":"what to change","constraints":["must keep X"]}}],"acceptance":["..."]}} +Rules: +- worker_tier simple = cheap text/ops (no file tools); medium_code = routine edits/scripts; hard = rare flagship only +- Prefer medium_code for code edits; use hard only for architecture, multi-repo, or ambiguous hard bugs +- Each prompt: one sentence, ASCII quotes only, max ~200 chars, self-contained +- Always fill paths[] with concrete file paths from context when known (absolute preferred for multi-root) +- edit_goal: short outcome; constraints: must-not-break items (empty array ok) +- Prefer 1 subtask when possible; never invent long copy-paste of the user request +- If context is truncated, plan from the visible goal only +- EventHub / *.calentiq.com / IFT: Traefik + Docker Swarm live in EventHubDevOps. + REAL files only (do NOT invent nginx /var/log or classic Traefik names): + - EventHubDevOps\\\\ift\\\\traefik\\\\dynamic_conf.yml (NOT traefik.yml / traefik.toml) + - EventHubDevOps\\\\ift\\\\docker-compose.core.yml (NO root docker-compose.yml) + Static Traefik config is NOT in this repo as traefik.yml — only dynamic_conf*.yml under ift|stage/traefik/. +""" + +PLAN_REPAIR_USER = """Your previous reply was not valid JSON ({error}). +Reply again with ONLY one compact JSON object in the required schema. +Keep each prompt under 200 characters. Include paths/edit_goal when known. No markdown.""" + +VERIFY_SYSTEM = """You verify worker results against acceptance criteria. +Reply with ONLY valid JSON: +{{"ok":true|false,"retry_ids":[],"notes":"short"}} +retry_ids lists failed subtask ids (empty if ok). +""" + +SYNTH_SYSTEM = """Merge worker outputs into one clear final answer for the user. +Respect acceptance criteria. Be concise and complete. Do not invent missing work.""" + + +@dataclass +class Subtask: + id: str + prompt: str + worker_tier: str = "simple" + depends_on: list[str] = field(default_factory=list) + paths: list[str] = field(default_factory=list) + edit_goal: str = "" + constraints: list[str] = field(default_factory=list) + + +@dataclass +class HierarchicalResult: + content: str + meta: dict[str, Any] + ok: bool = True + + +def _cfg() -> dict[str, Any]: + return load_orchestration().get("hierarchical", {}) or {} + + +async def _emit(on_progress: ProgressCallback | None, message: str) -> None: + if not on_progress: + return + result = on_progress(message) + if asyncio.iscoroutine(result): + await result + elif hasattr(result, "__await__"): + await result # type: ignore[misc] + + +def _progress_verbose() -> bool: + return bool(_cfg().get("progress_verbose", False)) + + +async def _emit_user( + on_progress: ProgressCallback | None, message: str, *, verbose_only: bool = False +) -> None: + """User-facing progress. verbose_only lines skipped unless progress_verbose.""" + if verbose_only and not _progress_verbose(): + return + await _emit(on_progress, message) + + +def _assistant_text(msg: dict[str, Any]) -> str: + """Qwen3 may put the answer in reasoning_content when thinking is on.""" + content = msg.get("content") + if isinstance(content, str) and content.strip(): + return content + for key in ("reasoning_content", "reasoning"): + val = msg.get(key) + if isinstance(val, str) and val.strip(): + return val + if isinstance(content, str): + return content + return "" + + +def should_run_hierarchical( + *, + tier_value: str, + header: str | None, + quality_mode: str | None, + enabled: bool | None = None, +) -> bool: + cfg = _cfg() + if enabled is None: + enabled = bool(cfg.get("enabled", True)) + if not enabled: + return False + mode = (header or "auto").strip().lower() + if mode in ("0", "false", "no"): + mode = "off" + if mode == "off": + return False + if mode == "force": + return True + # auto + if quality_mode == "economy": + return False + triggers = cfg.get("trigger_tiers") or ["COMPLEX", "REASONING"] + return tier_value in triggers + + +def _extract_json_blob(raw: str) -> str: + text = (raw or "").strip() + fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + if fence: + text = fence.group(1).strip() + start = text.find("{") + if start < 0: + raise ValueError("JSON object not found") + chunk = text[start:] + try: + obj, _end = json.JSONDecoder().raw_decode(chunk) + return json.dumps(obj, ensure_ascii=False) + except json.JSONDecodeError: + end = text.rfind("}") + if end <= start: + return chunk + return text[start : end + 1] + + +def loads_json_relaxed(raw: str) -> Any: + """Parse LLM JSON; tolerate fences, trailing commas, mild truncation.""" + blob = _extract_json_blob(raw) + candidates = [blob] + # trailing commas before } or ] + candidates.append(re.sub(r",\s*([}\]])", r"\1", blob)) + # common truncation: close open string + structures + for suffix in ( + '"}]}', + '"]}', + '"}]', + '"}', + "}", + '"}]}]}', + '"],"acceptance":[]}', + '"}],"acceptance":[]}', + ): + candidates.append(blob.rstrip() + suffix) + candidates.append(re.sub(r",\s*([}\]])", r"\1", blob.rstrip() + suffix)) + last_err: Exception | None = None + for cand in candidates: + try: + return json.loads(cand) + except Exception as exc: # noqa: BLE001 + last_err = exc + continue + raise ValueError(str(last_err) if last_err else "JSON parse failed") + + +def parse_plan_json(raw: str, max_subtasks: int) -> tuple[list[Subtask], list[str]]: + data = loads_json_relaxed(raw) + if not isinstance(data, dict): + raise ValueError("plan JSON root must be object") + raw_tasks = data.get("subtasks") or [] + acceptance = [str(x) for x in (data.get("acceptance") or [])] + tasks: list[Subtask] = [] + for i, item in enumerate(raw_tasks[:max_subtasks]): + if not isinstance(item, dict): + continue + tid = str(item.get("id") or str(i + 1)) + prompt = str(item.get("prompt") or "").strip() + if not prompt: + continue + tier = str(item.get("worker_tier") or "simple").strip().lower() + if tier not in ("simple", "medium_code", "hard"): + tier = "medium_code" if "code" in tier or "medium" in tier else "simple" + deps = item.get("depends_on") or [] + if not isinstance(deps, list): + deps = [] + paths_raw = item.get("paths") or item.get("path") or [] + if isinstance(paths_raw, str): + paths_list = [paths_raw] if paths_raw.strip() else [] + elif isinstance(paths_raw, list): + paths_list = [str(p).strip() for p in paths_raw if str(p).strip()] + else: + paths_list = [] + edit_goal = str(item.get("edit_goal") or item.get("goal") or "").strip() + constraints_raw = item.get("constraints") or [] + if isinstance(constraints_raw, str): + constraints_list = [constraints_raw] if constraints_raw.strip() else [] + elif isinstance(constraints_raw, list): + constraints_list = [ + str(c).strip() for c in constraints_raw if str(c).strip() + ] + else: + constraints_list = [] + tasks.append( + Subtask( + id=tid, + prompt=prompt, + worker_tier=tier, + depends_on=[str(d) for d in deps], + paths=paths_list, + edit_goal=edit_goal, + constraints=constraints_list, + ) + ) + if not tasks: + raise ValueError("empty subtasks") + return tasks, acceptance + + +def truncate_plan_input(text: str, max_chars: int) -> str: + """Optional head+tail trim. max_chars<=0 means no truncation. + + Planner (Llama 8B ~128k ctx) does not need a tight 6k char cap for typical + Zed prompts (~20k chars ≈ few k tokens). Keep only a soft safety ceiling + for pathological dumps so we do not burn latency/cost. + """ + text = text or "" + if max_chars <= 0 or len(text) <= max_chars: + return text + head = max(800, max_chars // 3) + tail = max_chars - head - 24 + if tail < 1200: + tail = max_chars - 24 + return "…[truncated head]…\n" + text[-tail:] + return text[:head] + "\n…[truncated]…\n" + text[-tail:] + + +def fallback_task_prompt(user_text: str, limit: int = 4000) -> str: + """Single-task fallback: prefer a clean tail of the user request.""" + text = (user_text or "").strip() + if len(text) <= limit: + return text + chunk = text[-limit:] + for sep in ("\n\n", "\n", ". ", "? ", "! "): + idx = chunk.find(sep) + if 0 <= idx < 240: + chunk = chunk[idx + len(sep) :] + break + return chunk.strip() or text[:limit] + + +PLAN_MARKER_RE = re.compile( + r"", + re.MULTILINE, +) + + +def tasks_to_payload( + tasks: list[Subtask], + acceptance: list[str], + *, + user_goal: str = "", +) -> dict[str, Any]: + return { + "subtasks": [ + { + "id": t.id, + "prompt": t.prompt, + "worker_tier": t.worker_tier, + "depends_on": t.depends_on, + "paths": t.paths, + "edit_goal": t.edit_goal, + "constraints": t.constraints, + } + for t in tasks + ], + "acceptance": acceptance, + "user_goal": digest_text(user_goal, 4000), + } + + +def payload_to_tasks( + data: dict[str, Any], max_subtasks: int +) -> tuple[list[Subtask], list[str], str]: + raw = json.dumps(data, ensure_ascii=False) + tasks, acceptance = parse_plan_json(raw, max_subtasks) + goal = str(data.get("user_goal") or "") + return tasks, acceptance, goal + + +def format_plan_confirm( + tasks: list[Subtask], + acceptance: list[str], + *, + user_goal: str = "", +) -> str: + lines = [ + "### План (ожидает утверждения)", + "", + "Проверь шаги. Ответь одним из вариантов:", + "- **утвердить** / `ok` / `да` — выполнить план", + "- **правка:** … — дополнить или изменить план (пересоберу)", + "- **отмена** — не выполнять", + "", + ] + for t in tasks: + deps = f" (после {', '.join(t.depends_on)})" if t.depends_on else "" + extra = "" + if t.paths: + extra += f" — `{', '.join(t.paths[:4])}`" + if t.edit_goal: + extra += f" → {t.edit_goal}" + lines.append( + f"{t.id}. [{t.worker_tier}] {t.prompt}{deps}{extra}" + ) + if acceptance: + lines.append("") + lines.append("Критерии: " + "; ".join(acceptance)) + payload = json.dumps( + tasks_to_payload(tasks, acceptance, user_goal=user_goal), + ensure_ascii=False, + separators=(",", ":"), + ) + lines.append("") + lines.append(f"") + return "\n".join(lines) + + +def find_pending_plan( + messages: list[dict[str, Any]] | None, +) -> dict[str, Any] | None: + """Latest assistant message with embedded hier-plan marker.""" + if not messages: + return None + for msg in reversed(messages): + if not isinstance(msg, dict): + continue + if str(msg.get("role") or "") != "assistant": + continue + content = msg.get("content") + if not isinstance(content, str): + continue + m = PLAN_MARKER_RE.search(content) + if not m: + continue + try: + data = json.loads(m.group(1)) + except json.JSONDecodeError: + continue + if isinstance(data, dict) and data.get("subtasks"): + return data + return None + + +def classify_plan_reply(text: str) -> str: + """Return approve | amend | cancel | unclear.""" + t = (text or "").strip() + if not t: + return "unclear" + low = t.lower() + if re.search(r"(отмен|cancel|\bstop\b|\bстоп\b)", low) and len(t) < 100: + return "cancel" + if ( + re.match( + r"^(ok|okay|lgtm|yes|y|да|ок|ага|go|approve|утверждаю|утвердить)\b", + low, + ) + and len(t) < 80 + ): + return "approve" + if ( + re.search( + r"утверд|выполняй|поехал[аи]?|можно\s+выполн|согласен|\blgtm\b|\bapprove\b", + low, + ) + and len(t) < 160 + and not re.search(r"добав|убери|измен|вместо|правк", low) + ): + return "approve" + if re.search( + r"добав|убери|измен|вместо|ещё|еще|правк|amend|change|instead|" + r"переплан|дополн", + low, + ): + return "amend" + if len(t) < 40 and re.search(r"\b(да|ок|ok|go|yes)\b", low): + return "approve" + # Default: treat non-trivial reply as plan amendment + if len(t) >= 12: + return "amend" + return "unclear" + + +def topological_waves(tasks: list[Subtask]) -> list[list[Subtask]]: + by_id = {t.id: t for t in tasks} + done: set[str] = set() + waves: list[list[Subtask]] = [] + remaining = list(tasks) + while remaining: + wave = [ + t + for t in remaining + if all(d in done or d not in by_id for d in t.depends_on) + ] + if not wave: + # cycle / bad deps — run rest sequentially as one wave + waves.append(remaining) + break + waves.append(wave) + for t in wave: + done.add(t.id) + remaining = [t for t in remaining if t.id not in done] + return waves + + +def worker_model_for(tier: str, *, quality_mode: str | None = None) -> str: + """Map subtask tier → LiteLLM model name. + + quality=max uses worker_map_max (e.g. hard → Qwen3.8-Max / c-complex). + """ + cfg = _cfg() + mapping = cfg.get("worker_map") or {} + if (quality_mode or "").strip().lower() == "max": + max_map = cfg.get("worker_map_max") or {} + if tier in max_map: + return str(max_map[tier]) + return str(mapping.get(tier) or mapping.get("medium_code") or "a-medium-code") + + +def digest_text(text: str, limit: int) -> str: + text = (text or "").strip() + if len(text) <= limit: + return text + return text[: limit - 20] + "\n…[truncated]" + + +def deterministic_checks( + results: dict[str, str], + acceptance: list[str], + *, + hard_used: bool, +) -> tuple[bool, list[str]]: + """Return (ok, fail_ids).""" + fail: list[str] = [] + for tid, content in results.items(): + if not (content or "").strip(): + fail.append(tid) + continue + low = content.lower() + if "error" in low[:80] and len(content) < 40: + fail.append(tid) + if hard_used: + # force Sonnet path for hard workers + return False, fail + if acceptance and not results: + return False, fail + return len(fail) == 0, fail + + +def should_verify( + *, + policy: str, + checks_ok: bool, + hard_used: bool, + quality_mode: str | None, +) -> bool: + if quality_mode == "max": + return True + pol = (policy or "on_fail_or_hard").lower() + if pol == "never": + return False + if pol == "always": + return True + # on_fail_or_hard + return (not checks_ok) or hard_used + + +class HierarchicalRunner: + def __init__( + self, + *, + litellm_url: str, + litellm_key: str, + client: Any = None, + ) -> None: + self.litellm_url = litellm_url.rstrip("/") + self.litellm_key = litellm_key + self._client = client + + async def _chat( + self, + model: str, + messages: list[dict[str, Any]], + *, + max_tokens: int, + temperature: float = 0, + on_progress: ProgressCallback | None = None, + stage: str = "call", + ) -> str: + import httpx + + from llm_cache import cache_key, run_cached + + cfg = _cfg() + timeout = float(cfg.get("call_timeout_sec", 180)) + ttl = int(cfg.get("llm_cache_ttl_sec", 3600)) + key = cache_key( + model=model, + messages=messages, + max_tokens=max_tokens, + temperature=temperature, + ) + + async def _hit(source: str) -> None: + await _emit_user( + on_progress, + f"{stage}: повторный запрос → {source} (`{model}`)", + verbose_only=True, + ) + + async def _live() -> str: + await _emit_user( + on_progress, + f"{stage}: ждём LiteLLM `{model}` (до {int(timeout)}с)…", + verbose_only=True, + ) + headers = { + "Authorization": f"Bearer {self.litellm_key}", + "Content-Type": "application/json", + } + payload = { + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + "stream": False, + "chat_template_kwargs": {"enable_thinking": False}, + "enable_thinking": False, + } + owns = self._client is None + # connect short; read must outlive LiteLLM upstream timeout + client = self._client or httpx.AsyncClient( + timeout=httpx.Timeout(timeout, connect=15.0) + ) + try: + resp = await client.post( + f"{self.litellm_url}/v1/chat/completions", + headers=headers, + json=payload, + ) + if resp.status_code >= 400: + raise RuntimeError( + f"litellm {model} HTTP {resp.status_code}: {resp.text[:300]}" + ) + data = resp.json() + choice = (data.get("choices") or [{}])[0] + msg = choice.get("message") or {} + return _assistant_text(msg) + finally: + if owns: + await client.aclose() + + text, source = await run_cached(key, _live, ttl_sec=ttl, on_hit=_hit) + if source == "live": + await _emit_user( + on_progress, + f"{stage}: ответ `{model}` получен", + verbose_only=True, + ) + return text + + async def _chat_with_fallback( + self, + primary: str, + fallback: str | None, + messages: list[dict[str, Any]], + *, + max_tokens: int, + on_progress: ProgressCallback | None = None, + stage: str = "call", + ) -> tuple[str, str]: + try: + return ( + await self._chat( + primary, + messages, + max_tokens=max_tokens, + on_progress=on_progress, + stage=stage, + ), + primary, + ) + except Exception as exc: # noqa: BLE001 + err = (str(exc) or type(exc).__name__)[:120] + if not fallback or fallback == primary: + raise + log.warning("primary %s failed (%s), trying %s", primary, err, fallback) + await _emit( + on_progress, + f"{stage}: `{primary}` → {err}, fallback `{fallback}`…", + ) + return ( + await self._chat( + fallback, + messages, + max_tokens=max_tokens, + on_progress=on_progress, + stage=stage, + ), + fallback, + ) + + async def run( + self, + user_text: str, + *, + quality_mode: str | None = None, + session_id: str = "", + messages: list[dict[str, Any]] | None = None, + on_progress: ProgressCallback | None = None, + agent_mode: bool = False, + ) -> HierarchicalResult: + cfg = _cfg() + t0 = time.perf_counter() + wall = float(cfg.get("wall_clock_sec", 180)) + max_subtasks = int(cfg.get("max_subtasks", 5)) + planner = str(cfg.get("planner_model", "claude-haiku-planner")) + planner_fb = cfg.get("planner_fallback") + verifier = str(cfg.get("verifier_model", "claude-sonnet-verifier")) + verifier_fb = cfg.get("verifier_fallback") + synth_model = str(cfg.get("synthesize_model", "a-medium-code")) + verify_policy = str(cfg.get("verify_policy", "on_fail_or_hard")) + verify_input = str(cfg.get("verify_input", "digests")) + digest_chars = int(cfg.get("digest_chars", 400)) + plan_max = int(cfg.get("plan_max_tokens", 1024)) + verify_max = int(cfg.get("verify_max_tokens", 1024)) + synth_max = int(cfg.get("synthesize_max_tokens", 4096)) + worker_max = int(cfg.get("worker_max_tokens", 2048)) + max_retries = int(cfg.get("max_verify_retries", 1)) + plan_confirm = bool(cfg.get("plan_confirm", True)) + + worker_calls: list[dict[str, Any]] = [] + progress_log: list[str] = [] + meta: dict[str, Any] = { + "mode": "hierarchical", + "session_id": session_id or None, + "planner_model": planner, + "verifier_model": None, + "synthesize_model": synth_model, + "verify_policy": verify_policy, + "worker_calls": worker_calls, + "verify_ok": None, + "verify_skipped": False, + "progress": progress_log, + "plan_confirm": plan_confirm, + "agent_mode": agent_mode, + } + + async def note(msg: str, *, verbose: bool = False) -> None: + progress_log.append(msg) + # Auto-quiet noisy lines unless progress_verbose + auto_verbose = ( + verbose + or msg.startswith(" •") + or msg.startswith("ctx") + or "ждём LiteLLM" in msg + or "payload:" in msg + or msg.startswith("worker #") + or msg.startswith("волна ") + or msg.startswith("verify:") + or msg.startswith("синтез") + or "параллельно" in msg + ) + await _emit_user(on_progress, msg, verbose_only=auto_verbose) + + if bool(cfg.get("show_context_fill", False)) and _progress_verbose(): + try: + from agent_hier import format_context_fill, estimate_tokens_from_chars + + raw = json.dumps(messages or [], ensure_ascii=False, default=str) + plan_budget = int(cfg.get("plan_input_chars") or 48000) + window = int(cfg.get("context_window_tokens") or 32768) + await note( + format_context_fill( + used_chars=len(raw), + budget_chars=plan_budget, + model_window_tokens=window, + label="ctx-in", + ), + verbose=True, + ) + except Exception: # noqa: BLE001 + pass + + tasks: list[Subtask] = [] + acceptance: list[str] = [] + skip_planning = False + pending = find_pending_plan(messages) if plan_confirm else None + if pending: + intent = classify_plan_reply(user_text) + meta["plan_reply_intent"] = intent + if intent == "cancel": + await note("план: отменён пользователем") + meta["awaiting_plan_confirm"] = False + meta["plan_cancelled"] = True + meta["elapsed_ms"] = int((time.perf_counter() - t0) * 1000) + return HierarchicalResult( + content="План отменён. Напиши новую задачу, когда будешь готов.", + meta=meta, + ok=True, + ) + if intent == "approve": + try: + tasks, acceptance, _goal = payload_to_tasks( + pending, max_subtasks + ) + skip_planning = True + meta["planner_model"] = "approved-plan" + await note( + f"план: утверждён пользователем ({len(tasks)} подзадач)" + ) + except Exception as exc: # noqa: BLE001 + log.warning("pending plan corrupt (%s); replan", exc) + pending = None + meta["plan_pending_error"] = str(exc)[:120] + elif intent in ("amend", "unclear"): + try: + _ot, _oa, old_goal = payload_to_tasks(pending, max_subtasks) + except Exception: # noqa: BLE001 + old_goal = "" + goal = old_goal or user_text + plan_text = ( + f"Original request:\n{truncate_plan_input(goal, 8000)}\n\n" + f"Previous plan JSON:\n" + f"{json.dumps(pending, ensure_ascii=False)[:3000]}\n\n" + f"User wants to change the plan:\n{user_text}\n\n" + "Produce an updated plan JSON only." + ) + await note("план: правки от пользователя → пересборка…") + plan_messages = [ + { + "role": "system", + "content": PLAN_SYSTEM.format(max_subtasks=max_subtasks), + }, + {"role": "user", "content": plan_text}, + ] + plan_raw, used_planner = await self._chat_with_fallback( + planner, + str(planner_fb) if planner_fb else None, + plan_messages, + max_tokens=plan_max, + on_progress=on_progress, + stage="план", + ) + meta["planner_model"] = used_planner + try: + tasks, acceptance = parse_plan_json(plan_raw, max_subtasks) + except Exception as exc: # noqa: BLE001 + await note(f"план: после правок JSON битый ({exc})") + tasks = [ + Subtask( + id="1", + prompt=fallback_task_prompt(goal), + worker_tier="simple", + ) + ] + acceptance = [] + meta["subtask_count"] = len(tasks) + meta["acceptance"] = acceptance + for t in tasks: + deps = f", deps={t.depends_on}" if t.depends_on else "" + await note( + f" • #{t.id} [{t.worker_tier}→{worker_model_for(t.worker_tier)}] " + f"{digest_text(t.prompt, 80)}{deps}" + ) + confirm = format_plan_confirm( + tasks, acceptance, user_goal=goal + ) + meta["awaiting_plan_confirm"] = True + meta["elapsed_ms"] = int((time.perf_counter() - t0) * 1000) + await note("план: ждём утверждения обновлённого плана") + return HierarchicalResult(content=confirm, meta=meta, ok=True) + + if not skip_planning: + await note(f"hierarchical: план через `{planner}`…") + max_plan_chars = int(cfg.get("plan_input_chars", 48000)) + plan_text = truncate_plan_input(user_text, max_plan_chars) + if max_plan_chars > 0 and len(user_text) > max_plan_chars: + await note( + f"план: вход обрезан до {max_plan_chars} символов " + f"(было {len(user_text)}; head+tail)" + ) + + plan_messages = [ + { + "role": "system", + "content": PLAN_SYSTEM.format(max_subtasks=max_subtasks), + }, + {"role": "user", "content": plan_text}, + ] + try: + plan_raw, used_planner = await self._chat_with_fallback( + planner, + str(planner_fb) if planner_fb else None, + plan_messages, + max_tokens=plan_max, + on_progress=on_progress, + stage="план", + ) + meta["planner_model"] = used_planner + try: + tasks, acceptance = parse_plan_json(plan_raw, max_subtasks) + except Exception as exc: # noqa: BLE001 + log.warning("plan parse failed (%s); trying repair", exc) + meta["plan_parse_error"] = (str(exc) or type(exc).__name__)[:160] + await note(f"план: JSON битый → repair ({exc})") + repair_model = str(planner_fb or planner) + try: + plan_raw2 = await self._chat( + repair_model, + [ + { + "role": "system", + "content": PLAN_SYSTEM.format( + max_subtasks=max_subtasks + ), + }, + {"role": "user", "content": plan_text}, + { + "role": "assistant", + "content": digest_text(plan_raw, 1200), + }, + { + "role": "user", + "content": PLAN_REPAIR_USER.format( + error=(str(exc) or type(exc).__name__)[:120] + ), + }, + ], + max_tokens=plan_max, + temperature=0.0, + on_progress=on_progress, + stage="план-repair", + ) + tasks, acceptance = parse_plan_json( + plan_raw2, max_subtasks + ) + used_planner = repair_model + meta["planner_model"] = used_planner + meta["plan_repaired"] = True + await note( + f"план готов после repair ({used_planner}): " + f"{len(tasks)} подзадач" + ) + except Exception as exc2: # noqa: BLE001 + log.warning( + "plan repair failed (%s); single-task fallback", exc2 + ) + meta["plan_repair_error"] = ( + str(exc2) or type(exc2).__name__ + )[:160] + tasks = [ + Subtask( + id="1", + prompt=fallback_task_prompt(user_text), + worker_tier="simple", + ) + ] + acceptance = [] + await note( + f"план: JSON не разобран → 1 задача-fallback ({exc2})" + ) + else: + await note( + f"план готов ({used_planner}): {len(tasks)} подзадач" + + ( + f", acceptance={len(acceptance)}" + if acceptance + else "" + ) + ) + except Exception as exc: # noqa: BLE001 + err = (str(exc) or type(exc).__name__)[:160] + log.warning("plan LLM failed (%s); single-task fallback", err) + meta["plan_llm_error"] = err + meta["planner_model"] = None + tasks = [ + Subtask( + id="1", + prompt=fallback_task_prompt(user_text), + worker_tier="simple", + ) + ] + acceptance = [] + await note( + f"план: LLM недоступен ({err}) → 1 задача без LLM-плана" + ) + + meta["subtask_count"] = len(tasks) + meta["acceptance"] = acceptance + + for t in tasks: + deps = f", deps={t.depends_on}" if t.depends_on else "" + await note( + f" • #{t.id} [{t.worker_tier}→{worker_model_for(t.worker_tier)}] " + f"{digest_text(t.prompt, 80)}{deps}" + ) + + if plan_confirm and not skip_planning: + confirm = format_plan_confirm( + tasks, acceptance, user_goal=user_text + ) + meta["awaiting_plan_confirm"] = True + meta["elapsed_ms"] = int((time.perf_counter() - t0) * 1000) + await note("план: ждём утверждения (ответь ok / правка / отмена)") + return HierarchicalResult(content=confirm, meta=meta, ok=True) + + meta["awaiting_plan_confirm"] = False + if agent_mode: + meta["agent_execute"] = True + meta["mode"] = "hierarchical_agent" + meta["plan_payload"] = tasks_to_payload( + tasks, acceptance, user_goal=user_text + ) + meta["elapsed_ms"] = int((time.perf_counter() - t0) * 1000) + await note("agent: план готов → executor с tools (Zed)") + return HierarchicalResult(content="", meta=meta, ok=True) + + results: dict[str, str] = {} + hard_used = any(t.worker_tier == "hard" for t in tasks) + + async def run_one(task: Subtask) -> None: + model = worker_model_for(task.worker_tier, quality_mode=quality_mode) + await note(f"worker #{task.id}: запрос `{model}`…") + dep_ctx = "" + if task.depends_on: + bits = [] + for d in task.depends_on: + if d in results: + bits.append( + f"[{d}]: {digest_text(results[d], digest_chars)}" + ) + if bits: + dep_ctx = "Prior results:\n" + "\n".join(bits) + "\n\n" + w_messages = [ + { + "role": "user", + "content": f"{dep_ctx}Task {task.id}:\n{task.prompt}", + } + ] + t_start = time.perf_counter() + try: + content = await self._chat( + model, + w_messages, + max_tokens=worker_max, + temperature=0.2, + on_progress=on_progress, + stage=f"worker #{task.id}", + ) + status = "ok" + except Exception as exc: # noqa: BLE001 + content = "" + status = f"error:{exc}" + log.warning("worker %s failed: %s", task.id, exc) + results[task.id] = content + latency_ms = int((time.perf_counter() - t_start) * 1000) + worker_calls.append( + { + "id": task.id, + "worker_tier": task.worker_tier, + "model": model, + "status": status, + "latency_ms": latency_ms, + "chars": len(content or ""), + } + ) + if status == "ok": + await note( + f"worker #{task.id}: ok `{model}` {latency_ms}ms, " + f"{len(content or '')} символов" + ) + else: + await note( + f"worker #{task.id}: ошибка `{model}` — {status[:120]}" + ) + + waves = topological_waves(tasks) + for wi, wave in enumerate(waves, 1): + if time.perf_counter() - t0 > wall: + raise TimeoutError("hierarchical wall clock exceeded") + ids = ", ".join(f"#{t.id}" for t in wave) + await note(f"волна {wi}/{len(waves)}: параллельно {ids}") + await asyncio.gather(*(run_one(t) for t in wave)) + + checks_ok, fail_ids = deterministic_checks( + results, acceptance, hard_used=hard_used + ) + meta["deterministic_ok"] = checks_ok + meta["fail_ids"] = fail_ids + + verify_ran = should_verify( + policy=verify_policy, + checks_ok=checks_ok, + hard_used=hard_used, + quality_mode=quality_mode, + ) + retry_ids: list[str] = list(fail_ids) + verify_notes = "" + + if verify_ran: + await note(f"verify: `{verifier}` (policy={verify_policy})…") + for attempt in range(max_retries + 1): + if time.perf_counter() - t0 > wall: + break + if verify_input == "full": + body_parts = [ + f"### {tid}\n{results.get(tid, '')}" for tid in results + ] + else: + body_parts = [ + f"### {tid}\n{digest_text(results.get(tid, ''), digest_chars)}" + for tid in results + ] + verify_user = ( + f"User request:\n{user_text}\n\n" + f"Acceptance:\n{json.dumps(acceptance, ensure_ascii=False)}\n\n" + f"Worker digests:\n" + "\n\n".join(body_parts) + ) + v_raw, used_v = await self._chat_with_fallback( + verifier, + str(verifier_fb) if verifier_fb else None, + [ + {"role": "system", "content": VERIFY_SYSTEM}, + {"role": "user", "content": verify_user}, + ], + max_tokens=verify_max, + on_progress=on_progress, + stage="verify", + ) + meta["verifier_model"] = used_v + try: + vdata = loads_json_relaxed(v_raw) + ok = bool(vdata.get("ok")) + retry_ids = [str(x) for x in (vdata.get("retry_ids") or [])] + verify_notes = str(vdata.get("notes") or "") + except Exception: # noqa: BLE001 + ok = False + retry_ids = fail_ids or list(results.keys())[:1] + verify_notes = "verify parse failed" + meta["verify_ok"] = ok + await note( + f"verify ({used_v}): ok={ok}" + + (f", retry={retry_ids}" if retry_ids and not ok else "") + + ( + f" — {digest_text(verify_notes, 100)}" + if verify_notes + else "" + ) + ) + if ok or not retry_ids or attempt >= max_retries: + break + await note(f"повтор workers: {retry_ids}") + to_retry = [t for t in tasks if t.id in set(retry_ids)] + await asyncio.gather(*(run_one(t) for t in to_retry)) + else: + meta["verify_skipped"] = True + meta["verify_ok"] = checks_ok + await note( + f"verify: пропущен (checks_ok={checks_ok}, hard={hard_used})" + ) + + skip_synth = bool(cfg.get("skip_synthesize_if_single", True)) + non_empty = [c for c in results.values() if (c or "").strip()] + max_plan_chars = int(cfg.get("plan_input_chars", 48000)) + if ( + skip_synth + and len(tasks) == 1 + and len(non_empty) == 1 + and not verify_notes + ): + final = non_empty[0] + meta["synthesize_skipped"] = True + meta["synthesize_model"] = None + await note("синтез: пропущен (1 успешный worker)") + else: + await note(f"синтез через `{synth_model}`…") + synth_parts = [] + for tid, content in results.items(): + synth_parts.append(f"### Subtask {tid}\n{content}") + if verify_notes: + synth_parts.append(f"Verifier notes: {verify_notes}") + cap = max_plan_chars if max_plan_chars > 0 else 4000 + orig = truncate_plan_input(user_text, min(4000, cap)) + synth_user = ( + f"Original request:\n{orig}\n\n" + f"Acceptance: {json.dumps(acceptance, ensure_ascii=False)}\n\n" + + "\n\n".join(synth_parts) + ) + final = await self._chat( + synth_model, + [ + {"role": "system", "content": SYNTH_SYSTEM}, + {"role": "user", "content": synth_user}, + ], + max_tokens=synth_max, + temperature=0.2, + on_progress=on_progress, + stage="синтез", + ) + meta["synthesize_skipped"] = False + meta["elapsed_ms"] = int((time.perf_counter() - t0) * 1000) + meta["hard_used"] = hard_used + await note(f"готово за {meta['elapsed_ms']}ms") + return HierarchicalResult( + content=final, + meta=meta, + ok=bool(meta.get("verify_ok", True)), + ) diff --git a/router/llm_cache.py b/router/llm_cache.py new file mode 100644 index 0000000..4ac8e27 --- /dev/null +++ b/router/llm_cache.py @@ -0,0 +1,134 @@ +"""Redis cache + in-flight coalescing for identical LiteLLM chat calls.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import os +from collections.abc import Awaitable, Callable +from typing import Any + +log = logging.getLogger("llm_cache") + +_redis = None +_redis_tried = False +_inflight: dict[str, asyncio.Future[str]] = {} +_inflight_lock = asyncio.Lock() + + +def _get_redis(): + global _redis, _redis_tried + if _redis_tried: + return _redis + _redis_tried = True + url = os.environ.get("REDIS_URL", "").strip() + if not url: + return None + try: + import redis + + client = redis.from_url(url, decode_responses=True) + client.ping() + _redis = client + except Exception as exc: # noqa: BLE001 + log.warning("llm cache redis unavailable: %s", exc) + _redis = None + return _redis + + +def cache_key( + *, + model: str, + messages: list[dict[str, Any]], + max_tokens: int, + temperature: float, +) -> str: + blob = json.dumps( + { + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + digest = hashlib.sha256(blob.encode("utf-8")).hexdigest() + return f"ai-router:llm-cache:{digest}" + + +def get_cached(key: str) -> str | None: + r = _get_redis() + if not r: + return None + try: + val = r.get(key) + return str(val) if val is not None else None + except Exception as exc: # noqa: BLE001 + log.warning("llm cache get failed: %s", exc) + return None + + +def set_cached(key: str, value: str, ttl_sec: int) -> None: + r = _get_redis() + if not r or ttl_sec <= 0: + return + try: + r.setex(key, ttl_sec, value) + except Exception as exc: # noqa: BLE001 + log.warning("llm cache set failed: %s", exc) + + +async def run_cached( + key: str, + factory: Callable[[], Awaitable[str]], + *, + ttl_sec: int, + on_hit: Callable[[str], Awaitable[None] | None] | None = None, +) -> tuple[str, str]: + """ + Return (text, source) where source is cache|coalesce|live. + Coalesce concurrent identical calls onto one upstream request. + """ + cached = get_cached(key) + if cached is not None: + if on_hit: + maybe = on_hit("cache") + if asyncio.iscoroutine(maybe): + await maybe + return cached, "cache" + + async with _inflight_lock: + existing = _inflight.get(key) + if existing is not None: + fut: asyncio.Future[str] = existing + mine = False + else: + fut = asyncio.get_running_loop().create_future() + _inflight[key] = fut + mine = True + + if not mine: + if on_hit: + maybe = on_hit("coalesce") + if asyncio.iscoroutine(maybe): + await maybe + return await fut, "coalesce" + + try: + text = await factory() + set_cached(key, text, ttl_sec) + if not fut.done(): + fut.set_result(text) + return text, "live" + except Exception as exc: + if not fut.done(): + fut.set_exception(exc) + raise + finally: + async with _inflight_lock: + if _inflight.get(key) is fut: + _inflight.pop(key, None) diff --git a/router/orchestrator.py b/router/orchestrator.py index 98853b2..2e32c56 100644 --- a/router/orchestrator.py +++ b/router/orchestrator.py @@ -115,6 +115,7 @@ class Classifier: self._complex = self._kw_re(gw.get("complex_keywords", [])) self._reasoning = self._kw_re(gw.get("reasoning_keywords", [])) self._medium_ops = self._kw_re(gw.get("medium_ops_keywords", [])) + self._medium_code = self._kw_re(gw.get("medium_code_keywords", [])) self._ocr = self._kw_re(gw.get("ocr_keywords", [])) self._escalation = self._kw_re(gw.get("escalation_keywords", [])) wt = gw.get("word_thresholds", {}) @@ -159,7 +160,7 @@ class Classifier: return Tier.REASONING, 0.88 if self._complex.search(text): return Tier.COMPLEX, 0.88 - if self._code_block.search(text): + if self._code_block.search(text) or self._medium_code.search(text): return Tier.MEDIUM_CODE, 0.85 if self._medium_ops.search(text): return Tier.MEDIUM_OPS, 0.82 diff --git a/router/path_resolve.py b/router/path_resolve.py new file mode 100644 index 0000000..a1b171b --- /dev/null +++ b/router/path_resolve.py @@ -0,0 +1,430 @@ +"""Path resolve: deterministic index first, cheap find_* tools if gaps remain.""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Any + +log = logging.getLogger("ai-router") + +# Absolute Windows root for EventHubDevOps (Zed multi-root). +_DEFAULT_DEVOPS_ROOT = r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps" + +# keyword groups → relative paths under DevOps root (first match wins per group). +# Order matters: more specific rules first. +_INDEX_RULES: list[tuple[tuple[str, ...], tuple[str, ...]]] = [ + ( + ("loadtest", "load-test", "load_test"), + (r"ift\traefik\dynamic_conf.loadtest.yml",), + ), + ( + ( + "traefik", + "dynamic_conf", + "router", + "middleware", + "calentiq", + "observer", + "host(", + "ift.calentiq", + "stage.calentiq", + ), + (r"ift\traefik\dynamic_conf.yml",), + ), + ( + ("portainer",), + (r"ift\docker-compose.portainer.yml",), + ), + ( + ("grafana", "dashboard"), + ( + r"ift\observability\grafana\provisioning\dashboards\dashboard.yml", + r"ift\observability\grafana\provisioning\datasources\prometheus.yml", + ), + ), + ( + ("prometheus", "observability"), + (r"ift\observability\prometheus.yml",), + ), + ( + ("admin", "compose.admin", "docker-compose.admin"), + (r"ift\docker-compose.admin.yml",), + ), + ( + ("client", "compose.client", "docker-compose.client"), + (r"ift\docker-compose.client.yml",), + ), + ( + ("compose", "swarm", "stack", "docker-compose", "service"), + (r"ift\docker-compose.core.yml",), + ), +] + + +def _join_under(root: str, rel: str) -> str: + root = root.rstrip("\\/") + rel = rel.replace("/", "\\").lstrip("\\") + return root + "\\" + rel + + +def _norm(path: str) -> str: + return path.replace("/", "\\").rstrip("\\").lower() + + +def _blob(plan: dict[str, Any] | None) -> str: + return json.dumps(plan or {}, ensure_ascii=False).lower() + + +def devops_root_from_plan(plan: dict[str, Any] | None) -> str: + blob = json.dumps(plan or {}, ensure_ascii=False) + m = re.search(r"([A-Za-z]:\\[^\"'\n\r]*?EventHubDevOps)", blob) + if m: + return m.group(1) + return _DEFAULT_DEVOPS_ROOT + + +def looks_devops(plan: dict[str, Any] | None) -> bool: + b = _blob(plan) + return any( + k in b + for k in ( + "traefik", + "calentiq", + "eventhubdevops", + "docker-compose", + "swarm", + "observer", + "ift.", + "devops", + ) + ) + + +def match_index_paths(plan: dict[str, Any] | None) -> list[str]: + """Return absolute paths from keyword index (may be empty).""" + if not looks_devops(plan): + return [] + root = devops_root_from_plan(plan) + text = _blob(plan) + found: list[str] = [] + seen: set[str] = set() + for keys, rels in _INDEX_RULES: + if not any(k in text for k in keys): + continue + for rel in rels: + abs_p = _join_under(root, rel) + nk = _norm(abs_p) + if nk in seen: + continue + seen.add(nk) + found.append(abs_p) + # one rule group is usually enough for a focused plan + if found: + break + return found + + +def apply_deterministic_path_index( + plan: dict[str, Any] | None, +) -> tuple[dict[str, Any] | None, list[str]]: + """Fill plan.paths from index. Returns (plan, unresolved_queries). + + unresolved_queries empty ⇒ index covered the goal; else cheap find_* needed. + """ + if not isinstance(plan, dict): + return plan, [] + out = dict(plan) + tasks = out.get("subtasks") + if not isinstance(tasks, list) or not tasks: + # still try goal-level match + indexed = match_index_paths(out) + if indexed: + out["subtasks"] = [ + { + "id": "1", + "prompt": str(out.get("user_goal") or "edit")[:200], + "worker_tier": "medium_code", + "paths": indexed, + "edit_goal": "", + "constraints": [], + } + ] + out["path_resolve"] = "index" + return out, [] + if looks_devops(out): + q = _queries_from_text(_blob(out)) + out["path_resolve_queries"] = q + return out, q + return out, [] + + indexed = match_index_paths(out) + unresolved: list[str] = [] + fixed: list[Any] = [] + for item in tasks: + if not isinstance(item, dict): + fixed.append(item) + continue + it = dict(item) + paths = [ + p + for p in (it.get("paths") or []) + if isinstance(p, str) and p.strip() + ] + # Prefer index hits when devops; replace empty/weak paths + if indexed: + it["paths"] = indexed + it["path_source"] = "index" + elif not paths: + q = _queries_from_text( + str(it.get("prompt") or "") + + " " + + str(it.get("edit_goal") or "") + + " " + + _blob(out) + ) + unresolved.extend(q) + it["paths"] = [] + it["path_source"] = "missing" + else: + it["path_source"] = "planner" + fixed.append(it) + out["subtasks"] = fixed + if indexed: + out["path_resolve"] = "index" + out["path_resolve_queries"] = [] + log.info("path_index hit → %s", indexed) + return out, [] + # Planner had paths but index missed — if devops, still verify via find + if looks_devops(out) and not indexed: + unresolved = unresolved or _queries_from_text(_blob(out)) + out["path_resolve_queries"] = unresolved + out["path_resolve"] = "needed" + return out, unresolved + out["path_resolve_queries"] = unresolved + if unresolved: + out["path_resolve"] = "needed" + return out, unresolved + + +def _queries_from_text(text: str) -> list[str]: + low = text.lower() + qs: list[str] = [] + if any(k in low for k in ("traefik", "router", "calentiq", "observer", "dynamic")): + qs.append("dynamic_conf.yml") + if any(k in low for k in ("compose", "swarm", "docker")): + qs.append("docker-compose.core.yml") + if "portainer" in low: + qs.append("docker-compose.portainer.yml") + if "grafana" in low: + qs.append("grafana") + if "prometheus" in low: + qs.append("prometheus.yml") + if not qs: + qs.append("dynamic_conf.yml") + # unique preserve order + out: list[str] = [] + for q in qs: + if q not in out: + out.append(q) + return out[:3] + + +def enrich_plan_from_discovery_tools( + plan: dict[str, Any] | None, + messages: list[dict[str, Any]] | None, +) -> dict[str, Any] | None: + """Pull absolute paths from find_path / list_directory tool results into plan.""" + if not isinstance(plan, dict): + return plan + # Never overwrite a good deterministic index hit with noisy find parses + if str(plan.get("path_resolve") or "") == "index": + return plan + tasks = plan.get("subtasks") or [] + if ( + isinstance(tasks, list) + and tasks + and isinstance(tasks[0], dict) + and str(tasks[0].get("path_source") or "") == "index" + ): + return plan + found = _paths_from_discovery_messages(messages) + if not found: + return plan + out = dict(plan) + tasks = list(out.get("subtasks") or []) + if not tasks: + out["subtasks"] = [ + { + "id": "1", + "prompt": "edit", + "worker_tier": "medium_code", + "paths": found[:3], + "path_source": "find", + } + ] + else: + fixed = [] + for item in tasks: + if not isinstance(item, dict): + fixed.append(item) + continue + it = dict(item) + it["paths"] = found[:3] + it["path_source"] = "find" + fixed.append(it) + out["subtasks"] = fixed + out["path_resolve"] = "find" + out["path_resolve_queries"] = [] + log.info("path_resolve find → %s", found[:3]) + return out + + +def _is_workspace_path(path: str) -> bool: + """Accept only real EventHub project paths — never /etc or null:/etc artifacts.""" + key = _norm(path) + if "eventhubdevops" not in key and "eventhub" not in key: + return False + if "\\etc\\" in key or key.startswith("etc\\"): + return False + if "\\nginx\\" in key: + return False + # Reject 1-letter drive artifacts from YAML like null:/etc → l:\etc + if re.match(r"^[a-z]:\\etc\\", key): + return False + base = key.rsplit("\\", 1)[-1] + if base in ("traefik.yml", "traefik.yaml", "docker-compose.yml"): + if "\\ift\\" not in key and "\\stage\\" not in key: + return False + return True + + +def _paths_from_discovery_messages( + messages: list[dict[str, Any]] | None, +) -> list[str]: + """Extract existing file paths from find/list tool results only.""" + call_ids: set[str] = set() + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if not isinstance(tc, dict): + continue + name = str((tc.get("function") or {}).get("name") or "").lower() + if not any(k in name for k in ("find", "list", "search", "grep", "glob")): + continue + # Do not treat read_file / edit_file as discovery + if any(k in name for k in ("read", "edit", "write", "create", "delete")): + continue + cid = str(tc.get("id") or "") + if cid: + call_ids.add(cid) + # No discovery tool calls → do not scrape read_file bodies (YAML has /etc/… paths) + if not call_ids: + return [] + paths: list[str] = [] + seen: set[str] = set() + for msg in messages or []: + if not isinstance(msg, dict) or msg.get("role") != "tool": + continue + if str(msg.get("tool_call_id") or "") not in call_ids: + continue + content = str(msg.get("content") or "") + low = content.lower() + if "not found" in low and len(content) < 200: + continue + for m in re.finditer( + r"([A-Za-z]:[\\/][^\s\"'<>\]\n\r]+\.(?:yml|yaml|toml|md|json|conf))", + content, + ): + p = m.group(1).replace("/", "\\") + if not _is_workspace_path(p): + continue + nk = _norm(p) + if nk in seen: + continue + seen.add(nk) + paths.append(p) + # Also accept plain relative hits that Zed returns under EventHubDevOps + for m in re.finditer( + r"((?:EventHubDevOps[\\/](?:ift|stage)[\\/][^\s\"'<>\]\n\r]+\.(?:yml|yaml)))", + content, + re.I, + ): + p = _join_under( + _DEFAULT_DEVOPS_ROOT, + m.group(1).split("EventHubDevOps", 1)[-1].lstrip("\\/"), + ) + if not _is_workspace_path(p): + continue + nk = _norm(p) + if nk in seen: + continue + seen.add(nk) + paths.append(p) + return paths + + +def discovery_tools_only(tools: list[Any], *, max_tools: int = 3) -> list[Any]: + prefer_keys = ("find_path", "find", "list_directory", "list_dir", "grep", "search", "glob") + ranked: list[Any] = [] + for key in prefer_keys: + for t in tools or []: + if not isinstance(t, dict): + continue + fn = t.get("function") if isinstance(t.get("function"), dict) else {} + name = str(fn.get("name") or t.get("name") or "").lower() + if key in name and t not in ranked: + ranked.append(t) + if len(ranked) >= max_tools: + break + return ranked[:max_tools] + + +def pick_path_resolve_model(cfg: dict[str, Any]) -> str: + """Cheap model for find_* only — allow a-simple (unlike edit executor).""" + return str(cfg.get("path_resolve_model") or "a-simple") + + +def path_resolve_needed(plan: dict[str, Any] | None) -> bool: + if not isinstance(plan, dict): + return False + qs = plan.get("path_resolve_queries") or [] + if qs: + return True + return str(plan.get("path_resolve") or "") == "needed" + + +def force_find_path_kickstart( + *, + model: str, + tools: list[Any], + queries: list[str], +) -> dict[str, Any]: + """Synthetic find_path so Zed searches instead of inventing read_file.""" + from agent_hier import _synthetic_tool_completion + + tool_name = "find_path" + for t in tools or []: + if not isinstance(t, dict): + continue + fn = t.get("function") if isinstance(t.get("function"), dict) else {} + name = str(fn.get("name") or "") + low = name.lower() + if "find" in low or "glob" in low or ("list" in low and "dir" in low): + tool_name = name + break + q = (queries[0] if queries else "dynamic_conf.yml").strip() + args: dict[str, Any] + if "list" in tool_name.lower(): + args = { + "path": _join_under(_DEFAULT_DEVOPS_ROOT, r"ift\traefik"), + } + else: + args = {"query": q, "path": _DEFAULT_DEVOPS_ROOT} + # Zed variants + args["glob"] = f"**/{q}" if "." in q else f"**/*{q}*" + return _synthetic_tool_completion( + model=model, tool_name=tool_name, arguments=args + ) diff --git a/router/progress_ui.py b/router/progress_ui.py new file mode 100644 index 0000000..9fed8eb --- /dev/null +++ b/router/progress_ui.py @@ -0,0 +1,201 @@ +"""Compact, readable progress lines for Zed agent chat.""" + +from __future__ import annotations + +import re +from typing import Any + +_MODEL_LABEL = { + "novita-planner": "Max", + "novita-verifier": "verify", + "a-medium-code": "Coder", + "a-simple": "Llama", + "b-complex": "DeepSeek", + "c-complex": "Max", + "approved-plan": "план", +} + +_NOISE = ( + "ждём LiteLLM", + "payload:", + "ctx ", + "ctx-", + "worker #", + "волна ", + "параллельно", + "bytes≈", + "upstream_stream", + "non-stream", +) + + +def model_label(model: str | None) -> str: + if not model: + return "модель" + m = str(model) + return _MODEL_LABEL.get(m, m.split("/")[-1][:18]) + + +def short_path(path: str | None, *, max_parts: int = 3) -> str: + if not path: + return "" + p = str(path).replace("/", "\\").rstrip("\\") + parts = [x for x in p.split("\\") if x] + if len(parts) <= max_parts: + return "\\".join(parts) + return "\\".join(parts[-max_parts:]) + + +def plan_paths(plan: dict[str, Any] | None) -> list[str]: + out: list[str] = [] + for item in (plan or {}).get("subtasks") or []: + if not isinstance(item, dict): + continue + for p in item.get("paths") or []: + if isinstance(p, str) and p.strip() and p not in out: + out.append(p.strip()) + return out + + +def is_noise(line: str) -> bool: + s = (line or "").strip() + if not s: + return True + low = s.lower() + return any(n.lower() in low for n in _NOISE) + + +def humanize_line(line: str) -> str | None: + """Map internal progress → short Russian status. None = drop.""" + s = (line or "").strip() + if not s or is_noise(s): + return None + low = s.lower() + + if s in ("план…", "план...", "план"): + return "Планирую задачу" + if "план через" in low or low.startswith("hierarchical:"): + return "Планирую задачу" + if "утверждён" in low or "утвержден" in low: + m = re.search(r"(\d+)\s*подзадач", s) + n = m.group(1) if m else "" + return f"План утверждён{f' · {n} шаг(а)' if n else ''}" + if "отменён" in low or "отменен" in low: + return "План отменён" + if "правки от пользователя" in low or "пересборк" in low: + return "Пересобираю план по правкам" + if "ожидает утверждения" in low or "ждём утверждения" in low or "awaiting" in low: + return "План готов — подтверди («ok»)" + if "agent: план готов" in low or "executor с tools" in low: + return "Перехожу к выполнению" + if "готово за" in low: + return "Готово" + if "json битый" in low or "repair" in low: + return "Чиню формат плана" + if low.startswith("path_index") or "path_index hit" in low: + return None # shown via dedicated path line + if "path_resolve" in low or "поиск файл" in low: + return "Ищу файлы в проекте" + if "runtime_probe" in low or "runtime:" in low: + return "Проверяю runtime" + if "force edit" in low: + return "Правлю файл" + if "kickstart" in low: + return "Читаю файл" + if "escalate" in low or "mid-loop" in low: + return None + if "408" in s or "timeout" in low: + return "Таймаут модели — продолжаю иначе" + if "400" in s and "bad" in low: + return None + # strip technical prefixes + s = re.sub(r"^план:\s*", "", s, flags=re.I) + s = re.sub(r"^agent:\s*", "", s, flags=re.I) + if len(s) > 90: + s = s[:87] + "…" + return s + + +def format_progress_block(lines: list[str]) -> str: + """Final/static progress block — numbered, no junk.""" + steps: list[str] = [] + seen: set[str] = set() + for raw in lines or []: + h = humanize_line(str(raw)) + if not h or h in seen: + continue + seen.add(h) + steps.append(h) + if not steps: + return "" + body = "\n".join(f"{i}. {t}" for i, t in enumerate(steps, 1)) + return f"**Ход**\n{body}\n\n---\n\n" + + +def stream_header() -> str: + return "**Agent**\n" + + +def stream_step(text: str) -> str: + return f"· {text}\n" + + +def execution_banner( + *, + path_mode: bool, + plan: dict[str, Any] | None, + model: str | None, +) -> str: + """Shown once after plan approve — what happens next.""" + paths = plan_paths(plan) + lines = ["", "---", "", "**Выполнение**"] + facts = (plan or {}).get("runtime_facts") if isinstance(plan, dict) else None + step = 1 + if isinstance(facts, dict) and (facts.get("host") or facts.get("hint")): + host = facts.get("host") or "?" + status = facts.get("http_status") + suggest = facts.get("suggested_backend_url") + bit = f"{host}" + if status is not None: + bit += f" → HTTP {status}" + if suggest: + bit += f" · fix `{suggest}`" + lines.append(f"{step}. Runtime: {bit}") + step += 1 + if paths: + shown = ", ".join(f"`{short_path(p)}`" for p in paths[:2]) + src = "" + tasks = (plan or {}).get("subtasks") or [] + if tasks and isinstance(tasks[0], dict): + ps = str(tasks[0].get("path_source") or "") + if ps == "index": + src = " · индекс" + elif ps == "find": + src = " · поиск" + lines.append(f"{step}. Файл{src}: {shown}") + step += 1 + if path_mode: + lines.append(f"{step}. Уточняю путь в репозитории…") + else: + lines.append(f"{step}. Читаю → правлю") + elif path_mode: + lines.append(f"{step}. Ищу нужные файлы…") + else: + lines.append(f"{step}. Запуск ({model_label(model)})") + lines.append("") + return "\n".join(lines) + "\n" + + +def tool_status(tool_names: list[str] | None, *, path: str | None = None) -> str: + names = [str(n).lower() for n in (tool_names or []) if n] + if any("edit" in n or "write" in n for n in names): + base = "Правлю" + elif any("read" in n for n in names): + base = "Читаю" + elif any("find" in n or "list" in n or "grep" in n for n in names): + base = "Ищу" + else: + base = "Инструмент" + if path: + return stream_step(f"{base} `{short_path(path)}`") + return stream_step(base) diff --git a/router/router.py b/router/router.py index 5d3ac2e..ff1232e 100644 --- a/router/router.py +++ b/router/router.py @@ -16,6 +16,30 @@ from fastapi import FastAPI, Header, HTTPException, Request, Response from fastapi.responses import JSONResponse, StreamingResponse from metrics import CLASSIFY, CLASSIFY_LLM, DURATION, ESCALATIONS, REQUESTS, metrics_payload +from hierarchical import HierarchicalRunner, should_run_hierarchical, find_pending_plan +from agent_stream import stream_agent_plan_then_act +from agent_hier import ( + completion_has_tool_calls, + completion_preview, + completion_to_sse_chunks, + context_fill_for_forward, + executor_fallback_completion, + force_edit_after_read_completion, + inject_plan_context, + merge_agent_meta, + meta_header, + messages_have_tool_activity, + pick_agent_executor_model, + plan_payload_from_meta, + prepare_agent_executor_forward, + request_has_tools, + rewrite_redundant_reread_completion, + sanitize_plan_paths, + should_force_edit_after_read, + should_stop_after_edit, + stop_after_edit_completion, + synthetic_first_tool_completion, +) from orchestrator import Orchestrator, SessionStore, Tier from rules_loader import reload_configs @@ -34,19 +58,38 @@ orchestrator = Orchestrator(session_store) app = FastAPI(title="EventHub AI Router", version="2.0.0") +def _message_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(str(block.get("text", ""))) + return "\n".join(parts) + return "" + + def _extract_text(messages: list[dict[str, Any]]) -> str: parts: list[str] = [] for msg in messages: - content = msg.get("content", "") - if isinstance(content, str): - parts.append(content) - elif isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - parts.append(str(block.get("text", ""))) + parts.append(_message_text(msg.get("content", ""))) return "\n".join(parts) +def _last_user_text(messages: list[dict[str, Any]]) -> str: + """Latest user turn only (plan approve/amend must not see prior transcript).""" + for msg in reversed(messages or []): + if not isinstance(msg, dict): + continue + if str(msg.get("role") or "") != "user": + continue + text = _message_text(msg.get("content", "")).strip() + if text: + return text + return _extract_text(messages) + + def _has_image(messages: list[dict[str, Any]]) -> bool: for msg in messages: content = msg.get("content", "") @@ -72,6 +115,299 @@ def _auth_or_403(authorization: str | None) -> None: raise HTTPException(status_code=403, detail="Invalid API key") +def _orchestrate_header(header: str | None, body: dict[str, Any]) -> str | None: + if header: + return header.strip().lower() + meta = body.get("metadata") or {} + if isinstance(meta, dict) and meta.get("orchestrate") is not None: + return str(meta.get("orchestrate")).strip().lower() + return None + + +def _completion_from_text(content: str, *, model: str, meta: dict[str, Any]) -> dict[str, Any]: + return { + "id": f"hier-{int(time.time() * 1000)}", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + "x_router_meta": meta, + } + + +def _meta_header(meta: dict[str, Any] | None) -> str: + """HTTP-header-safe meta (delegates to agent_hier.meta_header).""" + return meta_header(meta) + + +def _sse_completion(content: str, *, model: str, meta: dict[str, Any]) -> StreamingResponse: + chunk = { + "id": f"hier-{int(time.time() * 1000)}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + } + + async def gen(): + yield f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n".encode("utf-8") + yield b"data: [DONE]\n\n" + + return StreamingResponse( + gen(), + media_type="text/event-stream", + headers={"X-Router-Meta": _meta_header(meta)}, + ) + + +def _hier_cfg() -> dict[str, Any]: + from rules_loader import load_orchestration + + return load_orchestration().get("hierarchical", {}) or {} + + +def _format_progress_block(lines: list[str]) -> str: + from progress_ui import format_progress_block + + return format_progress_block(lines) + + +def _sse_chunk( + *, + cid: str, + model: str, + delta: dict[str, Any], + finish_reason: str | None = None, +) -> bytes: + payload = { + "id": cid, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": delta, + "finish_reason": finish_reason, + } + ], + } + return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n".encode("utf-8") + + +def _sse_hierarchical_progress( + runner: HierarchicalRunner, + *, + user_text: str, + quality_mode: str | None, + session_id: str, + prompt_hash: str, + base_meta: dict[str, Any], + fallback_model: str, + messages: list[dict[str, Any]] | None = None, +) -> StreamingResponse: + """Stream live stage updates, then final answer (Zed sees progress while waiting).""" + import asyncio + + cfg = _hier_cfg() + stream_progress = bool(cfg.get("stream_progress", True)) + progress_in_content = bool(cfg.get("progress_in_content", True)) + cid = f"hier-{int(time.time() * 1000)}" + model = "hierarchical" + tier = str(base_meta.get("tier", "UNKNOWN")) + lane = str(base_meta.get("lane", "?")) + t0 = time.perf_counter() + + async def gen(): + queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + + async def on_progress(msg: str) -> None: + if stream_progress: + await queue.put(("progress", msg)) + + async def work() -> None: + try: + hier = await runner.run( + user_text, + quality_mode=quality_mode, + session_id=session_id, + messages=messages, + on_progress=on_progress if stream_progress else None, + ) + await queue.put(("done", hier)) + except Exception as exc: # noqa: BLE001 + await queue.put(("error", exc)) + + intro = "Планирую задачу" + yield _sse_chunk( + cid=cid, + model=model, + delta={"role": "assistant", "content": "**Agent**\n"} + if progress_in_content + else {"role": "assistant"}, + ) + if progress_in_content: + yield _sse_chunk(cid=cid, model=model, delta={"content": f"· {intro}\n"}) + else: + yield _sse_chunk( + cid=cid, + model=model, + delta={"reasoning_content": f"{intro}\n"}, + ) + + task = asyncio.create_task(work()) + hier_meta = dict(base_meta) + final_text = "" + failed: Exception | None = None + + while True: + kind, payload = await queue.get() + if kind == "progress": + from progress_ui import humanize_line, stream_step + + line = humanize_line(str(payload)) + if not line: + continue + if progress_in_content: + yield _sse_chunk( + cid=cid, model=model, delta={"content": stream_step(line)} + ) + else: + yield _sse_chunk( + cid=cid, + model=model, + delta={"reasoning_content": stream_step(line)}, + ) + elif kind == "done": + hier = payload + hier_meta = {**base_meta, **hier.meta} + final_text = hier.content or "" + REQUESTS.labels( + tier=tier, lane=lane, model="hierarchical", status="200" + ).inc() + DURATION.labels(tier=tier, model="hierarchical").observe( + time.perf_counter() - t0 + ) + orchestrator.after_request( + session_id, + prompt_hash=prompt_hash, + success=hier.ok, + escalate=False, + ) + break + elif kind == "error": + failed = payload + break + + await task + + if failed is not None: + err = (str(failed) or type(failed).__name__)[:200] + hier_meta["hierarchical_error"] = err + note = ( + f"- hierarchical сорвался ({err}); " + f"отвечаю одной моделью `{fallback_model}`…\n" + ) + if progress_in_content: + yield _sse_chunk(cid=cid, model=model, delta={"content": note}) + yield _sse_chunk(cid=cid, model=model, delta={"content": "\n---\n\n"}) + try: + async with httpx.AsyncClient(timeout=120.0) as client: + resp = await client.post( + f"{LITELLM_URL}/v1/chat/completions", + headers={ + "Authorization": f"Bearer {LITELLM_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": fallback_model, + "messages": [ + {"role": "user", "content": user_text[-12000:]} + ], + "max_tokens": 2048, + "stream": False, + }, + ) + if resp.status_code >= 400: + raise RuntimeError(f"fallback HTTP {resp.status_code}") + data = resp.json() + msg = ((data.get("choices") or [{}])[0].get("message") or {}) + final_text = str( + msg.get("content") + or msg.get("reasoning_content") + or msg.get("reasoning") + or "" + ) + hier_meta["fallback_model"] = fallback_model + REQUESTS.labels( + tier=tier, lane=lane, model=fallback_model, status="200" + ).inc() + orchestrator.after_request( + session_id, + prompt_hash=prompt_hash, + success=bool(final_text), + escalate=False, + ) + except Exception as fb_exc: # noqa: BLE001 + fb_err = (str(fb_exc) or type(fb_exc).__name__)[:200] + yield _sse_chunk( + cid=cid, + model=model, + delta={ + "content": f"ошибка hierarchical: {err}; " + f"fallback тоже: {fb_err}\n" + }, + ) + REQUESTS.labels( + tier=tier, lane=lane, model="hierarchical", status="500" + ).inc() + orchestrator.after_request( + session_id, + prompt_hash=prompt_hash, + success=False, + escalate=False, + ) + yield _sse_chunk(cid=cid, model=model, delta={}, finish_reason="stop") + yield b"data: [DONE]\n\n" + return + + if progress_in_content and failed is None: + yield _sse_chunk(cid=cid, model=model, delta={"content": "\n---\n\n"}) + step = 240 + for i in range(0, len(final_text), step): + yield _sse_chunk( + cid=cid, model=model, delta={"content": final_text[i : i + step]} + ) + done_chunk = { + "id": cid, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model if failed is None else fallback_model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "x_router_meta": hier_meta, + } + yield f"data: {json.dumps(done_chunk, ensure_ascii=False)}\n\n".encode("utf-8") + yield b"data: [DONE]\n\n" + + return StreamingResponse( + gen(), + media_type="text/event-stream", + headers={"X-Router-Meta": _meta_header(base_meta)}, + ) + + def _quality_mode(header: str | None, body: dict[str, Any]) -> str | None: if header: mode = header.strip().lower() @@ -170,6 +506,84 @@ async def classify_debug( } +def _sanitize_reasoning_fields(obj: dict[str, Any]) -> bool: + """Normalize reasoning fields for Zed: one key, promote into content if empty.""" + changed = False + for choice in obj.get("choices") or []: + if not isinstance(choice, dict): + continue + for key in ("delta", "message"): + block = choice.get(key) + if not isinstance(block, dict): + continue + reasoning = block.get("reasoning") + reasoning_content = block.get("reasoning_content") + if reasoning is not None: + if reasoning_content is None: + block["reasoning_content"] = reasoning + reasoning_content = reasoning + changed = True + del block["reasoning"] + changed = True + if "reasoning_details" in block: + del block["reasoning_details"] + changed = True + psf = block.get("provider_specific_fields") + if isinstance(psf, dict) and "reasoning" in psf: + del psf["reasoning"] + changed = True + if not psf: + del block["provider_specific_fields"] + # Novita/Qwen3 often puts the visible answer only in reasoning_content + content = block.get("content") + rc = block.get("reasoning_content") + if (not isinstance(content, str) or not content.strip()) and isinstance( + rc, str + ) and rc.strip(): + block["content"] = rc + changed = True + return changed + + +def _sanitize_sse_line(line: bytes) -> bytes: + if not line.startswith(b"data:"): + return line + payload = line[5:].strip() + if payload == b"[DONE]": + return line + try: + obj = json.loads(payload) + except json.JSONDecodeError: + return line + if not isinstance(obj, dict) or not _sanitize_reasoning_fields(obj): + return line + return b"data: " + json.dumps(obj, ensure_ascii=False).encode("utf-8") + + +async def _iter_sanitized_sse(resp: httpx.Response): + buf = b"" + try: + async for chunk in resp.aiter_bytes(): + buf += chunk + while True: + split_at = buf.find(b"\n") + if split_at < 0: + break + line = buf[:split_at] + buf = buf[split_at + 1 :] + line = line.rstrip(b"\r") + if not line: + yield b"\n" + continue + yield _sanitize_sse_line(line) + b"\n" + if buf: + yield _sanitize_sse_line(buf.rstrip(b"\r")) + b"\n" + except httpx.HTTPError as exc: + # Upstream drop mid-SSE must not crash ASGI → Zed "unexpected EOF" + log.warning("upstream SSE aborted: %s", exc) + yield b"data: [DONE]\n\n" + + async def _forward_litellm( forward: dict[str, Any], *, @@ -183,9 +597,11 @@ async def _forward_litellm( start = time.perf_counter() tier = decision_meta.get("tier", "UNKNOWN") model = forward.get("model", DEFAULT_MODEL) + lane = decision_meta.get("lane", "?") - async with httpx.AsyncClient(timeout=180.0) as client: - if stream: + if stream: + client = httpx.AsyncClient(timeout=300.0) + try: req = client.build_request( "POST", f"{LITELLM_URL}/v1/chat/completions", @@ -194,19 +610,45 @@ async def _forward_litellm( ) resp = await client.send(req, stream=True) - async def event_stream(): - async for chunk in resp.aiter_bytes(): - yield chunk + if resp.status_code >= 400: + body = await resp.aread() + await resp.aclose() + await client.aclose() + status = str(resp.status_code) + REQUESTS.labels(tier=tier, lane=lane, model=model, status=status).inc() + DURATION.labels(tier=tier, model=model).observe(time.perf_counter() - start) + try: + content = json.loads(body) + except json.JSONDecodeError: + content = {"error": body.decode("utf-8", errors="replace")} + return JSONResponse(status_code=resp.status_code, content=content) - REQUESTS.labels(tier=tier, lane=decision_meta["lane"], model=model, status=str(resp.status_code)).inc() + async def event_stream(): + try: + async for chunk in _iter_sanitized_sse(resp): + yield chunk + finally: + await resp.aclose() + await client.aclose() + + REQUESTS.labels( + tier=tier, + lane=lane, + model=model, + status=str(resp.status_code), + ).inc() DURATION.labels(tier=tier, model=model).observe(time.perf_counter() - start) return StreamingResponse( event_stream(), status_code=resp.status_code, media_type=resp.headers.get("content-type", "text/event-stream"), - headers={"X-Router-Meta": json.dumps(decision_meta)}, + headers={"X-Router-Meta": _meta_header(decision_meta)}, ) + except Exception: + await client.aclose() + raise + async with httpx.AsyncClient(timeout=300.0) as client: resp = await client.post( f"{LITELLM_URL}/v1/chat/completions", headers=headers, @@ -214,7 +656,7 @@ async def _forward_litellm( ) status = str(resp.status_code) - REQUESTS.labels(tier=tier, lane=decision_meta["lane"], model=model, status=status).inc() + REQUESTS.labels(tier=tier, lane=lane, model=model, status=status).inc() DURATION.labels(tier=tier, model=model).observe(time.perf_counter() - start) if resp.status_code >= 400: @@ -222,15 +664,792 @@ async def _forward_litellm( data = resp.json() if isinstance(data, dict): + _sanitize_reasoning_fields(data) data["x_router_meta"] = decision_meta return data + +async def _forward_agent_executor( + body: dict[str, Any], + messages: list[dict[str, Any]], + *, + plan: dict[str, Any] | None, + base_meta: dict[str, Any], + stream: bool, + phase: str, +) -> Any: + from path_resolve import ( + apply_deterministic_path_index, + enrich_plan_from_discovery_tools, + force_find_path_kickstart, + path_resolve_needed, + pick_path_resolve_model, + ) + + cfg = _hier_cfg() + plan = sanitize_plan_paths(plan) if plan else plan + # Re-apply index every turn (plan from Zed transcript loses path_source) + if cfg.get("path_resolve_enabled", True): + plan, unresolved = apply_deterministic_path_index(plan) + if unresolved and not messages_have_tool_activity(messages): + log.info("path_index miss → queries=%s", unresolved) + if messages_have_tool_activity(messages): + plan = enrich_plan_from_discovery_tools(plan, messages) + + # DevOps: live TCP/HTTP facts before blind Traefik edits + if cfg.get("runtime_probe_enabled", True) and isinstance(plan, dict): + try: + from runtime_probe import attach_runtime_probe, runtime_facts_line + + plan = attach_runtime_probe( + plan, user_text=_last_user_text(messages), cfg=cfg + ) + line = runtime_facts_line(plan) + if line: + log.info("%s", line) + except Exception as exc: # noqa: BLE001 + log.warning("runtime_probe skipped: %s", exc) + + path_mode = bool(cfg.get("path_resolve_enabled", True)) and path_resolve_needed( + plan + ) + + if path_mode: + phase = "path_resolve" + executor = pick_path_resolve_model(cfg) + else: + executor = pick_agent_executor_model(cfg) + + # After edit_file already ran — stop the Zed loop (was infinite force-edit) + if ( + not path_mode + and phase == "tool_loop" + and should_stop_after_edit(messages, cfg) + ): + log.info("stop after edit_file (prevent edit loop)") + data = stop_after_edit_completion(model=str(executor), messages=messages) + meta = merge_agent_meta( + base_meta, executor=executor, plan=plan, phase="edit_done" + ) + meta["executor_stop_after_edit"] = True + REQUESTS.labels( + tier=meta.get("tier", "UNKNOWN"), + lane=meta.get("lane", "?"), + model=str(executor), + status="200", + ).inc() + if not stream: + data["x_router_meta"] = meta + return JSONResponse(content=data) + cid = f"hier-edit-done-{int(time.time() * 1000)}" + + async def edit_done_stream(): + for chunk in completion_to_sse_chunks( + data, cid=cid, model=str(data.get("model") or executor) + ): + yield chunk + + return StreamingResponse( + edit_done_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "X-Router-Meta": _meta_header(meta), + }, + ) + + # After successful read_file: skip Novita DeepSeek (408→400 loop) → edit_file + if ( + not path_mode + and phase == "tool_loop" + and should_force_edit_after_read(messages, cfg, plan=plan) + ): + log.info("force edit_file after successful read (skip DeepSeek)") + data = force_edit_after_read_completion( + plan=plan, + tools=body.get("tools") if isinstance(body.get("tools"), list) else [], + model=str(executor), + messages=messages, + ) + meta = merge_agent_meta( + base_meta, executor=executor, plan=plan, phase="force_edit" + ) + meta["executor_force_edit_after_read"] = True + facts = (plan or {}).get("runtime_facts") if isinstance(plan, dict) else None + if isinstance(facts, dict) and facts.get("suggested_backend_url"): + meta["runtime_targeted_edit"] = facts.get("suggested_backend_url") + REQUESTS.labels( + tier=meta.get("tier", "UNKNOWN"), + lane=meta.get("lane", "?"), + model=str(executor), + status="200", + ).inc() + if not stream: + data["x_router_meta"] = meta + return JSONResponse(content=data) + cid = f"hier-force-edit-{int(time.time() * 1000)}" + from progress_ui import short_path, stream_step + from agent_hier import _last_successful_read_path + + path = _last_successful_read_path(messages) + edit_label = ( + f"Правлю порт Traefik → `{facts.get('suggested_backend_url')}`" + if isinstance(facts, dict) and facts.get("suggested_backend_url") + else (f"Правлю `{short_path(path)}`" if path else "Правлю файл") + ) + + async def force_edit_stream(): + yield _sse_chunk( + cid=cid, + model=str(executor), + delta={ + "role": "assistant", + "content": stream_step(edit_label), + }, + ) + for chunk in completion_to_sse_chunks( + data, cid=cid, model=str(data.get("model") or executor) + ): + yield chunk + + return StreamingResponse( + force_edit_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "X-Router-Meta": _meta_header(meta), + }, + ) + + # Stream-to-Novita often aborts mid-SSE (Zed: unexpected EOF). Prefer non-stream. + use_upstream_stream = bool(cfg.get("executor_use_stream", False)) + forward = prepare_agent_executor_forward( + body, + messages, + plan, + cfg, + executor=executor, + stream=use_upstream_stream, + path_resolve=path_mode, + ) + + def _synth_fallback(reason: str) -> dict[str, Any]: + if path_mode: + return force_find_path_kickstart( + model=str(executor), + tools=forward.get("tools") or [], + queries=list((plan or {}).get("path_resolve_queries") or []) + or ["dynamic_conf.yml"], + ) + return executor_fallback_completion( + plan=plan, + tools=forward.get("tools") or [], + model=executor, + messages=messages, + cfg=cfg, + reason=reason, + ) + budget = int( + cfg.get("executor_tool_loop_chars") + if phase == "tool_loop" + else cfg.get("executor_input_chars") + or 16000 + ) + ctx_line, ctx_meta = context_fill_for_forward( + forward, cfg, budget_chars=budget + ) + meta = merge_agent_meta(base_meta, executor=executor, plan=plan, phase=phase) + meta.update( + { + "ctx_used_chars": ctx_meta["ctx_used_chars"], + "ctx_budget_chars": ctx_meta["ctx_budget_chars"], + "ctx_pct": ctx_meta["ctx_pct"], + "ctx_est_tokens": ctx_meta["ctx_est_tokens"], + } + ) + # prepare_agent_executor_forward may escalate model after edit failures + actual_model = str(forward.get("model") or executor) + if actual_model != str(executor): + meta["executor_escalated"] = True + meta["executor_model_requested"] = str(executor) + meta["executor_model"] = actual_model + meta["selected_model"] = actual_model + meta["planner_model"] = meta.get("planner_model") + meta["role_cost"] = { + "planner": meta.get("planner_model"), + "executor": actual_model, + "escalated": True, + } + executor = actual_model + log.info("hierarchical_agent escalate → %s", actual_model) + else: + meta["role_cost"] = { + "planner": meta.get("planner_model"), + "executor": str(executor), + "escalated": False, + } + log.info( + "hierarchical_agent phase=%s executor=%s tools=%s msgs=%s upstream_stream=%s %s", + phase, + executor, + len(forward.get("tools") or []), + len(forward["messages"]), + use_upstream_stream, + ctx_line, + ) + try: + if use_upstream_stream: + return await _forward_litellm( + forward, stream=stream, decision_meta=meta + ) + + # Gateway fails first via httpx; body timeout slightly under client so LiteLLM + # cancels cleanly — we still convert 408 → synthetic (never expose to Zed). + body_timeout = float( + forward.get("timeout") or cfg.get("executor_timeout_sec") or 55 + ) + client_timeout = body_timeout + 20.0 + # Ensure LiteLLM sees the same budget (otherwise deployment 120 hangs us) + forward = dict(forward) + forward["timeout"] = body_timeout + async with httpx.AsyncClient( + timeout=httpx.Timeout(client_timeout, connect=15.0) + ) as client: + resp = await client.post( + f"{LITELLM_URL}/v1/chat/completions", + headers={ + "Authorization": f"Bearer {LITELLM_KEY}", + "Content-Type": "application/json", + }, + json=forward, + ) + if resp.status_code >= 400: + body_l = (resp.text or "").lower() + is_timeout = resp.status_code in (408, 504, 502) or "timeout" in body_l + is_bad_req = resp.status_code == 400 or "invalid request" in body_l + # Mid tool_loop: one real model retry (no synthetic) before stop + if ( + is_timeout + and phase == "tool_loop" + and messages_have_tool_activity(messages) + and cfg.get("executor_midloop_model_retry", True) + ): + from agent_hier import pick_agent_escalate_model + + esc = pick_agent_escalate_model(cfg) + already_esc = str(forward.get("model") or executor) == str(esc) + if already_esc: + log.warning( + "mid-loop skip retry — already on escalate model %s → synth", + esc, + ) + else: + log.warning( + "litellm HTTP %s mid tool_loop → one model retry (no synthetic)", + resp.status_code, + ) + retry_fwd = dict(forward) + if cfg.get("executor_midloop_escalate", True): + retry_fwd["model"] = esc + meta["executor_midloop_escalate_model"] = esc + log.warning("mid-loop escalate → %s", esc) + tools_r = list(retry_fwd.get("tools") or []) + prefer = [] + for t in tools_r: + n = str( + ( + (t.get("function") or {}) + if isinstance(t, dict) + else {} + ).get("name") + or "" + ).lower() + if "read" in n or "edit" in n or "write" in n: + prefer.append(t) + if prefer: + retry_fwd["tools"] = prefer[:3] + retry_fwd["tool_choice"] = "required" + # Re-sanitize messages (null content → 400) + from agent_hier import sanitize_outbound_messages + + retry_fwd["messages"] = sanitize_outbound_messages( + retry_fwd.get("messages") or [] + ) + async with httpx.AsyncClient( + timeout=httpx.Timeout(client_timeout, connect=15.0) + ) as client2: + resp2 = await client2.post( + f"{LITELLM_URL}/v1/chat/completions", + headers={ + "Authorization": f"Bearer {LITELLM_KEY}", + "Content-Type": "application/json", + }, + json=retry_fwd, + ) + if resp2.status_code < 400: + data2 = resp2.json() + if isinstance(data2, dict): + _sanitize_reasoning_fields(data2) + if completion_has_tool_calls(data2): + log.info("mid-loop model retry OK → tool_calls") + meta["executor_midloop_retry"] = True + resp = resp2 + data = rewrite_redundant_reread_completion( + data2, + messages=messages, + plan=plan, + tools=forward.get("tools") or [], + model=str(retry_fwd.get("model") or executor), + cfg=cfg, + ) + choice0 = (data.get("choices") or [{}])[0] + msg0 = choice0.get("message") or {} + tcs = ( + msg0.get("tool_calls") + if isinstance(msg0, dict) + else None + ) + tc_names = [] + if isinstance(tcs, list): + for tc in tcs: + if isinstance(tc, dict): + tc_names.append( + str( + ( + (tc.get("function") or {}).get( + "name" + ) + ) + or "?" + ) + ) + log.info( + "executor result phase=%s finish=%s tools=%s", + phase, + choice0.get("finish_reason"), + tc_names, + ) + REQUESTS.labels( + tier=meta.get("tier", "UNKNOWN"), + lane=meta.get("lane", "?"), + model=executor, + status=str(resp.status_code), + ).inc() + if not stream: + data["x_router_meta"] = meta + return JSONResponse(content=data) + cid = f"hier-agent-{int(time.time() * 1000)}" + + async def event_stream_retry(): + if cfg.get("show_context_fill", False): + yield _sse_chunk( + cid=cid, + model=str(executor), + delta={ + "role": "assistant", + "content": ( + f"- {ctx_line}\n" + "- litellm mid-loop retry OK\n" + ), + }, + ) + for chunk in completion_to_sse_chunks( + data, + cid=cid, + model=str(data.get("model") or executor), + ): + yield chunk + + return StreamingResponse( + event_stream_retry(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "X-Router-Meta": _meta_header(meta), + }, + ) + log.warning( + "mid-loop model retry still failed status=%s", + getattr(resp2, "status_code", "?"), + ) + + if cfg.get("executor_synthetic_on_timeout", True) and ( + is_timeout or is_bad_req + ): + log.warning( + "litellm HTTP %s (timeout/400) → executor fallback", + resp.status_code, + ) + data = _synth_fallback(f"litellm_{resp.status_code}") + # Prefer force-edit if we already have file body + if ( + phase == "tool_loop" + and should_force_edit_after_read(messages, cfg, plan=plan) + and not completion_has_tool_calls(data) + ): + data = force_edit_after_read_completion( + plan=plan, + tools=forward.get("tools") or [], + model=str(executor), + messages=messages, + ) + meta["executor_force_edit_after_read"] = True + meta["executor_synthetic"] = True + fr0 = ((data.get("choices") or [{}])[0] or {}).get("finish_reason") + meta["executor_synthetic_reason"] = ( + f"litellm_{resp.status_code}_abort" + if fr0 == "stop" + else f"litellm_{resp.status_code}" + ) + if not stream: + data["x_router_meta"] = meta + return JSONResponse(content=data) + cid = f"hier-agent-synth-{int(time.time() * 1000)}" + + async def timeout_synth_stream(): + if cfg.get("show_context_fill", False): + label = ( + "stop (no mid-loop synthetic)\n" + if fr0 == "stop" + else "kickstart/edit\n" + ) + yield _sse_chunk( + cid=cid, + model=str(executor), + delta={ + "role": "assistant", + "content": ( + f"- {ctx_line}\n" + f"- litellm {resp.status_code} → {label}" + ), + }, + ) + for chunk in completion_to_sse_chunks( + data, cid=cid, model=executor + ): + yield chunk + + return StreamingResponse( + timeout_synth_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "X-Router-Meta": _meta_header(meta), + }, + ) + raise RuntimeError( + f"litellm HTTP {resp.status_code}: {resp.text[:300]}" + ) + data = resp.json() + if not isinstance(data, dict): + raise RuntimeError("litellm returned non-object JSON") + if isinstance(data, dict): + _sanitize_reasoning_fields(data) + + choice0 = (data.get("choices") or [{}])[0] + msg0 = choice0.get("message") or {} + tcs = msg0.get("tool_calls") if isinstance(msg0, dict) else None + tc_names = [] + if isinstance(tcs, list): + for tc in tcs: + if isinstance(tc, dict): + tc_names.append( + str(((tc.get("function") or {}).get("name")) or "?") + ) + if completion_has_tool_calls(data): + data = rewrite_redundant_reread_completion( + data, + messages=messages, + plan=plan, + tools=forward.get("tools") or [], + model=executor, + cfg=cfg, + ) + choice0 = (data.get("choices") or [{}])[0] + msg0 = choice0.get("message") or {} + tcs = msg0.get("tool_calls") if isinstance(msg0, dict) else None + tc_names = [] + if isinstance(tcs, list): + for tc in tcs: + if isinstance(tc, dict): + tc_names.append( + str(((tc.get("function") or {}).get("name")) or "?") + ) + log.info( + "executor result phase=%s finish=%s tools=%s", + phase, + choice0.get("finish_reason"), + tc_names, + ) + + # Model returned prose / empty tools — unblock Zed + if ( + cfg.get("executor_synthetic_on_timeout", True) + and not completion_has_tool_calls(data) + ): + log.warning( + "executor no tool_calls phase=%s %s", + phase, + completion_preview(data), + ) + # Mid-loop: escalate already tried on 408; for empty tools escalate once + if ( + phase == "tool_loop" + and messages_have_tool_activity(messages) + and cfg.get("executor_no_tools_escalate", True) + ): + from agent_hier import pick_agent_escalate_model + + esc = pick_agent_escalate_model(cfg) + retry_fwd = dict(forward) + retry_fwd["model"] = esc + retry_fwd["tool_choice"] = "required" + try: + async with httpx.AsyncClient( + timeout=httpx.Timeout(client_timeout, connect=15.0) + ) as client3: + resp3 = await client3.post( + f"{LITELLM_URL}/v1/chat/completions", + headers={ + "Authorization": f"Bearer {LITELLM_KEY}", + "Content-Type": "application/json", + }, + json=retry_fwd, + ) + if resp3.status_code < 400: + data3 = resp3.json() + if isinstance(data3, dict) and completion_has_tool_calls(data3): + data = data3 + _sanitize_reasoning_fields(data) + meta["executor_no_tools_escalated"] = True + except Exception as exc: # noqa: BLE001 + log.warning("no_tools escalate failed: %s", exc) + if not completion_has_tool_calls(data): + data = _synth_fallback( + f"no_tool_calls:{choice0.get('finish_reason')}" + ) + meta["executor_synthetic"] = True + fr_nt = ((data.get("choices") or [{}])[0] or {}).get("finish_reason") + meta["executor_synthetic_reason"] = ( + "no_tool_calls_abort" if fr_nt == "stop" else "no_tool_calls" + ) + if fr_nt == "stop": + log.warning("tool_loop fallback stop (kickstart_only / limit)") + else: + log.warning( + "no tool_calls (finish=%s) → kickstart/stop", + choice0.get("finish_reason"), + ) + + REQUESTS.labels( + tier=meta.get("tier", "UNKNOWN"), + lane=meta.get("lane", "?"), + model=executor, + status=str(resp.status_code), + ).inc() + + if not stream: + data["x_router_meta"] = meta + return JSONResponse(content=data) + + cid = f"hier-agent-{int(time.time() * 1000)}" + + async def event_stream(): + if cfg.get("show_context_fill", False): + yield _sse_chunk( + cid=cid, + model=str(executor), + delta={"role": "assistant", "content": f"- {ctx_line}\n"}, + ) + for chunk in completion_to_sse_chunks( + data, cid=cid, model=str(data.get("model") or executor) + ): + yield chunk + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "X-Router-Meta": _meta_header(meta), + }, + ) + except Exception as exc: # noqa: BLE001 + err = (str(exc) or type(exc).__name__)[:240] + log.exception("agent executor failed phase=%s: %s", phase, err) + meta["executor_error"] = err + # Always synthetic on timeout/errors when enabled — never raw 408 to Zed + if cfg.get("executor_synthetic_on_timeout", True): + data = _synth_fallback(f"exception:{err[:80]}") + cid = f"hier-agent-synth-{int(time.time() * 1000)}" + meta["executor_synthetic"] = True + fr_ex = ((data.get("choices") or [{}])[0] or {}).get("finish_reason") + meta["executor_synthetic_reason"] = ( + "exception_abort" if fr_ex == "stop" else "exception" + ) + if not stream: + data["x_router_meta"] = meta + return JSONResponse(content=data) + + async def synth_stream(): + if cfg.get("show_context_fill", False): + label = "stop" if fr_ex == "stop" else "kickstart" + yield _sse_chunk( + cid=cid, + model=str(executor), + delta={ + "role": "assistant", + "content": ( + f"- executor error → {label} ({err[:80]})\n" + ), + }, + ) + for chunk in completion_to_sse_chunks( + data, cid=cid, model=executor + ): + yield chunk + + return StreamingResponse( + synth_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "X-Router-Meta": _meta_header(meta), + }, + ) + msg = ( + f"Executor `{executor}` не ответил ({err}). " + "Роутер на IFT жив — это таймаут upstream, не «серверы недоступны»." + ) + if stream: + return _sse_completion(msg, model="hierarchical", meta=meta) + return _completion_from_text(msg, model="hierarchical", meta=meta) + + +async def _agent_hierarchical_chat( + *, + body: dict[str, Any], + messages: list[dict[str, Any]], + last_user: str, + decision: Any, + meta: dict[str, Any], + session_id: str, + prompt_hash: str, +) -> Any: + """Plan/confirm on gateway; executor model returns tool_calls for Zed. + + Critical: when stream=True we MUST return StreamingResponse immediately and + emit progress — never await the full plan before the first SSE byte. + """ + stream = bool(body.get("stream", False)) + cfg = _hier_cfg() + runner = HierarchicalRunner(litellm_url=LITELLM_URL, litellm_key=LITELLM_KEY) + + # Zed tool loop mid-flight: never text-synthesize + if messages_have_tool_activity(messages): + plan = find_pending_plan(messages) + return await _forward_agent_executor( + body, + messages, + plan=plan, + base_meta=meta, + stream=stream, + phase="tool_loop", + ) + + if stream: + return await stream_agent_plan_then_act( + runner=runner, + last_user=last_user, + quality_mode=decision.quality_mode, + session_id=session_id, + messages=messages, + body=body, + meta=meta, + decision=decision, + prompt_hash=prompt_hash, + litellm_url=LITELLM_URL, + litellm_key=LITELLM_KEY, + cfg=cfg, + find_pending_plan=find_pending_plan, + plan_payload_from_meta=plan_payload_from_meta, + inject_plan_context=inject_plan_context, + pick_agent_executor_model=pick_agent_executor_model, + prepare_agent_executor_forward=prepare_agent_executor_forward, + completion_to_sse_chunks=completion_to_sse_chunks, + synthetic_first_tool_completion=synthetic_first_tool_completion, + executor_fallback_completion=executor_fallback_completion, + meta_header=meta_header, + orchestrator=orchestrator, + requests_metric=REQUESTS, + ) + + # Non-stream: still catch errors — never raw 500 hang for Zed + try: + hier = await runner.run( + last_user, + quality_mode=decision.quality_mode, + session_id=session_id, + messages=messages, + agent_mode=True, + ) + except Exception as exc: # noqa: BLE001 + log.exception("agent hierarchical plan failed: %s", exc) + meta["hierarchical_error"] = (str(exc) or type(exc).__name__)[:200] + return await _forward_agent_executor( + body, + messages, + plan=None, + base_meta=meta, + stream=False, + phase="plan_error_fallback", + ) + + hier_meta = {**meta, **hier.meta} + if hier.meta.get("agent_execute"): + plan = plan_payload_from_meta(hier.meta) or find_pending_plan(messages) + orchestrator.after_request( + session_id, prompt_hash=prompt_hash, success=True, escalate=False + ) + return await _forward_agent_executor( + body, + messages, + plan=plan, + base_meta=hier_meta, + stream=False, + phase="execute", + ) + + content = hier.content or "" + if bool(cfg.get("progress_in_content", True)): + trail = hier.meta.get("progress") or [] + if isinstance(trail, list) and trail: + content = _format_progress_block([str(x) for x in trail]) + content + REQUESTS.labels( + tier=decision.tier.value, + lane=decision.lane, + model="hierarchical_agent", + status="200", + ).inc() + orchestrator.after_request( + session_id, prompt_hash=prompt_hash, success=True, escalate=False + ) + return _completion_from_text(content, model="hierarchical", meta=hier_meta) + + @app.post("/v1/chat/completions") async def chat_completions( request: Request, authorization: str | None = Header(default=None), x_ai_quality: str | None = Header(default=None, alias="X-AI-Quality"), + x_ai_orchestrate: str | None = Header(default=None, alias="X-AI-Orchestrate"), ) -> Any: _auth_or_403(authorization) @@ -239,25 +1458,103 @@ async def chat_completions( if not isinstance(messages, list): raise HTTPException(status_code=400, detail="messages must be a list") - text = _extract_text(messages) + text_all = _extract_text(messages) + last_user = _last_user_text(messages) has_image = _has_image(messages) session_id = _session_id(body) - prompt_hash = hashlib.sha256(text.encode()).hexdigest()[:16] + prompt_hash = hashlib.sha256(text_all.encode()).hexdigest()[:16] + quality = _quality_mode(x_ai_quality, body) + orch_hdr = _orchestrate_header(x_ai_orchestrate, body) requested_model = body.get("model", DEFAULT_MODEL) if requested_model in ("smart-router", "auto", ""): decision = await orchestrator.resolve( messages, - quality_mode=_quality_mode(x_ai_quality, body), + quality_mode=quality, session_id=session_id, - text=text, + text=text_all, has_image=has_image, - token_estimate=_estimate_tokens(text), + token_estimate=_estimate_tokens(text_all), ) if decision.classifier_source == "gigachat": CLASSIFY_LLM.labels(tier=decision.tier.value, status="ok").inc() + CLASSIFY.labels(tier=decision.tier.value).inc() target_model = decision.model meta = _router_meta(decision, requested=requested_model) + + if ( + not has_image + and should_run_hierarchical( + tier_value=decision.tier.value, + header=orch_hdr, + quality_mode=decision.quality_mode, + ) + ): + # Zed Agent Write: keep tool loop; hierarchical only plans/routes + if request_has_tools(body): + return await _agent_hierarchical_chat( + body=body, + messages=messages, + last_user=last_user, + decision=decision, + meta=meta, + session_id=session_id, + prompt_hash=prompt_hash, + ) + + runner = HierarchicalRunner( + litellm_url=LITELLM_URL, + litellm_key=LITELLM_KEY, + ) + stream = bool(body.get("stream", False)) + if stream: + return _sse_hierarchical_progress( + runner, + user_text=last_user, + quality_mode=decision.quality_mode, + session_id=session_id, + prompt_hash=prompt_hash, + base_meta=meta, + fallback_model=target_model, + messages=messages, + ) + + start = time.perf_counter() + try: + hier = await runner.run( + last_user, + quality_mode=decision.quality_mode, + session_id=session_id, + messages=messages, + ) + except Exception as exc: # noqa: BLE001 + log.exception("hierarchical failed, falling back to single route: %s", exc) + meta["hierarchical_error"] = (str(exc) or type(exc).__name__)[:200] + else: + hier_meta = {**meta, **hier.meta} + REQUESTS.labels( + tier=decision.tier.value, + lane=decision.lane, + model="hierarchical", + status="200", + ).inc() + DURATION.labels(tier=decision.tier.value, model="hierarchical").observe( + time.perf_counter() - start + ) + orchestrator.after_request( + session_id, + prompt_hash=prompt_hash, + success=hier.ok, + escalate=False, + ) + content = hier.content or "" + if bool(_hier_cfg().get("progress_in_content", True)): + trail = hier.meta.get("progress") or [] + if isinstance(trail, list) and trail: + content = _format_progress_block([str(x) for x in trail]) + content + return _completion_from_text( + content, model="hierarchical", meta=hier_meta + ) else: target_model = requested_model meta = {"selected_model": target_model, "requested_model": requested_model} diff --git a/router/runtime_probe.py b/router/runtime_probe.py new file mode 100644 index 0000000..c44a92b --- /dev/null +++ b/router/runtime_probe.py @@ -0,0 +1,348 @@ +"""DevOps runtime probe: gather live facts before editing Traefik/compose.""" + +from __future__ import annotations + +import json +import logging +import re +import socket +import subprocess +import urllib.error +import urllib.request +from typing import Any + +log = logging.getLogger("ai-router") + +_HOST_RE = re.compile( + r"\b([a-z0-9-]+(?:\.(?:ift|stage))?\.calentiq\.com)\b", + re.I, +) + +# hostname pattern → docker DNS name + candidate ports +_SERVICE_MAP: list[tuple[str, str, tuple[int, ...]]] = [ + (r"^observer\.", "observer_web", (80, 4000, 8080, 3000)), + (r"^grafana\.", "grafana", (3000, 80)), + (r"^prometheus\.", "prometheus", (9090, 80)), + (r"^portainer\.", "portainer", (9000, 9443, 80)), + (r"^ai-router\.", "ai-router", (8000, 80)), + (r"^litellm\.", "litellm", (4000, 80)), + (r"^logs\.|^loglynx\.", "loglynx", (6123, 80)), + (r"^kuma\.", "uptime-kuma", (3001, 80)), +] + + +def extract_hosts(*texts: str) -> list[str]: + found: list[str] = [] + seen: set[str] = set() + for text in texts: + if not text: + continue + for m in _HOST_RE.finditer(text): + h = m.group(1).lower() + if h not in seen: + seen.add(h) + found.append(h) + return found + + +def is_devops_request(plan: dict[str, Any] | None, *extra: str) -> bool: + blob = json.dumps(plan or {}, ensure_ascii=False).lower() + for t in extra: + blob += "\n" + str(t or "").lower() + keys = ( + "calentiq", + "traefik", + "bad gateway", + "502", + "504", + "eventhubdevops", + "docker-compose", + "swarm", + "observer", + "ift.", + "stage.", + "devops", + "gateway", + ) + if any(k in blob for k in keys): + return True + return bool(extract_hosts(blob)) + + +def service_for_host(host: str) -> tuple[str, tuple[int, ...]]: + h = host.lower() + for pat, name, ports in _SERVICE_MAP: + if re.search(pat, h): + return name, ports + # fallback: first label as service guess + label = h.split(".")[0] + return label.replace("-", "_"), (80, 443, 8080, 3000, 4000) + + +def _tcp_open(host: str, port: int, timeout: float = 2.0) -> bool: + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +def _http_status(url: str, timeout: float = 8.0) -> int | None: + try: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 + return int(getattr(resp, "status", 200) or 200) + except urllib.error.HTTPError as exc: + return int(exc.code) + except Exception: # noqa: BLE001 + return None + + +def probe_host(host: str, *, cfg: dict[str, Any] | None = None) -> dict[str, Any]: + """Live probe from AiRouter container (same Docker network as services).""" + cfg = cfg or {} + timeout = float(cfg.get("runtime_probe_timeout_sec") or 8) + service, ports = service_for_host(host) + https = f"https://{host}/" + http = f"http://{host}/" + status = _http_status(https, timeout=timeout) + if status is None: + status = _http_status(http, timeout=timeout) + + port_hits: dict[str, bool] = {} + open_ports: list[int] = [] + for p in ports: + ok = _tcp_open(service, p, timeout=min(2.0, timeout)) + port_hits[f"{service}:{p}"] = ok + if ok: + open_ports.append(p) + + closed_preferred = not port_hits.get(f"{service}:80", True) and bool(open_ports) + hint = "" + suggested_url = None + if status in (502, 503, 504) and open_ports: + # Classic misconfig: Traefik → :80 while app on :4000 + if 80 not in open_ports and open_ports: + suggested_url = f"http://{service}:{open_ports[0]}" + hint = ( + f"HTTP {status}: Traefik likely targets a closed port. " + f"{service}:80 open={port_hits.get(f'{service}:80')}; " + f"open ports={open_ports}. Prefer {suggested_url} in dynamic_conf.yml." + ) + else: + hint = ( + f"HTTP {status} but {service} has open ports {open_ports or 'none'}. " + "Check container health / upstream path." + ) + elif status in (502, 503, 504): + hint = ( + f"HTTP {status}: no open TCP on guessed service `{service}` " + f"ports {list(ports)}. Service down or wrong DNS name." + ) + elif status and status < 400: + hint = f"HTTP {status}: host responds; issue may be app-level." + else: + hint = f"HTTP status={status}; ports={port_hits}" + + facts: dict[str, Any] = { + "host": host, + "http_status": status, + "service": service, + "ports": port_hits, + "open_ports": open_ports, + "suggested_backend_url": suggested_url, + "hint": hint, + "source": "gateway_tcp", + "closed_port_80_but_alt_open": closed_preferred, + } + log.info( + "runtime_probe host=%s status=%s service=%s open=%s suggest=%s", + host, + status, + service, + open_ports, + suggested_url, + ) + return facts + + +def attach_runtime_probe( + plan: dict[str, Any] | None, + *, + user_text: str = "", + cfg: dict[str, Any] | None = None, +) -> dict[str, Any] | None: + """If DevOps task — probe hosts and attach runtime_facts to plan.""" + if not isinstance(plan, dict): + return plan + cfg = cfg or {} + if not cfg.get("runtime_probe_enabled", True): + return plan + if plan.get("runtime_facts"): + return plan + if not is_devops_request(plan, user_text): + return plan + + out = dict(plan) + hosts = extract_hosts( + user_text, + json.dumps(plan, ensure_ascii=False), + str(plan.get("user_goal") or ""), + ) + if not hosts: + # DevOps without host — still mark needed for terminal phase + out["runtime_probe"] = "needed" + out["runtime_facts"] = { + "hint": "DevOps task without clear host — curl/docker inspect before edit.", + "source": "none", + } + return out + + facts_list = [probe_host(h, cfg=cfg) for h in hosts[:3]] + primary = facts_list[0] + out["runtime_facts"] = primary + out["runtime_facts_all"] = facts_list + out["runtime_probe"] = "gateway" + # Strengthen paths toward Traefik dynamic conf for ift hosts + if "ift." in primary.get("host", "") or primary.get("suggested_backend_url"): + root = r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps" + yml = root + r"\ift\traefik\dynamic_conf.yml" + tasks = list(out.get("subtasks") or []) + if tasks and isinstance(tasks[0], dict): + t0 = dict(tasks[0]) + paths = list(t0.get("paths") or []) + if yml not in paths: + paths = [yml] + paths + t0["paths"] = paths[:4] + if primary.get("suggested_backend_url"): + t0["edit_goal"] = ( + f"Set Traefik service backend to {primary['suggested_backend_url']} " + f"(HTTP {primary.get('http_status')}: wrong upstream port)." + ) + tasks[0] = t0 + out["subtasks"] = tasks + return out + + +def runtime_facts_line(plan: dict[str, Any] | None) -> str: + facts = (plan or {}).get("runtime_facts") if isinstance(plan, dict) else None + if not isinstance(facts, dict) or not facts: + return "" + parts = [ + f"host={facts.get('host')}", + f"http={facts.get('http_status')}", + f"service={facts.get('service')}", + ] + if facts.get("open_ports"): + parts.append(f"open={facts.get('open_ports')}") + if facts.get("suggested_backend_url"): + parts.append(f"fix→{facts.get('suggested_backend_url')}") + hint = str(facts.get("hint") or "")[:180] + return "RUNTIME: " + "; ".join(str(p) for p in parts) + (f" | {hint}" if hint else "") + + +def devops_blocks_blind_force_edit(plan: dict[str, Any] | None) -> bool: + """Do not blind-edit DevOps YAML until runtime facts exist.""" + if not isinstance(plan, dict): + return False + if not is_devops_request(plan): + return False + facts = plan.get("runtime_facts") + if not facts: + return True + # Facts present — allow targeted edit (including force with goal from facts) + return False + + +def suggested_traefik_edit( + plan: dict[str, Any] | None, +) -> dict[str, str] | None: + """If probe found closed:80 + open:alt — return old/new URL snippets.""" + if not isinstance(plan, dict): + return None + facts = plan.get("runtime_facts") + if not isinstance(facts, dict): + return None + url = facts.get("suggested_backend_url") + service = facts.get("service") + if not url or not service: + return None + # Common misconfig in our dynamic_conf + old = f'url: "http://{service}:80"' + new = f'url: "{url}"' + if old == new: + return None + return { + "path": r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps\ift\traefik\dynamic_conf.yml", + "old_text": old, + "new_text": new, + "reason": str(facts.get("hint") or "port mismatch"), + } + + +def force_runtime_fix_edit( + *, + plan: dict[str, Any] | None, + tools: list[Any], + model: str, +) -> dict[str, Any] | None: + """Synthetic edit_file for clear Traefik port mismatch — skip blind marker edits.""" + fix = suggested_traefik_edit(plan) + if not fix: + return None + # late import to avoid cycles + from agent_hier import _pick_edit_tool, _synthetic_tool_completion + + edit_name = _pick_edit_tool(tools) + if not edit_name: + return None + log.info( + "runtime_probe → targeted edit %s => %s", + fix["old_text"], + fix["new_text"], + ) + return _synthetic_tool_completion( + model=model, + tool_name=edit_name, + arguments={ + "path": fix["path"], + "edits": [{"old_text": fix["old_text"], "new_text": fix["new_text"]}], + }, + ) + + +def ssh_probe_fallback(host: str, cfg: dict[str, Any]) -> dict[str, Any] | None: + """Optional: ssh to host and docker inspect (if gateway has keys).""" + ssh_host = str(cfg.get("runtime_probe_ssh_host") or "").strip() + if not ssh_host: + return None + service, _ports = service_for_host(host) + script = ( + f"echo HTTP=$(curl -sk -o /dev/null -w '%{{http_code}}' --max-time 5 https://{host}/ || echo err); " + f"CID=$(docker ps -q --filter name={service} | head -1); " + f"echo CID=$CID; " + f"if [ -n \"$CID\" ]; then docker inspect \"$CID\" --format 'PORTS={{{{json .NetworkSettings.Ports}}}}'; fi" + ) + try: + proc = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=5", + ssh_host, + "bash", + "-lc", + script, + ], + capture_output=True, + text=True, + timeout=20, + check=False, + ) + out = (proc.stdout or "") + "\n" + (proc.stderr or "") + return {"host": host, "ssh_raw": out[:1500], "source": "ssh", "service": service} + except Exception as exc: # noqa: BLE001 + log.warning("runtime ssh probe failed: %s", exc) + return None diff --git a/scripts/check-litellm-master.sh b/scripts/check-litellm-master.sh new file mode 100644 index 0000000..934d95a --- /dev/null +++ b/scripts/check-litellm-master.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +PG=$(docker ps -q -f name=ai-router_postgres | head -1) + +echo "== Tokens/users now ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT count(*) AS tokens FROM \"LiteLLM_VerificationToken\"; +SELECT count(*) AS users FROM \"LiteLLM_UserTable\"; +SELECT \"user_id\", \"user_role\" FROM \"LiteLLM_UserTable\" LIMIT 5; +" + +echo "== Master key models probe ==" +code=$(curl -sS -o /tmp/m.json -w "%{http_code}" --max-time 20 \ + -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \ + https://litellm.ift.calentiq.com/v1/models) +echo "models HTTP $code" +head -c 120 /tmp/m.json; echo + +echo "== Flush redis key cache (best effort) ==" +RID=$(docker ps -q -f name=ai-router_redis | head -1) +if [[ -n "$RID" ]]; then + docker exec "$RID" redis-cli KEYS '*token*' 2>/dev/null | head -20 || true + docker exec "$RID" redis-cli KEYS '*litellm*' 2>/dev/null | head -20 || true + # do not FLUSHALL — may kill router session cache; only clear litellm-ish if safe +fi +echo DONE +REMOTE diff --git a/scripts/check-observer-live.sh b/scripts/check-observer-live.sh new file mode 100644 index 0000000..53e63a5 --- /dev/null +++ b/scripts/check-observer-live.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +echo "== curl observer ==" +curl -sS -o /tmp/obs.body -w "http=%{http_code} time=%{time_total}\n" \ + --max-time 15 -k https://observer.ift.calentiq.com/ || true +head -c 400 /tmp/obs.body; echo +echo "== swarm observer_web ==" +docker service ls 2>/dev/null | grep -i observer || true +docker service ps $(docker service ls -q --filter name=observer 2>/dev/null | head -1) --no-trunc 2>/dev/null | head -8 || \ + docker ps -a --filter name=observer --format '{{.Names}} {{.Status}}' | head -10 +echo "== network aliases ==" +docker network inspect eventhub-ift-net --format '{{range .Containers}}{{.Name}} {{.IPv4Address}}{{"\n"}}{{end}}' 2>/dev/null | grep -i observer || true +REMOTE diff --git a/scripts/clean-spurious-litellm-fails.sh b/scripts/clean-spurious-litellm-fails.sh new file mode 100644 index 0000000..6dc5134 --- /dev/null +++ b/scripts/clean-spurious-litellm-fails.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Explain + delete spurious LiteLLM failure logs from restart disconnects +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +USER="${POSTGRES_USER:-litellm}" +DB="${POSTGRES_DB:-litellm}" + +echo "== Error message sample ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "$USER" -d "$DB" -t -A -c " +SELECT left(metadata->'error_information'->>'error_message', 200) +FROM \"LiteLLM_SpendLogs\" +WHERE status='failure' +LIMIT 3; +" + +echo "== Delete empty-model restart failures ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "$USER" -d "$DB" -c " +DELETE FROM \"LiteLLM_SpendLogs\" +WHERE status = 'failure' + AND (model IS NULL OR model = '') + AND (call_type IS NULL OR call_type = ''); +" + +echo "== Remaining ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "$USER" -d "$DB" -c " +SELECT status, count(*) FROM \"LiteLLM_SpendLogs\" GROUP BY 1; +" +echo CLEANED_SPURIOUS_FAILS_OK +REMOTE diff --git a/scripts/clear-litellm-db-ift.sh b/scripts/clear-litellm-db-ift.sh new file mode 100644 index 0000000..e6cebdb --- /dev/null +++ b/scripts/clear-litellm-db-ift.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Clear LiteLLM Postgres spend/logs on IFT (keep schema + keys if present) +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +if [[ -z "$PG" ]]; then + echo "ERROR: postgres container not running" >&2 + exit 1 +fi +USER="${POSTGRES_USER:-litellm}" +DB="${POSTGRES_DB:-litellm}" +echo "== Tables before ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "$USER" -d "$DB" -c "\dt" + +# LiteLLM Prisma tables — truncate data, keep schema +# Prefer spend/usage; also clear invite/audit-ish if present +SQL=$(cat <<'EOS' +DO $$ +DECLARE + r RECORD; + n bigint; +BEGIN + FOR r IN + SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename NOT IN ('_prisma_migrations') + LOOP + EXECUTE format('SELECT count(*) FROM %I', r.tablename) INTO n; + RAISE NOTICE 'truncate % (% rows)', r.tablename, n; + EXECUTE format('TRUNCATE TABLE %I RESTART IDENTITY CASCADE', r.tablename); + END LOOP; +END $$; +EOS +) + +echo "== Truncate all public tables (except _prisma_migrations) ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$PG" \ + psql -U "$USER" -d "$DB" -v ON_ERROR_STOP=1 <<< "$SQL" + +echo "== Counts after ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "$USER" -d "$DB" -c " +SELECT relname AS table, n_live_tup AS approx_rows +FROM pg_stat_user_tables +ORDER BY relname; +" + +echo "== Restart litellm (refresh Admin UI caches) ==" +docker service update --force ai-router_litellm >/dev/null +for i in $(seq 1 40); do + curl -sf --max-time 5 https://litellm.ift.calentiq.com/health/liveliness >/dev/null && break + sleep 2 +done +curl -sf --max-time 5 https://litellm.ift.calentiq.com/health/liveliness && echo +echo LITELLM_DB_CLEARED_OK +REMOTE diff --git a/scripts/deploy-cursor-parity-ift.sh b/scripts/deploy-cursor-parity-ift.sh new file mode 100644 index 0000000..ee84c21 --- /dev/null +++ b/scripts/deploy-cursor-parity-ift.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Deploy Cursor-parity AiRouter (Max plan / Coder routine / DeepSeek hard) to IFT +set -euo pipefail +AIR=/mnt/c/Users/alexc/IdeaProjects/eventHub/EventHubAiRouter +cd "$AIR" + +echo "== Local unit + compile ==" +python3 -m py_compile \ + router/agent_hier.py router/router.py router/hierarchical.py router/orchestrator.py +python3 -m unittest discover -s test/unit -q +PRIMARY_PROVIDER=hybrid python3 scripts/gen-litellm-config.py + +echo "== Sync to eventhub-ift ==" +scp \ + router/agent_hier.py \ + router/router.py \ + router/hierarchical.py \ + router/orchestrator.py \ + eventhub-ift:/opt/ai-router-stack/router/ +scp \ + config/orchestration.yaml \ + config/providers.yaml \ + config/routing_rules.yaml \ + eventhub-ift:/opt/ai-router-stack/config/ +scp litellm_config.yaml eventhub-ift:/opt/ai-router-stack/litellm_config.yaml +scp README.md AGENTS.md eventhub-ift:/opt/ai-router-stack/ 2>/dev/null || true + +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +export PRIMARY_PROVIDER="${PRIMARY_PROVIDER:-hybrid}" + +sed -i 's/\r$//' \ + router/agent_hier.py router/router.py router/hierarchical.py router/orchestrator.py \ + config/orchestration.yaml config/providers.yaml config/routing_rules.yaml \ + litellm_config.yaml || true + +echo "== Build gateway image ==" +docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" . + +TS=$(date +%Y%m%d%H%M%S) +ORCH="orchestration_config_${TS}" +LITC="litellm_config_${TS}" +docker config create "$ORCH" ./config/orchestration.yaml +docker config create "$LITC" ./litellm_config.yaml + +OLD_ORCH=$(docker service inspect ai-router_router --format '{{range .Spec.TaskTemplate.ContainerSpec.Configs}}{{println .ConfigName}}{{end}}' | grep orchestration | tail -1 || true) +OLD_LIT=$(docker service inspect ai-router_litellm --format '{{range .Spec.TaskTemplate.ContainerSpec.Configs}}{{println .ConfigName}}{{end}}' | grep litellm_config | grep -v entrypoint | tail -1 || true) + +echo "== Update ai-router_router ==" +RARGS=(--image "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" --force) +RARGS+=(--config-add "source=${ORCH},target=/app/config/orchestration.yaml") +if [ -n "${OLD_ORCH:-}" ]; then + RARGS+=(--config-rm "$OLD_ORCH") +fi +docker service update "${RARGS[@]}" ai-router_router >/dev/null + +echo "== Update ai-router_litellm ==" +LARGS=(--force) +LARGS+=(--config-add "source=${LITC},target=/app/config.yaml") +if [ -n "${OLD_LIT:-}" ]; then + LARGS+=(--config-rm "$OLD_LIT") +fi +docker service update "${LARGS[@]}" ai-router_litellm >/dev/null + +echo "== Wait health ==" +ok=0 +for i in $(seq 1 60); do + if curl -sf --max-time 5 https://ai-router.ift.calentiq.com/health >/dev/null \ + && curl -sf --max-time 5 https://litellm.ift.calentiq.com/health/liveliness >/dev/null; then + ok=1 + break + fi + sleep 3 +done +if [ "$ok" != 1 ]; then + echo "ERROR: health not ready" >&2 + docker service ps ai-router_router --no-trunc | head -8 + docker service ps ai-router_litellm --no-trunc | head -8 + exit 1 +fi + +RID=$(docker ps -q -f name=ai-router_router | head -1) +LID=$(docker ps -q -f name=ai-router_litellm | head -1) +echo "== Verify orchestration ==" +docker exec "$RID" grep -E 'planner_model:|verifier_model:|agent_executor_model:|executor_synthetic_mode:|worker_map:|hard:' /app/config/orchestration.yaml +echo "== Verify litellm novita-planner ==" +docker exec "$LID" sh -c 'grep -A6 "model_name: novita-planner" /app/config.yaml | head -8' + +echo "== Smoke classify + chat ==" +curl -sf --max-time 30 -H "Authorization: Bearer ${ROUTER_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{"text":"поправь timeout в router.py"}' \ + https://ai-router.ift.calentiq.com/classify | head -c 400 +echo +curl -sf --max-time 90 -H "Authorization: Bearer ${ROUTER_API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{"model":"smart-router","messages":[{"role":"user","content":"ping"}],"max_tokens":8}' \ + https://ai-router.ift.calentiq.com/v1/chat/completions | head -c 500 +echo +echo CURSOR_PARITY_DEPLOY_OK +REMOTE diff --git a/scripts/deploy-fast-after-read.sh b/scripts/deploy-fast-after-read.sh new file mode 100644 index 0000000..c582702 --- /dev/null +++ b/scripts/deploy-fast-after-read.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +PYTHONPATH=router python3 -m py_compile router/agent_hier.py router/router.py +PYTHONPATH=router python3 -m unittest discover -s test/unit -q +scp router/agent_hier.py router/router.py \ + eventhub-ift:/opt/ai-router-stack/router/ +scp config/orchestration.yaml eventhub-ift:/opt/ai-router-stack/config/ +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +sed -i 's/\r$//' router/agent_hier.py router/router.py config/orchestration.yaml || true +docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" . +TS=$(date +%Y%m%d%H%M%S) +ORCH="orchestration_config_${TS}" +docker config create "$ORCH" ./config/orchestration.yaml +OLD_ORCH=$(docker service inspect ai-router_router --format '{{range .Spec.TaskTemplate.ContainerSpec.Configs}}{{println .ConfigName}}{{end}}' | grep orchestration | tail -1 || true) +RARGS=(--image "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" --force) +RARGS+=(--config-add "source=${ORCH},target=/app/config/orchestration.yaml") +[[ -n "${OLD_ORCH:-}" ]] && RARGS+=(--config-rm "$OLD_ORCH") +docker service update "${RARGS[@]}" ai-router_router >/dev/null +for i in $(seq 1 40); do + curl -sf --max-time 5 https://ai-router.ift.calentiq.com/health >/dev/null && break + sleep 2 +done +docker exec "$(docker ps -q -f name=ai-router_router | head -1)" \ + grep -E 'executor_tool_loop_timeout_sec|executor_escalate_after_read|executor_rewrite_reread|executor_newest_tool' \ + /app/config/orchestration.yaml +echo FAST_AFTER_READ_OK +REMOTE diff --git a/scripts/deploy-kickstart-no-tools.sh b/scripts/deploy-kickstart-no-tools.sh new file mode 100644 index 0000000..cb87baa --- /dev/null +++ b/scripts/deploy-kickstart-no-tools.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +AIR=/mnt/c/Users/alexc/IdeaProjects/eventHub/EventHubAiRouter +cd "$AIR" +python3 -m py_compile router/agent_hier.py router/agent_stream.py router/router.py +python3 -m unittest discover -s test/unit -q +scp router/agent_hier.py router/agent_stream.py router/router.py \ + eventhub-ift:/opt/ai-router-stack/router/ +scp config/orchestration.yaml eventhub-ift:/opt/ai-router-stack/config/ +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +sed -i 's/\r$//' router/agent_hier.py router/agent_stream.py router/router.py config/orchestration.yaml || true +docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" . +TS=$(date +%Y%m%d%H%M%S) +ORCH="orchestration_config_${TS}" +docker config create "$ORCH" ./config/orchestration.yaml +OLD_ORCH=$(docker service inspect ai-router_router --format '{{range .Spec.TaskTemplate.ContainerSpec.Configs}}{{println .ConfigName}}{{end}}' | grep orchestration | tail -1 || true) +RARGS=(--image "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" --force) +RARGS+=(--config-add "source=${ORCH},target=/app/config/orchestration.yaml") +[[ -n "${OLD_ORCH:-}" ]] && RARGS+=(--config-rm "$OLD_ORCH") +docker service update "${RARGS[@]}" ai-router_router >/dev/null +for i in $(seq 1 40); do + curl -sf --max-time 5 https://ai-router.ift.calentiq.com/health >/dev/null && break + sleep 2 +done +docker exec "$(docker ps -q -f name=ai-router_router | head -1)" \ + grep -E 'executor_synthetic_mode|executor_no_tools_escalate' /app/config/orchestration.yaml +echo KICKSTART_NO_TOOLS_OK +REMOTE diff --git a/scripts/deploy-path-resolve.sh b/scripts/deploy-path-resolve.sh new file mode 100644 index 0000000..89eb2b9 --- /dev/null +++ b/scripts/deploy-path-resolve.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +PYTHONPATH=router python3 -m py_compile \ + router/agent_hier.py router/router.py router/agent_stream.py \ + router/path_resolve.py router/hierarchical.py +PYTHONPATH=router python3 -m unittest discover -s test/unit -q +scp router/agent_hier.py router/router.py router/agent_stream.py \ + router/path_resolve.py router/hierarchical.py \ + eventhub-ift:/opt/ai-router-stack/router/ +scp config/orchestration.yaml eventhub-ift:/opt/ai-router-stack/config/ +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +sed -i 's/\r$//' router/*.py config/orchestration.yaml || true +docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" . +TS=$(date +%Y%m%d%H%M%S) +ORCH="orchestration_config_${TS}" +docker config create "$ORCH" ./config/orchestration.yaml +OLD_ORCH=$(docker service inspect ai-router_router --format '{{range .Spec.TaskTemplate.ContainerSpec.Configs}}{{println .ConfigName}}{{end}}' | grep orchestration | tail -1 || true) +RARGS=(--image "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" --force) +RARGS+=(--config-add "source=${ORCH},target=/app/config/orchestration.yaml") +[[ -n "${OLD_ORCH:-}" ]] && RARGS+=(--config-rm "$OLD_ORCH") +docker service update "${RARGS[@]}" ai-router_router >/dev/null +for i in $(seq 1 40); do + curl -sf --max-time 5 https://ai-router.ift.calentiq.com/health >/dev/null && break + sleep 2 +done +docker exec "$(docker ps -q -f name=ai-router_router | head -1)" \ + grep -E 'path_resolve_' /app/config/orchestration.yaml +test -f /opt/ai-router-stack/router/path_resolve.py +echo PATH_RESOLVE_OK +REMOTE diff --git a/scripts/deploy-progress-ui.sh b/scripts/deploy-progress-ui.sh new file mode 100644 index 0000000..247860e --- /dev/null +++ b/scripts/deploy-progress-ui.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +PYTHONPATH=router python3 -m py_compile \ + router/agent_hier.py router/router.py router/agent_stream.py \ + router/path_resolve.py router/hierarchical.py router/progress_ui.py +PYTHONPATH=router python3 -m unittest discover -s test/unit -q +scp router/agent_hier.py router/router.py router/agent_stream.py \ + router/path_resolve.py router/hierarchical.py router/progress_ui.py \ + eventhub-ift:/opt/ai-router-stack/router/ +scp config/orchestration.yaml eventhub-ift:/opt/ai-router-stack/config/ +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +sed -i 's/\r$//' router/*.py config/orchestration.yaml || true +docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" . +TS=$(date +%Y%m%d%H%M%S) +ORCH="orchestration_config_${TS}" +docker config create "$ORCH" ./config/orchestration.yaml +OLD_ORCH=$(docker service inspect ai-router_router --format '{{range .Spec.TaskTemplate.ContainerSpec.Configs}}{{println .ConfigName}}{{end}}' | grep orchestration | tail -1 || true) +RARGS=(--image "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" --force) +RARGS+=(--config-add "source=${ORCH},target=/app/config/orchestration.yaml") +[[ -n "${OLD_ORCH:-}" ]] && RARGS+=(--config-rm "$OLD_ORCH") +docker service update "${RARGS[@]}" ai-router_router >/dev/null +for i in $(seq 1 40); do + curl -sf --max-time 5 https://ai-router.ift.calentiq.com/health >/dev/null && break + sleep 2 +done +docker exec "$(docker ps -q -f name=ai-router_router | head -1)" \ + grep -E 'executor_force_edit_after_read|path_resolve_enabled' /app/config/orchestration.yaml +test -f /opt/ai-router-stack/router/progress_ui.py +echo PROGRESS_UI_OK +REMOTE diff --git a/scripts/deploy-quiet-progress-ift.sh b/scripts/deploy-quiet-progress-ift.sh new file mode 100644 index 0000000..2afbb62 --- /dev/null +++ b/scripts/deploy-quiet-progress-ift.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Hotfix: quiet progress + executor timeout/escalate +set -euo pipefail +AIR=/mnt/c/Users/alexc/IdeaProjects/eventHub/EventHubAiRouter +cd "$AIR" +python3 -m py_compile router/hierarchical.py router/agent_stream.py router/router.py router/agent_hier.py +python3 -m unittest discover -s test/unit -q +PRIMARY_PROVIDER=hybrid LITELLM_MODEL_TIMEOUT=180 python3 scripts/gen-litellm-config.py + +scp router/hierarchical.py router/agent_stream.py router/router.py router/agent_hier.py \ + eventhub-ift:/opt/ai-router-stack/router/ +scp config/orchestration.yaml eventhub-ift:/opt/ai-router-stack/config/ +scp litellm_config.yaml eventhub-ift:/opt/ai-router-stack/litellm_config.yaml + +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +sed -i 's/\r$//' router/*.py config/orchestration.yaml litellm_config.yaml || true +docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" . +TS=$(date +%Y%m%d%H%M%S) +ORCH="orchestration_config_${TS}" +LITC="litellm_config_${TS}" +docker config create "$ORCH" ./config/orchestration.yaml +docker config create "$LITC" ./litellm_config.yaml +OLD_ORCH=$(docker service inspect ai-router_router --format '{{range .Spec.TaskTemplate.ContainerSpec.Configs}}{{println .ConfigName}}{{end}}' | grep orchestration | tail -1 || true) +OLD_LIT=$(docker service inspect ai-router_litellm --format '{{range .Spec.TaskTemplate.ContainerSpec.Configs}}{{println .ConfigName}}{{end}}' | grep litellm_config | grep -v entrypoint | tail -1 || true) +RARGS=(--image "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" --force) +RARGS+=(--config-add "source=${ORCH},target=/app/config/orchestration.yaml") +[[ -n "${OLD_ORCH:-}" ]] && RARGS+=(--config-rm "$OLD_ORCH") +docker service update "${RARGS[@]}" ai-router_router >/dev/null +LARGS=(--force --config-add "source=${LITC},target=/app/config.yaml") +[[ -n "${OLD_LIT:-}" ]] && LARGS+=(--config-rm "$OLD_LIT") +docker service update "${LARGS[@]}" ai-router_litellm >/dev/null +for i in $(seq 1 50); do + curl -sf --max-time 5 https://ai-router.ift.calentiq.com/health >/dev/null && \ + curl -sf --max-time 5 https://litellm.ift.calentiq.com/health/liveliness >/dev/null && break + sleep 3 +done +RID=$(docker ps -q -f name=ai-router_router | head -1) +LID=$(docker ps -q -f name=ai-router_litellm | head -1) +docker exec "$RID" grep -E 'progress_verbose|executor_timeout|executor_midloop_escalate|show_context_fill' /app/config/orchestration.yaml +docker exec "$LID" sh -c 'grep -A5 "model_name: a-medium-code" /app/config.yaml | head -8' +echo QUIET_PROGRESS_TIMEOUT_OK +REMOTE diff --git a/scripts/deploy-runtime-probe.sh b/scripts/deploy-runtime-probe.sh new file mode 100644 index 0000000..4113115 --- /dev/null +++ b/scripts/deploy-runtime-probe.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +PYTHONPATH=router python3 -m py_compile \ + router/agent_hier.py router/router.py router/agent_stream.py \ + router/path_resolve.py router/hierarchical.py router/progress_ui.py \ + router/runtime_probe.py +PYTHONPATH=router python3 -m unittest discover -s test/unit -q +scp router/agent_hier.py router/router.py router/agent_stream.py \ + router/path_resolve.py router/hierarchical.py router/progress_ui.py \ + router/runtime_probe.py \ + eventhub-ift:/opt/ai-router-stack/router/ +scp config/orchestration.yaml eventhub-ift:/opt/ai-router-stack/config/ +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +sed -i 's/\r$//' router/*.py config/orchestration.yaml || true +docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" . +TS=$(date +%Y%m%d%H%M%S) +ORCH="orchestration_config_${TS}" +docker config create "$ORCH" ./config/orchestration.yaml +OLD_ORCH=$(docker service inspect ai-router_router --format '{{range .Spec.TaskTemplate.ContainerSpec.Configs}}{{println .ConfigName}}{{end}}' | grep orchestration | tail -1 || true) +RARGS=(--image "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" --force) +RARGS+=(--config-add "source=${ORCH},target=/app/config/orchestration.yaml") +[[ -n "${OLD_ORCH:-}" ]] && RARGS+=(--config-rm "$OLD_ORCH") +docker service update "${RARGS[@]}" ai-router_router >/dev/null +for i in $(seq 1 40); do + curl -sf --max-time 5 https://ai-router.ift.calentiq.com/health >/dev/null && break + sleep 2 +done +docker exec "$(docker ps -q -f name=ai-router_router | head -1)" \ + grep -E 'runtime_probe_enabled|path_resolve_enabled' /app/config/orchestration.yaml +test -f /opt/ai-router-stack/router/runtime_probe.py +echo RUNTIME_PROBE_OK +REMOTE diff --git a/scripts/deploy.sh b/scripts/deploy.sh index fec068f..e7c7d07 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -23,21 +23,48 @@ ensure_secret() { if docker secret inspect "$name" >/dev/null 2>&1; then echo "secret exists: $name" else + if [[ -z "$value" ]]; then + value="_" + fi echo -n "$value" | docker secret create "$name" - echo "created secret: $name" fi } echo "== Ensure Swarm secrets ==" -ensure_secret novita_api_key "${NOVITA_API_KEY:?NOVITA_API_KEY required}" +PRIMARY="${PRIMARY_PROVIDER:-hybrid}" +if [[ "$PRIMARY" == "hybrid" || "$PRIMARY" == "novita" ]]; then + ensure_secret novita_api_key "${NOVITA_API_KEY:?NOVITA_API_KEY required for PRIMARY_PROVIDER=${PRIMARY}}" +else + ensure_secret novita_api_key "${NOVITA_API_KEY:-}" +fi +ensure_secret anthropic_api_key "${ANTHROPIC_API_KEY:-}" ensure_secret litellm_master_key "${LITELLM_MASTER_KEY:?LITELLM_MASTER_KEY required}" ensure_secret litellm_salt_key "${LITELLM_SALT_KEY:?LITELLM_SALT_KEY required}" ensure_secret router_api_key "${ROUTER_API_KEY:?ROUTER_API_KEY required}" ensure_secret postgres_password "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD required}" ensure_secret groq_api_key "${GROQ_API_KEY:-}" +ensure_secret xai_api_key "${XAI_API_KEY:-}" ensure_secret gemini_api_key "${GEMINI_API_KEY:-}" +ensure_secret together_api_key "${TOGETHER_API_KEY:-}" +ensure_secret openrouter_api_key "${OPENROUTER_API_KEY:-}" ensure_secret gigachat_credentials "${GIGACHAT_CREDENTIALS:-}" +vless_conf_is_stub() { + local f="${1:-vless/vless.conf}" + [[ ! -f "$f" ]] && return 0 + grep -qE "0\.0\.0\.0|example\.com|UUID@host|^#" "$f" && return 0 + grep -qE "^vless://" "$f" || return 0 + return 1 +} + +if [[ "${VPN_ENABLED:-false}" == "true" ]] && [[ -n "${VLESS_SUB_URL:-}" ]]; then + if vless_conf_is_stub "vless/vless.conf"; then + echo "== Fetch vless.conf from subscription ==" + bash scripts/fetch-vless-subscription.sh vless/vless.conf + fi +fi + if [[ ! -f vless/vless.conf ]]; then echo "WARN: vless/vless.conf missing — stub for secret (VPN off until configured)" cp vless/vless.conf.example vless/vless.conf 2>/dev/null || echo "# stub" > vless/vless.conf @@ -46,11 +73,17 @@ ensure_secret vless_conf "$(cat vless/vless.conf)" echo "== Build images ==" docker build -f router/Dockerfile -t "${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}" . -docker build -t ai-router/vless-proxy:local ./vless -docker build -t ai-router/vpn-watchdog:local ./watchdog +if [[ "${VPN_ENABLED:-false}" == "true" ]]; then + docker build -t ai-router/vless-proxy:local ./vless + docker build -t ai-router/vpn-watchdog:local ./watchdog +else + echo "VPN off — stub images (replicas=0)" + docker build -f vless/Dockerfile.stub -t ai-router/vless-proxy:local ./vless + docker build -t ai-router/vpn-watchdog:local ./watchdog +fi -echo "== Sync routing config (optional regen) ==" -bash scripts/sync-routing-config.sh || true +echo "== Sync routing config (PRIMARY_PROVIDER=${PRIMARY_PROVIDER:-hybrid}) ==" +python3 scripts/gen-litellm-config.py echo "== Deploy stack: ${STACK_NAME} ==" docker stack deploy -c docker-stack.yml --with-registry-auth "${STACK_NAME}" diff --git a/scripts/diag-400-body.sh b/scripts/diag-400-body.sh new file mode 100644 index 0000000..aa7f385 --- /dev/null +++ b/scripts/diag-400-body.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +echo "== litellm 400/408 detail ==" +docker service logs --since 40m --raw ai-router_litellm 2>&1 \ + | grep -iE 'BadRequest|invalid request|400|deepseek|tool|error|trace_id' \ + | tail -80 +echo +echo "== router 400 bodies ==" +docker service logs --since 40m --raw ai-router_router 2>&1 \ + | grep -iE '400|BadRequest|invalid|executor failed|mid-loop' \ + | tail -40 +REMOTE diff --git a/scripts/diag-408-after-index.sh b/scripts/diag-408-after-index.sh new file mode 100644 index 0000000..c8e718b --- /dev/null +++ b/scripts/diag-408-after-index.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +echo "== Spend 15m ==" +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT \"startTime\", status, model, total_tokens, completion_tokens, + coalesce(\"request_duration_ms\",0) AS ms, + left(coalesce(metadata->'error_information'->>'error_message',''), 100) AS er +FROM \"LiteLLM_SpendLogs\" +WHERE \"startTime\" >= now() - interval '20 minutes' +ORDER BY \"startTime\" DESC LIMIT 25; +" +echo "== Router ==" +docker service logs --since 20m --raw ai-router_router 2>&1 \ + | grep -iE 'path_index|path_resolve|hierarchical_agent phase|executor result|408|escalate|mid-loop|rewrite|kickstart|loadtest|dynamic_conf' \ + | tail -60 +REMOTE diff --git a/scripts/diag-compose-path.sh b/scripts/diag-compose-path.sh new file mode 100644 index 0000000..afac31a --- /dev/null +++ b/scripts/diag-compose-path.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +docker service logs --since 20m --raw ai-router_router 2>&1 \ + | grep -iE 'docker-compose|kickstart|tool_calls|executor result|rewrite|hierarchical_agent phase|not found' \ + | tail -50 +REMOTE diff --git a/scripts/diag-edit-loop.sh b/scripts/diag-edit-loop.sh new file mode 100644 index 0000000..ec79b20 --- /dev/null +++ b/scripts/diag-edit-loop.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +echo "== Spend 20m ==" +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT to_char(\"startTime\", 'HH24:MI:SS') AS t, status, + CASE WHEN model LIKE '%qwen3.8-max%' THEN 'Max' + WHEN model LIKE '%qwen3-coder%' THEN 'Coder' + WHEN model LIKE '%deepseek%' THEN 'DeepSeek' + ELSE left(model,28) END AS role, + total_tokens AS tok, completion_tokens AS out, + coalesce(\"request_duration_ms\",0) AS ms +FROM \"LiteLLM_SpendLogs\" +WHERE \"startTime\" >= now() - interval '20 minutes' +ORDER BY \"startTime\" ASC; +" +echo +echo "== Router (edit/force/loop) ==" +docker service logs --since 20m --raw ai-router_router 2>&1 \ + | grep -iE 'force edit|edit_file|rewrite|path_index|hierarchical_agent phase|executor result|synth|sticky|Стоп|fallback|tool_calls|calentiq-ift' \ + | grep -vE 'metrics|health|Waiting|Uvicorn|Started|Finished|Shutting' \ + | tail -80 +REMOTE diff --git a/scripts/diag-executor-fail.sh b/scripts/diag-executor-fail.sh new file mode 100644 index 0000000..0ac850b --- /dev/null +++ b/scripts/diag-executor-fail.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +PG=$(docker ps -q -f name=ai-router_postgres | head -1) + +echo "== Spend last 30m ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT \"startTime\", status, model, call_type, total_tokens, + left(coalesce(metadata->'error_information'->>'error_message',''), 180) AS er +FROM \"LiteLLM_SpendLogs\" +ORDER BY \"startTime\" DESC +LIMIT 25; +" + +echo "== Router logs (executor/plan) ==" +docker service logs --since 25m ai-router_router 2>&1 \ + | grep -iE 'hierarchical|executor|novita-planner|a-medium-code|timeout|408|error|fail|tool' \ + | tail -50 + +echo "== LiteLLM logs errors ==" +docker service logs --since 25m ai-router_litellm 2>&1 \ + | grep -iE 'ERROR|Exception|timeout|408|502|qwen3-coder|novita-planner|a-medium-code|Give Feedback' \ + | tail -40 +REMOTE diff --git a/scripts/diag-fake-paths.sh b/scripts/diag-fake-paths.sh new file mode 100644 index 0000000..8c5f866 --- /dev/null +++ b/scripts/diag-fake-paths.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +docker service logs --since 25m --raw ai-router_router 2>&1 \ + | grep -iE 'traefik\.yml|rewrite|bogus|kickstart|executor result|hierarchical_agent phase|paths:|Plan approved|tool_calls' \ + | tail -60 +REMOTE diff --git a/scripts/diag-hang2.sh b/scripts/diag-hang2.sh new file mode 100644 index 0000000..bded013 --- /dev/null +++ b/scripts/diag-hang2.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +echo "== live router logs (last 5m, full) ==" +docker service logs --since 8m --raw ai-router_router 2>&1 | tail -80 +echo +echo "== litellm last ==" +docker service logs --since 8m --raw ai-router_litellm 2>&1 | grep -iE 'qwen|deepseek|timeout|408|error|POST' | tail -40 +echo +echo "== in-flight? ==" +docker service ps ai-router_router --no-trunc | head -5 +REMOTE diff --git a/scripts/diag-hang3.sh b/scripts/diag-hang3.sh new file mode 100644 index 0000000..ae802cc --- /dev/null +++ b/scripts/diag-hang3.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +echo "== Spend after 19:06 ==" +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT \"startTime\", status, model, total_tokens, completion_tokens, + coalesce(\"request_duration_ms\",0) AS ms, + left(coalesce(metadata->'error_information'->>'error_message',''), 120) AS er +FROM \"LiteLLM_SpendLogs\" +WHERE \"startTime\" >= '2026-08-12 19:06:00' +ORDER BY \"startTime\" DESC; +" +echo "== router since 19:06 full agent lines ==" +docker service logs --since 15m --raw ai-router_router 2>&1 \ + | grep -iE '19:0[6-9]|19:1|executor|tool_loop|408|HTTP Request|kickstart|escalate|fallback|synth|no tool|POST /v1' \ + | tail -60 +REMOTE diff --git a/scripts/diag-litellm-fails.sh b/scripts/diag-litellm-fails.sh new file mode 100644 index 0000000..1eaf4ba --- /dev/null +++ b/scripts/diag-litellm-fails.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a + +echo "== Services ==" +docker stack services ai-router --format '{{.Name}} {{.Replicas}} {{.Image}}' | grep -E 'litellm|postgres|router' || true + +echo "== Health ==" +curl -sS -o /tmp/llh.txt -w "liveliness:%{http_code}\n" --max-time 10 https://litellm.ift.calentiq.com/health/liveliness || true +curl -sS -o /tmp/llr.txt -w "readiness:%{http_code}\n" --max-time 10 https://litellm.ift.calentiq.com/health/readiness || true +head -c 300 /tmp/llh.txt; echo; head -c 500 /tmp/llr.txt; echo + +echo "== Recent litellm logs (errors) ==" +docker service logs --tail 120 ai-router_litellm 2>&1 \ + | grep -iE 'error|exception|fail|traceback|500|401|prisma|database|budget|key|warning' \ + | tail -60 || true + +echo "== Last 40 lines raw ==" +docker service logs --tail 40 ai-router_litellm 2>&1 | tail -40 + +echo "== Postgres quick ==" +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c \ + "SELECT count(*) AS spend FROM \"LiteLLM_SpendLogs\"; SELECT count(*) AS users FROM \"LiteLLM_UserTable\"; SELECT count(*) AS tokens FROM \"LiteLLM_VerificationToken\";" + +echo "== UI/home probe ==" +curl -sS -o /tmp/ui.txt -w "ui:%{http_code}\n" --max-time 15 https://litellm.ift.calentiq.com/ui/ || true +curl -sS -o /tmp/models.txt -w "models:%{http_code}\n" --max-time 20 \ + -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \ + https://litellm.ift.calentiq.com/v1/models || true +head -c 200 /tmp/models.txt; echo +REMOTE diff --git a/scripts/diag-litellm-fails2.sh b/scripts/diag-litellm-fails2.sh new file mode 100644 index 0000000..336a820 --- /dev/null +++ b/scripts/diag-litellm-fails2.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +PG=$(docker ps -q -f name=ai-router_postgres | head -1) + +echo "== SpendLogs status breakdown ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT status, count(*) FROM \"LiteLLM_SpendLogs\" GROUP BY 1 ORDER BY 2 DESC; +" + +echo "== Recent spend rows ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT \"startTime\", status, model, \"api_key\", + left(coalesce(\"error_information\"::text, ''), 120) AS err, + left(coalesce(request_tags::text, ''), 60) AS tags +FROM \"LiteLLM_SpendLogs\" +ORDER BY \"startTime\" DESC +LIMIT 20; +" + +echo "== ErrorLogs ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c \ + "SELECT count(*) FROM \"LiteLLM_ErrorLogs\";" + +echo "== Body-read errors around restart ==" +docker service logs --since 15m ai-router_litellm 2>&1 \ + | grep -iE 'Unexpected error reading request body|ClientDisconnect|ConnectionReset|502|504|failed' \ + | tail -30 || true + +echo "== Uptime-kuma / probes hitting litellm ==" +docker service logs --tail 30 ai-router_uptime-kuma 2>&1 | tail -20 || true +REMOTE diff --git a/scripts/diag-litellm-fails3.sh b/scripts/diag-litellm-fails3.sh new file mode 100644 index 0000000..78c6fb7 --- /dev/null +++ b/scripts/diag-litellm-fails3.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +USER="${POSTGRES_USER:-litellm}" +DB="${POSTGRES_DB:-litellm}" + +echo "== SpendLogs columns ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "$USER" -d "$DB" -c "\d+ \"LiteLLM_SpendLogs\"" | head -80 + +echo "== Recent failures ==" +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "$USER" -d "$DB" -c " +SELECT \"startTime\", status, model, + left(coalesce(messages::text,''), 80) AS messages, + left(coalesce(metadata::text,''), 200) AS metadata, + left(coalesce(\"proxyServerRequest\"::text,''), 120) AS req, + left(coalesce(response::text,''), 200) AS response +FROM \"LiteLLM_SpendLogs\" +ORDER BY \"startTime\" DESC +LIMIT 15; +" +REMOTE diff --git a/scripts/diag-litellm-fails4.sh b/scripts/diag-litellm-fails4.sh new file mode 100644 index 0000000..16d658e --- /dev/null +++ b/scripts/diag-litellm-fails4.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT \"startTime\", status, model, call_type, + left(coalesce(metadata::text,''), 400) AS metadata, + left(coalesce(response::text,''), 300) AS response +FROM \"LiteLLM_SpendLogs\" +ORDER BY \"startTime\" DESC +LIMIT 15; +" +REMOTE diff --git a/scripts/diag-no-tool-calls.sh b/scripts/diag-no-tool-calls.sh new file mode 100644 index 0000000..8281216 --- /dev/null +++ b/scripts/diag-no-tool-calls.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +echo "== Spend last 15m ==" +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT \"startTime\", status, model, call_type, total_tokens, completion_tokens, + left(coalesce(metadata->'error_information'->>'error_message',''), 120) AS er +FROM \"LiteLLM_SpendLogs\" +ORDER BY \"startTime\" DESC LIMIT 15; +" +echo "== Router ==" +docker service logs --since 15m ai-router_router 2>&1 \ + | grep -iE 'executor|tool_calls|finish|no_tool|408|escalate|hierarchical_agent|a-medium-code|b-complex' \ + | tail -40 +REMOTE diff --git a/scripts/diag-observer-502.sh b/scripts/diag-observer-502.sh new file mode 100644 index 0000000..92bcad4 --- /dev/null +++ b/scripts/diag-observer-502.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +echo "== curl inside from traefik net ==" +# find traefik containe +TR=$(docker ps --filter name=traefik -q | head -1) +OW=$(docker ps --filter name=observer_web -q | head -1) +echo "traefik=$TR observer=$OW" +if [[ -n "$OW" ]]; then + echo "-- observer logs --" + docker logs --tail 30 "$OW" 2>&1 | tail -30 + echo "-- wget localhost from observer --" + docker exec "$OW" sh -c 'wget -qO- --timeout=3 http://127.0.0.1/ 2>&1 | head -c 200; echo; wget -qO- --timeout=3 http://127.0.0.1:80/ 2>&1 | head -c 200; echo' || true + echo "-- inspect ports --" + docker inspect "$OW" --format 'IP={{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}} Ports={{json .NetworkSettings.Ports}}' +fi +if [[ -n "$TR" && -n "$OW" ]]; then + echo "-- traefik -> observer_web:80 --" + docker exec "$TR" wget -qO- --timeout=5 http://observer_web:80/ 2>&1 | head -c 300 || \ + docker exec "$TR" wget -qO- --timeout=5 http://eventhub-ift-core_observer_web:80/ 2>&1 | head -c 300 || true + echo + # DNS from traefik + docker exec "$TR" getent hosts observer_web 2>&1 || docker exec "$TR" nslookup observer_web 2>&1 | head -10 || true +fi +echo "== dynamic_conf observer snippet ==" +grep -n -A6 'observer' /opt/ai-router-stack/../EventHubDevOps/ift/traefik/dynamic_conf.yml 2>/dev/null | head -5 || \ + grep -n -A5 'observer:' /home/*/IdeaProjects/eventHub/EventHubDevOps/ift/traefik/dynamic_conf.yml 2>/dev/null | head || true +# try find compose on host +find /opt -name 'dynamic_conf.yml' 2>/dev/null | head -5 +REMOTE diff --git a/scripts/diag-slow-read.sh b/scripts/diag-slow-read.sh new file mode 100644 index 0000000..1bb0d6a --- /dev/null +++ b/scripts/diag-slow-read.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +echo "== Spend last 20m ==" +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT \"startTime\", status, model, total_tokens, completion_tokens, + coalesce(\"request_duration_ms\",0) AS ms, + left(coalesce(metadata->'error_information'->>'error_message',''), 100) AS er +FROM \"LiteLLM_SpendLogs\" +ORDER BY \"startTime\" DESC LIMIT 20; +" +echo "== Router recent ==" +docker service logs --since 20m ai-router_router 2>&1 \ + | grep -iE 'executor|tool_loop|kickstart|escalate|408|no tool|hierarchical_agent|phase=|waiting|timeout' \ + | tail -50 +REMOTE diff --git a/scripts/explain-logs.sh b/scripts/explain-logs.sh new file mode 100644 index 0000000..4899280 --- /dev/null +++ b/scripts/explain-logs.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +echo "== Spend last 25m ==" +PG=$(docker ps -q -f name=ai-router_postgres | head -1) +docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$PG" \ + psql -U "${POSTGRES_USER:-litellm}" -d "${POSTGRES_DB:-litellm}" -c " +SELECT to_char(\"startTime\", 'HH24:MI:SS') AS t, status, + CASE + WHEN model LIKE '%qwen3.8-max%' THEN 'planner-Max' + WHEN model LIKE '%qwen3-coder%' THEN 'coder-30b' + WHEN model LIKE '%deepseek%' THEN 'DeepSeek' + WHEN model LIKE '%llama%' THEN 'Llama' + ELSE left(model, 40) + END AS role, + total_tokens AS tok, completion_tokens AS out, + coalesce(\"request_duration_ms\",0) AS ms, + left(coalesce(metadata->'error_information'->>'error_message',''), 80) AS er +FROM \"LiteLLM_SpendLogs\" +WHERE \"startTime\" >= now() - interval '25 minutes' +ORDER BY \"startTime\" ASC; +" +echo +echo "== Router agent timeline ==" +docker service logs --since 25m --raw ai-router_router 2>&1 \ + | grep -iE 'path_index|path_resolve|hierarchical_agent phase|executor forward|executor result|escalate|408|400|rewrite|kickstart|no tool|mid-loop|synth|Выполн|plan' \ + | grep -vE 'metrics|health|Waiting for|Application|Uvicorn|Finished server|Started server|Shutting' \ + | tail -80 +REMOTE diff --git a/scripts/fetch-vless-subscription.sh b/scripts/fetch-vless-subscription.sh new file mode 100644 index 0000000..04eb7e8 --- /dev/null +++ b/scripts/fetch-vless-subscription.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Fetch vless.conf from Happ/mireon subscription (JSON or plain vless:// lines). +# Env: VLESS_SUB_URL, VLESS_UA, VLESS_HWID (Happ → Settings / app id) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +if [[ -f "${ROOT}/.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${ROOT}/.env" + set +a +fi +SUB_URL="${VLESS_SUB_URL:?set VLESS_SUB_URL}" +OUT="${1:-${ROOT}/vless/vless.conf}" +UA="${VLESS_UA:-Happ/3.3.6/Windows/2607171516600}" +HWID="${VLESS_HWID:-}" + +TMP="$(mktemp)" +trap 'rm -f "${TMP}"' EXIT + +CURL=(curl -fsSL -A "$UA") +[[ -n "$HWID" ]] && CURL+=(-H "x-hwid: ${HWID}") + +"${CURL[@]}" "$SUB_URL" | tr -d '\n\r' | base64 -d >"${TMP}" 2>/dev/null || "${CURL[@]}" "$SUB_URL" >"${TMP}" + +pick="$(grep -E '^vless://' "${TMP}" | grep -v '@0\.0\.0\.0:1' | head -1 || true)" +if [[ -z "$pick" ]]; then + pick="$(python3 "${SCRIPT_DIR}/vless-from-subscription-json.py" "${TMP}")" || true +fi + +if [[ -z "$pick" ]]; then + echo "fetch-vless-subscription: no valid vless URI" >&2 + head -c 400 "${TMP}" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$OUT")" +printf '%s\n' "$pick" >"$OUT" +chmod 600 "$OUT" +echo "Wrote $(wc -c <"$OUT") bytes -> $OUT" diff --git a/scripts/gen-litellm-config.py b/scripts/gen-litellm-config.py new file mode 100644 index 0000000..b610456 --- /dev/null +++ b/scripts/gen-litellm-config.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Generate litellm_config.yaml from PRIMARY_PROVIDER + config/*.yaml.""" +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parent.parent +PROFILE = os.environ.get("PRIMARY_PROVIDER", "hybrid").strip().lower() + + +def load(path: Path) -> dict: + with path.open(encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +def api_key_ref(name: str) -> str: + return f"os.environ/{name}" + + +def _apply_thinking(params: dict, spec: dict) -> None: + """Qwen3 defaults to thinking; empty content + hang unless disabled.""" + model = str(params.get("model") or spec.get("model") or "").lower() + force = bool(spec.get("disable_thinking")) + auto = ("qwen3" in model) and not spec.get("enable_thinking") + if force or auto: + params["extra_body"] = { + "chat_template_kwargs": {"enable_thinking": False}, + "enable_thinking": False, + } + + +def lane_entry(name: str, spec: dict, rpm: int | None) -> dict: + params: dict = { + "model": spec["model"], + "api_key": api_key_ref(spec["api_key"]), + # Per-deployment timeout: litellm_settings.request_timeout often shows as + # Deployment Info timeout: None on stream/tool hangs. + "timeout": int(os.environ.get("LITELLM_MODEL_TIMEOUT", "180")), + } + if rpm: + params["rpm"] = rpm + _apply_thinking(params, spec) + return {"model_name": name, "litellm_params": params} + + +def fixed_entry(name: str, spec: dict) -> dict: + params: dict = { + "model": spec["model"], + "api_key": api_key_ref(spec["api_key"]), + "timeout": int(os.environ.get("LITELLM_MODEL_TIMEOUT", "180")), + } + for key in ("ssl_verify", "max_tokens", "temperature"): + if key in spec: + params[key] = spec[key] + _apply_thinking(params, spec) + return {"model_name": name, "litellm_params": params} + + +def smart_router_internal(rules: dict) -> dict: + litellm_rules = rules.get("litellm", {}) + return { + "model_name": "smart-router-internal", + "litellm_params": { + "model": "auto_router/complexity_router", + "drop_params": True, + "complexity_router_default_model": "a-medium-ops", + "complexity_router_config": { + "tiers": { + "SIMPLE": "a-simple", + "MEDIUM": "a-medium-ops", + "MEDIUM_CODE": "a-medium-code", + "COMPLEX": "a-complex", + "REASONING": "a-reasoning", + }, + "classifier_fallback": "heuristic", + "keyword_tier_rules": litellm_rules.get("keyword_tier_rules", []), + "custom_technical_keywords": litellm_rules.get( + "custom_technical_keywords", [] + ), + "token_thresholds": {"simple": 20, "complex": 500}, + "tier_boundaries": { + "simple_medium": 0.18, + "medium_complex": 0.38, + "complex_reasoning": 0.62, + }, + "session_affinity": True, + "session_affinity_ttl_seconds": 1800, + }, + }, + } + + +def main() -> int: + providers = load(ROOT / "config/providers.yaml") + profiles = providers.get("profiles", {}) + if PROFILE not in profiles: + print(f"ERROR: unknown PRIMARY_PROVIDER={PROFILE!r}", file=sys.stderr) + print(f"Available: {', '.join(sorted(profiles))}", file=sys.stderr) + return 1 + + profile = profiles[PROFILE] + base = load(ROOT / "config/litellm.base.yaml") + matrix = load(ROOT / "config/model_matrix.yaml") + rules = load(ROOT / "config/routing_rules.yaml") + meta = matrix.get("models", {}) + + model_list: list[dict] = [] + + # Classifier first + fixed = providers.get("fixed_models", {}) + if "gigachat-classifier" in fixed: + model_list.append(fixed_entry("gigachat-classifier", fixed["gigachat-classifier"])) + + # Lane models from active profile + lanes = profile.get("lanes", {}) + for name, spec in lanes.items(): + rpm = meta.get(name, {}).get("rpm") + model_list.append(lane_entry(name, spec, rpm)) + + # Fixed models (Claude / OR fallbacks / GigaChat / optional free tiers) + for name in ( + "novita-planner", + "novita-verifier", + "claude-haiku-planner", + "claude-sonnet-verifier", + "groq-llama-8b", + "groq-qwen-coder", + "gemini-flash", + "grok-3", + "gigachat-pro", + ): + if name in fixed: + model_list.append(fixed_entry(name, fixed[name])) + + model_list.append(smart_router_internal(rules)) + + # smart-router alias (Zed default) + sr = profile.get("smart_router", {}) + if sr: + sr_params: dict = { + "model": sr["model"], + "api_key": api_key_ref(sr["api_key"]), + } + _apply_thinking(sr_params, sr) + model_list.append({"model_name": "smart-router", "litellm_params": sr_params}) + + router = base.setdefault("router_settings", {}) + router["fallbacks"] = [ + {k: v for k, v in row.items()} + for row in _fallback_list(profile.get("fallbacks", {})) + ] + router["default_fallbacks"] = profile.get( + "default_fallbacks", ["a-medium-code", "groq-qwen-coder"] + ) + + out_cfg = {**base, "model_list": model_list} + out_path = ROOT / "litellm_config.yaml" + header = ( + f"# LiteLLM — generated for PRIMARY_PROVIDER={PROFILE}\n" + f"# Profile: {profile.get('label', PROFILE)}\n" + f"# Regenerate: PRIMARY_PROVIDER={PROFILE} bash scripts/gen-litellm-config.py\n\n" + ) + body = yaml.dump(out_cfg, allow_unicode=True, sort_keys=False) + out_path.write_text(header + body, encoding="utf-8") + print(f"Wrote {out_path} (profile={PROFILE}, {len(model_list)} models)") + return 0 + + +def _fallback_list(fallbacks: dict) -> list[dict]: + return [{model: targets} for model, targets in fallbacks.items()] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/litellm-entrypoint.sh b/scripts/litellm-entrypoint.sh index ca1e762..1fcd991 100644 --- a/scripts/litellm-entrypoint.sh +++ b/scripts/litellm-entrypoint.sh @@ -20,9 +20,21 @@ fi if [ -f /run/secrets/groq_api_key ]; then export GROQ_API_KEY="$(read_secret /run/secrets/groq_api_key)" fi +if [ -f /run/secrets/xai_api_key ]; then + export XAI_API_KEY="$(read_secret /run/secrets/xai_api_key)" +fi if [ -f /run/secrets/gemini_api_key ]; then export GEMINI_API_KEY="$(read_secret /run/secrets/gemini_api_key)" fi +if [ -f /run/secrets/together_api_key ]; then + export TOGETHER_API_KEY="$(read_secret /run/secrets/together_api_key)" +fi +if [ -f /run/secrets/openrouter_api_key ]; then + export OPENROUTER_API_KEY="$(read_secret /run/secrets/openrouter_api_key)" +fi +if [ -f /run/secrets/anthropic_api_key ]; then + export ANTHROPIC_API_KEY="$(read_secret /run/secrets/anthropic_api_key)" +fi if [ -f /run/secrets/gigachat_credentials ]; then export GIGACHAT_CREDENTIALS="$(read_secret /run/secrets/gigachat_credentials)" fi diff --git a/scripts/no-proxy-default.sh b/scripts/no-proxy-default.sh new file mode 100644 index 0000000..d162240 --- /dev/null +++ b/scripts/no-proxy-default.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Canonical NO_PROXY when litellm uses HTTP_PROXY (VLESS VPN). +# +# Direct (bypass VPN): Novita, Groq, Gemini, GigaChat, internal/Calentiq. +# VPN-only (NOT listed): Anthropic → api.anthropic.com; OpenRouter → openrouter.ai; +# xAI Grok → api.x.ai; Together → api.together.xyz +# +# shellcheck disable=SC2034 +NO_PROXY_DEFAULT='localhost,127.0.0.1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,api.novita.ai,novita.ai,api.groq.com,groq.com,generativelanguage.googleapis.com,ngw.devices.sberbank.ru,gigachat.devices.sberbank.ru,git.sabilin.com,*.ift.calentiq.com,ift.calentiq.com,*.ift.eventhub.local,*.eventhub.local' diff --git a/scripts/refresh-vless-subscription.sh b/scripts/refresh-vless-subscription.sh new file mode 100644 index 0000000..d2f5284 --- /dev/null +++ b/scripts/refresh-vless-subscription.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Refresh vless.conf from subscription and reload vless-proxy on Swarm (IFT). +set -euo pipefail + +STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACK_NAME="${STACK_NAME:-ai-router}" +cd "$STACK_DIR" + +set -a +# shellcheck disable=SC1091 +source .env +set +a + +bash "${STACK_DIR}/scripts/fetch-vless-subscription.sh" "${STACK_DIR}/vless/vless.conf" + +CONF="$(cat "${STACK_DIR}/vless/vless.conf")" +SVC="${STACK_NAME}_vless-proxy" + +rotate_vless_secret() { + if docker secret inspect vless_conf >/dev/null 2>&1; then + echo "== Detach old vless_conf secret ==" + docker service scale "${SVC}=0" >/dev/null || true + sleep 3 + docker service update --secret-rm vless_conf "${SVC}" >/dev/null 2>&1 || true + sleep 2 + docker secret rm vless_conf >/dev/null 2>&1 || true + fi + echo -n "$CONF" | docker secret create vless_conf - +} + +if docker service inspect "${SVC}" >/dev/null 2>&1; then + rotate_vless_secret + docker service update \ + --secret-add "source=vless_conf,target=/app/vless.conf,mode=0444" \ + "${SVC}" >/dev/null + docker service update --force "${SVC}" >/dev/null + TARGET="${VPN_ENABLED:-false}" + if [[ "$TARGET" == "true" ]]; then + docker service scale "${SVC}=1" >/dev/null + fi + echo "refresh-vless: vless-proxy reloaded" +else + rotate_vless_secret + echo "refresh-vless: secret vless_conf updated (service ${SVC} missing — run deploy.sh)" +fi diff --git a/scripts/setup-vless-refresh-cron.sh b/scripts/setup-vless-refresh-cron.sh new file mode 100644 index 0000000..bff81e4 --- /dev/null +++ b/scripts/setup-vless-refresh-cron.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Cron: refresh VPN subscription every 6h (config может меняться на стороне провайдера). +set -euo pipefail + +STACK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CRON_FILE="/etc/cron.d/ai-router-vless-refresh" +LOG="/var/log/ai-router-vless-refresh.log" + +sudo tee "$CRON_FILE" >/dev/null <>${LOG} 2>&1 +EOF +sudo chmod 644 "$CRON_FILE" +echo "Installed ${CRON_FILE} (every 6h)" diff --git a/scripts/smoke-observer-approve.sh b/scripts/smoke-observer-approve.sh new file mode 100644 index 0000000..f1cc17e --- /dev/null +++ b/scripts/smoke-observer-approve.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +BASE="${PUBLIC_URL:-https://ai-router.ift.calentiq.com}" +KEY="${ROUTER_API_KEY:-$LITELLM_MASTER_KEY}" +PLAN=$(python3 - <<'PY' +import json +d=json.load(open("/tmp/agent-plan.json")) +msg=(d.get("choices") or [{}])[0].get("message") or {} +print(msg.get("content") or "") +PY +) +# Approve with tools like Zed agent +python3 - <<'PY' > /tmp/agent-approve-req.json +import json +plan=open("/tmp/agent-plan.json").read() +d=json.loads(plan) +content=((d.get("choices") or [{}])[0].get("message") or {}).get("content") or "" +tools=[{ + "type":"function", + "function":{ + "name":"read_file", + "description":"Read a file", + "parameters":{"type":"object","properties":{"path":{"type":"string"}},"required":["path"]} + } +},{ + "type":"function", + "function":{ + "name":"edit_file", + "description":"Edit a file", + "parameters":{"type":"object","properties":{ + "path":{"type":"string"}, + "edits":{"type":"array","items":{"type":"object","properties":{ + "old_text":{"type":"string"},"new_text":{"type":"string"} + }}} + },"required":["path","edits"]} + } +}] +body={ + "model":"smart-router", + "stream": False, + "tools": tools, + "tool_choice":"auto", + "messages":[ + {"role":"user","content":"observer.ift.calentiq.com не работает - Bad Gateway почини"}, + {"role":"assistant","content":content}, + {"role":"user","content":"ok"} + ] +} +json.dump(body, open("/tmp/agent-approve-req.json","w"), ensure_ascii=False) +print("req bytes", len(json.dumps(body))) +PY +echo "== approve+tools ==" +curl -sS --max-time 180 "$BASE/v1/chat/completions" \ + -H "Authorization: Bearer $KEY" \ + -H "Content-Type: application/json" \ + -H "X-AI-Orchestrate: force" \ + -H "X-AI-Quality: balanced" \ + -d @/tmp/agent-approve-req.json | tee /tmp/agent-exec1.json | python3 -c ' +import json,sys +d=json.load(sys.stdin) +ch=(d.get("choices") or [{}])[0] +msg=ch.get("message") or {} +print("finish", ch.get("finish_reason")) +print("content", (msg.get("content") or "")[:1200]) +tcs=msg.get("tool_calls") or [] +print("tool_calls", len(tcs)) +for tc in tcs: + fn=tc.get("function") or {} + print(" TOOL", fn.get("name"), (fn.get("arguments") or "")[:300]) +meta=d.get("x_router_meta") or {} +print("meta phase", meta.get("agent_phase"), "force_edit", meta.get("executor_force_edit_after_read"), "model", meta.get("executor_model") or meta.get("selected_model")) +' +echo "== router log since approve ==" +docker service logs --since 3m --raw ai-router_router 2>&1 \ + | grep -iE 'path_index|force edit|stop after|hierarchical_agent|executor result|kickstart|phase=' \ + | tail -30 +REMOTE diff --git a/scripts/smoke-observer-approve2.sh b/scripts/smoke-observer-approve2.sh new file mode 100644 index 0000000..40004b7 --- /dev/null +++ b/scripts/smoke-observer-approve2.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift 'bash -s' <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +BASE="${PUBLIC_URL:-https://ai-router.ift.calentiq.com}" +KEY="${ROUTER_API_KEY:-$LITELLM_MASTER_KEY}" +test -f /tmp/agent-plan.json && echo "plan_ok $(wc -c &1 \ + | grep -iE 'path_index|force edit|stop after|phase=|executor result|kickstart|Выполн' \ + | tail -25 +REMOTE diff --git a/scripts/smoke-observer-request.sh b/scripts/smoke-observer-request.sh new file mode 100644 index 0000000..359c462 --- /dev/null +++ b/scripts/smoke-observer-request.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +ssh -o BatchMode=yes eventhub-ift bash -s <<'REMOTE' +set -euo pipefail +cd /opt/ai-router-stack +set -a; source .env; set +a +BASE="${PUBLIC_URL:-https://ai-router.ift.calentiq.com}" +KEY="${ROUTER_API_KEY:-$LITELLM_MASTER_KEY}" +echo "BASE=$BASE" +# 1) Plan request (no tools) — hierarchical plan confirm +curl -sS --max-time 120 "$BASE/v1/chat/completions" \ + -H "Authorization: Bearer $KEY" \ + -H "Content-Type: application/json" \ + -H "X-AI-Orchestrate: force" \ + -H "X-AI-Quality: balanced" \ + -d '{ + "model": "smart-router", + "stream": false, + "messages": [ + {"role": "user", "content": "observer.ift.calentiq.com не работает - Bad Gateway почини"} + ] + }' | tee /tmp/agent-plan.json | python3 -c ' +import json,sys +d=json.load(sys.stdin) +msg=(d.get("choices") or [{}])[0].get("message") or {} +print("=== PLAN CONTENT ===") +print((msg.get("content") or "")[:2500]) +meta=d.get("x_router_meta") or {} +print("=== META keys ===", sorted(meta.keys())[:30]) +print("mode", meta.get("mode"), "awaiting", meta.get("awaiting_plan_confirm"), "planner", meta.get("planner_model")) +' +REMOTE diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index e8b6af6..911622d 100644 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -14,9 +14,28 @@ BASE="${TEST_BASE_URL:-http://127.0.0.1:8000}" LITELLM_BASE="${TEST_LITELLM_URL:-http://127.0.0.1:4000}" ROUTER_KEY="${ROUTER_API_KEY:?ROUTER_API_KEY required}" -CURL=(curl -sf) -if [[ "$BASE" == https:* ]]; then - CURL+=( -k ) +curl_common() { + local url="$1" + CURL=(curl -sf) + if [[ "$url" == https:* ]]; then + CURL+=( -k ) + fi + if [[ -n "${TEST_RESOLVE_IP:-}" ]]; then + local host="${url#*://}" + host="${host%%/*}" + host="${host%%:*}" + CURL+=( --resolve "${host}:443:${TEST_RESOLVE_IP}" --resolve "${host}:80:${TEST_RESOLVE_IP}" ) + fi +} + +curl_common "${BASE}" +if [[ "$LITELLM_BASE" == https:* && -n "${TEST_RESOLVE_IP:-}" ]]; then + LITELLM_CURL=(curl -sf -k) + lh="${LITELLM_BASE#*://}"; lh="${lh%%/*}"; lh="${lh%%:*}" + LITELLM_CURL+=( --resolve "${lh}:443:${TEST_RESOLVE_IP}" --resolve "${lh}:80:${TEST_RESOLVE_IP}" ) +else + LITELLM_CURL=(curl -sf) + [[ "$LITELLM_BASE" == https:* ]] && LITELLM_CURL+=( -k ) fi echo "== Router health ==" @@ -34,20 +53,31 @@ echo "== Classify COMPLEX ==" -H "Content-Type: application/json" \ -d '{"messages":[{"role":"user","content":"Спроектируй архитектуру microservices"}]}' | jq . -echo "== Chat smart-router max_tokens=16 ==" +echo "== Chat smart-router max_tokens=16 (single-shot / no force hierarchical) ==" "${CURL[@]}" "${BASE}/v1/chat/completions" \ -H "Authorization: Bearer ${ROUTER_KEY}" \ -H "Content-Type: application/json" \ + -H "X-AI-Orchestrate: off" \ -d '{"model":"smart-router","max_tokens":16,"messages":[{"role":"user","content":"bash docker service ls"}]}' \ | jq '.choices[0].message.content, .x_router_meta // empty' +if [[ "${SMOKE_HIERARCHICAL:-}" == "1" ]]; then + echo "== Hierarchical force (Novita plan/verify + workers; VPN not required) ==" + "${CURL[@]}" "${BASE}/v1/chat/completions" \ + -H "Authorization: Bearer ${ROUTER_KEY}" \ + -H "Content-Type: application/json" \ + -H "X-AI-Orchestrate: force" \ + -d '{"model":"smart-router","max_tokens":256,"messages":[{"role":"user","content":"Разбей на 2 шага: 1) что такое Docker 2) одна команда docker ps. Кратко."}]}' \ + | jq '.x_router_meta.mode, .x_router_meta.planner_model, .x_router_meta.worker_calls, .x_router_meta.verify_skipped, (.choices[0].message.content|.[0:200])' +fi + if [[ -n "${SKIP_LITELLM_SMOKE:-}" ]]; then echo "SKIP_LITELLM_SMOKE set — skipping LiteLLM checks" else echo "== LiteLLM liveliness ==" - "${CURL[@]}" "${LITELLM_BASE}/health/liveliness" && echo + "${LITELLM_CURL[@]}" "${LITELLM_BASE}/health/liveliness" && echo echo "== LiteLLM UI ==" - curl -sfI -k "${LITELLM_BASE}/ui" | head -3 || true + "${LITELLM_CURL[@]}" -I "${LITELLM_BASE}/ui" | head -3 || true fi echo "All smoke checks passed." diff --git a/scripts/switch-provider.sh b/scripts/switch-provider.sh new file mode 100644 index 0000000..409ab2a --- /dev/null +++ b/scripts/switch-provider.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Switch PRIMARY_PROVIDER and regenerate litellm_config.yaml +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +PROFILE="${1:-}" +if [[ -z "$PROFILE" ]]; then + echo "Usage: bash scripts/switch-provider.sh " >&2 + echo "Profiles:" >&2 + python3 -c " +import yaml +from pathlib import Path +p = yaml.safe_load(Path('config/providers.yaml').read_text(encoding='utf-8')) +for k, v in sorted(p.get('profiles', {}).items()): + wc = ' [welcome credit]' if v.get('welcome_credit') else '' + print(f' {k:12} {v.get(\"label\", \"\")}{wc}') +" + exit 1 +fi + +if [[ ! -f .env ]]; then + echo "ERROR: .env missing — copy from .env.example" >&2 + exit 1 +fi + +if grep -q '^PRIMARY_PROVIDER=' .env; then + sed -i "s/^PRIMARY_PROVIDER=.*/PRIMARY_PROVIDER=${PROFILE}/" .env +else + echo "PRIMARY_PROVIDER=${PROFILE}" >> .env +fi + +set -a +# shellcheck disable=SC1091 +source .env +set +a + +python3 scripts/gen-litellm-config.py +echo "" +echo "Switched to PRIMARY_PROVIDER=${PROFILE}" +echo "Redeploy on IFT: bash scripts/deploy.sh (Swarm configs are immutable — stack rm if deploy fails)" diff --git a/scripts/sync-routing-config.sh b/scripts/sync-routing-config.sh index f4009ba..c501a02 100644 --- a/scripts/sync-routing-config.sh +++ b/scripts/sync-routing-config.sh @@ -1,78 +1,12 @@ #!/usr/bin/env bash -# Sync model_list fragment from config/*.yaml +# Regenerate litellm_config.yaml from PRIMARY_PROVIDER + config/providers.yaml set -euo pipefail - ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -export ROOT - -python3 <<'PY' -import os -import yaml -from pathlib import Path - -root = Path(os.environ["ROOT"]) -rules = yaml.safe_load((root / "config/routing_rules.yaml").read_text(encoding="utf-8")) -matrix = yaml.safe_load((root / "config/model_matrix.yaml").read_text(encoding="utf-8")) - -entries = [] -for name, cfg in matrix.get("models", {}).items(): - novita = cfg.get("novita") - if not novita: - continue - entry = { - "model_name": name, - "litellm_params": { - "model": novita, - "api_key": "os.environ/NOVITA_API_KEY", - }, - } - if cfg.get("rpm"): - entry["litellm_params"]["rpm"] = cfg["rpm"] - entries.append(entry) - -for name, cfg in matrix.get("optional_providers", {}).items(): - entries.append({ - "model_name": name, - "litellm_params": { - "model": cfg["model"], - "api_key": cfg.get("api_key", "os.environ/GROQ_API_KEY"), - }, - }) - -litellm_rules = rules.get("litellm", {}) -entries.append({ - "model_name": "smart-router-internal", - "litellm_params": { - "model": "auto_router/complexity_router", - "drop_params": True, - "complexity_router_default_model": "a-medium-ops", - "complexity_router_config": { - "tiers": { - "SIMPLE": "a-simple", - "MEDIUM": "a-medium-ops", - "MEDIUM_CODE": "a-medium-code", - "COMPLEX": "a-complex", - "REASONING": "a-reasoning", - }, - "classifier_fallback": "heuristic", - "keyword_tier_rules": litellm_rules.get("keyword_tier_rules", []), - "custom_technical_keywords": litellm_rules.get("custom_technical_keywords", []), - "token_thresholds": {"simple": 20, "complex": 500}, - "tier_boundaries": { - "simple_medium": 0.18, - "medium_complex": 0.38, - "complex_reasoning": 0.62, - }, - "session_affinity": True, - "session_affinity_ttl_seconds": 1800, - }, - }, -}) - -out = root / "litellm_config.generated.yaml" -out.write_text( - yaml.dump({"model_list": entries}, allow_unicode=True, sort_keys=False), - encoding="utf-8", -) -print(f"Wrote {out} ({len(entries)} models)") -PY +cd "$ROOT" +if [[ -f .env ]]; then + set -a + # shellcheck disable=SC1091 + source .env + set +a +fi +python3 scripts/gen-litellm-config.py diff --git a/scripts/vless-from-subscription-json.py b/scripts/vless-from-subscription-json.py new file mode 100644 index 0000000..36a861d --- /dev/null +++ b/scripts/vless-from-subscription-json.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Extract first working vless:// URI from Happ/mireon subscription JSON.""" +import json +import sys +import urllib.parse + +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +configs = data if isinstance(data, list) else [data] +for j in configs: + if not isinstance(j, dict): + continue + for ob in j.get("outbounds", []): + if ob.get("protocol") != "vless": + continue + tag = ob.get("tag") or "proxy" + st = ob.get("settings") or {} + vnext = (st.get("vnext") or [{}])[0] + addr, port = vnext.get("address"), vnext.get("port") + users = (vnext.get("users") or [{}])[0] + uid = users.get("id") + if not all([addr, port, uid]) or addr in ("0.0.0.0", "127.0.0.1"): + continue + flow = users.get("flow") or "" + stream = ob.get("streamSettings") or {} + net = stream.get("network") or "tcp" + sec = stream.get("security") or "none" + rs = stream.get("realitySettings") or {} + ts = stream.get("tlsSettings") or {} + sni = rs.get("serverName") or ts.get("serverName") or "" + params = {"encryption": "none", "security": sec, "type": net} + if sni: + params["sni"] = sni + if rs.get("publicKey"): + params["pbk"] = rs["publicKey"] + if rs.get("shortId"): + params["sid"] = rs["shortId"] + if rs.get("fingerprint"): + params["fp"] = rs["fingerprint"] + if flow: + params["flow"] = flow + q = urllib.parse.urlencode(params) + print(f"vless://{uid}@{addr}:{port}?{q}#{tag}") + sys.exit(0) +sys.exit(1) diff --git a/scripts/vpn-disable.sh b/scripts/vpn-disable.sh index 465f1c4..3b163c8 100644 --- a/scripts/vpn-disable.sh +++ b/scripts/vpn-disable.sh @@ -10,7 +10,9 @@ set -a source .env set +a -NO_PROXY_VAL="${NO_PROXY:-localhost,127.0.0.1,api.novita.ai,novita.ai}" +# shellcheck disable=SC1091 +source "${STACK_DIR}/scripts/no-proxy-default.sh" +NO_PROXY_VAL="${NO_PROXY:-${NO_PROXY_DEFAULT}}" echo "== Scale vless-proxy and vpn-watchdog to 0 ==" docker service scale "${STACK_NAME}_vless-proxy=0" "${STACK_NAME}_vpn-watchdog=0" || true diff --git a/scripts/vpn-enable.sh b/scripts/vpn-enable.sh index 1ecfbc7..2d4fb36 100644 --- a/scripts/vpn-enable.sh +++ b/scripts/vpn-enable.sh @@ -11,7 +11,9 @@ source .env set +a VLESS_URL="${VLESS_PROXY_URL:-http://vless-proxy:8080}" -NO_PROXY_VAL="${NO_PROXY:-localhost,127.0.0.1,api.novita.ai,novita.ai}" +# shellcheck disable=SC1091 +source "${STACK_DIR}/scripts/no-proxy-default.sh" +NO_PROXY_VAL="${NO_PROXY:-${NO_PROXY_DEFAULT}}" echo "== Scale vless-proxy and vpn-watchdog to 1 ==" docker service scale "${STACK_NAME}_vless-proxy=1" "${STACK_NAME}_vpn-watchdog=1" @@ -42,4 +44,4 @@ docker service update \ --env-add "NO_PROXY=${NO_PROXY_VAL}" \ "${STACK_NAME}_litellm" -echo "VPN enabled. Novita bypasses via NO_PROXY." +echo "VPN enabled. Direct: Novita/Groq/Gemini/GigaChat. VPN: Anthropic/OpenRouter/Grok/Together." diff --git a/test/unit/test_hierarchical.py b/test/unit/test_hierarchical.py new file mode 100644 index 0000000..ca8b76a --- /dev/null +++ b/test/unit/test_hierarchical.py @@ -0,0 +1,1044 @@ +#!/usr/bin/env python3 +"""Unit tests for hierarchical helpers (no network).""" + +from __future__ import annotations + +import json +import os +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +os.environ["CONFIG_DIR"] = str(ROOT / "config") +sys.path.insert(0, str(ROOT / "router")) + +from hierarchical import ( # noqa: E402 + Subtask, + classify_plan_reply, + deterministic_checks, + digest_text, + find_pending_plan, + format_plan_confirm, + parse_plan_json, + should_run_hierarchical, + should_verify, + topological_waves, + worker_model_for, +) +from agent_hier import ( # noqa: E402 + _first_path_hint, + context_fill_for_forward, + format_context_fill, + inject_plan_context, + messages_have_tool_activity, + request_has_tools, + synthetic_first_tool_completion, +) + + +class HierarchicalHelpersTest(unittest.TestCase): + def test_parse_plan_json(self) -> None: + raw = """ + { + "subtasks": [ + {"id": "1", "prompt": "do a", "worker_tier": "simple", "depends_on": []}, + {"id": "2", "prompt": "do b", "worker_tier": "medium_code", "depends_on": ["1"]}, + {"id": "3", "prompt": "do c", "worker_tier": "hard", "depends_on": ["2"]} + ], + "acceptance": ["works"] + } + """ + tasks, acc = parse_plan_json(raw, max_subtasks=5) + self.assertEqual(len(tasks), 3) + self.assertEqual(acc, ["works"]) + self.assertEqual(tasks[1].depends_on, ["1"]) + self.assertEqual(tasks[2].worker_tier, "hard") + + def test_parse_plan_caps_subtasks(self) -> None: + items = [ + { + "id": str(i), + "prompt": f"p{i}", + "worker_tier": "simple", + "depends_on": [], + } + for i in range(1, 10) + ] + raw = json.dumps({"subtasks": items, "acceptance": []}) + tasks, _ = parse_plan_json(raw, max_subtasks=5) + self.assertEqual(len(tasks), 5) + + def test_topological_waves(self) -> None: + tasks = [ + Subtask("1", "a", "simple", []), + Subtask("2", "b", "simple", ["1"]), + Subtask("3", "c", "simple", []), + ] + waves = topological_waves(tasks) + self.assertEqual({t.id for t in waves[0]}, {"1", "3"}) + self.assertEqual([t.id for t in waves[1]], ["2"]) + + def test_should_run_hierarchical(self) -> None: + self.assertTrue( + should_run_hierarchical( + tier_value="COMPLEX", + header="auto", + quality_mode="auto", + enabled=True, + ) + ) + self.assertFalse( + should_run_hierarchical( + tier_value="SIMPLE", + header="auto", + quality_mode="auto", + enabled=True, + ) + ) + self.assertTrue( + should_run_hierarchical( + tier_value="SIMPLE", + header="force", + quality_mode="auto", + enabled=True, + ) + ) + self.assertFalse( + should_run_hierarchical( + tier_value="COMPLEX", + header="off", + quality_mode="auto", + enabled=True, + ) + ) + self.assertFalse( + should_run_hierarchical( + tier_value="COMPLEX", + header="auto", + quality_mode="economy", + enabled=True, + ) + ) + + def test_should_verify(self) -> None: + self.assertFalse( + should_verify( + policy="on_fail_or_hard", + checks_ok=True, + hard_used=False, + quality_mode="auto", + ) + ) + self.assertTrue( + should_verify( + policy="on_fail_or_hard", + checks_ok=False, + hard_used=False, + quality_mode="auto", + ) + ) + self.assertTrue( + should_verify( + policy="on_fail_or_hard", + checks_ok=True, + hard_used=True, + quality_mode="auto", + ) + ) + self.assertTrue( + should_verify( + policy="never", + checks_ok=False, + hard_used=True, + quality_mode="max", + ) + ) + + def test_deterministic_checks(self) -> None: + ok, fail = deterministic_checks({"1": "hello"}, [], hard_used=False) + self.assertTrue(ok) + self.assertEqual(fail, []) + ok2, fail2 = deterministic_checks({"1": ""}, [], hard_used=False) + self.assertFalse(ok2) + self.assertEqual(fail2, ["1"]) + ok3, _ = deterministic_checks({"1": "hello"}, [], hard_used=True) + self.assertFalse(ok3) + + def test_digest_and_worker_map(self) -> None: + self.assertIn("truncated", digest_text("x" * 1000, 50)) + # hard → DeepSeek (b-complex); quality=max → Max (c-complex) + self.assertEqual(worker_model_for("hard"), "b-complex") + self.assertEqual( + worker_model_for("hard", quality_mode="max"), "c-complex" + ) + self.assertEqual(worker_model_for("simple"), "a-simple") + self.assertEqual(worker_model_for("medium_code"), "a-medium-code") + + def test_parse_plan_paths_and_goals(self) -> None: + raw = json.dumps( + { + "subtasks": [ + { + "id": "1", + "prompt": "fix router", + "worker_tier": "medium_code", + "depends_on": [], + "paths": [r"C:\Users\alexc\IdeaProjects\eventHub\EventHubAiRouter\router\router.py"], + "edit_goal": "bump timeout", + "constraints": ["keep SSE"], + } + ], + "acceptance": ["timeout higher"], + } + ) + tasks, acc = parse_plan_json(raw, max_subtasks=3) + self.assertEqual(len(tasks), 1) + self.assertTrue(tasks[0].paths[0].endswith("router.py")) + self.assertEqual(tasks[0].edit_goal, "bump timeout") + self.assertEqual(tasks[0].constraints, ["keep SSE"]) + self.assertEqual(acc, ["timeout higher"]) + payload = format_plan_confirm(tasks, acc, user_goal="bump") + self.assertIn("bump timeout", payload) + pending = find_pending_plan([{"role": "assistant", "content": payload}]) + self.assertIsNotNone(pending) + self.assertEqual(pending["subtasks"][0]["edit_goal"], "bump timeout") + + def test_executor_never_synthetic_and_no_llama(self) -> None: + from agent_hier import ( + executor_fallback_completion, + pick_agent_executor_model, + should_escalate_executor, + ) + from rules_loader import load_orchestration + + hier = load_orchestration().get("hierarchical") or {} + self.assertEqual(hier.get("planner_model"), "novita-planner") + self.assertEqual(hier.get("agent_executor_model"), "a-medium-code") + self.assertEqual(hier.get("executor_synthetic_mode"), "kickstart_only") + self.assertEqual(pick_agent_executor_model({"agent_executor_model": "a-simple"}), "a-medium-code") + self.assertEqual(pick_agent_executor_model(hier), "a-medium-code") + stop = executor_fallback_completion( + plan={"subtasks": [{"prompt": "edit x", "paths": ["a.py"]}]}, + tools=[ + { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object"}}, + } + ], + model="a-medium-code", + messages=[{"role": "user", "content": "go"}], + cfg={"executor_synthetic_mode": "never"}, + reason="litellm_408", + ) + choice = (stop.get("choices") or [{}])[0] + self.assertEqual(choice.get("finish_reason"), "stop") + msg = choice.get("message") or {} + self.assertFalse(msg.get("tool_calls")) + # escalate after edit failures + fail_msgs = [ + { + "role": "tool", + "content": "Error: expected struct Edit", + "tool_call_id": "1", + }, + { + "role": "tool", + "content": "Error: expected struct Edit", + "tool_call_id": "2", + }, + ] + self.assertTrue( + should_escalate_executor( + fail_msgs, {"agent_escalate_after_edit_failures": 2} + ) + ) + self.assertTrue(hier.get("executor_escalate_after_read", False)) + self.assertEqual(int(hier.get("executor_tool_loop_timeout_sec") or 0), 45) + yml = r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps\ift\traefik\dynamic_conf.yml" + read_ok = [ + { + "role": "assistant", + "tool_calls": [ + { + "id": "r1", + "type": "function", + "function": { + "name": "read_file", + "arguments": json.dumps({"path": yml}), + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "r1", + "content": "http:\n routers:\n api:\n rule: Host(`x`)\n", + }, + ] + self.assertTrue( + should_escalate_executor( + read_ok, {"executor_escalate_after_read": True} + ) + ) + from agent_hier import ( + force_edit_after_read_completion, + sanitize_outbound_messages, + should_force_edit_after_read, + ) + + self.assertTrue( + should_force_edit_after_read( + read_ok, {"executor_force_edit_after_read": True} + ) + ) + # After an edit was already issued — never force another (edit loop bug) + after_edit = read_ok + [ + { + "role": "assistant", + "tool_calls": [ + { + "id": "e1", + "type": "function", + "function": { + "name": "edit_file", + "arguments": json.dumps( + { + "path": yml, + "edits": [ + {"old_text": "x", "new_text": "y"} + ], + } + ), + }, + } + ], + }, + {"role": "tool", "tool_call_id": "e1", "content": "ok"}, + ] + self.assertFalse( + should_force_edit_after_read( + after_edit, {"executor_force_edit_after_read": True} + ) + ) + from agent_hier import should_stop_after_edit, stop_after_edit_completion + + self.assertTrue(should_stop_after_edit(after_edit, {})) + done = stop_after_edit_completion(model="b-complex", messages=after_edit) + self.assertEqual(done["choices"][0]["finish_reason"], "stop") + self.assertIn("Готово", done["choices"][0]["message"]["content"]) + forced = force_edit_after_read_completion( + plan={"subtasks": [{"prompt": "traefik", "paths": [yml]}]}, + tools=[ + { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object"}}, + }, + { + "type": "function", + "function": {"name": "edit_file", "parameters": {"type": "object"}}, + }, + ], + model="b-complex", + messages=read_ok, + ) + self.assertEqual( + forced["choices"][0]["message"]["tool_calls"][0]["function"]["name"], + "edit_file", + ) + dirty = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "r1", + "type": "function", + "function": {"name": "read_file", "arguments": {"path": yml}}, + } + ], + "reasoning_content": "secret", + }, + {"role": "tool", "tool_call_id": "orphan", "content": "x"}, + {"role": "tool", "tool_call_id": "r1", "content": "ok"}, + ] + clean = sanitize_outbound_messages(dirty) + self.assertEqual(clean[0]["content"], "") + self.assertNotIn("reasoning_content", clean[0]) + self.assertIsInstance(clean[0]["tool_calls"][0]["function"]["arguments"], str) + self.assertEqual(len([m for m in clean if m.get("role") == "tool"]), 1) + + from agent_hier import ( + compress_messages_for_tool_loop, + rewrite_redundant_reread_completion, + ) + + fat = "x" * 9000 + compressed = compress_messages_for_tool_loop( + [{"role": "tool", "tool_call_id": "r1", "content": fat}], + newest_tool_result_chars=3500, + ) + self.assertLessEqual(len(compressed[0]["content"]), 3520) + self.assertIn("TRUNCATED", compressed[0]["content"]) + tools = [ + { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object"}}, + }, + { + "type": "function", + "function": {"name": "edit_file", "parameters": {"type": "object"}}, + }, + ] + reread = { + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "tool_calls": [ + { + "id": "r2", + "type": "function", + "function": { + "name": "read_file", + "arguments": json.dumps({"path": yml}), + }, + } + ], + }, + } + ] + } + rewritten = rewrite_redundant_reread_completion( + reread, + messages=read_ok, + plan={ + "subtasks": [ + { + "prompt": "add comment", + "paths": [yml], + "edit_goal": "marker", + } + ] + }, + tools=tools, + model="b-complex", + cfg={"executor_rewrite_reread": True}, + ) + name = rewritten["choices"][0]["message"]["tool_calls"][0]["function"]["name"] + self.assertEqual(name, "edit_file") + # Root docker-compose.yml does not exist in EventHubDevOps + from agent_hier import _is_bogus_devops_path + + self.assertTrue( + _is_bogus_devops_path( + r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps\docker-compose.yml" + ) + ) + self.assertTrue( + _is_bogus_devops_path( + r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps\ift\traefik\traefik.yml" + ) + ) + self.assertFalse( + _is_bogus_devops_path( + r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps\ift\docker-compose.core.yml" + ) + ) + self.assertFalse( + _is_bogus_devops_path( + r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps\ift\traefik\dynamic_conf.yml" + ) + ) + bogus_read = { + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "tool_calls": [ + { + "id": "r3", + "type": "function", + "function": { + "name": "read_file", + "arguments": json.dumps( + { + "path": r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps\docker-compose.yml" + } + ), + }, + } + ], + }, + } + ] + } + fixed = rewrite_redundant_reread_completion( + bogus_read, + messages=read_ok, + plan={ + "subtasks": [ + {"prompt": "traefik EventHubDevOps", "paths": [yml]} + ] + }, + tools=tools, + model="b-complex", + cfg={"executor_rewrite_reread": True}, + ) + # Already have traefik body → must edit, not chase root compose + self.assertEqual( + fixed["choices"][0]["message"]["tool_calls"][0]["function"]["name"], + "edit_file", + ) + + + def test_plan_confirm_helpers(self) -> None: + tasks = [Subtask("1", "fix auth", "simple", [])] + text = format_plan_confirm(tasks, ["ok"], user_goal="fix login") + self.assertIn("ожидает утверждения", text) + self.assertIn("