aleksey
a2d238d92e
Make Zed Agent closer to Cursor: deterministic DevOps path index, live Traefik port probe before blind edits, stop-after-edit, and quieter Russian progress.
349 lines
11 KiB
Python
349 lines
11 KiB
Python
"""DevOps runtime probe: gather live facts before editing Traefik/compose."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import socket
|
|
import subprocess
|
|
import urllib.error
|
|
import urllib.request
|
|
from typing import Any
|
|
|
|
log = logging.getLogger("ai-router")
|
|
|
|
_HOST_RE = re.compile(
|
|
r"\b([a-z0-9-]+(?:\.(?:ift|stage))?\.calentiq\.com)\b",
|
|
re.I,
|
|
)
|
|
|
|
# hostname pattern → docker DNS name + candidate ports
|
|
_SERVICE_MAP: list[tuple[str, str, tuple[int, ...]]] = [
|
|
(r"^observer\.", "observer_web", (80, 4000, 8080, 3000)),
|
|
(r"^grafana\.", "grafana", (3000, 80)),
|
|
(r"^prometheus\.", "prometheus", (9090, 80)),
|
|
(r"^portainer\.", "portainer", (9000, 9443, 80)),
|
|
(r"^ai-router\.", "ai-router", (8000, 80)),
|
|
(r"^litellm\.", "litellm", (4000, 80)),
|
|
(r"^logs\.|^loglynx\.", "loglynx", (6123, 80)),
|
|
(r"^kuma\.", "uptime-kuma", (3001, 80)),
|
|
]
|
|
|
|
|
|
def extract_hosts(*texts: str) -> list[str]:
|
|
found: list[str] = []
|
|
seen: set[str] = set()
|
|
for text in texts:
|
|
if not text:
|
|
continue
|
|
for m in _HOST_RE.finditer(text):
|
|
h = m.group(1).lower()
|
|
if h not in seen:
|
|
seen.add(h)
|
|
found.append(h)
|
|
return found
|
|
|
|
|
|
def is_devops_request(plan: dict[str, Any] | None, *extra: str) -> bool:
|
|
blob = json.dumps(plan or {}, ensure_ascii=False).lower()
|
|
for t in extra:
|
|
blob += "\n" + str(t or "").lower()
|
|
keys = (
|
|
"calentiq",
|
|
"traefik",
|
|
"bad gateway",
|
|
"502",
|
|
"504",
|
|
"eventhubdevops",
|
|
"docker-compose",
|
|
"swarm",
|
|
"observer",
|
|
"ift.",
|
|
"stage.",
|
|
"devops",
|
|
"gateway",
|
|
)
|
|
if any(k in blob for k in keys):
|
|
return True
|
|
return bool(extract_hosts(blob))
|
|
|
|
|
|
def service_for_host(host: str) -> tuple[str, tuple[int, ...]]:
|
|
h = host.lower()
|
|
for pat, name, ports in _SERVICE_MAP:
|
|
if re.search(pat, h):
|
|
return name, ports
|
|
# fallback: first label as service guess
|
|
label = h.split(".")[0]
|
|
return label.replace("-", "_"), (80, 443, 8080, 3000, 4000)
|
|
|
|
|
|
def _tcp_open(host: str, port: int, timeout: float = 2.0) -> bool:
|
|
try:
|
|
with socket.create_connection((host, port), timeout=timeout):
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _http_status(url: str, timeout: float = 8.0) -> int | None:
|
|
try:
|
|
req = urllib.request.Request(url, method="GET")
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310
|
|
return int(getattr(resp, "status", 200) or 200)
|
|
except urllib.error.HTTPError as exc:
|
|
return int(exc.code)
|
|
except Exception: # noqa: BLE001
|
|
return None
|
|
|
|
|
|
def probe_host(host: str, *, cfg: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
"""Live probe from AiRouter container (same Docker network as services)."""
|
|
cfg = cfg or {}
|
|
timeout = float(cfg.get("runtime_probe_timeout_sec") or 8)
|
|
service, ports = service_for_host(host)
|
|
https = f"https://{host}/"
|
|
http = f"http://{host}/"
|
|
status = _http_status(https, timeout=timeout)
|
|
if status is None:
|
|
status = _http_status(http, timeout=timeout)
|
|
|
|
port_hits: dict[str, bool] = {}
|
|
open_ports: list[int] = []
|
|
for p in ports:
|
|
ok = _tcp_open(service, p, timeout=min(2.0, timeout))
|
|
port_hits[f"{service}:{p}"] = ok
|
|
if ok:
|
|
open_ports.append(p)
|
|
|
|
closed_preferred = not port_hits.get(f"{service}:80", True) and bool(open_ports)
|
|
hint = ""
|
|
suggested_url = None
|
|
if status in (502, 503, 504) and open_ports:
|
|
# Classic misconfig: Traefik → :80 while app on :4000
|
|
if 80 not in open_ports and open_ports:
|
|
suggested_url = f"http://{service}:{open_ports[0]}"
|
|
hint = (
|
|
f"HTTP {status}: Traefik likely targets a closed port. "
|
|
f"{service}:80 open={port_hits.get(f'{service}:80')}; "
|
|
f"open ports={open_ports}. Prefer {suggested_url} in dynamic_conf.yml."
|
|
)
|
|
else:
|
|
hint = (
|
|
f"HTTP {status} but {service} has open ports {open_ports or 'none'}. "
|
|
"Check container health / upstream path."
|
|
)
|
|
elif status in (502, 503, 504):
|
|
hint = (
|
|
f"HTTP {status}: no open TCP on guessed service `{service}` "
|
|
f"ports {list(ports)}. Service down or wrong DNS name."
|
|
)
|
|
elif status and status < 400:
|
|
hint = f"HTTP {status}: host responds; issue may be app-level."
|
|
else:
|
|
hint = f"HTTP status={status}; ports={port_hits}"
|
|
|
|
facts: dict[str, Any] = {
|
|
"host": host,
|
|
"http_status": status,
|
|
"service": service,
|
|
"ports": port_hits,
|
|
"open_ports": open_ports,
|
|
"suggested_backend_url": suggested_url,
|
|
"hint": hint,
|
|
"source": "gateway_tcp",
|
|
"closed_port_80_but_alt_open": closed_preferred,
|
|
}
|
|
log.info(
|
|
"runtime_probe host=%s status=%s service=%s open=%s suggest=%s",
|
|
host,
|
|
status,
|
|
service,
|
|
open_ports,
|
|
suggested_url,
|
|
)
|
|
return facts
|
|
|
|
|
|
def attach_runtime_probe(
|
|
plan: dict[str, Any] | None,
|
|
*,
|
|
user_text: str = "",
|
|
cfg: dict[str, Any] | None = None,
|
|
) -> dict[str, Any] | None:
|
|
"""If DevOps task — probe hosts and attach runtime_facts to plan."""
|
|
if not isinstance(plan, dict):
|
|
return plan
|
|
cfg = cfg or {}
|
|
if not cfg.get("runtime_probe_enabled", True):
|
|
return plan
|
|
if plan.get("runtime_facts"):
|
|
return plan
|
|
if not is_devops_request(plan, user_text):
|
|
return plan
|
|
|
|
out = dict(plan)
|
|
hosts = extract_hosts(
|
|
user_text,
|
|
json.dumps(plan, ensure_ascii=False),
|
|
str(plan.get("user_goal") or ""),
|
|
)
|
|
if not hosts:
|
|
# DevOps without host — still mark needed for terminal phase
|
|
out["runtime_probe"] = "needed"
|
|
out["runtime_facts"] = {
|
|
"hint": "DevOps task without clear host — curl/docker inspect before edit.",
|
|
"source": "none",
|
|
}
|
|
return out
|
|
|
|
facts_list = [probe_host(h, cfg=cfg) for h in hosts[:3]]
|
|
primary = facts_list[0]
|
|
out["runtime_facts"] = primary
|
|
out["runtime_facts_all"] = facts_list
|
|
out["runtime_probe"] = "gateway"
|
|
# Strengthen paths toward Traefik dynamic conf for ift hosts
|
|
if "ift." in primary.get("host", "") or primary.get("suggested_backend_url"):
|
|
root = r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps"
|
|
yml = root + r"\ift\traefik\dynamic_conf.yml"
|
|
tasks = list(out.get("subtasks") or [])
|
|
if tasks and isinstance(tasks[0], dict):
|
|
t0 = dict(tasks[0])
|
|
paths = list(t0.get("paths") or [])
|
|
if yml not in paths:
|
|
paths = [yml] + paths
|
|
t0["paths"] = paths[:4]
|
|
if primary.get("suggested_backend_url"):
|
|
t0["edit_goal"] = (
|
|
f"Set Traefik service backend to {primary['suggested_backend_url']} "
|
|
f"(HTTP {primary.get('http_status')}: wrong upstream port)."
|
|
)
|
|
tasks[0] = t0
|
|
out["subtasks"] = tasks
|
|
return out
|
|
|
|
|
|
def runtime_facts_line(plan: dict[str, Any] | None) -> str:
|
|
facts = (plan or {}).get("runtime_facts") if isinstance(plan, dict) else None
|
|
if not isinstance(facts, dict) or not facts:
|
|
return ""
|
|
parts = [
|
|
f"host={facts.get('host')}",
|
|
f"http={facts.get('http_status')}",
|
|
f"service={facts.get('service')}",
|
|
]
|
|
if facts.get("open_ports"):
|
|
parts.append(f"open={facts.get('open_ports')}")
|
|
if facts.get("suggested_backend_url"):
|
|
parts.append(f"fix→{facts.get('suggested_backend_url')}")
|
|
hint = str(facts.get("hint") or "")[:180]
|
|
return "RUNTIME: " + "; ".join(str(p) for p in parts) + (f" | {hint}" if hint else "")
|
|
|
|
|
|
def devops_blocks_blind_force_edit(plan: dict[str, Any] | None) -> bool:
|
|
"""Do not blind-edit DevOps YAML until runtime facts exist."""
|
|
if not isinstance(plan, dict):
|
|
return False
|
|
if not is_devops_request(plan):
|
|
return False
|
|
facts = plan.get("runtime_facts")
|
|
if not facts:
|
|
return True
|
|
# Facts present — allow targeted edit (including force with goal from facts)
|
|
return False
|
|
|
|
|
|
def suggested_traefik_edit(
|
|
plan: dict[str, Any] | None,
|
|
) -> dict[str, str] | None:
|
|
"""If probe found closed:80 + open:alt — return old/new URL snippets."""
|
|
if not isinstance(plan, dict):
|
|
return None
|
|
facts = plan.get("runtime_facts")
|
|
if not isinstance(facts, dict):
|
|
return None
|
|
url = facts.get("suggested_backend_url")
|
|
service = facts.get("service")
|
|
if not url or not service:
|
|
return None
|
|
# Common misconfig in our dynamic_conf
|
|
old = f'url: "http://{service}:80"'
|
|
new = f'url: "{url}"'
|
|
if old == new:
|
|
return None
|
|
return {
|
|
"path": r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps\ift\traefik\dynamic_conf.yml",
|
|
"old_text": old,
|
|
"new_text": new,
|
|
"reason": str(facts.get("hint") or "port mismatch"),
|
|
}
|
|
|
|
|
|
def force_runtime_fix_edit(
|
|
*,
|
|
plan: dict[str, Any] | None,
|
|
tools: list[Any],
|
|
model: str,
|
|
) -> dict[str, Any] | None:
|
|
"""Synthetic edit_file for clear Traefik port mismatch — skip blind marker edits."""
|
|
fix = suggested_traefik_edit(plan)
|
|
if not fix:
|
|
return None
|
|
# late import to avoid cycles
|
|
from agent_hier import _pick_edit_tool, _synthetic_tool_completion
|
|
|
|
edit_name = _pick_edit_tool(tools)
|
|
if not edit_name:
|
|
return None
|
|
log.info(
|
|
"runtime_probe → targeted edit %s => %s",
|
|
fix["old_text"],
|
|
fix["new_text"],
|
|
)
|
|
return _synthetic_tool_completion(
|
|
model=model,
|
|
tool_name=edit_name,
|
|
arguments={
|
|
"path": fix["path"],
|
|
"edits": [{"old_text": fix["old_text"], "new_text": fix["new_text"]}],
|
|
},
|
|
)
|
|
|
|
|
|
def ssh_probe_fallback(host: str, cfg: dict[str, Any]) -> dict[str, Any] | None:
|
|
"""Optional: ssh to host and docker inspect (if gateway has keys)."""
|
|
ssh_host = str(cfg.get("runtime_probe_ssh_host") or "").strip()
|
|
if not ssh_host:
|
|
return None
|
|
service, _ports = service_for_host(host)
|
|
script = (
|
|
f"echo HTTP=$(curl -sk -o /dev/null -w '%{{http_code}}' --max-time 5 https://{host}/ || echo err); "
|
|
f"CID=$(docker ps -q --filter name={service} | head -1); "
|
|
f"echo CID=$CID; "
|
|
f"if [ -n \"$CID\" ]; then docker inspect \"$CID\" --format 'PORTS={{{{json .NetworkSettings.Ports}}}}'; fi"
|
|
)
|
|
try:
|
|
proc = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"ConnectTimeout=5",
|
|
ssh_host,
|
|
"bash",
|
|
"-lc",
|
|
script,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=20,
|
|
check=False,
|
|
)
|
|
out = (proc.stdout or "") + "\n" + (proc.stderr or "")
|
|
return {"host": host, "ssh_raw": out[:1500], "source": "ssh", "service": service}
|
|
except Exception as exc: # noqa: BLE001
|
|
log.warning("runtime ssh probe failed: %s", exc)
|
|
return None
|