"""DevOps runtime probe: gather live facts before editing Traefik/compose.""" from __future__ import annotations import json import logging import re import socket import subprocess import urllib.error import urllib.request from typing import Any log = logging.getLogger("ai-router") _HOST_RE = re.compile( r"\b([a-z0-9-]+(?:\.(?:ift|stage))?\.calentiq\.com)\b", re.I, ) # hostname pattern → docker DNS name + candidate ports _SERVICE_MAP: list[tuple[str, str, tuple[int, ...]]] = [ (r"^observer\.", "observer_web", (80, 4000, 8080, 3000)), (r"^grafana\.", "grafana", (3000, 80)), (r"^prometheus\.", "prometheus", (9090, 80)), (r"^portainer\.", "portainer", (9000, 9443, 80)), (r"^ai-router\.", "ai-router", (8000, 80)), (r"^litellm\.", "litellm", (4000, 80)), (r"^logs\.|^loglynx\.", "loglynx", (6123, 80)), (r"^kuma\.", "uptime-kuma", (3001, 80)), ] def extract_hosts(*texts: str) -> list[str]: found: list[str] = [] seen: set[str] = set() for text in texts: if not text: continue for m in _HOST_RE.finditer(text): h = m.group(1).lower() if h not in seen: seen.add(h) found.append(h) return found def is_devops_request(plan: dict[str, Any] | None, *extra: str) -> bool: blob = json.dumps(plan or {}, ensure_ascii=False).lower() for t in extra: blob += "\n" + str(t or "").lower() keys = ( "calentiq", "traefik", "bad gateway", "502", "504", "eventhubdevops", "docker-compose", "swarm", "observer", "ift.", "stage.", "devops", "gateway", ) if any(k in blob for k in keys): return True return bool(extract_hosts(blob)) def service_for_host(host: str) -> tuple[str, tuple[int, ...]]: h = host.lower() for pat, name, ports in _SERVICE_MAP: if re.search(pat, h): return name, ports # fallback: first label as service guess label = h.split(".")[0] return label.replace("-", "_"), (80, 443, 8080, 3000, 4000) def _tcp_open(host: str, port: int, timeout: float = 2.0) -> bool: try: with socket.create_connection((host, port), timeout=timeout): return True except OSError: return False def _http_status(url: str, timeout: float = 8.0) -> int | None: try: req = urllib.request.Request(url, method="GET") with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 return int(getattr(resp, "status", 200) or 200) except urllib.error.HTTPError as exc: return int(exc.code) except Exception: # noqa: BLE001 return None def probe_host(host: str, *, cfg: dict[str, Any] | None = None) -> dict[str, Any]: """Live probe from AiRouter container (same Docker network as services).""" cfg = cfg or {} timeout = float(cfg.get("runtime_probe_timeout_sec") or 8) service, ports = service_for_host(host) https = f"https://{host}/" http = f"http://{host}/" status = _http_status(https, timeout=timeout) if status is None: status = _http_status(http, timeout=timeout) port_hits: dict[str, bool] = {} open_ports: list[int] = [] for p in ports: ok = _tcp_open(service, p, timeout=min(2.0, timeout)) port_hits[f"{service}:{p}"] = ok if ok: open_ports.append(p) closed_preferred = not port_hits.get(f"{service}:80", True) and bool(open_ports) hint = "" suggested_url = None if status in (502, 503, 504) and open_ports: # Classic misconfig: Traefik → :80 while app on :4000 if 80 not in open_ports and open_ports: suggested_url = f"http://{service}:{open_ports[0]}" hint = ( f"HTTP {status}: Traefik likely targets a closed port. " f"{service}:80 open={port_hits.get(f'{service}:80')}; " f"open ports={open_ports}. Prefer {suggested_url} in dynamic_conf.yml." ) else: hint = ( f"HTTP {status} but {service} has open ports {open_ports or 'none'}. " "Check container health / upstream path." ) elif status in (502, 503, 504): hint = ( f"HTTP {status}: no open TCP on guessed service `{service}` " f"ports {list(ports)}. Service down or wrong DNS name." ) elif status and status < 400: hint = f"HTTP {status}: host responds; issue may be app-level." else: hint = f"HTTP status={status}; ports={port_hits}" facts: dict[str, Any] = { "host": host, "http_status": status, "service": service, "ports": port_hits, "open_ports": open_ports, "suggested_backend_url": suggested_url, "hint": hint, "source": "gateway_tcp", "closed_port_80_but_alt_open": closed_preferred, } log.info( "runtime_probe host=%s status=%s service=%s open=%s suggest=%s", host, status, service, open_ports, suggested_url, ) return facts def attach_runtime_probe( plan: dict[str, Any] | None, *, user_text: str = "", cfg: dict[str, Any] | None = None, ) -> dict[str, Any] | None: """If DevOps task — probe hosts and attach runtime_facts to plan.""" if not isinstance(plan, dict): return plan cfg = cfg or {} if not cfg.get("runtime_probe_enabled", True): return plan if plan.get("runtime_facts"): return plan if not is_devops_request(plan, user_text): return plan out = dict(plan) hosts = extract_hosts( user_text, json.dumps(plan, ensure_ascii=False), str(plan.get("user_goal") or ""), ) if not hosts: # DevOps without host — still mark needed for terminal phase out["runtime_probe"] = "needed" out["runtime_facts"] = { "hint": "DevOps task without clear host — curl/docker inspect before edit.", "source": "none", } return out facts_list = [probe_host(h, cfg=cfg) for h in hosts[:3]] primary = facts_list[0] out["runtime_facts"] = primary out["runtime_facts_all"] = facts_list out["runtime_probe"] = "gateway" # Strengthen paths toward Traefik dynamic conf for ift hosts if "ift." in primary.get("host", "") or primary.get("suggested_backend_url"): root = r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps" yml = root + r"\ift\traefik\dynamic_conf.yml" tasks = list(out.get("subtasks") or []) if tasks and isinstance(tasks[0], dict): t0 = dict(tasks[0]) paths = list(t0.get("paths") or []) if yml not in paths: paths = [yml] + paths t0["paths"] = paths[:4] if primary.get("suggested_backend_url"): t0["edit_goal"] = ( f"Set Traefik service backend to {primary['suggested_backend_url']} " f"(HTTP {primary.get('http_status')}: wrong upstream port)." ) tasks[0] = t0 out["subtasks"] = tasks return out def runtime_facts_line(plan: dict[str, Any] | None) -> str: facts = (plan or {}).get("runtime_facts") if isinstance(plan, dict) else None if not isinstance(facts, dict) or not facts: return "" parts = [ f"host={facts.get('host')}", f"http={facts.get('http_status')}", f"service={facts.get('service')}", ] if facts.get("open_ports"): parts.append(f"open={facts.get('open_ports')}") if facts.get("suggested_backend_url"): parts.append(f"fix→{facts.get('suggested_backend_url')}") hint = str(facts.get("hint") or "")[:180] return "RUNTIME: " + "; ".join(str(p) for p in parts) + (f" | {hint}" if hint else "") def devops_blocks_blind_force_edit(plan: dict[str, Any] | None) -> bool: """Do not blind-edit DevOps YAML until runtime facts exist.""" if not isinstance(plan, dict): return False if not is_devops_request(plan): return False facts = plan.get("runtime_facts") if not facts: return True # Facts present — allow targeted edit (including force with goal from facts) return False def suggested_traefik_edit( plan: dict[str, Any] | None, ) -> dict[str, str] | None: """If probe found closed:80 + open:alt — return old/new URL snippets.""" if not isinstance(plan, dict): return None facts = plan.get("runtime_facts") if not isinstance(facts, dict): return None url = facts.get("suggested_backend_url") service = facts.get("service") if not url or not service: return None # Common misconfig in our dynamic_conf old = f'url: "http://{service}:80"' new = f'url: "{url}"' if old == new: return None return { "path": r"C:\Users\alexc\IdeaProjects\eventHub\EventHubDevOps\ift\traefik\dynamic_conf.yml", "old_text": old, "new_text": new, "reason": str(facts.get("hint") or "port mismatch"), } def force_runtime_fix_edit( *, plan: dict[str, Any] | None, tools: list[Any], model: str, ) -> dict[str, Any] | None: """Synthetic edit_file for clear Traefik port mismatch — skip blind marker edits.""" fix = suggested_traefik_edit(plan) if not fix: return None # late import to avoid cycles from agent_hier import _pick_edit_tool, _synthetic_tool_completion edit_name = _pick_edit_tool(tools) if not edit_name: return None log.info( "runtime_probe → targeted edit %s => %s", fix["old_text"], fix["new_text"], ) return _synthetic_tool_completion( model=model, tool_name=edit_name, arguments={ "path": fix["path"], "edits": [{"old_text": fix["old_text"], "new_text": fix["new_text"]}], }, ) def ssh_probe_fallback(host: str, cfg: dict[str, Any]) -> dict[str, Any] | None: """Optional: ssh to host and docker inspect (if gateway has keys).""" ssh_host = str(cfg.get("runtime_probe_ssh_host") or "").strip() if not ssh_host: return None service, _ports = service_for_host(host) script = ( f"echo HTTP=$(curl -sk -o /dev/null -w '%{{http_code}}' --max-time 5 https://{host}/ || echo err); " f"CID=$(docker ps -q --filter name={service} | head -1); " f"echo CID=$CID; " f"if [ -n \"$CID\" ]; then docker inspect \"$CID\" --format 'PORTS={{{{json .NetworkSettings.Ports}}}}'; fi" ) try: proc = subprocess.run( [ "ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", ssh_host, "bash", "-lc", script, ], capture_output=True, text=True, timeout=20, check=False, ) out = (proc.stdout or "") + "\n" + (proc.stderr or "") return {"host": host, "ssh_raw": out[:1500], "source": "ssh", "service": service} except Exception as exc: # noqa: BLE001 log.warning("runtime ssh probe failed: %s", exc) return None