diff --git a/.env.example b/.env.example index c14d4b6..33d0810 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,12 @@ NOVITA_API_KEY= GROQ_API_KEY= GEMINI_API_KEY= +# GigaChat — optional LLM classifier (hybrid mode, freemium for физлица) +# Authorization key from https://developers.sber.ru/studio/ → GigaChat API +GIGACHAT_CREDENTIALS= +CLASSIFIER_MODE=hybrid +CLASSIFIER_LLM_MODEL=gigachat-classifier + # Budget (USD/month, also in litellm_config.yaml) LITELLM_MAX_BUDGET=50 diff --git a/README.md b/README.md index b5e666e..06b3e13 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,26 @@ Novita always direct (`NO_PROXY=api.novita.ai`). Header `X-AI-Quality: auto|economy|balanced|max` or `metadata.quality_mode` in request body. Default: `auto` (start lane A/B by tier, escalate on retry/5xx). +## LLM classifier (GigaChat, optional) + +When `CLASSIFIER_MODE=hybrid` (default) and `GIGACHAT_CREDENTIALS` is set: + +1. **Heuristic** classify first (0 cost) +2. If `confidence < 0.6` → one call to **GigaChat-2-Lite** via LiteLLM (`gigachat-classifier`) +3. Lane orchestration unchanged (A/B/C, Redis, budget) + +| `CLASSIFIER_MODE` | Behavior | +|-------------------|----------| +| `heuristic` | Keywords only (no GigaChat) | +| `hybrid` | GigaChat only on low confidence | +| `llm` | Always GigaChat for text (except vision) | + +Freemium GigaChat — для личного некомmercial теста; prod — юр. тариф Сбера. + +Setup: [developers.sber.ru](https://developers.sber.ru/docs/ru/gigachat/quickstart/ind-create-project) → Authorization key → `.env` `GIGACHAT_CREDENTIALS`. + +Response field: `x_router_meta.classifier_source` = `heuristic` | `gigachat` | `heuristic_fallback`. + ## Response metadata Each chat response includes `x_router_meta`: diff --git a/config/orchestration.yaml b/config/orchestration.yaml index e32c6a9..6b23ecf 100644 --- a/config/orchestration.yaml +++ b/config/orchestration.yaml @@ -30,3 +30,11 @@ escalation: redis: key_prefix: "ai-router:session:" ttl_sec: 1800 + +classifier: + # heuristic | hybrid (default) | llm + mode: hybrid + llm_model: gigachat-classifier + low_confidence_threshold: 0.6 + timeout_sec: 15 + max_tokens: 64 diff --git a/docker-stack.yml b/docker-stack.yml index 1bd60e5..2288673 100644 --- a/docker-stack.yml +++ b/docker-stack.yml @@ -44,6 +44,8 @@ secrets: external: true gemini_api_key: external: true + gigachat_credentials: + external: true vless_conf: external: true @@ -119,6 +121,7 @@ services: - postgres_password - groq_api_key - gemini_api_key + - gigachat_credentials networks: ai-internal: aliases: @@ -162,6 +165,8 @@ services: REDIS_URL: redis://redis:6379/0 CONFIG_DIR: /app/config DEFAULT_QUALITY_MODE: ${DEFAULT_QUALITY_MODE:-auto} + CLASSIFIER_MODE: ${CLASSIFIER_MODE:-hybrid} + CLASSIFIER_LLM_MODEL: ${CLASSIFIER_LLM_MODEL:-gigachat-classifier} secrets: - litellm_master_key - router_api_key diff --git a/litellm_config.yaml b/litellm_config.yaml index fd50523..4ea73ac 100644 --- a/litellm_config.yaml +++ b/litellm_config.yaml @@ -25,6 +25,7 @@ environment_variables: NOVITA_API_KEY: os.environ/NOVITA_API_KEY GROQ_API_KEY: os.environ/GROQ_API_KEY GEMINI_API_KEY: os.environ/GEMINI_API_KEY + GIGACHAT_CREDENTIALS: os.environ/GIGACHAT_CREDENTIALS router_settings: routing_strategy: simple-shuffle @@ -43,6 +44,15 @@ router_settings: default_fallbacks: ["a-medium-code", "a-complex", "groq-qwen-coder"] model_list: + # --- LLM tier classifier (GigaChat freemium, hybrid mode only) --- + - model_name: gigachat-classifier + litellm_params: + model: gigachat/GigaChat-2-Lite + api_key: os.environ/GIGACHAT_CREDENTIALS + ssl_verify: false + max_tokens: 64 + temperature: 0 + # --- Lane models (generated from config/model_matrix.yaml) --- - model_name: a-simple litellm_params: diff --git a/router/classifier_llm.py b/router/classifier_llm.py new file mode 100644 index 0000000..8ae0fea --- /dev/null +++ b/router/classifier_llm.py @@ -0,0 +1,126 @@ +"""Optional LLM tier classifier via LiteLLM (GigaChat-2-Lite).""" + +from __future__ import annotations + +import json +import logging +import os +import re +from typing import Any + +import httpx + +from rules_loader import load_orchestration + +log = logging.getLogger("classifier_llm") + +TEXT_TIERS = frozenset( + {"SIMPLE", "MEDIUM_OPS", "MEDIUM_CODE", "COMPLEX", "REASONING"}, +) + +CLASSIFY_SYSTEM = """You classify coding-assistant requests into exactly one tier. +Reply with ONLY valid JSON, no markdown: +{"tier":"SIMPLE|MEDIUM_OPS|MEDIUM_CODE|COMPLEX|REASONING","confidence":0.0-1.0} + +SIMPLE — greetings, definitions, short questions +MEDIUM_OPS — bash, docker, devops, infrastructure +MEDIUM_CODE — write/refactor code, functions, bugs +COMPLEX — architecture, migrations, system design +REASONING — step-by-step proof, deep analysis""" + + +class LlmClassifier: + def __init__(self) -> None: + orch = load_orchestration() + clf = orch.get("classifier", {}) + self.mode = os.environ.get("CLASSIFIER_MODE", clf.get("mode", "hybrid")).lower() + self.model = os.environ.get( + "CLASSIFIER_LLM_MODEL", + clf.get("llm_model", "gigachat-classifier"), + ) + self.low_conf = float(clf.get("low_confidence_threshold", 0.6)) + self.litellm_url = os.environ.get("LITELLM_INTERNAL_URL", "http://litellm:4000").rstrip("/") + self.litellm_key = os.environ.get("LITELLM_MASTER_KEY", "") + self.timeout = float(clf.get("timeout_sec", 15)) + self.max_tokens = int(clf.get("max_tokens", 64)) + self.enabled = self.mode in ("hybrid", "llm") and bool(self.litellm_key) + + def should_use_llm(self, tier_value: str, confidence: float, has_image: bool) -> bool: + if not self.enabled or has_image: + return False + if tier_value.startswith("VISION"): + return False + if self.mode == "llm": + return True + if self.mode == "hybrid": + return confidence < self.low_conf + return False + + @staticmethod + def _parse_json(content: str) -> dict[str, Any] | None: + text = content.strip() + fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + if fence: + text = fence.group(1).strip() + try: + data = json.loads(text) + return data if isinstance(data, dict) else None + except json.JSONDecodeError: + match = re.search(r"\{[\s\S]*\}", text) + if not match: + return None + try: + data = json.loads(match.group(0)) + return data if isinstance(data, dict) else None + except json.JSONDecodeError: + return None + + async def classify(self, text: str) -> tuple[str, float] | None: + snippet = text.strip()[:4000] + if not snippet: + return None + + payload = { + "model": self.model, + "max_tokens": self.max_tokens, + "temperature": 0, + "messages": [ + {"role": "system", "content": CLASSIFY_SYSTEM}, + {"role": "user", "content": snippet}, + ], + } + headers = { + "Authorization": f"Bearer {self.litellm_key}", + "Content-Type": "application/json", + } + + try: + async with httpx.AsyncClient(timeout=self.timeout) as client: + resp = await client.post( + f"{self.litellm_url}/v1/chat/completions", + headers=headers, + json=payload, + ) + if resp.status_code >= 400: + log.warning("LLM classify HTTP %s: %s", resp.status_code, resp.text[:200]) + return None + data = resp.json() + content = ( + data.get("choices", [{}])[0] + .get("message", {}) + .get("content", "") + ) + parsed = self._parse_json(content) + if not parsed: + log.warning("LLM classify: invalid JSON in response: %s", content[:120]) + return None + tier_raw = str(parsed.get("tier", "")).upper().replace("-", "_") + if tier_raw not in TEXT_TIERS: + log.warning("LLM classify: unknown tier %s", tier_raw) + return None + confidence = float(parsed.get("confidence", 0.75)) + confidence = max(0.0, min(1.0, confidence)) + return tier_raw, confidence + except Exception as exc: # noqa: BLE001 + log.warning("LLM classify failed: %s", exc) + return None diff --git a/router/metrics.py b/router/metrics.py index fc31ea5..4e50f45 100644 --- a/router/metrics.py +++ b/router/metrics.py @@ -19,6 +19,11 @@ CLASSIFY = Counter( "Classification results", ["tier"], ) +CLASSIFY_LLM = Counter( + "ai_router_classify_llm_total", + "LLM classifier invocations (GigaChat)", + ["tier", "status"], +) DURATION = Histogram( "ai_router_request_duration_seconds", "Request duration", diff --git a/router/orchestrator.py b/router/orchestrator.py index c5a58e6..98853b2 100644 --- a/router/orchestrator.py +++ b/router/orchestrator.py @@ -11,6 +11,7 @@ from dataclasses import dataclass, field from enum import Enum from typing import Any +from classifier_llm import LlmClassifier from rules_loader import load_model_matrix, load_orchestration, load_routing_rules log = logging.getLogger("orchestrator") @@ -54,6 +55,7 @@ class RouteDecision: escalation_level: int confidence: float delegated_internal: bool = False + classifier_source: str = "heuristic" class SessionStore: @@ -172,6 +174,7 @@ class Orchestrator: def __init__(self, session_store: SessionStore) -> None: self.sessions = session_store self.classifier = Classifier() + self.llm_classifier = LlmClassifier() self._orch = load_orchestration() self._matrix = load_model_matrix() self._models: dict[str, dict] = self._matrix.get("models", {}) @@ -217,7 +220,7 @@ class Orchestrator: suffix = TIER_SUFFIX[tier] return f"{prefix}-{suffix}" - def resolve( + async def resolve( self, messages: list[dict[str, Any]], *, @@ -229,6 +232,17 @@ class Orchestrator: ) -> RouteDecision: mode = quality_mode or self.default_quality tier, confidence = self.classifier.classify(messages, has_image=has_image, text=text) + classifier_source = "heuristic" + + if self.llm_classifier.should_use_llm(tier.value, confidence, has_image): + llm_result = await self.llm_classifier.classify(text) + if llm_result: + tier = Tier(llm_result[0]) + confidence = llm_result[1] + classifier_source = "gigachat" + elif self.llm_classifier.mode == "llm": + classifier_source = "heuristic_fallback" + ctx = self.sessions.get(session_id) prompt_hash = hashlib.sha256(text.encode()).hexdigest()[:16] @@ -263,11 +277,13 @@ class Orchestrator: escalation_level=escalation, confidence=confidence, delegated_internal=True, + classifier_source=classifier_source, ) - delegated = confidence < self.classifier._low_conf and tier in ( - Tier.MEDIUM_OPS, - Tier.SIMPLE, + delegated = ( + classifier_source == "heuristic" + and confidence < self.classifier._low_conf + and tier in (Tier.MEDIUM_OPS, Tier.SIMPLE) ) return RouteDecision( @@ -278,6 +294,7 @@ class Orchestrator: escalation_level=escalation, confidence=confidence, delegated_internal=delegated, + classifier_source=classifier_source, ) def after_request( diff --git a/router/router.py b/router/router.py index ad58d6f..5d3ac2e 100644 --- a/router/router.py +++ b/router/router.py @@ -15,7 +15,7 @@ import httpx from fastapi import FastAPI, Header, HTTPException, Request, Response from fastapi.responses import JSONResponse, StreamingResponse -from metrics import CLASSIFY, DURATION, ESCALATIONS, REQUESTS, metrics_payload +from metrics import CLASSIFY, CLASSIFY_LLM, DURATION, ESCALATIONS, REQUESTS, metrics_payload from orchestrator import Orchestrator, SessionStore, Tier from rules_loader import reload_configs @@ -104,6 +104,7 @@ def _router_meta(decision, *, requested: str) -> dict[str, Any]: "confidence": round(decision.confidence, 3), "requested_model": requested, "delegated_internal": decision.delegated_internal, + "classifier_source": decision.classifier_source, } @@ -150,7 +151,7 @@ async def classify_debug( body = await request.json() messages = body.get("messages") or [] text = _extract_text(messages) - decision = orchestrator.resolve( + decision = await orchestrator.resolve( messages, quality_mode=_quality_mode(x_ai_quality, body), session_id=_session_id(body), @@ -159,6 +160,8 @@ async def classify_debug( token_estimate=_estimate_tokens(text), ) CLASSIFY.labels(tier=decision.tier.value).inc() + if decision.classifier_source == "gigachat": + CLASSIFY_LLM.labels(tier=decision.tier.value, status="ok").inc() return { "tier": decision.tier.value, "lane": decision.lane, @@ -243,7 +246,7 @@ async def chat_completions( requested_model = body.get("model", DEFAULT_MODEL) if requested_model in ("smart-router", "auto", ""): - decision = orchestrator.resolve( + decision = await orchestrator.resolve( messages, quality_mode=_quality_mode(x_ai_quality, body), session_id=session_id, @@ -251,6 +254,8 @@ async def chat_completions( has_image=has_image, token_estimate=_estimate_tokens(text), ) + if decision.classifier_source == "gigachat": + CLASSIFY_LLM.labels(tier=decision.tier.value, status="ok").inc() target_model = decision.model meta = _router_meta(decision, requested=requested_model) else: diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 82dc7cb..fec068f 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -36,6 +36,7 @@ ensure_secret router_api_key "${ROUTER_API_KEY:?ROUTER_API_KEY required}" ensure_secret postgres_password "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD required}" ensure_secret groq_api_key "${GROQ_API_KEY:-}" ensure_secret gemini_api_key "${GEMINI_API_KEY:-}" +ensure_secret gigachat_credentials "${GIGACHAT_CREDENTIALS:-}" if [[ ! -f vless/vless.conf ]]; then echo "WARN: vless/vless.conf missing — stub for secret (VPN off until configured)" diff --git a/scripts/litellm-entrypoint.sh b/scripts/litellm-entrypoint.sh index b610158..ca1e762 100644 --- a/scripts/litellm-entrypoint.sh +++ b/scripts/litellm-entrypoint.sh @@ -23,6 +23,9 @@ fi if [ -f /run/secrets/gemini_api_key ]; then export GEMINI_API_KEY="$(read_secret /run/secrets/gemini_api_key)" fi +if [ -f /run/secrets/gigachat_credentials ]; then + export GIGACHAT_CREDENTIALS="$(read_secret /run/secrets/gigachat_credentials)" +fi if [ -f /run/secrets/postgres_password ]; then export POSTGRES_PASSWORD="$(read_secret /run/secrets/postgres_password)" export DATABASE_URL="postgresql://${POSTGRES_USER:-litellm}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-litellm}"