feat(ift): EventHub AI Router stack for Zed
FastAPI gateway with A/B/C lane orchestration, LiteLLM proxy config, Swarm stack (postgres, redis, VPN off-by-default), deploy/smoke/audit scripts.
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
# LiteLLM Proxy — Novita lanes A/B/C + Auto Router v2 fallback
|
||||
# Regenerate model_list tail: bash scripts/sync-routing-config.sh
|
||||
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
store_model_in_db: false
|
||||
max_budget: 50
|
||||
budget_duration: 30d
|
||||
|
||||
litellm_settings:
|
||||
drop_params: true
|
||||
set_verbose: false
|
||||
request_timeout: 120
|
||||
num_retries: 2
|
||||
cache: true
|
||||
cache_params:
|
||||
type: redis
|
||||
host: redis
|
||||
port: 6379
|
||||
ttl: 3600
|
||||
callbacks: ["prometheus"]
|
||||
require_auth_for_metrics_endpoint: false
|
||||
|
||||
environment_variables:
|
||||
NOVITA_API_KEY: os.environ/NOVITA_API_KEY
|
||||
GROQ_API_KEY: os.environ/GROQ_API_KEY
|
||||
GEMINI_API_KEY: os.environ/GEMINI_API_KEY
|
||||
|
||||
router_settings:
|
||||
routing_strategy: simple-shuffle
|
||||
num_retries: 2
|
||||
timeout: 120
|
||||
allowed_fails: 2
|
||||
cooldown_time: 30
|
||||
fallbacks:
|
||||
- a-simple: ["b-simple", "groq-llama-8b"]
|
||||
- a-medium-code: ["a-medium-ops", "groq-qwen-coder"]
|
||||
- a-complex: ["b-complex", "a-reasoning"]
|
||||
- b-complex: ["a-complex", "c-complex"]
|
||||
- a-reasoning: ["b-reasoning", "gemini-flash"]
|
||||
- a-vision: ["a-vision-ocr"]
|
||||
- smart-router-internal: ["a-medium-ops", "a-medium-code", "a-complex"]
|
||||
default_fallbacks: ["a-medium-code", "a-complex", "groq-qwen-coder"]
|
||||
|
||||
model_list:
|
||||
# --- Lane models (generated from config/model_matrix.yaml) ---
|
||||
- model_name: a-simple
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen3-4b-fp8
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 60
|
||||
- model_name: b-simple
|
||||
litellm_params:
|
||||
model: novita/meta-llama/llama-3.1-8b-instruct
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 60
|
||||
- model_name: c-simple
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen3-8b-fp8
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 60
|
||||
|
||||
- model_name: a-medium-ops
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen3-8b-fp8
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 40
|
||||
- model_name: b-medium-ops
|
||||
litellm_params:
|
||||
model: novita/deepseek/deepseek-v3.2
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 40
|
||||
- model_name: c-medium-ops
|
||||
litellm_params:
|
||||
model: novita/deepseek/deepseek-v3.2
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 40
|
||||
|
||||
- model_name: a-medium-code
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen3-coder-30b-a3b-instruct
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 40
|
||||
- model_name: b-medium-code
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen3-coder-30b-a3b-instruct
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 40
|
||||
- model_name: c-medium-code
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen3-coder-30b-a3b-instruct
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 30
|
||||
|
||||
- model_name: a-complex
|
||||
litellm_params:
|
||||
model: novita/deepseek/deepseek-v3.2
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 30
|
||||
- model_name: b-complex
|
||||
litellm_params:
|
||||
model: novita/deepseek/deepseek-r1-0528
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 20
|
||||
- model_name: c-complex
|
||||
litellm_params:
|
||||
model: novita/deepseek/deepseek-r1-turbo
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 15
|
||||
|
||||
- model_name: a-reasoning
|
||||
litellm_params:
|
||||
model: novita/deepseek/deepseek-r1-0528-qwen3-8b
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 30
|
||||
- model_name: b-reasoning
|
||||
litellm_params:
|
||||
model: novita/deepseek/deepseek-r1-0528
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 20
|
||||
- model_name: c-reasoning
|
||||
litellm_params:
|
||||
model: novita/deepseek/deepseek-r1-turbo
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 15
|
||||
|
||||
- model_name: a-vision-ocr
|
||||
litellm_params:
|
||||
model: novita/paddlepaddle/paddleocr-vl
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 30
|
||||
- model_name: a-vision
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen3-vl-30b-a3b-instruct
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 20
|
||||
- model_name: b-vision
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen2.5-vl-72b-instruct
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 15
|
||||
- model_name: c-vision
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen3-vl-235b-a22b-instruct
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
rpm: 10
|
||||
|
||||
# --- Optional fallbacks ---
|
||||
- model_name: groq-llama-8b
|
||||
litellm_params:
|
||||
model: groq/llama-3.1-8b-instant
|
||||
api_key: os.environ/GROQ_API_KEY
|
||||
- model_name: groq-qwen-coder
|
||||
litellm_params:
|
||||
model: groq/qwen-qwen-2.5-coder-32b
|
||||
api_key: os.environ/GROQ_API_KEY
|
||||
- model_name: gemini-flash
|
||||
litellm_params:
|
||||
model: gemini/gemini-2.0-flash
|
||||
api_key: os.environ/GEMINI_API_KEY
|
||||
|
||||
# --- Auto Router v2 fallback when gateway confidence low ---
|
||||
- model_name: smart-router-internal
|
||||
litellm_params:
|
||||
model: auto_router/complexity_router
|
||||
drop_params: true
|
||||
complexity_router_default_model: a-medium-ops
|
||||
complexity_router_config:
|
||||
tiers:
|
||||
SIMPLE: a-simple
|
||||
MEDIUM: a-medium-ops
|
||||
MEDIUM_CODE: a-medium-code
|
||||
COMPLEX: a-complex
|
||||
REASONING: a-reasoning
|
||||
classifier_fallback: heuristic
|
||||
keyword_tier_rules:
|
||||
- keywords: ["привет", "hello", "hi", "thanks", "спасибо", "что такое", "what is", "define"]
|
||||
tier: SIMPLE
|
||||
- keywords: ["bash", "docker", "swarm", "ci/cd", "gitea", "rebar", "erlang", "devops", "kubectl", "terraform"]
|
||||
tier: MEDIUM
|
||||
- keywords: ["рефакторинг", "refactor", "архитектура", "architecture", "спроектируй", "migration plan"]
|
||||
tier: COMPLEX
|
||||
- keywords: ["step by step", "prove", "analyze deeply", "think through"]
|
||||
tier: REASONING
|
||||
custom_technical_keywords: [erlang, rebar3, traefik, mnesia, litellm, novita]
|
||||
token_thresholds:
|
||||
simple: 20
|
||||
complex: 500
|
||||
tier_boundaries:
|
||||
simple_medium: 0.18
|
||||
medium_complex: 0.38
|
||||
complex_reasoning: 0.62
|
||||
session_affinity: true
|
||||
session_affinity_ttl_seconds: 1800
|
||||
|
||||
# Alias for Zed default model name (gateway resolves before LiteLLM)
|
||||
- model_name: smart-router
|
||||
litellm_params:
|
||||
model: novita/qwen/qwen3-8b-fp8
|
||||
api_key: os.environ/NOVITA_API_KEY
|
||||
Reference in New Issue
Block a user