EventHubAiRouter/litellm_config.yaml

# LiteLLM Proxy — Novita lanes A/B/C + Auto Router v2 fallback
# Regenerate model_list tail: bash scripts/sync-routing-config.sh

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  store_model_in_db: false
  max_budget: 50
  budget_duration: 30d

litellm_settings:
  drop_params: true
  set_verbose: false
  request_timeout: 120
  num_retries: 2
  cache: true
  cache_params:
    type: redis
    host: redis
    port: 6379
    ttl: 3600
  callbacks: ["prometheus"]
  require_auth_for_metrics_endpoint: false

environment_variables:
  NOVITA_API_KEY: os.environ/NOVITA_API_KEY
  GROQ_API_KEY: os.environ/GROQ_API_KEY
  GEMINI_API_KEY: os.environ/GEMINI_API_KEY

router_settings:
  routing_strategy: simple-shuffle
  num_retries: 2
  timeout: 120
  allowed_fails: 2
  cooldown_time: 30
  fallbacks:
    - a-simple: ["b-simple", "groq-llama-8b"]
    - a-medium-code: ["a-medium-ops", "groq-qwen-coder"]
    - a-complex: ["b-complex", "a-reasoning"]
    - b-complex: ["a-complex", "c-complex"]
    - a-reasoning: ["b-reasoning", "gemini-flash"]
    - a-vision: ["a-vision-ocr"]
    - smart-router-internal: ["a-medium-ops", "a-medium-code", "a-complex"]
  default_fallbacks: ["a-medium-code", "a-complex", "groq-qwen-coder"]

model_list:
  # --- Lane models (generated from config/model_matrix.yaml) ---
  - model_name: a-simple
    litellm_params:
      model: novita/qwen/qwen3-4b-fp8
      api_key: os.environ/NOVITA_API_KEY
      rpm: 60
  - model_name: b-simple
    litellm_params:
      model: novita/meta-llama/llama-3.1-8b-instruct
      api_key: os.environ/NOVITA_API_KEY
      rpm: 60
  - model_name: c-simple
    litellm_params:
      model: novita/qwen/qwen3-8b-fp8
      api_key: os.environ/NOVITA_API_KEY
      rpm: 60

  - model_name: a-medium-ops
    litellm_params:
      model: novita/qwen/qwen3-8b-fp8
      api_key: os.environ/NOVITA_API_KEY
      rpm: 40
  - model_name: b-medium-ops
    litellm_params:
      model: novita/deepseek/deepseek-v3.2
      api_key: os.environ/NOVITA_API_KEY
      rpm: 40
  - model_name: c-medium-ops
    litellm_params:
      model: novita/deepseek/deepseek-v3.2
      api_key: os.environ/NOVITA_API_KEY
      rpm: 40

  - model_name: a-medium-code
    litellm_params:
      model: novita/qwen/qwen3-coder-30b-a3b-instruct
      api_key: os.environ/NOVITA_API_KEY
      rpm: 40
  - model_name: b-medium-code
    litellm_params:
      model: novita/qwen/qwen3-coder-30b-a3b-instruct
      api_key: os.environ/NOVITA_API_KEY
      rpm: 40
  - model_name: c-medium-code
    litellm_params:
      model: novita/qwen/qwen3-coder-30b-a3b-instruct
      api_key: os.environ/NOVITA_API_KEY
      rpm: 30

  - model_name: a-complex
    litellm_params:
      model: novita/deepseek/deepseek-v3.2
      api_key: os.environ/NOVITA_API_KEY
      rpm: 30
  - model_name: b-complex
    litellm_params:
      model: novita/deepseek/deepseek-r1-0528
      api_key: os.environ/NOVITA_API_KEY
      rpm: 20
  - model_name: c-complex
    litellm_params:
      model: novita/deepseek/deepseek-r1-turbo
      api_key: os.environ/NOVITA_API_KEY
      rpm: 15

  - model_name: a-reasoning
    litellm_params:
      model: novita/deepseek/deepseek-r1-0528-qwen3-8b
      api_key: os.environ/NOVITA_API_KEY
      rpm: 30
  - model_name: b-reasoning
    litellm_params:
      model: novita/deepseek/deepseek-r1-0528
      api_key: os.environ/NOVITA_API_KEY
      rpm: 20
  - model_name: c-reasoning
    litellm_params:
      model: novita/deepseek/deepseek-r1-turbo
      api_key: os.environ/NOVITA_API_KEY
      rpm: 15

  - model_name: a-vision-ocr
    litellm_params:
      model: novita/paddlepaddle/paddleocr-vl
      api_key: os.environ/NOVITA_API_KEY
      rpm: 30
  - model_name: a-vision
    litellm_params:
      model: novita/qwen/qwen3-vl-30b-a3b-instruct
      api_key: os.environ/NOVITA_API_KEY
      rpm: 20
  - model_name: b-vision
    litellm_params:
      model: novita/qwen/qwen2.5-vl-72b-instruct
      api_key: os.environ/NOVITA_API_KEY
      rpm: 15
  - model_name: c-vision
    litellm_params:
      model: novita/qwen/qwen3-vl-235b-a22b-instruct
      api_key: os.environ/NOVITA_API_KEY
      rpm: 10

  # --- Optional fallbacks ---
  - model_name: groq-llama-8b
    litellm_params:
      model: groq/llama-3.1-8b-instant
      api_key: os.environ/GROQ_API_KEY
  - model_name: groq-qwen-coder
    litellm_params:
      model: groq/qwen-qwen-2.5-coder-32b
      api_key: os.environ/GROQ_API_KEY
  - model_name: gemini-flash
    litellm_params:
      model: gemini/gemini-2.0-flash
      api_key: os.environ/GEMINI_API_KEY

  # --- Auto Router v2 fallback when gateway confidence low ---
  - model_name: smart-router-internal
    litellm_params:
      model: auto_router/complexity_router
      drop_params: true
      complexity_router_default_model: a-medium-ops
      complexity_router_config:
        tiers:
          SIMPLE: a-simple
          MEDIUM: a-medium-ops
          MEDIUM_CODE: a-medium-code
          COMPLEX: a-complex
          REASONING: a-reasoning
        classifier_fallback: heuristic
        keyword_tier_rules:
          - keywords: ["привет", "hello", "hi", "thanks", "спасибо", "что такое", "what is", "define"]
            tier: SIMPLE
          - keywords: ["bash", "docker", "swarm", "ci/cd", "gitea", "rebar", "erlang", "devops", "kubectl", "terraform"]
            tier: MEDIUM
          - keywords: ["рефакторинг", "refactor", "архитектура", "architecture", "спроектируй", "migration plan"]
            tier: COMPLEX
          - keywords: ["step by step", "prove", "analyze deeply", "think through"]
            tier: REASONING
        custom_technical_keywords: [erlang, rebar3, traefik, mnesia, litellm, novita]
        token_thresholds:
          simple: 20
          complex: 500
        tier_boundaries:
          simple_medium: 0.18
          medium_complex: 0.38
          complex_reasoning: 0.62
        session_affinity: true
        session_affinity_ttl_seconds: 1800

  # Alias for Zed default model name (gateway resolves before LiteLLM)
  - model_name: smart-router
    litellm_params:
      model: novita/qwen/qwen3-8b-fp8
      api_key: os.environ/NOVITA_API_KEY