EventHubAiRouter/docker-stack.yml

version: "3.8"

# AI Router stack — IFT Docker Swarm
# VPN (vless-proxy, vpn-watchdog) scaled to 0 by default

networks:
  ai-internal:
    driver: overlay
    internal: true
  eventhub-ift:
    external: true
    name: eventhub-ift-net

volumes:
  litellm-postgres-data:
  uptime-kuma-data:

configs:
  litellm_config:
    file: ./litellm_config.yaml
  litellm_entrypoint:
    file: ./scripts/litellm-entrypoint.sh
  router_entrypoint:
    file: ./scripts/router-entrypoint.sh
  routing_config:
    file: ./config/routing_rules.yaml
  model_matrix:
    file: ./config/model_matrix.yaml
  orchestration_config:
    file: ./config/orchestration.yaml

secrets:
  novita_api_key:
    external: true
  litellm_master_key:
    external: true
  litellm_salt_key:
    external: true
  router_api_key:
    external: true
  postgres_password:
    external: true
  groq_api_key:
    external: true
  gemini_api_key:
    external: true
  vless_conf:
    external: true

services:
  postgres:
    image: postgres:16-alpine
    environment:
      POSTGRES_DB: ${POSTGRES_DB:-litellm}
      POSTGRES_USER: ${POSTGRES_USER:-litellm}
      POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
    secrets:
      - postgres_password
    volumes:
      - litellm-postgres-data:/var/lib/postgresql/data
    networks:
      - ai-internal
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
      placement:
        constraints:
          - node.role == manager
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-litellm} -d ${POSTGRES_DB:-litellm}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 20s

  redis:
    image: redis:7-alpine
    command: ["redis-server", "--save", "", "--appendonly", "no"]
    networks:
      - ai-internal
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
      placement:
        constraints:
          - node.role == manager
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 3s
      retries: 3

  litellm:
    image: ${LITELLM_IMAGE:-ghcr.io/berriai/litellm:main-v1.96.0-stable}
    entrypoint: ["/bin/sh", "/entrypoint/litellm-entrypoint.sh"]
    configs:
      - source: litellm_config
        target: /app/config.yaml
      - source: litellm_entrypoint
        target: /entrypoint/litellm-entrypoint.sh
        mode: 0555
    environment:
      STORE_MODEL_IN_DB: "False"
      PROXY_BASE_URL: ${PROXY_BASE_URL:-https://litellm.ift.calentiq.com}
      FORWARDED_ALLOW_IPS: "*"
      DISABLE_AIOHTTP_TRANSPORT: ${DISABLE_AIOHTTP_TRANSPORT:-True}
      USE_AIOHTTP_TRANSPORT: ${USE_AIOHTTP_TRANSPORT:-False}
      HTTP_PROXY: ${HTTP_PROXY:-}
      HTTPS_PROXY: ${HTTPS_PROXY:-}
      NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1,api.novita.ai,novita.ai}
      POSTGRES_USER: ${POSTGRES_USER:-litellm}
      POSTGRES_DB: ${POSTGRES_DB:-litellm}
    secrets:
      - novita_api_key
      - litellm_master_key
      - litellm_salt_key
      - postgres_password
      - groq_api_key
      - gemini_api_key
    networks:
      ai-internal:
        aliases:
          - litellm
      eventhub-ift:
        aliases:
          - litellm
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
        delay: 5s
      update_config:
        parallelism: 1
        failure_action: rollback
      placement:
        constraints:
          - node.role == manager
    healthcheck:
      test: ["CMD-SHELL", "wget -q -O- http://127.0.0.1:4000/health/liveliness || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s

  router:
    image: ${ROUTER_IMAGE:-git.sabilin.com/eventhub/ai-router-gateway:ift}
    entrypoint: ["/bin/sh", "/entrypoint/router-entrypoint.sh"]
    configs:
      - source: router_entrypoint
        target: /entrypoint/router-entrypoint.sh
        mode: 0555
      - source: routing_config
        target: /app/config/routing_rules.yaml
      - source: model_matrix
        target: /app/config/model_matrix.yaml
      - source: orchestration_config
        target: /app/config/orchestration.yaml
    environment:
      LITELLM_INTERNAL_URL: http://litellm:4000
      REDIS_URL: redis://redis:6379/0
      CONFIG_DIR: /app/config
      DEFAULT_QUALITY_MODE: ${DEFAULT_QUALITY_MODE:-auto}
    secrets:
      - litellm_master_key
      - router_api_key
    networks:
      ai-internal:
      eventhub-ift:
        aliases:
          - ai-router
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
      placement:
        constraints:
          - node.role == manager
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8000/health || exit 1"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 20s

  vless-proxy:
    image: ${VLESS_IMAGE:-ai-router/vless-proxy:local}
    secrets:
      - source: vless_conf
        target: /app/vless.conf
    networks:
      - ai-internal
    deploy:
      replicas: 0
      restart_policy:
        condition: any
        delay: 5s
        max_attempts: 0
      placement:
        constraints:
          - node.role == manager
    healthcheck:
      test: ["CMD-SHELL", "wget -q -O- http://127.0.0.1:8080 || exit 1"]
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 20s

  vpn-watchdog:
    image: ${WATCHDOG_IMAGE:-ai-router/vpn-watchdog:local}
    environment:
      STACK_NAME: ${STACK_NAME:-ai-router}
      VLESS_PROXY_URL: ${VLESS_PROXY_URL:-http://vless-proxy:8080}
      CHECK_INTERVAL: "60"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
    networks:
      - ai-internal
    deploy:
      replicas: 0
      restart_policy:
        condition: any
      placement:
        constraints:
          - node.role == manager

  uptime-kuma:
    image: louislam/uptime-kuma:1
    volumes:
      - uptime-kuma-data:/app/data
    networks:
      ai-internal:
      eventhub-ift:
        aliases:
          - uptime-kuma
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
      placement:
        constraints:
          - node.role == manager