Add observer, prometheus and grafana

Makefile

@@ -162,18 +162,6 @@ test-all: eunit ## Запустить ВСЕ тесты (EUnit + API)
 
 # ============================================================================
 # LOAD TESTING
-#3. Мониторинг во время нагрузочного теста
-#Во время теста полезно следить за состоянием ноды:
-#
-#Через Docker (если приложение в контейнере):
-#bash
-#docker stats eventhub
-#docker exec eventhub /app/bin/eventhub remote_console
-#Внутри консоли Erlang можно выполнить:
-#
-#erlang
-#observer:start().   % графический мониторинг
-#recon:proc_count(5). % топ-5 процессов по памяти (если установлен recon)
 # ============================================================================
 tsung-test: ## Запустить нагрузочный тест Tsung
 	@echo "Запуск нагрузочного теста Tsung..."

docker/Dockerfile

@@ -17,6 +17,7 @@ COPY src/ src/
 
 # Копируем sys.config из src/config/ в config/
 COPY src/config/sys.config ./config/sys.config
+COPY src/config/vm.args ./config/vm.args
 
 RUN rebar3 as prod release
 RUN rebar3 as prod tar
@@ -38,7 +39,9 @@ RUN mkdir -p /app/data && chmod 777 /app/data
 WORKDIR /app
 EXPOSE 8080 8081 8445 8446
 
+ENV PATH="/app/erts-16.3.1/bin:$PATH"
+
 ENV RELX_REPLACE_OS_VARS=true
 ENV MNESIA_DIR=/app/data
 
-CMD ["/app/bin/eventhub", "foreground"]
+CMD /app/bin/eventhub foreground

docker/ObserverWeb.Dockerfile

@@ -0,0 +1,27 @@
+FROM erlang:28-alpine
+
+RUN apk add --no-cache \
+    elixir \
+    nodejs \
+    npm \
+    git \
+    inotify-tools
+
+WORKDIR /app
+
+RUN git clone https://github.com/thiagoesteves/observer_web.git .
+
+RUN mix local.hex --force && mix local.rebar --force && mix deps.get
+
+RUN mkdir -p priv/static && \
+    touch priv/static/app.css && touch priv/static/app.js && \
+    cd assets && npm install && cd .. && \
+    mix assets.build
+
+# Копируем наш dev.exs с автоподключением к нодам EventHub
+COPY docker/observer_web/dev.exs .
+
+EXPOSE 4000
+
+# Только запуск сервера, без ручного Node.connect
+CMD elixir --sname observer_web@observer_web --cookie eventhub_cookie -S mix run --no-halt dev.exs

docker/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   eventhub-node1:
     build:
@@ -18,6 +16,7 @@ services:
       - ADMIN_HTTP_PORT=8445
       - ADMIN_WS_PORT=8446
       - MNESIA_DIR=/app/data
+      - RELEASE_COOKIE=eventhub_cookie
     volumes:
       - eventhub-node1-data:/app/data
     networks:
@@ -41,6 +40,7 @@ services:
       - ADMIN_HTTP_PORT=8445
       - ADMIN_WS_PORT=8446
       - MNESIA_DIR=/app/data
+      - RELEASE_COOKIE=eventhub_cookie
       - JOIN_NODES=eventhub-node1@eventhub-node1
     volumes:
       - eventhub-node2-data:/app/data
@@ -67,6 +67,7 @@ services:
       - ADMIN_HTTP_PORT=8445
       - ADMIN_WS_PORT=8446
       - MNESIA_DIR=/app/data
+      - RELEASE_COOKIE=eventhub_cookie
       - JOIN_NODES=eventhub-node1@eventhub-node1
     volumes:
       - eventhub-node3-data:/app/data
@@ -76,10 +77,57 @@ services:
       - eventhub-node1
     restart: unless-stopped
 
+  observer_web:
+    build:
+      context: ..
+      dockerfile: docker/ObserverWeb.Dockerfile
+    container_name: observer_web
+    ports:
+      - "4000:4000"
+    environment:
+      - RELEASE_COOKIE=eventhub_cookie
+    networks:
+      - eventhub-net
+    restart: unless-stopped
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus-data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+    ports:
+      - "9090:9090"
+    networks:
+      - eventhub-net
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    depends_on:
+      - prometheus
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=zxs45gvHB
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning
+    networks:
+      - eventhub-net
+    restart: unless-stopped
+
 volumes:
   eventhub-node1-data:
   eventhub-node2-data:
   eventhub-node3-data:
+  prometheus-data:
+  grafana-data:
 
 networks:
   eventhub-net:

docker/docker-entrypoint.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-set -e
-
-# Создаём директорию Mnesia, если не существует
-mkdir -p ${MNESIA_DIR}
-
-# Запускаем приложение
-exec /app/bin/eventhub foreground

docker/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+providers:
+  - name: 'EventHub'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: true
+    updateIntervalSeconds: 10
+    allowUiUpdates: false
+    options:
+      path: /etc/grafana/dashboards
+      foldersFromFilesStructure: true

docker/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090   # имя сервиса Prometheus в docker-compose
+    isDefault: true
+    editable: false

docker/observer_web/dev.exs

@@ -0,0 +1,115 @@
+# Development server for Observer Web
+
+# Phoenix
+
+defmodule WebDev.Router do
+  use Phoenix.Router, helpers: false
+
+  import Observer.Web.Router
+
+  pipeline :browser do
+    plug(:fetch_session)
+  end
+
+  scope "/" do
+    pipe_through(:browser)
+
+    observer_dashboard("/observer")
+  end
+end
+
+defmodule WebDev.Endpoint do
+  use Phoenix.Endpoint, otp_app: :observer_web
+
+  socket("/live", Phoenix.LiveView.Socket)
+  socket("/phoenix/live_reload/socket", Phoenix.LiveReloader.Socket)
+
+  plug(Phoenix.LiveReloader)
+  plug(Phoenix.CodeReloader)
+
+  plug(Plug.Session,
+    store: :cookie,
+    key: "_observer_web_key",
+    signing_salt: "/VEDsdfsffMnp5"
+  )
+
+  plug(WebDev.Router)
+end
+
+defmodule WebDev.ErrorHTML do
+  use Phoenix.Component
+
+  def render(template, _assigns) do
+    Phoenix.Controller.status_message_from_template(template)
+  end
+end
+
+# Configuration
+
+port = "PORT" |> System.get_env("4000") |> String.to_integer()
+
+Application.put_env(:observer_web, WebDev.Endpoint,
+  adapter: Bandit.PhoenixAdapter,
+  check_origin: false,
+  debug_errors: true,
+  http: [port: port],
+  live_view: [signing_salt: "eX7TFPY6Y/+XQ1o2pOUW3DjgAoXGTAdX"],
+  pubsub_server: ObserverWeb.PubSub,
+  render_errors: [formats: [html: WebDev.ErrorHTML], layout: false],
+  secret_key_base: "jAu3udxm+8tIRDXLLKo+EupAlEvdLsnNG82O8e9nqylpBM9gP8AjUnZ4PWNttztU",
+  url: [host: "localhost"],
+  watchers: [
+    esbuild: {Esbuild, :install_and_run, [:default, ~w(--sourcemap=inline --watch)]},
+    tailwind: {Tailwind, :install_and_run, [:default, ~w(--watch)]}
+  ],
+  live_reload: [
+    patterns: [
+      ~r"priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$",
+      ~r"lib/web/components/.*(ex)$",
+      ~r"lib/web/live/.*(ex)$"
+    ]
+  ]
+)
+
+Application.put_env(:phoenix, :serve_endpoints, true)
+Application.put_env(:phoenix, :persistent, true)
+
+Task.async(fn ->
+  # Stop the default Telemetry server to start a new one with new defaults
+  mode = "OBSERVER_WEB_TELEMETRY_MODE" |> System.get_env("local") |> String.to_atom()
+
+  retention_period =
+    "OBSERVER_WEB_TELEMETRY_RETENTION_PERIOD" |> System.get_env("1800000") |> String.to_integer()
+
+  telemetry_module = ObserverWeb.Telemetry.Storage
+  :ok = Supervisor.terminate_child(ObserverWeb.Application, telemetry_module)
+  :ok = Supervisor.delete_child(ObserverWeb.Application, telemetry_module)
+
+  {:ok, _} =
+    Supervisor.start_child(
+      ObserverWeb.Application,
+      {telemetry_module, [mode: mode, data_retention_period: retention_period]}
+    )
+
+  {:ok, _} = Supervisor.start_child(ObserverWeb.Application, WebDev.Endpoint)
+
+  Process.sleep(:infinity)
+end)
+
+# ============================================================
+# Автоподключение к узлам EventHub с задержкой
+# ============================================================
+Task.start(fn ->
+  :timer.sleep(7000)   # даём время на полный старт Phoenix
+  nodes = [
+    :"eventhub-node1@eventhub-node1",
+    :"eventhub-node2@eventhub-node2",
+    :"eventhub-node3@eventhub-node3"
+  ]
+  Enum.each(nodes, fn node ->
+    case Node.connect(node) do
+      true  -> IO.puts("Connected to #{node}")
+      false -> IO.puts("Failed to connect to #{node} (will retry manually)")
+    end
+  end)
+end)

docker/prometheus/prometheus.yml

@@ -0,0 +1,22 @@
+global:
+  scrape_interval: 5s
+
+scrape_configs:
+  - job_name: 'eventhub-node1'
+    static_configs:
+      - targets: ['eventhub-node1:8080'] # http://localhost:8080/metrics/default
+        labels:
+          node: 'node1'
+    metrics_path: '/metrics/default'
+  - job_name: 'eventhub-node2'
+    static_configs:
+      - targets: ['eventhub-node2:8080']
+        labels:
+          node: 'node2'
+    metrics_path: '/metrics/default'
+  - job_name: 'eventhub-node3'
+    static_configs:
+      - targets: ['eventhub-node3:8080']
+        labels:
+          node: 'node3'
+    metrics_path: '/metrics/default'

rebar.config

@@ -8,7 +8,8 @@
     {jose, "1.11.10"},
     {argon2, "1.2.0"},
     {meck, "0.9.2"},
-    {gun, "2.0.0"}
+    {gun, "2.0.0"},
+    {prometheus_cowboy, "0.2.0"}
 ]}.
 
 {shell, [
@@ -25,7 +26,7 @@
 {profiles, [
     {prod, [
         {relx, [
-            {release, {eventhub, "0.0.1"}, [eventhub, sasl]},
+            {release, {eventhub, "0.0.1"}, [eventhub, sasl, runtime_tools, os_mon, prometheus_cowboy]},
             {include_erts, true},
             {extended_start_script, true},
             {sys_config, "./src/config/sys.config"}

src/config/vm.args

@@ -0,0 +1,3 @@
+-sname ${NODE_NAME}
+-setcookie ${RELEASE_COOKIE}
+-kernel inet_dist_use_interface {0,0,0,0}

src/eventhub_app.erl

@@ -14,6 +14,9 @@ start(_StartType, _StartArgs) ->
       ok = infra_mnesia:wait_for_tables(),
       start_http(),
       start_admin_http(),
+      % Запускаем сборщик метрик Prometheus
+      application:ensure_all_started(prometheus),
+      application:ensure_all_started(prometheus_cowboy),
       {ok, Pid};
     Error ->
       Error
@@ -27,6 +30,7 @@ start_http() ->
 
   Dispatch = cowboy_router:compile([
     {'_', [
+      {"/metrics/[:registry]", prometheus_cowboy2_handler, []},
       {"/health", handler_health, []},
       {"/v1/register", handler_register, []},
       {"/v1/login", handler_login, []},