EventHubBack/src/infra/stats_collector.erl

%%%-------------------------------------------------------------------
%%% @doc Сбор admin-статистики через Mnesia detailed-subscribe.
%%% ETS держит абсолютные upsert-счётчики и дневные бакеты;
%%% периодический flush пишет в stats_counter / stats_daily.
%%% @end
%%%-------------------------------------------------------------------
-module(stats_collector).
-behaviour(gen_server).

-include("records.hrl").

-export([start_link/0, subscribe/0, rebuild/0, is_ready/0, call/1,
  get_stats/0]).
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
  terminate/2, code_change/3]).

-define(FLUSH_INTERVAL, 300000).      %% 5 min
-define(CLEANUP_INTERVAL, 3600000).   %% 1 h
-define(DAILY_RETENTION_DAYS, 730).
-define(COUNTER_ETS, stats_counter_ets).
-define(DAILY_ETS, stats_daily_ets).
-define(SUB_TABLES, [
  user, event, calendar, booking, review, report, ticket, subscription
]).

-record(state, {
  ready = false :: boolean(),
  subscribed = false :: boolean()
}).

%%%===================================================================
%%% API
%%%===================================================================

start_link() ->
  gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).

subscribe() ->
  gen_server:call(?MODULE, subscribe, infinity).

rebuild() ->
  gen_server:call(?MODULE, rebuild, infinity).

is_ready() ->
  gen_server:call(?MODULE, is_ready).

call(Msg) ->
  gen_server:call(?MODULE, Msg).

%% @doc Совместимость: снимок counter ETS.
get_stats() ->
  gen_server:call(?MODULE, get_stats).

%%%===================================================================
%%% gen_server
%%%===================================================================

init([]) ->
  ets:new(?COUNTER_ETS, [named_table, public, set, {keypos, 1},
    {write_concurrency, true}]),
  ets:new(?DAILY_ETS, [named_table, public, set, {keypos, 1},
    {write_concurrency, true}]),
  stats_tops:init_tables(),
  erlang:send_after(?FLUSH_INTERVAL, self(), flush),
  erlang:send_after(?CLEANUP_INTERVAL, self(), cleanup_daily),
  {ok, #state{}}.

handle_call(subscribe, _From, State) ->
  NewState = do_subscribe(State),
  {reply, ok, NewState};
handle_call(rebuild, _From, State) ->
  do_rebuild(),
  {reply, ok, State#state{ready = true}};
handle_call(is_ready, _From, State) ->
  {reply, State#state.ready, State};
handle_call(get_stats, _From, State) ->
  {reply, ets:tab2list(?COUNTER_ETS), State};
handle_call({get_counter, _Key}, _From, #state{ready = false} = State) ->
  {reply, {error, not_ready}, State};
handle_call({get_counter, Key}, _From, State) ->
  {reply, {ok, ets_get(?COUNTER_ETS, Key)}, State};
handle_call({get_dim, _E, _F}, _From, #state{ready = false} = State) ->
  {reply, {error, not_ready}, State};
handle_call({get_dim, Entity, Field}, _From, State) ->
  {reply, {ok, read_dim(Entity, Field)}, State};
handle_call({get_daily, _M, _F, _T}, _From, #state{ready = false} = State) ->
  {reply, {error, not_ready}, State};
handle_call({get_daily, Metric, From, To}, _From, State) ->
  {reply, {ok, read_daily(Metric, From, To)}, State};
handle_call(_Msg, _From, State) ->
  {reply, {error, unknown}, State}.

handle_cast(_Msg, State) ->
  {noreply, State}.

handle_info(flush, State) ->
  flush_to_mnesia(),
  erlang:send_after(?FLUSH_INTERVAL, self(), flush),
  {noreply, State};
handle_info(cleanup_daily, State) ->
  cleanup_old_daily(?DAILY_RETENTION_DAYS),
  erlang:send_after(?CLEANUP_INTERVAL, self(), cleanup_daily),
  {noreply, State};
handle_info({mnesia_table_event, Event}, State) ->
  handle_mnesia_event(Event),
  {noreply, State};
handle_info(_Info, State) ->
  {noreply, State}.

terminate(_Reason, _State) ->
  flush_to_mnesia(),
  ok.

code_change(_OldVsn, State, _Extra) ->
  {ok, State}.

%%%===================================================================
%%% Bootstrap
%%%===================================================================

do_subscribe(#state{subscribed = true} = State) ->
  State;
do_subscribe(State) ->
  load_from_mnesia(),
  case ets:info(?COUNTER_ETS, size) =:= 0 andalso ets:info(?DAILY_ETS, size) =:= 0 of
    true -> do_rebuild();
    false -> stats_tops:rebuild()
  end,
  lists:foreach(fun(Table) ->
    case lists:member(Table, mnesia:system_info(tables)) of
      true ->
        case mnesia:wait_for_tables([Table], 60000) of
          ok -> mnesia:subscribe({table, Table, detailed});
          {timeout, _} ->
            error_logger:warning_msg(
              "stats_collector: skip subscribe ~p (wait timeout)~n", [Table])
        end;
      false ->
        error_logger:warning_msg(
          "stats_collector: skip subscribe ~p (no table)~n", [Table])
    end
  end, ?SUB_TABLES),
  State#state{ready = true, subscribed = true}.

do_rebuild() ->
  ets:delete_all_objects(?COUNTER_ETS),
  ets:delete_all_objects(?DAILY_ETS),
  backfill_all(),
  stats_tops:rebuild(),
  flush_to_mnesia(),
  ok.

load_from_mnesia() ->
  case lists:member(stats_counter, mnesia:system_info(tables)) of
    true ->
      lists:foreach(fun(#stats_counter{key = K, value = V}) ->
        ets:insert(?COUNTER_ETS, {K, V})
      end, mnesia:dirty_match_object(#stats_counter{_ = '_'}));
    false -> ok
  end,
  case lists:member(stats_daily, mnesia:system_info(tables)) of
    true ->
      lists:foreach(fun(#stats_daily{key = K, value = V}) ->
        ets:insert(?DAILY_ETS, {K, V})
      end, mnesia:dirty_match_object(#stats_daily{_ = '_'}));
    false -> ok
  end,
  ok.

%%%===================================================================
%%% Mnesia events
%%%===================================================================

handle_mnesia_event({write, Table, New, Olds, _ActivityId}) ->
  Old = case Olds of
          [O | _] -> O;
          [] -> undefined
        end,
  process_write(Table, New, Old);
handle_mnesia_event({delete, _Table, _Key, Olds, _ActivityId}) ->
  lists:foreach(fun(Old) -> process_delete(element(1, Old), Old) end, Olds);
handle_mnesia_event({delete_object, Table, Old, _Olds, _ActivityId}) ->
  process_delete(Table, Old);
handle_mnesia_event(_) ->
  ok.

%%%===================================================================
%%% Write / delete handlers
%%%===================================================================

process_write(user, New, undefined) ->
  #user{status = St, role = Role, created_at = Created} = New,
  add_dim(user, status, St, 1),
  add_dim(user, role, Role, 1),
  add_daily(users_created, date_of(Created), 1),
  case St of
    pending -> add_daily(pending_users_created, date_of(Created), 1);
    _ -> ok
  end;
process_write(user, New, Old) ->
  #user{status = StNew, role = RoleNew, created_at = Created} = New,
  #user{status = StOld, role = RoleOld} = Old,
  move_dim(user, status, StOld, StNew),
  move_dim(user, role, RoleOld, RoleNew),
  case {StOld, StNew} of
    {pending, pending} -> ok;
    {pending, _} -> add_daily(pending_users_created, date_of(Created), -1);
    {_, pending} -> add_daily(pending_users_created, date_of(Created), 1);
    _ -> ok
  end;

process_write(event, New, undefined) ->
  #event{status = St, event_type = Type, created_at = Created} = New,
  add_dim(event, status, St, 1),
  add_dim(event, type, Type, 1),
  add_daily(events_created, date_of(Created), 1),
  stats_tops:on_event(New, undefined);
process_write(event, New, Old) ->
  #event{status = StNew, event_type = TypeNew} = New,
  #event{status = StOld, event_type = TypeOld} = Old,
  move_dim(event, status, StOld, StNew),
  move_dim(event, type, TypeOld, TypeNew),
  stats_tops:on_event(New, Old);

process_write(calendar, New, undefined) ->
  #calendar{status = St, type = Type, created_at = Created} = New,
  add_dim(calendar, status, St, 1),
  add_dim(calendar, type, Type, 1),
  add_daily(calendars_created, date_of(Created), 1),
  stats_tops:on_calendar(New, undefined);
process_write(calendar, New, Old) ->
  #calendar{status = StNew, type = TypeNew} = New,
  #calendar{status = StOld, type = TypeOld} = Old,
  move_dim(calendar, status, StOld, StNew),
  move_dim(calendar, type, TypeOld, TypeNew),
  stats_tops:on_calendar(New, Old);

process_write(booking, New, undefined) ->
  #booking{status = St, created_at = Created} = New,
  add_dim(booking, status, St, 1),
  add_daily(bookings_created, date_of(Created), 1);
process_write(booking, New, Old) ->
  move_dim(booking, status, Old#booking.status, New#booking.status);

process_write(review, New, undefined) ->
  #review{status = St, target_type = TT, created_at = Created} = New,
  add_dim(review, status, St, 1),
  add_dim(review, target_type, TT, 1),
  add_daily(reviews_created, date_of(Created), 1),
  stats_tops:on_review(New, undefined);
process_write(review, New, Old) ->
  #review{status = StNew, target_type = TTNew} = New,
  #review{status = StOld, target_type = TTOld} = Old,
  move_dim(review, status, StOld, StNew),
  move_dim(review, target_type, TTOld, TTNew),
  stats_tops:on_review(New, Old);

process_write(report, New, undefined) ->
  #report{status = St, target_type = TT, created_at = Created} = New,
  add_dim(report, status, St, 1),
  add_dim(report, target_type, TT, 1),
  add_daily(reports_created, date_of(Created), 1),
  apply_report_resolution(report_resolution(New)),
  stats_tops:on_report(New, undefined);
process_write(report, New, Old) ->
  #report{status = StNew, target_type = TTNew} = New,
  #report{status = StOld, target_type = TTOld} = Old,
  move_dim(report, status, StOld, StNew),
  move_dim(report, target_type, TTOld, TTNew),
  move_resolution(report_resolution(Old), report_resolution(New)),
  stats_tops:on_report(New, Old);

process_write(ticket, New, undefined) ->
  #ticket{status = St, count = Count, first_seen = First} = New,
  add_dim(ticket, status, St, 1),
  add_counter(ticket_errors_sum, Count),
  add_daily(tickets_created, date_of(First), 1),
  apply_ticket_resolution(ticket_resolution(New));
process_write(ticket, New, Old) ->
  #ticket{status = StNew, count = CountNew} = New,
  #ticket{status = StOld, count = CountOld} = Old,
  move_dim(ticket, status, StOld, StNew),
  add_counter(ticket_errors_sum, CountNew - CountOld),
  move_resolution(ticket_resolution(Old), ticket_resolution(New));

process_write(subscription, New, undefined) ->
  #subscription{status = St, plan = Plan, trial_used = Trial,
    created_at = Created} = New,
  add_dim(subscription, status, St, 1),
  add_dim(subscription, plan, Plan, 1),
  case Trial of
    false -> add_counter(trial_subscriptions, 1);
    _ -> ok
  end,
  add_daily(subscriptions_created, date_of(Created), 1);
process_write(subscription, New, Old) ->
  #subscription{status = StNew, plan = PlanNew, trial_used = TrialNew} = New,
  #subscription{status = StOld, plan = PlanOld, trial_used = TrialOld} = Old,
  move_dim(subscription, status, StOld, StNew),
  move_dim(subscription, plan, PlanOld, PlanNew),
  case {TrialOld, TrialNew} of
    {false, false} -> ok;
    {false, _} -> add_counter(trial_subscriptions, -1);
    {_, false} -> add_counter(trial_subscriptions, 1);
    _ -> ok
  end;

process_write(_, _, _) ->
  ok.

process_delete(user, #user{status = St, role = Role, created_at = Created}) ->
  add_dim(user, status, St, -1),
  add_dim(user, role, Role, -1),
  add_daily(users_created, date_of(Created), -1),
  case St of
    pending -> add_daily(pending_users_created, date_of(Created), -1);
    _ -> ok
  end;
process_delete(event, #event{status = St, event_type = Type, created_at = Created} = E) ->
  add_dim(event, status, St, -1),
  add_dim(event, type, Type, -1),
  add_daily(events_created, date_of(Created), -1),
  stats_tops:remove_event(E);
process_delete(calendar, #calendar{status = St, type = Type, created_at = Created} = C) ->
  add_dim(calendar, status, St, -1),
  add_dim(calendar, type, Type, -1),
  add_daily(calendars_created, date_of(Created), -1),
  stats_tops:remove_calendar(C);
process_delete(booking, #booking{status = St, created_at = Created}) ->
  add_dim(booking, status, St, -1),
  add_daily(bookings_created, date_of(Created), -1);
process_delete(review, #review{status = St, target_type = TT, created_at = Created} = R) ->
  add_dim(review, status, St, -1),
  add_dim(review, target_type, TT, -1),
  add_daily(reviews_created, date_of(Created), -1),
  stats_tops:remove_review(R);
process_delete(report, #report{status = St, target_type = TT, created_at = Created} = R) ->
  add_dim(report, status, St, -1),
  add_dim(report, target_type, TT, -1),
  add_daily(reports_created, date_of(Created), -1),
  remove_report_resolution(report_resolution(R)),
  stats_tops:remove_report(R);
process_delete(ticket, #ticket{status = St, count = Count, first_seen = First} = T) ->
  add_dim(ticket, status, St, -1),
  add_counter(ticket_errors_sum, -Count),
  add_daily(tickets_created, date_of(First), -1),
  remove_ticket_resolution(ticket_resolution(T));
process_delete(subscription, #subscription{status = St, plan = Plan,
    trial_used = Trial, created_at = Created}) ->
  add_dim(subscription, status, St, -1),
  add_dim(subscription, plan, Plan, -1),
  case Trial of
    false -> add_counter(trial_subscriptions, -1);
    _ -> ok
  end,
  add_daily(subscriptions_created, date_of(Created), -1);
process_delete(_, _) ->
  ok.

%%%===================================================================
%%% Backfill
%%%===================================================================

backfill_all() ->
  lists:foreach(fun(U) -> process_write(user, U, undefined) end,
    safe_match(user, #user{_ = '_'})),
  lists:foreach(fun(E) -> process_write(event, E, undefined) end,
    safe_match(event, #event{_ = '_'})),
  lists:foreach(fun(C) -> process_write(calendar, C, undefined) end,
    safe_match(calendar, #calendar{_ = '_'})),
  lists:foreach(fun(B) -> process_write(booking, B, undefined) end,
    safe_match(booking, #booking{_ = '_'})),
  lists:foreach(fun(R) -> process_write(review, R, undefined) end,
    safe_match(review, #review{_ = '_'})),
  lists:foreach(fun(R) -> process_write(report, R, undefined) end,
    safe_match(report, #report{_ = '_'})),
  lists:foreach(fun(T) -> process_write(ticket, T, undefined) end,
    safe_match(ticket, #ticket{_ = '_'})),
  lists:foreach(fun(S) -> process_write(subscription, S, undefined) end,
    safe_match(subscription, #subscription{_ = '_'})),
  ok.

safe_match(Table, Pattern) ->
  case lists:member(Table, mnesia:system_info(tables)) of
    true ->
      try mnesia:dirty_match_object(Pattern) of
        Rows when is_list(Rows) -> Rows
      catch
        exit:{aborted, {no_exists, _}} -> [];
        error:{aborted, {no_exists, _}} -> []
      end;
    false -> []
  end.

%%%===================================================================
%%% ETS helpers
%%%===================================================================

dim_key(Entity, Field, Value) ->
  {list_to_atom(atom_to_list(Entity) ++ "_by_" ++ atom_to_list(Field)), Value}.

add_dim(Entity, Field, Value, Delta) when is_atom(Value) ->
  add_counter(dim_key(Entity, Field, Value), Delta);
add_dim(_, _, _, _) ->
  ok.

move_dim(_E, _F, Same, Same) ->
  ok;
move_dim(Entity, Field, OldVal, NewVal) ->
  add_dim(Entity, Field, OldVal, -1),
  add_dim(Entity, Field, NewVal, 1).

add_counter(_Key, 0) ->
  ok;
add_counter(Key, Delta) ->
  ensure_key(?COUNTER_ETS, Key),
  ets:update_counter(?COUNTER_ETS, Key, {2, Delta}),
  %% clamp at 0 for safety under races
  case ets:lookup(?COUNTER_ETS, Key) of
    [{Key, V}] when V < 0 -> ets:insert(?COUNTER_ETS, {Key, 0});
    _ -> ok
  end.

add_daily(_Metric, undefined, _) ->
  ok;
add_daily(_Metric, _Date, 0) ->
  ok;
add_daily(Metric, Date, Delta) ->
  Key = {Metric, Date},
  ensure_key(?DAILY_ETS, Key),
  ets:update_counter(?DAILY_ETS, Key, {2, Delta}),
  case ets:lookup(?DAILY_ETS, Key) of
    [{Key, V}] when V < 0 -> ets:insert(?DAILY_ETS, {Key, 0});
    _ -> ok
  end.

ensure_key(Tab, Key) ->
  case ets:member(Tab, Key) of
    true -> ok;
    false -> ets:insert(Tab, {Key, 0})
  end.

ets_get(Tab, Key) ->
  case ets:lookup(Tab, Key) of
    [{Key, V}] -> V;
    [] -> 0
  end.

read_dim(Entity, Field) ->
  Prefix = list_to_atom(atom_to_list(Entity) ++ "_by_" ++ atom_to_list(Field)),
  lists:filtermap(fun
    ({{P, Val}, V}) when P =:= Prefix, V > 0, is_atom(Val) -> {true, {Val, V}};
    (_) -> false
  end, ets:tab2list(?COUNTER_ETS)).

read_daily(Metric, From, To) ->
  FromDate = date_of(From),
  ToDate = date_of(To),
  Matches = lists:filtermap(fun
    ({{M, Day}, V}) when M =:= Metric, V > 0,
        Day >= FromDate, Day =< ToDate ->
      {true, {Day, V}};
    (_) -> false
  end, ets:tab2list(?DAILY_ETS)),
  lists:sort(Matches).

date_of({{Y, M, D}, _}) -> {Y, M, D};
date_of({Y, M, D} = Date) when is_integer(Y), is_integer(M), is_integer(D) -> Date;
date_of(_) -> undefined.

%%%===================================================================
%%% Resolution time (avg)
%%%===================================================================

-define(TICKET_EPOCH, {{1970, 1, 1}, {0, 0, 0}}).

%% Ticket: same semantics as core_ticket:avg_resolution_time — closed_at set.
ticket_resolution(#ticket{first_seen = First, closed_at = Closed}) ->
  case is_set_datetime(Closed) of
    true ->
      case resolution_seconds(First, Closed) of
        undefined -> undefined;
        Sec -> {ticket, Sec}
      end;
    false -> undefined
  end.

%% Report: only reviewed|dismissed with resolved_at.
report_resolution(#report{status = St, created_at = Created, resolved_at = Resolved})
    when St =:= reviewed; St =:= dismissed ->
  case is_set_datetime(Resolved) of
    true ->
      case resolution_seconds(Created, Resolved) of
        undefined -> undefined;
        Sec -> {{report, St}, Sec}
      end;
    false -> undefined
  end;
report_resolution(_) ->
  undefined.

move_resolution(Same, Same) ->
  ok;
move_resolution(Old, New) ->
  remove_resolution(Old),
  apply_resolution(New).

apply_resolution(undefined) -> ok;
apply_resolution({ticket, Sec}) -> apply_ticket_resolution({ticket, Sec});
apply_resolution({{report, St}, Sec}) -> apply_report_resolution({{report, St}, Sec}).

remove_resolution(undefined) -> ok;
remove_resolution({ticket, Sec}) -> remove_ticket_resolution({ticket, Sec});
remove_resolution({{report, St}, Sec}) -> remove_report_resolution({{report, St}, Sec}).

apply_ticket_resolution(undefined) -> ok;
apply_ticket_resolution({ticket, Sec}) ->
  add_counter(ticket_resolved_count, 1),
  add_counter(ticket_resolved_seconds_sum, Sec).

remove_ticket_resolution(undefined) -> ok;
remove_ticket_resolution({ticket, Sec}) ->
  add_counter(ticket_resolved_count, -1),
  add_counter(ticket_resolved_seconds_sum, -Sec).

apply_report_resolution(undefined) -> ok;
apply_report_resolution({{report, St}, Sec}) ->
  add_counter({report_resolved_count, St}, 1),
  add_counter({report_resolved_seconds_sum, St}, Sec).

remove_report_resolution(undefined) -> ok;
remove_report_resolution({{report, St}, Sec}) ->
  add_counter({report_resolved_count, St}, -1),
  add_counter({report_resolved_seconds_sum, St}, -Sec).

is_set_datetime(undefined) -> false;
is_set_datetime(?TICKET_EPOCH) -> false;
is_set_datetime({{_, _, _}, {_, _, _}}) -> true;
is_set_datetime(_) -> false.

resolution_seconds(Start, End) ->
  case {is_set_datetime(Start), is_set_datetime(End)} of
    {true, true} ->
      max(0, calendar:datetime_to_gregorian_seconds(End) -
        calendar:datetime_to_gregorian_seconds(Start));
    _ -> undefined
  end.

%%%===================================================================
%%% Persist / cleanup
%%%===================================================================

flush_to_mnesia() ->
  Now = calendar:universal_time(),
  case lists:member(stats_counter, mnesia:system_info(tables)) of
    true ->
      lists:foreach(fun({Key, Value}) ->
        mnesia:dirty_write(#stats_counter{key = Key, value = Value, updated_at = Now})
      end, ets:tab2list(?COUNTER_ETS));
    false -> ok
  end,
  case lists:member(stats_daily, mnesia:system_info(tables)) of
    true ->
      lists:foreach(fun({Key, Value}) ->
        mnesia:dirty_write(#stats_daily{key = Key, value = Value, updated_at = Now})
      end, ets:tab2list(?DAILY_ETS));
    false -> ok
  end,
  ok.

cleanup_old_daily(RetentionDays) ->
  case lists:member(stats_daily, mnesia:system_info(tables)) of
    false -> ok;
    true ->
      {Date, _} = calendar:universal_time(),
      CutoffSec = calendar:datetime_to_gregorian_seconds({Date, {0, 0, 0}})
        - RetentionDays * 86400,
      Cutoff = element(1, calendar:gregorian_seconds_to_datetime(CutoffSec)),
      lists:foreach(fun
        (#stats_daily{key = {_, Day}} = Row) when Day < Cutoff ->
          mnesia:dirty_delete_object(Row),
          ets:delete(?DAILY_ETS, Row#stats_daily.key);
        (_) -> ok
      end, mnesia:dirty_match_object(#stats_daily{_ = '_'})),
      ok
  end.