%%%------------------------------------------------------------------- %%% @doc Сбор admin-статистики через Mnesia detailed-subscribe. %%% ETS держит абсолютные upsert-счётчики и дневные бакеты; %%% периодический flush пишет в stats_counter / stats_daily. %%% @end %%%------------------------------------------------------------------- -module(stats_collector). -behaviour(gen_server). -include("records.hrl"). -export([start_link/0, subscribe/0, rebuild/0, is_ready/0, call/1, get_stats/0]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -define(FLUSH_INTERVAL, 300000). %% 5 min -define(CLEANUP_INTERVAL, 3600000). %% 1 h -define(DAILY_RETENTION_DAYS, 730). -define(COUNTER_ETS, stats_counter_ets). -define(DAILY_ETS, stats_daily_ets). -define(SUB_TABLES, [ user, event, calendar, booking, review, report, ticket, subscription ]). -record(state, { ready = false :: boolean(), subscribed = false :: boolean() }). %%%=================================================================== %%% API %%%=================================================================== start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). subscribe() -> gen_server:call(?MODULE, subscribe, infinity). rebuild() -> gen_server:call(?MODULE, rebuild, infinity). is_ready() -> gen_server:call(?MODULE, is_ready). call(Msg) -> gen_server:call(?MODULE, Msg). %% @doc Совместимость: снимок counter ETS. get_stats() -> gen_server:call(?MODULE, get_stats). %%%=================================================================== %%% gen_server %%%=================================================================== init([]) -> ets:new(?COUNTER_ETS, [named_table, public, set, {keypos, 1}, {write_concurrency, true}]), ets:new(?DAILY_ETS, [named_table, public, set, {keypos, 1}, {write_concurrency, true}]), stats_tops:init_tables(), erlang:send_after(?FLUSH_INTERVAL, self(), flush), erlang:send_after(?CLEANUP_INTERVAL, self(), cleanup_daily), {ok, #state{}}. handle_call(subscribe, _From, State) -> NewState = do_subscribe(State), {reply, ok, NewState}; handle_call(rebuild, _From, State) -> do_rebuild(), {reply, ok, State#state{ready = true}}; handle_call(is_ready, _From, State) -> {reply, State#state.ready, State}; handle_call(get_stats, _From, State) -> {reply, ets:tab2list(?COUNTER_ETS), State}; handle_call({get_counter, _Key}, _From, #state{ready = false} = State) -> {reply, {error, not_ready}, State}; handle_call({get_counter, Key}, _From, State) -> {reply, {ok, ets_get(?COUNTER_ETS, Key)}, State}; handle_call({get_dim, _E, _F}, _From, #state{ready = false} = State) -> {reply, {error, not_ready}, State}; handle_call({get_dim, Entity, Field}, _From, State) -> {reply, {ok, read_dim(Entity, Field)}, State}; handle_call({get_daily, _M, _F, _T}, _From, #state{ready = false} = State) -> {reply, {error, not_ready}, State}; handle_call({get_daily, Metric, From, To}, _From, State) -> {reply, {ok, read_daily(Metric, From, To)}, State}; handle_call(_Msg, _From, State) -> {reply, {error, unknown}, State}. handle_cast(_Msg, State) -> {noreply, State}. handle_info(flush, State) -> flush_to_mnesia(), erlang:send_after(?FLUSH_INTERVAL, self(), flush), {noreply, State}; handle_info(cleanup_daily, State) -> cleanup_old_daily(?DAILY_RETENTION_DAYS), erlang:send_after(?CLEANUP_INTERVAL, self(), cleanup_daily), {noreply, State}; handle_info({mnesia_table_event, Event}, State) -> handle_mnesia_event(Event), {noreply, State}; handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, _State) -> flush_to_mnesia(), ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. %%%=================================================================== %%% Bootstrap %%%=================================================================== do_subscribe(#state{subscribed = true} = State) -> State; do_subscribe(State) -> load_from_mnesia(), case ets:info(?COUNTER_ETS, size) =:= 0 andalso ets:info(?DAILY_ETS, size) =:= 0 of true -> do_rebuild(); false -> stats_tops:rebuild() end, lists:foreach(fun(Table) -> case lists:member(Table, mnesia:system_info(tables)) of true -> case mnesia:wait_for_tables([Table], 60000) of ok -> mnesia:subscribe({table, Table, detailed}); {timeout, _} -> error_logger:warning_msg( "stats_collector: skip subscribe ~p (wait timeout)~n", [Table]) end; false -> error_logger:warning_msg( "stats_collector: skip subscribe ~p (no table)~n", [Table]) end end, ?SUB_TABLES), State#state{ready = true, subscribed = true}. do_rebuild() -> ets:delete_all_objects(?COUNTER_ETS), ets:delete_all_objects(?DAILY_ETS), backfill_all(), stats_tops:rebuild(), flush_to_mnesia(), ok. load_from_mnesia() -> case lists:member(stats_counter, mnesia:system_info(tables)) of true -> lists:foreach(fun(#stats_counter{key = K, value = V}) -> ets:insert(?COUNTER_ETS, {K, V}) end, mnesia:dirty_match_object(#stats_counter{_ = '_'})); false -> ok end, case lists:member(stats_daily, mnesia:system_info(tables)) of true -> lists:foreach(fun(#stats_daily{key = K, value = V}) -> ets:insert(?DAILY_ETS, {K, V}) end, mnesia:dirty_match_object(#stats_daily{_ = '_'})); false -> ok end, ok. %%%=================================================================== %%% Mnesia events %%%=================================================================== handle_mnesia_event({write, Table, New, Olds, _ActivityId}) -> Old = case Olds of [O | _] -> O; [] -> undefined end, process_write(Table, New, Old); handle_mnesia_event({delete, _Table, _Key, Olds, _ActivityId}) -> lists:foreach(fun(Old) -> process_delete(element(1, Old), Old) end, Olds); handle_mnesia_event({delete_object, Table, Old, _Olds, _ActivityId}) -> process_delete(Table, Old); handle_mnesia_event(_) -> ok. %%%=================================================================== %%% Write / delete handlers %%%=================================================================== process_write(user, New, undefined) -> #user{status = St, role = Role, created_at = Created} = New, add_dim(user, status, St, 1), add_dim(user, role, Role, 1), add_daily(users_created, date_of(Created), 1), case St of pending -> add_daily(pending_users_created, date_of(Created), 1); _ -> ok end; process_write(user, New, Old) -> #user{status = StNew, role = RoleNew, created_at = Created} = New, #user{status = StOld, role = RoleOld} = Old, move_dim(user, status, StOld, StNew), move_dim(user, role, RoleOld, RoleNew), case {StOld, StNew} of {pending, pending} -> ok; {pending, _} -> add_daily(pending_users_created, date_of(Created), -1); {_, pending} -> add_daily(pending_users_created, date_of(Created), 1); _ -> ok end; process_write(event, New, undefined) -> #event{status = St, event_type = Type, created_at = Created} = New, add_dim(event, status, St, 1), add_dim(event, type, Type, 1), add_daily(events_created, date_of(Created), 1), stats_tops:on_event(New, undefined); process_write(event, New, Old) -> #event{status = StNew, event_type = TypeNew} = New, #event{status = StOld, event_type = TypeOld} = Old, move_dim(event, status, StOld, StNew), move_dim(event, type, TypeOld, TypeNew), stats_tops:on_event(New, Old); process_write(calendar, New, undefined) -> #calendar{status = St, type = Type, created_at = Created} = New, add_dim(calendar, status, St, 1), add_dim(calendar, type, Type, 1), add_daily(calendars_created, date_of(Created), 1), stats_tops:on_calendar(New, undefined); process_write(calendar, New, Old) -> #calendar{status = StNew, type = TypeNew} = New, #calendar{status = StOld, type = TypeOld} = Old, move_dim(calendar, status, StOld, StNew), move_dim(calendar, type, TypeOld, TypeNew), stats_tops:on_calendar(New, Old); process_write(booking, New, undefined) -> #booking{status = St, created_at = Created} = New, add_dim(booking, status, St, 1), add_daily(bookings_created, date_of(Created), 1); process_write(booking, New, Old) -> move_dim(booking, status, Old#booking.status, New#booking.status); process_write(review, New, undefined) -> #review{status = St, target_type = TT, created_at = Created} = New, add_dim(review, status, St, 1), add_dim(review, target_type, TT, 1), add_daily(reviews_created, date_of(Created), 1), stats_tops:on_review(New, undefined); process_write(review, New, Old) -> #review{status = StNew, target_type = TTNew} = New, #review{status = StOld, target_type = TTOld} = Old, move_dim(review, status, StOld, StNew), move_dim(review, target_type, TTOld, TTNew), stats_tops:on_review(New, Old); process_write(report, New, undefined) -> #report{status = St, target_type = TT, created_at = Created} = New, add_dim(report, status, St, 1), add_dim(report, target_type, TT, 1), add_daily(reports_created, date_of(Created), 1), apply_report_resolution(report_resolution(New)), stats_tops:on_report(New, undefined); process_write(report, New, Old) -> #report{status = StNew, target_type = TTNew} = New, #report{status = StOld, target_type = TTOld} = Old, move_dim(report, status, StOld, StNew), move_dim(report, target_type, TTOld, TTNew), move_resolution(report_resolution(Old), report_resolution(New)), stats_tops:on_report(New, Old); process_write(ticket, New, undefined) -> #ticket{status = St, count = Count, first_seen = First} = New, add_dim(ticket, status, St, 1), add_counter(ticket_errors_sum, Count), add_daily(tickets_created, date_of(First), 1), apply_ticket_resolution(ticket_resolution(New)); process_write(ticket, New, Old) -> #ticket{status = StNew, count = CountNew} = New, #ticket{status = StOld, count = CountOld} = Old, move_dim(ticket, status, StOld, StNew), add_counter(ticket_errors_sum, CountNew - CountOld), move_resolution(ticket_resolution(Old), ticket_resolution(New)); process_write(subscription, New, undefined) -> #subscription{status = St, plan = Plan, trial_used = Trial, created_at = Created} = New, add_dim(subscription, status, St, 1), add_dim(subscription, plan, Plan, 1), case Trial of false -> add_counter(trial_subscriptions, 1); _ -> ok end, add_daily(subscriptions_created, date_of(Created), 1); process_write(subscription, New, Old) -> #subscription{status = StNew, plan = PlanNew, trial_used = TrialNew} = New, #subscription{status = StOld, plan = PlanOld, trial_used = TrialOld} = Old, move_dim(subscription, status, StOld, StNew), move_dim(subscription, plan, PlanOld, PlanNew), case {TrialOld, TrialNew} of {false, false} -> ok; {false, _} -> add_counter(trial_subscriptions, -1); {_, false} -> add_counter(trial_subscriptions, 1); _ -> ok end; process_write(_, _, _) -> ok. process_delete(user, #user{status = St, role = Role, created_at = Created}) -> add_dim(user, status, St, -1), add_dim(user, role, Role, -1), add_daily(users_created, date_of(Created), -1), case St of pending -> add_daily(pending_users_created, date_of(Created), -1); _ -> ok end; process_delete(event, #event{status = St, event_type = Type, created_at = Created} = E) -> add_dim(event, status, St, -1), add_dim(event, type, Type, -1), add_daily(events_created, date_of(Created), -1), stats_tops:remove_event(E); process_delete(calendar, #calendar{status = St, type = Type, created_at = Created} = C) -> add_dim(calendar, status, St, -1), add_dim(calendar, type, Type, -1), add_daily(calendars_created, date_of(Created), -1), stats_tops:remove_calendar(C); process_delete(booking, #booking{status = St, created_at = Created}) -> add_dim(booking, status, St, -1), add_daily(bookings_created, date_of(Created), -1); process_delete(review, #review{status = St, target_type = TT, created_at = Created} = R) -> add_dim(review, status, St, -1), add_dim(review, target_type, TT, -1), add_daily(reviews_created, date_of(Created), -1), stats_tops:remove_review(R); process_delete(report, #report{status = St, target_type = TT, created_at = Created} = R) -> add_dim(report, status, St, -1), add_dim(report, target_type, TT, -1), add_daily(reports_created, date_of(Created), -1), remove_report_resolution(report_resolution(R)), stats_tops:remove_report(R); process_delete(ticket, #ticket{status = St, count = Count, first_seen = First} = T) -> add_dim(ticket, status, St, -1), add_counter(ticket_errors_sum, -Count), add_daily(tickets_created, date_of(First), -1), remove_ticket_resolution(ticket_resolution(T)); process_delete(subscription, #subscription{status = St, plan = Plan, trial_used = Trial, created_at = Created}) -> add_dim(subscription, status, St, -1), add_dim(subscription, plan, Plan, -1), case Trial of false -> add_counter(trial_subscriptions, -1); _ -> ok end, add_daily(subscriptions_created, date_of(Created), -1); process_delete(_, _) -> ok. %%%=================================================================== %%% Backfill %%%=================================================================== backfill_all() -> lists:foreach(fun(U) -> process_write(user, U, undefined) end, safe_match(user, #user{_ = '_'})), lists:foreach(fun(E) -> process_write(event, E, undefined) end, safe_match(event, #event{_ = '_'})), lists:foreach(fun(C) -> process_write(calendar, C, undefined) end, safe_match(calendar, #calendar{_ = '_'})), lists:foreach(fun(B) -> process_write(booking, B, undefined) end, safe_match(booking, #booking{_ = '_'})), lists:foreach(fun(R) -> process_write(review, R, undefined) end, safe_match(review, #review{_ = '_'})), lists:foreach(fun(R) -> process_write(report, R, undefined) end, safe_match(report, #report{_ = '_'})), lists:foreach(fun(T) -> process_write(ticket, T, undefined) end, safe_match(ticket, #ticket{_ = '_'})), lists:foreach(fun(S) -> process_write(subscription, S, undefined) end, safe_match(subscription, #subscription{_ = '_'})), ok. safe_match(Table, Pattern) -> case lists:member(Table, mnesia:system_info(tables)) of true -> try mnesia:dirty_match_object(Pattern) of Rows when is_list(Rows) -> Rows catch exit:{aborted, {no_exists, _}} -> []; error:{aborted, {no_exists, _}} -> [] end; false -> [] end. %%%=================================================================== %%% ETS helpers %%%=================================================================== dim_key(Entity, Field, Value) -> {list_to_atom(atom_to_list(Entity) ++ "_by_" ++ atom_to_list(Field)), Value}. add_dim(Entity, Field, Value, Delta) when is_atom(Value) -> add_counter(dim_key(Entity, Field, Value), Delta); add_dim(_, _, _, _) -> ok. move_dim(_E, _F, Same, Same) -> ok; move_dim(Entity, Field, OldVal, NewVal) -> add_dim(Entity, Field, OldVal, -1), add_dim(Entity, Field, NewVal, 1). add_counter(_Key, 0) -> ok; add_counter(Key, Delta) -> ensure_key(?COUNTER_ETS, Key), ets:update_counter(?COUNTER_ETS, Key, {2, Delta}), %% clamp at 0 for safety under races case ets:lookup(?COUNTER_ETS, Key) of [{Key, V}] when V < 0 -> ets:insert(?COUNTER_ETS, {Key, 0}); _ -> ok end. add_daily(_Metric, undefined, _) -> ok; add_daily(_Metric, _Date, 0) -> ok; add_daily(Metric, Date, Delta) -> Key = {Metric, Date}, ensure_key(?DAILY_ETS, Key), ets:update_counter(?DAILY_ETS, Key, {2, Delta}), case ets:lookup(?DAILY_ETS, Key) of [{Key, V}] when V < 0 -> ets:insert(?DAILY_ETS, {Key, 0}); _ -> ok end. ensure_key(Tab, Key) -> case ets:member(Tab, Key) of true -> ok; false -> ets:insert(Tab, {Key, 0}) end. ets_get(Tab, Key) -> case ets:lookup(Tab, Key) of [{Key, V}] -> V; [] -> 0 end. read_dim(Entity, Field) -> Prefix = list_to_atom(atom_to_list(Entity) ++ "_by_" ++ atom_to_list(Field)), lists:filtermap(fun ({{P, Val}, V}) when P =:= Prefix, V > 0, is_atom(Val) -> {true, {Val, V}}; (_) -> false end, ets:tab2list(?COUNTER_ETS)). read_daily(Metric, From, To) -> FromDate = date_of(From), ToDate = date_of(To), Matches = lists:filtermap(fun ({{M, Day}, V}) when M =:= Metric, V > 0, Day >= FromDate, Day =< ToDate -> {true, {Day, V}}; (_) -> false end, ets:tab2list(?DAILY_ETS)), lists:sort(Matches). date_of({{Y, M, D}, _}) -> {Y, M, D}; date_of({Y, M, D} = Date) when is_integer(Y), is_integer(M), is_integer(D) -> Date; date_of(_) -> undefined. %%%=================================================================== %%% Resolution time (avg) %%%=================================================================== -define(TICKET_EPOCH, {{1970, 1, 1}, {0, 0, 0}}). %% Ticket: same semantics as core_ticket:avg_resolution_time — closed_at set. ticket_resolution(#ticket{first_seen = First, closed_at = Closed}) -> case is_set_datetime(Closed) of true -> case resolution_seconds(First, Closed) of undefined -> undefined; Sec -> {ticket, Sec} end; false -> undefined end. %% Report: only reviewed|dismissed with resolved_at. report_resolution(#report{status = St, created_at = Created, resolved_at = Resolved}) when St =:= reviewed; St =:= dismissed -> case is_set_datetime(Resolved) of true -> case resolution_seconds(Created, Resolved) of undefined -> undefined; Sec -> {{report, St}, Sec} end; false -> undefined end; report_resolution(_) -> undefined. move_resolution(Same, Same) -> ok; move_resolution(Old, New) -> remove_resolution(Old), apply_resolution(New). apply_resolution(undefined) -> ok; apply_resolution({ticket, Sec}) -> apply_ticket_resolution({ticket, Sec}); apply_resolution({{report, St}, Sec}) -> apply_report_resolution({{report, St}, Sec}). remove_resolution(undefined) -> ok; remove_resolution({ticket, Sec}) -> remove_ticket_resolution({ticket, Sec}); remove_resolution({{report, St}, Sec}) -> remove_report_resolution({{report, St}, Sec}). apply_ticket_resolution(undefined) -> ok; apply_ticket_resolution({ticket, Sec}) -> add_counter(ticket_resolved_count, 1), add_counter(ticket_resolved_seconds_sum, Sec). remove_ticket_resolution(undefined) -> ok; remove_ticket_resolution({ticket, Sec}) -> add_counter(ticket_resolved_count, -1), add_counter(ticket_resolved_seconds_sum, -Sec). apply_report_resolution(undefined) -> ok; apply_report_resolution({{report, St}, Sec}) -> add_counter({report_resolved_count, St}, 1), add_counter({report_resolved_seconds_sum, St}, Sec). remove_report_resolution(undefined) -> ok; remove_report_resolution({{report, St}, Sec}) -> add_counter({report_resolved_count, St}, -1), add_counter({report_resolved_seconds_sum, St}, -Sec). is_set_datetime(undefined) -> false; is_set_datetime(?TICKET_EPOCH) -> false; is_set_datetime({{_, _, _}, {_, _, _}}) -> true; is_set_datetime(_) -> false. resolution_seconds(Start, End) -> case {is_set_datetime(Start), is_set_datetime(End)} of {true, true} -> max(0, calendar:datetime_to_gregorian_seconds(End) - calendar:datetime_to_gregorian_seconds(Start)); _ -> undefined end. %%%=================================================================== %%% Persist / cleanup %%%=================================================================== flush_to_mnesia() -> Now = calendar:universal_time(), case lists:member(stats_counter, mnesia:system_info(tables)) of true -> lists:foreach(fun({Key, Value}) -> mnesia:dirty_write(#stats_counter{key = Key, value = Value, updated_at = Now}) end, ets:tab2list(?COUNTER_ETS)); false -> ok end, case lists:member(stats_daily, mnesia:system_info(tables)) of true -> lists:foreach(fun({Key, Value}) -> mnesia:dirty_write(#stats_daily{key = Key, value = Value, updated_at = Now}) end, ets:tab2list(?DAILY_ETS)); false -> ok end, ok. cleanup_old_daily(RetentionDays) -> case lists:member(stats_daily, mnesia:system_info(tables)) of false -> ok; true -> {Date, _} = calendar:universal_time(), CutoffSec = calendar:datetime_to_gregorian_seconds({Date, {0, 0, 0}}) - RetentionDays * 86400, Cutoff = element(1, calendar:gregorian_seconds_to_datetime(CutoffSec)), lists:foreach(fun (#stats_daily{key = {_, Day}} = Row) when Day < Cutoff -> mnesia:dirty_delete_object(Row), ets:delete(?DAILY_ETS, Row#stats_daily.key); (_) -> ok end, mnesia:dirty_match_object(#stats_daily{_ = '_'})), ok end.