fix: raise Mnesia wait_for_tables beyond gen_server 5s default
CI / test (push) Failing after 27m9s
CI / deploy-ift (push) Has been skipped
CI / e2e-ift (push) Has been skipped
CI / deploy-stage (push) Has been skipped
CI / e2e-stage (push) Has been skipped

Caller timed out while mnesia:wait_for_tables ran the full 5s on IFT disc load; use infinity call timeout, 120s wait, and one repair retry.
This commit is contained in:
2026-07-19 10:45:25 +03:00
parent a6dadcb9d5
commit e8addae3b7
2 changed files with 33 additions and 7 deletions
+4 -1
View File
@@ -50,7 +50,10 @@ start_application() ->
application:ensure_all_started(mnesia), application:ensure_all_started(mnesia),
ok = infra_mnesia:verify_dump_log(), ok = infra_mnesia:verify_dump_log(),
ok = infra_mnesia:init_tables(), ok = infra_mnesia:init_tables(),
ok = infra_mnesia:wait_for_tables(), case infra_mnesia:wait_for_tables() of
ok -> ok;
{error, WaitReason} -> error({mnesia_tables_not_ready, WaitReason})
end,
ok = migration_engine:ensure_applied(), ok = migration_engine:ensure_applied(),
%% После wait + migrations: иначе backfill ловит {no_exists, user} на ещё не загруженных таблицах. %% После wait + migrations: иначе backfill ловит {no_exists, user} на ещё не загруженных таблицах.
ok = stats_collector:subscribe(), ok = stats_collector:subscribe(),
+29 -6
View File
@@ -26,7 +26,8 @@
-define(DISC_TABLES, ?TABLES -- [session, verification, admin_session, node_metric]). -define(DISC_TABLES, ?TABLES -- [session, verification, admin_session, node_metric]).
%% ram_copies: joining nodes must add_table_copy — create_table already_exists skips it. %% ram_copies: joining nodes must add_table_copy — create_table already_exists skips it.
-define(RAM_TABLES, [session, verification, admin_session]). -define(RAM_TABLES, [session, verification, admin_session]).
-define(TABLE_WAIT_TIMEOUT, 5000). %% Disc load on IFT after crash-loop can exceed default gen_server:call 5s.
-define(TABLE_WAIT_TIMEOUT, 120000).
-define(CLEANUP_INTERVAL, 30000). % 30 секунд -define(CLEANUP_INTERVAL, 30000). % 30 секунд
%% =================================================================== %% ===================================================================
@@ -39,13 +40,13 @@ start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
init_tables() -> init_tables() ->
gen_server:call(?MODULE, init_tables). gen_server:call(?MODULE, init_tables, infinity).
wait_for_tables() -> wait_for_tables() ->
gen_server:call(?MODULE, wait_for_tables). gen_server:call(?MODULE, wait_for_tables, infinity).
add_cluster_nodes(Nodes) -> add_cluster_nodes(Nodes) ->
gen_server:call(?MODULE, {add_nodes, Nodes}). gen_server:call(?MODULE, {add_nodes, Nodes}, infinity).
%% @doc Soften dump_log overload under write-heavy load (IFT stress). %% @doc Soften dump_log overload under write-heavy load (IFT stress).
%% OTP warns {dump_log, write_threshold|time_threshold} when a new dump %% OTP warns {dump_log, write_threshold|time_threshold} when a new dump
@@ -134,8 +135,10 @@ handle_call({add_nodes, Nodes}, _From, State) ->
{reply, ok, State}; {reply, ok, State};
handle_call(wait_for_tables, _From, State) -> handle_call(wait_for_tables, _From, State) ->
mnesia:wait_for_tables(?TABLES, ?TABLE_WAIT_TIMEOUT), case do_wait_for_tables() of
{reply, ok, State}. ok -> {reply, ok, State};
{error, Reason} -> {reply, {error, Reason}, State}
end;
handle_cast(_Msg, State) -> {noreply, State}. handle_cast(_Msg, State) -> {noreply, State}.
handle_info({cleanup}, State) -> handle_info({cleanup}, State) ->
@@ -231,6 +234,26 @@ add_local_ram_copy(Tab) ->
wait_for_tables_available() -> wait_for_tables_available() ->
lists:foreach(fun(Tab) -> wait_for_table(Tab) end, ?DISC_TABLES ++ ?RAM_TABLES). lists:foreach(fun(Tab) -> wait_for_table(Tab) end, ?DISC_TABLES ++ ?RAM_TABLES).
do_wait_for_tables() ->
case mnesia:wait_for_tables(?TABLES, ?TABLE_WAIT_TIMEOUT) of
ok -> ok;
{timeout, Remaining} ->
io:format("Mnesia wait_for_tables timeout, repairing ~p~n", [Remaining]),
lists:foreach(fun(Tab) ->
catch create_table(Tab),
case lists:member(Tab, ?RAM_TABLES) of
true -> catch add_local_ram_copy(Tab);
false -> catch add_local_disc_copy(Tab)
end
end, Remaining),
case mnesia:wait_for_tables(?TABLES, ?TABLE_WAIT_TIMEOUT) of
ok -> ok;
{timeout, Still} ->
io:format("Mnesia tables still not ready: ~p~n", [Still]),
{error, {tables_not_ready, Still}}
end
end.
wait_for_table(Tab) -> wait_for_table(Tab) ->
case lists:member(Tab, mnesia:system_info(tables)) of case lists:member(Tab, mnesia:system_info(tables)) of
true -> ok; true -> ok;