fix: raise Mnesia wait_for_tables beyond gen_server 5s default
Caller timed out while mnesia:wait_for_tables ran the full 5s on IFT disc load; use infinity call timeout, 120s wait, and one repair retry.
This commit is contained in:
@@ -50,7 +50,10 @@ start_application() ->
|
|||||||
application:ensure_all_started(mnesia),
|
application:ensure_all_started(mnesia),
|
||||||
ok = infra_mnesia:verify_dump_log(),
|
ok = infra_mnesia:verify_dump_log(),
|
||||||
ok = infra_mnesia:init_tables(),
|
ok = infra_mnesia:init_tables(),
|
||||||
ok = infra_mnesia:wait_for_tables(),
|
case infra_mnesia:wait_for_tables() of
|
||||||
|
ok -> ok;
|
||||||
|
{error, WaitReason} -> error({mnesia_tables_not_ready, WaitReason})
|
||||||
|
end,
|
||||||
ok = migration_engine:ensure_applied(),
|
ok = migration_engine:ensure_applied(),
|
||||||
%% После wait + migrations: иначе backfill ловит {no_exists, user} на ещё не загруженных таблицах.
|
%% После wait + migrations: иначе backfill ловит {no_exists, user} на ещё не загруженных таблицах.
|
||||||
ok = stats_collector:subscribe(),
|
ok = stats_collector:subscribe(),
|
||||||
|
|||||||
@@ -26,7 +26,8 @@
|
|||||||
-define(DISC_TABLES, ?TABLES -- [session, verification, admin_session, node_metric]).
|
-define(DISC_TABLES, ?TABLES -- [session, verification, admin_session, node_metric]).
|
||||||
%% ram_copies: joining nodes must add_table_copy — create_table already_exists skips it.
|
%% ram_copies: joining nodes must add_table_copy — create_table already_exists skips it.
|
||||||
-define(RAM_TABLES, [session, verification, admin_session]).
|
-define(RAM_TABLES, [session, verification, admin_session]).
|
||||||
-define(TABLE_WAIT_TIMEOUT, 5000).
|
%% Disc load on IFT after crash-loop can exceed default gen_server:call 5s.
|
||||||
|
-define(TABLE_WAIT_TIMEOUT, 120000).
|
||||||
-define(CLEANUP_INTERVAL, 30000). % 30 секунд
|
-define(CLEANUP_INTERVAL, 30000). % 30 секунд
|
||||||
|
|
||||||
%% ===================================================================
|
%% ===================================================================
|
||||||
@@ -39,13 +40,13 @@ start_link() ->
|
|||||||
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
|
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
|
||||||
|
|
||||||
init_tables() ->
|
init_tables() ->
|
||||||
gen_server:call(?MODULE, init_tables).
|
gen_server:call(?MODULE, init_tables, infinity).
|
||||||
|
|
||||||
wait_for_tables() ->
|
wait_for_tables() ->
|
||||||
gen_server:call(?MODULE, wait_for_tables).
|
gen_server:call(?MODULE, wait_for_tables, infinity).
|
||||||
|
|
||||||
add_cluster_nodes(Nodes) ->
|
add_cluster_nodes(Nodes) ->
|
||||||
gen_server:call(?MODULE, {add_nodes, Nodes}).
|
gen_server:call(?MODULE, {add_nodes, Nodes}, infinity).
|
||||||
|
|
||||||
%% @doc Soften dump_log overload under write-heavy load (IFT stress).
|
%% @doc Soften dump_log overload under write-heavy load (IFT stress).
|
||||||
%% OTP warns {dump_log, write_threshold|time_threshold} when a new dump
|
%% OTP warns {dump_log, write_threshold|time_threshold} when a new dump
|
||||||
@@ -134,8 +135,10 @@ handle_call({add_nodes, Nodes}, _From, State) ->
|
|||||||
{reply, ok, State};
|
{reply, ok, State};
|
||||||
|
|
||||||
handle_call(wait_for_tables, _From, State) ->
|
handle_call(wait_for_tables, _From, State) ->
|
||||||
mnesia:wait_for_tables(?TABLES, ?TABLE_WAIT_TIMEOUT),
|
case do_wait_for_tables() of
|
||||||
{reply, ok, State}.
|
ok -> {reply, ok, State};
|
||||||
|
{error, Reason} -> {reply, {error, Reason}, State}
|
||||||
|
end;
|
||||||
|
|
||||||
handle_cast(_Msg, State) -> {noreply, State}.
|
handle_cast(_Msg, State) -> {noreply, State}.
|
||||||
handle_info({cleanup}, State) ->
|
handle_info({cleanup}, State) ->
|
||||||
@@ -231,6 +234,26 @@ add_local_ram_copy(Tab) ->
|
|||||||
wait_for_tables_available() ->
|
wait_for_tables_available() ->
|
||||||
lists:foreach(fun(Tab) -> wait_for_table(Tab) end, ?DISC_TABLES ++ ?RAM_TABLES).
|
lists:foreach(fun(Tab) -> wait_for_table(Tab) end, ?DISC_TABLES ++ ?RAM_TABLES).
|
||||||
|
|
||||||
|
do_wait_for_tables() ->
|
||||||
|
case mnesia:wait_for_tables(?TABLES, ?TABLE_WAIT_TIMEOUT) of
|
||||||
|
ok -> ok;
|
||||||
|
{timeout, Remaining} ->
|
||||||
|
io:format("Mnesia wait_for_tables timeout, repairing ~p~n", [Remaining]),
|
||||||
|
lists:foreach(fun(Tab) ->
|
||||||
|
catch create_table(Tab),
|
||||||
|
case lists:member(Tab, ?RAM_TABLES) of
|
||||||
|
true -> catch add_local_ram_copy(Tab);
|
||||||
|
false -> catch add_local_disc_copy(Tab)
|
||||||
|
end
|
||||||
|
end, Remaining),
|
||||||
|
case mnesia:wait_for_tables(?TABLES, ?TABLE_WAIT_TIMEOUT) of
|
||||||
|
ok -> ok;
|
||||||
|
{timeout, Still} ->
|
||||||
|
io:format("Mnesia tables still not ready: ~p~n", [Still]),
|
||||||
|
{error, {tables_not_ready, Still}}
|
||||||
|
end
|
||||||
|
end.
|
||||||
|
|
||||||
wait_for_table(Tab) ->
|
wait_for_table(Tab) ->
|
||||||
case lists:member(Tab, mnesia:system_info(tables)) of
|
case lists:member(Tab, mnesia:system_info(tables)) of
|
||||||
true -> ok;
|
true -> ok;
|
||||||
|
|||||||
Reference in New Issue
Block a user