diff --git a/src/eventhub_app.erl b/src/eventhub_app.erl index 051901a..7558122 100644 --- a/src/eventhub_app.erl +++ b/src/eventhub_app.erl @@ -50,7 +50,10 @@ start_application() -> application:ensure_all_started(mnesia), ok = infra_mnesia:verify_dump_log(), ok = infra_mnesia:init_tables(), - ok = infra_mnesia:wait_for_tables(), + case infra_mnesia:wait_for_tables() of + ok -> ok; + {error, WaitReason} -> error({mnesia_tables_not_ready, WaitReason}) + end, ok = migration_engine:ensure_applied(), %% После wait + migrations: иначе backfill ловит {no_exists, user} на ещё не загруженных таблицах. ok = stats_collector:subscribe(), diff --git a/src/infra/infra_mnesia.erl b/src/infra/infra_mnesia.erl index 59b6670..a166dee 100644 --- a/src/infra/infra_mnesia.erl +++ b/src/infra/infra_mnesia.erl @@ -26,7 +26,8 @@ -define(DISC_TABLES, ?TABLES -- [session, verification, admin_session, node_metric]). %% ram_copies: joining nodes must add_table_copy — create_table already_exists skips it. -define(RAM_TABLES, [session, verification, admin_session]). --define(TABLE_WAIT_TIMEOUT, 5000). +%% Disc load on IFT after crash-loop can exceed default gen_server:call 5s. +-define(TABLE_WAIT_TIMEOUT, 120000). -define(CLEANUP_INTERVAL, 30000). % 30 секунд %% =================================================================== @@ -39,13 +40,13 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). init_tables() -> - gen_server:call(?MODULE, init_tables). + gen_server:call(?MODULE, init_tables, infinity). wait_for_tables() -> - gen_server:call(?MODULE, wait_for_tables). + gen_server:call(?MODULE, wait_for_tables, infinity). add_cluster_nodes(Nodes) -> - gen_server:call(?MODULE, {add_nodes, Nodes}). + gen_server:call(?MODULE, {add_nodes, Nodes}, infinity). %% @doc Soften dump_log overload under write-heavy load (IFT stress). %% OTP warns {dump_log, write_threshold|time_threshold} when a new dump @@ -134,8 +135,10 @@ handle_call({add_nodes, Nodes}, _From, State) -> {reply, ok, State}; handle_call(wait_for_tables, _From, State) -> - mnesia:wait_for_tables(?TABLES, ?TABLE_WAIT_TIMEOUT), - {reply, ok, State}. + case do_wait_for_tables() of + ok -> {reply, ok, State}; + {error, Reason} -> {reply, {error, Reason}, State} + end; handle_cast(_Msg, State) -> {noreply, State}. handle_info({cleanup}, State) -> @@ -231,6 +234,26 @@ add_local_ram_copy(Tab) -> wait_for_tables_available() -> lists:foreach(fun(Tab) -> wait_for_table(Tab) end, ?DISC_TABLES ++ ?RAM_TABLES). +do_wait_for_tables() -> + case mnesia:wait_for_tables(?TABLES, ?TABLE_WAIT_TIMEOUT) of + ok -> ok; + {timeout, Remaining} -> + io:format("Mnesia wait_for_tables timeout, repairing ~p~n", [Remaining]), + lists:foreach(fun(Tab) -> + catch create_table(Tab), + case lists:member(Tab, ?RAM_TABLES) of + true -> catch add_local_ram_copy(Tab); + false -> catch add_local_disc_copy(Tab) + end + end, Remaining), + case mnesia:wait_for_tables(?TABLES, ?TABLE_WAIT_TIMEOUT) of + ok -> ok; + {timeout, Still} -> + io:format("Mnesia tables still not ready: ~p~n", [Still]), + {error, {tables_not_ready, Still}} + end + end. + wait_for_table(Tab) -> case lists:member(Tab, mnesia:system_info(tables)) of true -> ok;