diff --git a/rebar.config b/rebar.config index fc75d84f..78049cea 100644 --- a/rebar.config +++ b/rebar.config @@ -35,8 +35,10 @@ , {"closure(E) | mria_mnesia : Mod || [mria, mria_lb, mria_schema, mria_node_monitor, mria_membership, mria_rlog] : Mod", [{{mria_mnesia,join_cluster,1}, {mria_rlog,role,1}}]} , {"closure(E) | mria_membership : Mod || [mria, mria_lb, mria_schema, mria_node_monitor, mria_rlog] : Mod", - [{{mria_membership,handle_cast,2}, {mria_rlog,role,1}}, - {{mria_membership,role,1},{mria_rlog,role,1}}]} + [{{mria_membership,handle_cast,2}, {mria_lb,notify_core_node_down,1}}, + {{mria_membership,handle_cast,2}, {mria_rlog,role,1}}, + {{mria_membership,handle_cast,2}, {mria_schema,shards,0}}, + {{mria_membership,role,1}, {mria_rlog,role,1}}]} ]}. {xref_checks, diff --git a/src/mria_lb.erl b/src/mria_lb.erl index 3ca61487..88d14d1f 100644 --- a/src/mria_lb.erl +++ b/src/mria_lb.erl @@ -25,6 +25,7 @@ , core_nodes/0 , join_cluster/1 , leave_cluster/0 + , notify_core_node_down/1 ]). %% gen_server callbacks @@ -92,6 +93,28 @@ leave_cluster() -> error(Err) end. +%% Immediately unset shard core nodes matching the down node to reduce the risk of +%% trying to communicate with the failed node, for example delegating +%% a write operation (RPC) to that failed core node +%% TODO: race conditions with mria_lb gen_server, which can be the first to update +%% a core node per shard +-spec notify_core_node_down(node()) -> ok. +notify_core_node_down(DownNode) -> + case mria_config:role() of + replicant -> + lists:foreach( + fun(Shard) -> + case mria_status:get_core_node(Shard, 0) of + {ok, DownNode} -> + mria_status:notify_core_node_down(Shard); + _ -> ok + end + end, + mria_schema:shards()); + _ -> + ok + end. + %%================================================================================ %% gen_server callbacks %%================================================================================ diff --git a/src/mria_membership.erl b/src/mria_membership.erl index 00f1f9d2..6f0ab576 100644 --- a/src/mria_membership.erl +++ b/src/mria_membership.erl @@ -341,6 +341,7 @@ handle_cast({node_down, Node}, State) -> insert(Member#member{status = down}); [] -> ignore end, + mria_lb:notify_core_node_down(Node), notify({node, down, Node}, State), {noreply, State}; @@ -384,6 +385,7 @@ handle_cast({leaving, Node}, State) -> insert(Member#member{status = leaving}); [] -> ignore end, + mria_lb:notify_core_node_down(Node), notify({node, leaving, Node}, State), {noreply, State}; @@ -409,6 +411,7 @@ handle_cast({mnesia_down, Node}, State) -> [] -> ignore end, ?tp(mria_membership_mnesia_down, #{node => Node}), + mria_lb:notify_core_node_down(Node), notify({mnesia, down, Node}, State), {noreply, State};