fix: immediately unset a core node for all shards, when that core node is down

rebar.config

@@ -35,8 +35,10 @@
  , {"closure(E) | mria_mnesia : Mod || [mria, mria_lb, mria_schema, mria_node_monitor, mria_membership, mria_rlog] : Mod",
     [{{mria_mnesia,join_cluster,1}, {mria_rlog,role,1}}]}
  , {"closure(E) | mria_membership : Mod || [mria, mria_lb, mria_schema, mria_node_monitor, mria_rlog] : Mod",
-    [{{mria_membership,handle_cast,2}, {mria_rlog,role,1}},
-     {{mria_membership,role,1},{mria_rlog,role,1}}]}
+    [{{mria_membership,handle_cast,2}, {mria_lb,notify_core_node_down,1}},
+     {{mria_membership,handle_cast,2}, {mria_rlog,role,1}},
+     {{mria_membership,handle_cast,2}, {mria_schema,shards,0}},
+     {{mria_membership,role,1}, {mria_rlog,role,1}}]}
  ]}.
 
 {xref_checks,

src/mria_lb.erl

@@ -25,6 +25,7 @@
         , core_nodes/0
         , join_cluster/1
         , leave_cluster/0
+        , notify_core_node_down/1
         ]).
 
 %% gen_server callbacks
@@ -92,6 +93,28 @@ leave_cluster() ->
             error(Err)
     end.
 
+%% Immediately unset shard core nodes matching the down node to reduce the risk of
+%% trying to communicate with the failed node, for example delegating
+%% a write operation (RPC) to that failed core node
+%% TODO: race conditions with mria_lb gen_server, which can be the first to update
+%% a core node per shard
+-spec notify_core_node_down(node()) -> ok.
+notify_core_node_down(DownNode) ->
+    case mria_config:role() of
+        replicant ->
+            lists:foreach(
+              fun(Shard) ->
+                      case mria_status:get_core_node(Shard, 0) of
+                          {ok, DownNode} ->
+                              mria_status:notify_core_node_down(Shard);
+                          _ -> ok
+                      end
+              end,
+              mria_schema:shards());
+        _ ->
+            ok
+    end.
+
 %%================================================================================
 %% gen_server callbacks
 %%================================================================================

src/mria_membership.erl

@@ -341,6 +341,7 @@ handle_cast({node_down, Node}, State) ->
             insert(Member#member{status = down});
         [] -> ignore
     end,
+    mria_lb:notify_core_node_down(Node),
     notify({node, down, Node}, State),
     {noreply, State};
 
@@ -384,6 +385,7 @@ handle_cast({leaving, Node}, State) ->
             insert(Member#member{status = leaving});
         [] -> ignore
     end,
+    mria_lb:notify_core_node_down(Node),
     notify({node, leaving, Node}, State),
     {noreply, State};
 
@@ -409,6 +411,7 @@ handle_cast({mnesia_down, Node}, State) ->
         [] -> ignore
     end,
     ?tp(mria_membership_mnesia_down, #{node => Node}),
+    mria_lb:notify_core_node_down(Node),
     notify({mnesia, down, Node}, State),
     {noreply, State};