DAOS-11134 control: Add leadership check to doGroupUpdate() (#9737) (#9870)

In rare circumstances, this method could be called after
leadership is lost but before the worker loop is exited.
Add a final check for leadership before invoking the group
update dRPC in order to avoid trying to make the group
update on a non-leader replica.

Also fixes a bug where a sync group update could be followed
by an unnecessary async group update.

Signed-off-by: Michael MacDonald <[email protected]>

Author: mjmac

src/control/server/mgmt_system.go

@@ -190,8 +190,8 @@ func (svc *mgmtSvc) joinLoop(parent context.Context) {
 					svc.log.Errorf("sync GroupUpdate failed: %s", err)
 					continue
 				}
+				groupUpdateNeeded = false
 			}
-			groupUpdateNeeded = false
 		case <-groupUpdateTimer.C:
 			if !groupUpdateNeeded {
 				continue
@@ -364,6 +364,11 @@ func (svc *mgmtSvc) doGroupUpdate(ctx context.Context, forced bool) error {
 		rankSet.Add(rank)
 	}
 
+	// Final check to make sure we're still leader.
+	if err := svc.sysdb.CheckLeader(); err != nil {
+		return err
+	}
+
 	svc.log.Debugf("group update request: version: %d, ranks: %s", req.MapVersion, rankSet)
 	dResp, err := svc.harness.CallDrpc(ctx, drpc.MethodGroupUpdate, req)
 	if err != nil {

src/control/system/raft.go

@@ -318,7 +318,10 @@ func (db *Database) submitRaftUpdate(data []byte) error {
 		// signal some callers to retry the operation on the
 		// new leader.
 		if IsRaftLeadershipError(err) {
-			return ErrRaftUnavail
+			return &ErrNotLeader{
+				LeaderHint: db.leaderHint(),
+				Replicas:   db.cfg.stringReplicas(db.getReplica()),
+			}
 		}
 
 		return err