node: add skew enforcement prometheus alert

Author: djoshy

cmd/machine-config-controller/start.go

@@ -272,6 +272,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle
 			ctx.OCLInformerFactory.Machineconfiguration().V1().MachineOSBuilds(),
 			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigNodes(),
 			ctx.ConfigInformerFactory.Config().V1().Schedulers(),
+			ctx.OperatorInformerFactory.Operator().V1().MachineConfigurations(),
 			ctx.ClientBuilder.KubeClientOrDie("node-update-controller"),
 			ctx.ClientBuilder.MachineConfigClientOrDie("node-update-controller"),
 			ctx.FeatureGatesHandler,

install/0000_90_machine-config_01_prometheus-rules.yaml

@@ -38,7 +38,19 @@ spec:
           annotations:
             summary: "Triggers when nodes in a pool have overlapping labels such as master, worker, and a custom label therefore a choice must be made as to which is honored."
             description: "Node {{ $labels.exported_node }} has triggered a pool alert due to a label change. For more details check MachineConfigController pod logs: oc logs -f -n {{ $labels.namespace }} machine-config-controller-xxxxx -c machine-config-controller"
-            runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/machine-config-operator/MachineConfigControllerPoolAlert.md 
+            runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/machine-config-operator/MachineConfigControllerPoolAlert.md
+    - name: mcc-boot-image-skew-enforcement-none
+      rules:
+        - alert: MCCBootImageSkewEnforcementNone
+          expr: |
+            mcc_boot_image_skew_enforcement_none == 1
+          labels:
+            namespace: openshift-machine-config-operator
+            severity: info
+          annotations:
+            summary: "Boot image skew enforcement is disabled. Scaling operations may not be successful."
+            description: "Boot image skew enforcement mode is set to None. When scaling up, new nodes may be provisioned with older boot images that could introduce compatibility issues. Consider manually updating boot images to match the cluster version."
+            runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/machine-config-operator/MCCBootImageSkewEnforcementNone.md
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule

pkg/controller/common/metrics.go

@@ -25,6 +25,15 @@ var (
 			Help: "state of OS image override",
 		}, []string{"pool"})
 
+	// MCCBootImageSkewEnforcementNone indicates when boot image skew enforcement is disabled.
+	// Set to 1 when mode is "None", 0 otherwise. A value of 1 indicates scaling operations may
+	// not be successful
+	MCCBootImageSkewEnforcementNone = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "mcc_boot_image_skew_enforcement_none",
+			Help: "Set to 1 when boot image skew enforcement mode is None, indicating scaling may not be successful as bootimages are out of date",
+		})
+
 	// MCCDrainErr logs failed drain
 	MCCDrainErr = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
@@ -94,6 +103,7 @@ func RegisterMCCMetrics() error {
 		MCCUpdatedMachineCount,
 		MCCDegradedMachineCount,
 		MCCUnavailableMachineCount,
+		MCCBootImageSkewEnforcementNone,
 	})
 
 	if err != nil {

pkg/controller/node/node_controller.go

@@ -15,12 +15,15 @@ import (
 	configv1 "github.com/openshift/api/config/v1"
 	features "github.com/openshift/api/features"
 	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
+	opv1 "github.com/openshift/api/operator/v1"
 
 	cligoinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
 	cligolistersv1 "github.com/openshift/client-go/config/listers/config/v1"
 	mcfgclientset "github.com/openshift/client-go/machineconfiguration/clientset/versioned"
 	"github.com/openshift/client-go/machineconfiguration/clientset/versioned/scheme"
 	mcfginformersv1 "github.com/openshift/client-go/machineconfiguration/informers/externalversions/machineconfiguration/v1"
+	mcopinformersv1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1"
+	mcoplistersv1 "github.com/openshift/client-go/operator/listers/operator/v1"
 
 	mcfglistersv1 "github.com/openshift/client-go/machineconfiguration/listers/machineconfiguration/v1"
 	"github.com/openshift/library-go/pkg/operator/v1helpers"
@@ -108,6 +111,9 @@ type Controller struct {
 	schedulerList         cligolistersv1.SchedulerLister
 	schedulerListerSynced cache.InformerSynced
 
+	mcopLister       mcoplistersv1.MachineConfigurationLister
+	mcopListerSynced cache.InformerSynced
+
 	queue workqueue.TypedRateLimitingInterface[string]
 
 	fgHandler ctrlcommon.FeatureGatesHandler
@@ -127,6 +133,7 @@ func New(
 	mosbInformer mcfginformersv1.MachineOSBuildInformer,
 	mcnInformer mcfginformersv1.MachineConfigNodeInformer,
 	schedulerInformer cligoinformersv1.SchedulerInformer,
+	mcopInformer mcopinformersv1.MachineConfigurationInformer,
 	kubeClient clientset.Interface,
 	mcfgClient mcfgclientset.Interface,
 	fgHandler ctrlcommon.FeatureGatesHandler,
@@ -141,6 +148,7 @@ func New(
 		podInformer,
 		mcnInformer,
 		schedulerInformer,
+		mcopInformer,
 		kubeClient,
 		mcfgClient,
 		defaultUpdateDelay,
@@ -158,6 +166,7 @@ func NewWithCustomUpdateDelay(
 	mosbInformer mcfginformersv1.MachineOSBuildInformer,
 	mcnInformer mcfginformersv1.MachineConfigNodeInformer,
 	schedulerInformer cligoinformersv1.SchedulerInformer,
+	mcopInformer mcopinformersv1.MachineConfigurationInformer,
 	kubeClient clientset.Interface,
 	mcfgClient mcfgclientset.Interface,
 	updateDelay time.Duration,
@@ -173,6 +182,7 @@ func NewWithCustomUpdateDelay(
 		podInformer,
 		mcnInformer,
 		schedulerInformer,
+		mcopInformer,
 		kubeClient,
 		mcfgClient,
 		updateDelay,
@@ -191,6 +201,7 @@ func newController(
 	podInformer coreinformersv1.PodInformer,
 	mcnInformer mcfginformersv1.MachineConfigNodeInformer,
 	schedulerInformer cligoinformersv1.SchedulerInformer,
+	mcopInformer mcopinformersv1.MachineConfigurationInformer,
 	kubeClient clientset.Interface,
 	mcfgClient mcfgclientset.Interface,
 	updateDelay time.Duration,
@@ -263,6 +274,9 @@ func newController(
 	ctrl.schedulerList = schedulerInformer.Lister()
 	ctrl.schedulerListerSynced = schedulerInformer.Informer().HasSynced
 
+	ctrl.mcopLister = mcopInformer.Lister()
+	ctrl.mcopListerSynced = mcopInformer.Informer().HasSynced
+
 	return ctrl
 }
 
@@ -271,7 +285,7 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
 	defer ctrl.queue.ShutDown()
 
-	if !cache.WaitForCacheSync(stopCh, ctrl.ccListerSynced, ctrl.mcListerSynced, ctrl.mcpListerSynced, ctrl.moscListerSynced, ctrl.mosbListerSynced, ctrl.nodeListerSynced, ctrl.schedulerListerSynced) {
+	if !cache.WaitForCacheSync(stopCh, ctrl.ccListerSynced, ctrl.mcListerSynced, ctrl.mcpListerSynced, ctrl.moscListerSynced, ctrl.mosbListerSynced, ctrl.nodeListerSynced, ctrl.schedulerListerSynced, ctrl.mcopListerSynced) {
 		return
 	}
 
@@ -1847,5 +1861,31 @@ func (ctrl *Controller) syncMetrics() error {
 		ctrlcommon.MCCDegradedMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.DegradedMachineCount))
 		ctrlcommon.MCCUnavailableMachineCount.WithLabelValues(pool.Name).Set(float64(pool.Status.UnavailableMachineCount))
 	}
+
+	// Update boot image skew enforcement metric
+	ctrl.syncBootImageSkewEnforcementMetric()
+
 	return nil
 }
+
+// syncBootImageSkewEnforcementMetric updates the mcc_boot_image_skew_enforcement_none metric
+// based on the current BootImageSkewEnforcementStatus mode in MachineConfiguration.
+// The metric is set to 1 when mode is "None", indicating that scaling operations may
+// not be successful.
+func (ctrl *Controller) syncBootImageSkewEnforcementMetric() {
+	if ctrl.fgHandler == nil || !ctrl.fgHandler.Enabled(features.FeatureGateBootImageSkewEnforcement) {
+		return
+	}
+
+	mcop, err := ctrl.mcopLister.Get(ctrlcommon.MCOOperatorKnobsObjectName)
+	if err != nil {
+		klog.V(4).Infof("Failed to get MachineConfiguration for boot image skew enforcement metric: %v", err)
+		return
+	}
+
+	if mcop.Status.BootImageSkewEnforcementStatus.Mode == opv1.BootImageSkewEnforcementModeStatusNone {
+		ctrlcommon.MCCBootImageSkewEnforcementNone.Set(1)
+	} else {
+		ctrlcommon.MCCBootImageSkewEnforcementNone.Set(0)
+	}
+}

pkg/controller/node/node_controller_test.go

@@ -34,6 +34,8 @@ import (
 	configv1informer "github.com/openshift/client-go/config/informers/externalversions"
 	"github.com/openshift/client-go/machineconfiguration/clientset/versioned/fake"
 	informers "github.com/openshift/client-go/machineconfiguration/informers/externalversions"
+	fakeoperatorclient "github.com/openshift/client-go/operator/clientset/versioned/fake"
+	operatorinformer "github.com/openshift/client-go/operator/informers/externalversions"
 	"github.com/openshift/machine-config-operator/pkg/constants"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	daemonconsts "github.com/openshift/machine-config-operator/pkg/daemon/constants"
@@ -102,17 +104,20 @@ func (f *fixture) newControllerWithStopChan(stopCh <-chan struct{}) *Controller
 	f.client = fake.NewSimpleClientset(f.objects...)
 	f.kubeclient = k8sfake.NewSimpleClientset(f.kubeobjects...)
 	f.schedulerClient = fakeconfigv1client.NewSimpleClientset(f.schedulerObjects...)
+	operatorClient := fakeoperatorclient.NewSimpleClientset()
 
 	i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc())
 	k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc())
 	ci := configv1informer.NewSharedInformerFactory(f.schedulerClient, noResyncPeriodFunc())
+	oi := operatorinformer.NewSharedInformerFactory(operatorClient, noResyncPeriodFunc())
 	c := NewWithCustomUpdateDelay(i.Machineconfiguration().V1().ControllerConfigs(), i.Machineconfiguration().V1().MachineConfigs(), i.Machineconfiguration().V1().MachineConfigPools(), k8sI.Core().V1().Nodes(),
-		k8sI.Core().V1().Pods(), i.Machineconfiguration().V1().MachineOSConfigs(), i.Machineconfiguration().V1().MachineOSBuilds(), i.Machineconfiguration().V1().MachineConfigNodes(), ci.Config().V1().Schedulers(), f.kubeclient, f.client, time.Millisecond, f.fgHandler)
+		k8sI.Core().V1().Pods(), i.Machineconfiguration().V1().MachineOSConfigs(), i.Machineconfiguration().V1().MachineOSBuilds(), i.Machineconfiguration().V1().MachineConfigNodes(), ci.Config().V1().Schedulers(), oi.Operator().V1().MachineConfigurations(), f.kubeclient, f.client, time.Millisecond, f.fgHandler)
 
 	c.ccListerSynced = alwaysReady
 	c.mcpListerSynced = alwaysReady
 	c.nodeListerSynced = alwaysReady
 	c.schedulerListerSynced = alwaysReady
+	c.mcopListerSynced = alwaysReady
 	c.eventRecorder = &record.FakeRecorder{}
 
 	i.Start(stopCh)

test/e2e-bootstrap/bootstrap_test.go

@@ -624,6 +624,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle
 			ctx.InformerFactory.Machineconfiguration().V1().MachineOSBuilds(),
 			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigNodes(),
 			ctx.ConfigInformerFactory.Config().V1().Schedulers(),
+			ctx.OperatorInformerFactory.Operator().V1().MachineConfigurations(),
 			ctx.ClientBuilder.KubeClientOrDie("node-update-controller"),
 			ctx.ClientBuilder.MachineConfigClientOrDie("node-update-controller"),
 			ctx.FeatureGatesHandler,