Skip to content

Commit 6d48b64

Browse files
Merge pull request #5547 from djoshy/implement-skew-enforcement-e2e
MCO-1984: Add Component Readiness tests for boot image skew enforcement
2 parents 56e63ae + ed0b5ca commit 6d48b64

14 files changed

+311
-52
lines changed
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package extended
2+
3+
import (
4+
exutil "github.com/openshift/machine-config-operator/test/extended-priv/util"
5+
)
6+
7+
// ClusterOperator struct is used to handle ClusterOperator resources in OCP
8+
type ClusterOperator struct {
9+
template string
10+
Resource
11+
}
12+
13+
// ClusterOperatorList struct handles list of COs
14+
type ClusterOperatorList struct {
15+
ResourceList
16+
}
17+
18+
// NewClusterOperator create a ClusterOperator struct
19+
func NewClusterOperator(oc *exutil.CLI, name string) *ClusterOperator {
20+
return &ClusterOperator{Resource: *NewResource(oc, "co", name)}
21+
}
22+
23+
// NewClusterOperatorList create a ClusterOperatorList struct
24+
func NewClusterOperatorList(oc *exutil.CLI) *ClusterOperatorList {
25+
return &ClusterOperatorList{*NewResourceList(oc, "co")}
26+
}

test/extended-priv/const.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,4 +95,16 @@ const (
9595
NodeDisruptionPolicyActionRestart = "Restart"
9696
NodeDisruptionPolicyActionDrain = "Drain"
9797
NodeDisruptionPolicyActionDaemonReload = "DaemonReload"
98+
// BootImageSkewEnforcement feature constants
99+
100+
// RHCOSVersionMode is used to signify that the cluster boot image is described via RHCOS version
101+
RHCOSVersionMode = "RHCOSVersion"
102+
// OCPVersionMode is used to signify that the cluster boot image is described via OCP version
103+
OCPVersionMode = "OCPVersion"
104+
// SkewEnforcementManualMode indicates boot image updates require manual intervention
105+
SkewEnforcementManualMode = "Manual"
106+
// SkewEnforcementAutomaticMode indicates boot image updates are applied automatically
107+
SkewEnforcementAutomaticMode = "Automatic"
108+
// SkewEnforcementNoneMode indicates boot image skew enforcement is disabled
109+
SkewEnforcementNoneMode = "None"
98110
)

test/extended-priv/gomega_matchers.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,3 +171,8 @@ func (matcher *AvailableMatcher) NegatedFailureMessage(actual interface{}) (mess
171171
func BeAvailable() types.GomegaMatcher {
172172
return &DegradedMatcher{&conditionMatcher{conditionType: "Available", field: "status", expected: "True"}}
173173
}
174+
175+
// BeUpgradeable returns the gomega matcher to check if a resource is upgradeable or not.
176+
func BeUpgradeable() types.GomegaMatcher {
177+
return &conditionMatcher{conditionType: "Upgradeable", field: "status", expected: "True"}
178+
}

test/extended-priv/machineconfiguration.go

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package extended
22

33
import (
4+
"fmt"
45
"strings"
56

7+
o "github.com/onsi/gomega"
68
exutil "github.com/openshift/machine-config-operator/test/extended-priv/util"
79
logger "github.com/openshift/machine-config-operator/test/extended-priv/util/logext"
810
)
@@ -83,18 +85,18 @@ func (mc MachineConfiguration) GetAllManagedBootImagesResources() ([]string, err
8385
return strings.Fields(result), nil
8486
}
8587

86-
// SetManualSkew configures bootImageSkewEnforcement to Manual mode with the specified mode type and version.
88+
// SetManualSkew configures bootImageSkewEnforcement to Manual mode with the specified mode type and version./
8789
// mode should be "RHCOSVersion" or "OCPVersion", version is the corresponding version string.
8890
func (mc MachineConfiguration) SetManualSkew(mode, version string) error {
8991
logger.Infof("Setting .spec.bootImageSkewEnforcement to Manual mode (%s: %s) on %s", mode, version, mc)
9092
var versionField string
9193
switch mode {
92-
case "RHCOSVersion":
94+
case RHCOSVersionMode:
9395
versionField = `"rhcosVersion":"` + version + `"`
94-
case "OCPVersion":
96+
case OCPVersionMode:
9597
versionField = `"ocpVersion":"` + version + `"`
9698
default:
97-
versionField = `"rhcosVersion":"` + version + `"`
99+
return fmt.Errorf("unsupported manual skew mode: %s", mode)
98100
}
99101
return mc.Patch("merge", `{"spec":{"bootImageSkewEnforcement":{"mode":"Manual","manual":{"mode":"`+mode+`",`+versionField+`}}}}`)
100102
}
@@ -118,3 +120,31 @@ func (mc MachineConfiguration) RemoveSkew() error {
118120
}
119121
return mc.Patch("json", `[{ "op": "remove", "path": "/spec/bootImageSkewEnforcement"}]`)
120122
}
123+
124+
// GetBootImageSkewEnforcementStatusMode returns the .status.bootImageSkewEnforcementStatus.mode field
125+
func (mc MachineConfiguration) GetBootImageSkewEnforcementStatusMode() (string, error) {
126+
return mc.Get(`{.status.bootImageSkewEnforcementStatus.mode}`)
127+
}
128+
129+
// WaitForBootImageSkewEnforcementStatusMode waits for the bootImageSkewEnforcementStatus.mode to reach the expected value
130+
func (mc MachineConfiguration) WaitForBootImageSkewEnforcementStatusMode(expectedMode string) {
131+
o.Eventually(mc.GetBootImageSkewEnforcementStatusMode, "2m", "10s").Should(o.Equal(expectedMode))
132+
}
133+
134+
// WaitForBootImageControllerComplete waits for the boot image controller to finish processing
135+
// (BootImageUpdateProgressing=False)
136+
func (mc MachineConfiguration) WaitForBootImageControllerComplete() {
137+
o.Eventually(mc.IsConditionStatusTrue, "2m", "2s").WithArguments("BootImageUpdateProgressing").
138+
Should(o.BeFalse(), "Expected %s BootImageUpdateProgressing to be False.\n%s", mc, mc.PrettyString())
139+
}
140+
141+
// WaitForBootImageControllerDegradedState waits for the boot image controller to be in the expected degraded state
142+
func (mc MachineConfiguration) WaitForBootImageControllerDegradedState(degraded bool) {
143+
if degraded {
144+
o.Eventually(mc.IsConditionStatusTrue, "2m", "2s").WithArguments("BootImageUpdateDegraded").
145+
Should(o.BeTrue(), "Expected %s BootImageUpdateDegraded to be True.\n%s", mc, mc.PrettyString())
146+
} else {
147+
o.Eventually(mc.IsConditionStatusTrue, "2m", "2s").WithArguments("BootImageUpdateDegraded").
148+
Should(o.BeFalse(), "Expected %s BootImageUpdateDegraded to be False.\n%s", mc, mc.PrettyString())
149+
}
150+
}

test/extended-priv/mco_bootimages.go

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati
5454
var (
5555
duplicatedMachinesetName = fmt.Sprintf("cloned-tc-%s", GetCurrentTestPolarionIDNumber())
5656
firstMachineSet = NewMachineSetList(oc.AsAdmin(), MachineAPINamespace).GetAllOrFail()[0]
57-
fakeImageName = GetValidUpdateBootImageValue(oc.AsAdmin())
57+
fakeImageName = getBackdatedBootImage(oc.AsAdmin())
5858
)
5959

6060
exutil.By("Duplicate machineset for testing")
@@ -112,7 +112,7 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati
112112
g.It("[PolarionID:74240][OTP] ManagedBootImages. Restore All MachineSet images", g.Label("Platform:aws", "Platform:gcp", "Platform:vsphere", "Platform:azure"), func() {
113113
var (
114114
machineSet = NewMachineSetList(oc.AsAdmin(), MachineAPINamespace).GetAllOrFail()[0]
115-
fakeImageName = GetValidUpdateBootImageValue(oc.AsAdmin())
115+
fakeImageName = getBackdatedBootImage(oc.AsAdmin())
116116
clonedMSName = "cloned-tc-74240"
117117
clonedWrongBootImageMSName = "cloned-tc-74240-wrong-boot-image"
118118
clonedOwnedMSName = "cloned-tc-74240-owned"
@@ -219,7 +219,7 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati
219219
g.It("[PolarionID:74239][OTP] ManagedBootImages. Restore Partial MachineSet images", g.Label("Platform:aws", "Platform:gcp", "Platform:vsphere", "Platform:azure"), func() {
220220
var (
221221
machineSet = NewMachineSetList(oc.AsAdmin(), MachineAPINamespace).GetAllOrFail()[0]
222-
fakeImageName = GetValidUpdateBootImageValue(oc.AsAdmin())
222+
fakeImageName = getBackdatedBootImage(oc.AsAdmin())
223223
clonedMSLabelName = "cloned-tc-74239-label"
224224
clonedMSNoLabelName = "cloned-tc-74239-no-label"
225225
clonedMSLabelOwnedName = "cloned-tc-74239-label-owned"
@@ -316,7 +316,7 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati
316316
var (
317317
machineConfiguration = GetMachineConfiguration(oc.AsAdmin())
318318
machineSet = NewMachineSetList(oc.AsAdmin(), MachineAPINamespace).GetAllOrFail()[0]
319-
fakeImageName = GetValidUpdateBootImageValue(oc.AsAdmin())
319+
fakeImageName = getBackdatedBootImage(oc.AsAdmin())
320320
clonedMSName = "cloned-tc-74751-copy"
321321
labelName = "test"
322322
labelValue = "update"
@@ -508,7 +508,7 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati
508508

509509
machineConfiguration = GetMachineConfiguration(oc.AsAdmin())
510510
machineSet = NewMachineSetList(oc.AsAdmin(), MachineAPINamespace).GetAllOrFail()[0]
511-
fakeImageName = GetValidUpdateBootImageValue(oc.AsAdmin())
511+
fakeImageName = getBackdatedBootImage(oc.AsAdmin())
512512
labelName = "test"
513513
labelValue = "update"
514514

@@ -788,13 +788,27 @@ func getCoreOsBootImageFromConfigMapOrFail(platform, region string, arch archite
788788
return image
789789
}
790790

791+
// GetRHCOSVersionFromConfigMap retrieves the RHCOS release version from the coreos-bootimages ConfigMap
792+
func GetRHCOSVersionFromConfigMap(oc *exutil.CLI) string {
793+
coreosBootimagesCM := NewConfigMap(oc.AsAdmin(), MachineConfigNamespace, "coreos-bootimages")
794+
streamJSON, err := coreosBootimagesCM.GetDataValue("stream")
795+
o.Expect(err).NotTo(o.HaveOccurred(), "Error getting stream data from coreos-bootimages configmap")
796+
797+
parsedStream := gjson.Parse(streamJSON)
798+
// Get the release version from aws artifacts
799+
rhcosVersion := parsedStream.Get("architectures.x86_64.artifacts.aws.release").String()
800+
o.Expect(rhcosVersion).NotTo(o.BeEmpty(), "RHCOS version not found in coreos-bootimages configmap")
801+
802+
return rhcosVersion
803+
}
804+
791805
// testUserDataUpdateFailure function that executes the common parts of the update spec v3 negative test cases
792806
func testUserDataUpdateFailure(oc *exutil.CLI, clonedMSName, clonedSecretName, expectedFailedMessageRegexp string, userDataModifyFunc func(userData string) (string, error)) {
793807

794808
var (
795809
machineConfiguration = GetMachineConfiguration(oc.AsAdmin())
796810
machineSet = NewMachineSetList(oc.AsAdmin(), MachineAPINamespace).GetAllOrFail()[0]
797-
fakeImageName = GetValidUpdateBootImageValue(oc.AsAdmin())
811+
fakeImageName = getBackdatedBootImage(oc.AsAdmin())
798812
labelName = "test"
799813
labelValue = "update"
800814
secondLabelValue = "update2"
@@ -887,9 +901,9 @@ func checkManagedBootImagesStatus(mc *MachineConfiguration, mode string) {
887901
Should(o.Equal(mode), "Error: The %s mode does not match even after patched", mode)
888902
}
889903

890-
// GetValidUpdateBootImageValue returns a valid boot image value for testing based on platform
904+
// getBackdatedBootImage returns a valid boot image value for testing based on platform
891905
// MCO will only update images previously published in the installer. This function returns one of those valid images
892-
func GetValidUpdateBootImageValue(oc *exutil.CLI) string {
906+
func getBackdatedBootImage(oc *exutil.CLI) string {
893907
var (
894908
platform = exutil.CheckPlatform(oc)
895909
)
Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
package extended
2+
3+
import (
4+
"context"
5+
6+
g "github.com/onsi/ginkgo/v2"
7+
o "github.com/onsi/gomega"
8+
constants "github.com/openshift/machine-config-operator/pkg/controller/common"
9+
exutil "github.com/openshift/machine-config-operator/test/extended-priv/util"
10+
logger "github.com/openshift/machine-config-operator/test/extended-priv/util/logext"
11+
)
12+
13+
const (
14+
// Test RHCOS & OCP versions for skew enforcement tests
15+
rhcosVersionExceedsSkew = "48.84.202208021106-0"
16+
ocpVersionExceedsSkew = "4.12.0"
17+
)
18+
19+
var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/disruptive][Serial][Disruptive][OCPFeatureGate:BootImageSkewEnforcement]", func() {
20+
defer g.GinkgoRecover()
21+
22+
var (
23+
oc = exutil.NewCLI("mco-bootimage", exutil.KubeConfigPath()).AsAdmin()
24+
machineConfiguration *MachineConfiguration
25+
originalSpec string
26+
mcoCO *ClusterOperator
27+
)
28+
29+
g.BeforeEach(func() {
30+
// Skip on single-node topologies
31+
exutil.SkipOnSingleNodeTopology(oc)
32+
machineConfiguration = GetMachineConfiguration(oc)
33+
// Save initial state to restore after each test
34+
originalSpec = machineConfiguration.GetSpecOrFail()
35+
mcoCO = NewClusterOperator(oc, "machine-config")
36+
})
37+
38+
g.AfterEach(func() {
39+
exutil.By("Restoring MachineConfiguration to original state")
40+
o.Expect(machineConfiguration.SetSpec(originalSpec)).To(o.Succeed())
41+
})
42+
43+
g.It("Verify Manual mode with RHCOSVersion and Upgradeable (Happy case) [apigroup:machineconfiguration.openshift.io]", func() {
44+
// Get the current RHCOS version from the coreos-bootimages configmap (guaranteed to be within skew)
45+
rhcosVersionWithinSkew := GetRHCOSVersionFromConfigMap(oc)
46+
logger.Infof("Using RHCOS version from configmap: %s", rhcosVersionWithinSkew)
47+
48+
// Set manual mode with a boot image version that is within skew limits
49+
o.Expect(machineConfiguration.SetManualSkew(RHCOSVersionMode, rhcosVersionWithinSkew)).To(o.Succeed())
50+
51+
// Wait for the controller to reflect Manual mode in skew enforcement status
52+
machineConfiguration.WaitForBootImageSkewEnforcementStatusMode(SkewEnforcementManualMode)
53+
54+
// Check machine-config CO upgradeable status, should be set to true
55+
o.Eventually(mcoCO, "1m", "10s").Should(BeUpgradeable(), "co/machine-config should be upgradeable when manual skew version is within limits")
56+
})
57+
58+
g.It("Verify Manual mode with RHCOSVersion and Upgradeable (Sad case) [apigroup:machineconfiguration.openshift.io]", func() {
59+
// Set manual mode with a boot image version that is NOT within skew limits
60+
o.Expect(machineConfiguration.SetManualSkew(RHCOSVersionMode, rhcosVersionExceedsSkew)).To(o.Succeed())
61+
62+
// Wait for the controller to reflect Manual mode in skew enforcement status
63+
machineConfiguration.WaitForBootImageSkewEnforcementStatusMode(SkewEnforcementManualMode)
64+
65+
// Check machine-config CO upgradeable status, should be set to false
66+
o.Eventually(mcoCO, "1m", "10s").ShouldNot(BeUpgradeable(), "co/machine-config should not be upgradeable when manual skew version exceeds limits")
67+
})
68+
69+
g.It("Verify Manual mode with OCPVersion and Upgradeable (Happy case) [apigroup:machineconfiguration.openshift.io]", func() {
70+
// Set manual mode with a boot image version that is within skew limits
71+
o.Expect(machineConfiguration.SetManualSkew(OCPVersionMode, constants.OCPVersionBootImageSkewLimit)).To(o.Succeed())
72+
73+
// Wait for the controller to reflect Manual mode in skew enforcement status
74+
machineConfiguration.WaitForBootImageSkewEnforcementStatusMode(SkewEnforcementManualMode)
75+
76+
// Check machine-config CO upgradeable status, should be set to true
77+
o.Eventually(mcoCO, "1m", "10s").Should(BeUpgradeable(), "co/machine-config should be upgradeable when manual skew version is within limits")
78+
})
79+
80+
g.It("Verify Manual mode with OCPVersion and Upgradeable (Sad case) [apigroup:machineconfiguration.openshift.io]", func() {
81+
// Set manual mode with a boot image version that is NOT within skew limits
82+
o.Expect(machineConfiguration.SetManualSkew(OCPVersionMode, ocpVersionExceedsSkew)).To(o.Succeed())
83+
84+
// Wait for the controller to reflect Manual mode in skew enforcement status
85+
machineConfiguration.WaitForBootImageSkewEnforcementStatusMode(SkewEnforcementManualMode)
86+
87+
// Check machine-config CO upgradeable status, should be set to false
88+
o.Eventually(mcoCO, "1m", "10s").ShouldNot(BeUpgradeable(), "co/machine-config should not be upgradeable when manual skew version exceeds limits")
89+
})
90+
91+
g.It("Verify Automatic mode and Upgradeable (Happy Case) [apigroup:machineconfiguration.openshift.io]", func() {
92+
// only applicable on GCP, AWS clusters
93+
skipTestIfSupportedPlatformNotMatched(oc, GCPPlatform, AWSPlatform)
94+
95+
// No opinion on skew enforcement for these platforms will result in Automatic mode
96+
o.Expect(machineConfiguration.RemoveSkew()).To(o.Succeed())
97+
98+
// Wait for the controller to reflect Automatic mode in skew enforcement status
99+
machineConfiguration.WaitForBootImageSkewEnforcementStatusMode(SkewEnforcementAutomaticMode)
100+
101+
// Pick a random machineset to test
102+
machineSetUnderTest := NewMachineSetList(oc.AsAdmin(), MachineAPINamespace).GetAllOrFail()[0]
103+
logger.Infof("MachineSet under test: %s", machineSetUnderTest.name)
104+
105+
// Save and restore full spec to ensure cleanup regardless of what we modify
106+
originalMachineSetSpec := machineSetUnderTest.GetSpecOrFail()
107+
defer func() {
108+
o.Expect(machineSetUnderTest.SetSpec(originalMachineSetSpec)).To(o.Succeed())
109+
}()
110+
111+
// Patch the boot image to an older version to trigger an update loop
112+
backdatedBootImage := getBackdatedBootImage(oc)
113+
o.Expect(machineSetUnderTest.SetCoreOsBootImage(backdatedBootImage)).To(o.Succeed())
114+
logger.Infof("Set backdated boot image '%s' in MachineSet %s to trigger update loop", backdatedBootImage, machineSetUnderTest.name)
115+
116+
// Verify that the boot image controller has finished processing
117+
machineConfiguration.WaitForBootImageControllerComplete()
118+
119+
// Verify that the boot image controller is not degraded
120+
machineConfiguration.WaitForBootImageControllerDegradedState(false)
121+
122+
// Check machine-config CO upgradeable status, should be set to True
123+
o.Eventually(mcoCO, "1m", "10s").Should(BeUpgradeable(), "co/machine-config should be upgradeable with restored machineset")
124+
})
125+
126+
g.It("Verify Automatic mode and Upgradeable (Sad Case) [apigroup:machineconfiguration.openshift.io]", func(_ context.Context) {
127+
// only applicable on GCP, AWS clusters
128+
skipTestIfSupportedPlatformNotMatched(oc, GCPPlatform, AWSPlatform)
129+
130+
// No opinion on skew enforcement for these platforms will result in Automatic mode
131+
o.Expect(machineConfiguration.RemoveSkew()).To(o.Succeed())
132+
133+
// Wait for the controller to reflect Automatic mode in skew enforcement status
134+
machineConfiguration.WaitForBootImageSkewEnforcementStatusMode(SkewEnforcementAutomaticMode)
135+
136+
// Pick a random machineset to test
137+
machineSetUnderTest := NewMachineSetList(oc.AsAdmin(), MachineAPINamespace).GetAllOrFail()[0]
138+
logger.Infof("MachineSet under test: %s", machineSetUnderTest.name)
139+
140+
// Save and restore full spec to ensure cleanup regardless of what we modify
141+
originalMachineSetSpec := machineSetUnderTest.GetSpecOrFail()
142+
defer func() {
143+
o.Expect(machineSetUnderTest.SetSpec(originalMachineSetSpec)).To(o.Succeed())
144+
}()
145+
146+
// Set a non-existent user data secret in the machineset's providerSpec; this will cause a boot image controller degrade
147+
nonExistentSecret := "non-existent-user-data"
148+
o.Expect(machineSetUnderTest.SetUserDataSecret(nonExistentSecret)).To(o.Succeed())
149+
logger.Infof("Set non-existent user data secret '%s' in MachineSet %s", nonExistentSecret, machineSetUnderTest.name)
150+
151+
// Patch the boot image to an older version to trigger an update loop
152+
backdatedBootImage := getBackdatedBootImage(oc)
153+
o.Expect(machineSetUnderTest.SetCoreOsBootImage(backdatedBootImage)).To(o.Succeed())
154+
logger.Infof("Set backdated boot image '%s' in MachineSet %s to trigger update loop", backdatedBootImage, machineSetUnderTest.name)
155+
156+
// Verify that the boot image controller has finished processing
157+
machineConfiguration.WaitForBootImageControllerComplete()
158+
159+
// Verify that the boot image controller is degraded
160+
machineConfiguration.WaitForBootImageControllerDegradedState(true)
161+
162+
// Check machine-config CO upgradeable status, should be set to false due to the degrade
163+
o.Eventually(mcoCO, "1m", "10s").ShouldNot(BeUpgradeable(), "co/machine-config should not be upgradeable with broken machineset and update loop")
164+
165+
// Restore machineset to original spec
166+
o.Expect(machineSetUnderTest.SetSpec(originalMachineSetSpec)).To(o.Succeed())
167+
168+
// Verify that the boot image controller has finished processing
169+
machineConfiguration.WaitForBootImageControllerComplete()
170+
171+
// Verify that the boot image controller is not degraded
172+
machineConfiguration.WaitForBootImageControllerDegradedState(false)
173+
174+
// Check machine-config CO upgradeable status, should be set back to true
175+
o.Eventually(mcoCO, "1m", "10s").Should(BeUpgradeable(), "co/machine-config should be upgradeable with restored machineset")
176+
})
177+
178+
g.It("Verify None mode [apigroup:machineconfiguration.openshift.io]", func() {
179+
// Set None mode, effectively disabling skew enforcement
180+
o.Expect(machineConfiguration.SetNoneSkew()).To(o.Succeed())
181+
182+
// Wait for the controller to reflect None mode in skew enforcement status
183+
machineConfiguration.WaitForBootImageSkewEnforcementStatusMode(SkewEnforcementNoneMode)
184+
185+
// Check machine-config CO upgradeable status, should be set to true
186+
o.Eventually(mcoCO, "1m", "10s").Should(BeUpgradeable(), "co/machine-config should be upgradeable when skew enforcement is disabled (None mode)")
187+
})
188+
})

0 commit comments

Comments
 (0)