diff --git a/cluster/examples/hook-sidecar-consumer.yml b/cluster/examples/hook-sidecar-consumer.yml index 5caecb4213d2..75efb68d01ae 100644 --- a/cluster/examples/hook-sidecar-consumer.yml +++ b/cluster/examples/hook-sidecar-consumer.yml @@ -1,3 +1,4 @@ +status: {} apiVersion: kubevirt.io/v1alpha1 kind: VirtualMachine metadata: @@ -18,14 +19,24 @@ spec: bus: virtio name: registrydisk volumeName: registryvolume + - disk: + bus: virtio + name: cloudinitdisk + volumeName: cloudinitvolume machine: type: "" resources: requests: - memory: 64M + memory: 1024M terminationGracePeriodSeconds: 0 volumes: - name: registryvolume registryDisk: - image: registry:5000/kubevirt/cirros-registry-disk-demo:devel + image: registry:5000/kubevirt/fedora-cloud-registry-disk-demo:devel + - cloudInitNoCloud: + userData: |- + #cloud-config + password: fedora + chpasswd: { expire: False } + name: cloudinitvolume status: {} \ No newline at end of file diff --git a/cmd/example-hook-sidecar/README.md b/cmd/example-hook-sidecar/README.md index de1c873cbc0b..398a77d5190c 100644 --- a/cmd/example-hook-sidecar/README.md +++ b/cmd/example-hook-sidecar/README.md @@ -1,37 +1,28 @@ # KubeVirt SMBIOS hook sidecar -Example VM definition: +To use this hook, use following annotations: ```yaml -apiVersion: kubevirt.io/v1alpha2 -kind: VirtualMachineInstance -metadata: - creationTimestamp: null - labels: - special: vm-hook-sidecar-consumer - name: vm-hook-sidecar-consumer - annotations: - # Request the hook sidecar - hooks.kubevirt.io/hookSidecars: '[{"image": "registry:5000/kubevirt/example-hook-sidecar:devel"}]' - # Overwrite base board manufacturer name - smbios.vm.kubevirt.io/baseBoardManufacturer: "Radical Edward" -spec: - domain: - devices: - disks: - - disk: - bus: virtio - name: registrydisk - volumeName: registryvolume - machine: - type: "" - resources: - requests: - memory: 64M - terminationGracePeriodSeconds: 0 - volumes: - - name: registryvolume - registryDisk: - image: registry:5000/kubevirt/cirros-registry-disk-demo:devel -status: {} +annotations: + # Request the hook sidecar + hooks.kubevirt.io/hookSidecars: '[{"image": "registry:5000/kubevirt/example-hook-sidecar:devel"}]' + # Overwrite base board manufacturer name + smbios.vm.kubevirt.io/baseBoardManufacturer: "Radical Edward" ``` + +## Example + +```shell +# Create a VM requesting the hook sidecar +cluster/kubectl.sh create -f cluster/examples/hook-sidecar-consumer.yml + +# Once the VM is ready, connect to its display and login using name and password "fedora" +cluster/virtctl.sh vnc vm-hook-sidecar-consumer + +# Install dmidecode +sudo dnf install -y dmidecode + +# Check whether the base board manufacturer value was successfully overwritten +sudo dmidecode -s baseboard-manufacturer +``` + diff --git a/cmd/example-hook-sidecar/smbios.go b/cmd/example-hook-sidecar/smbios.go index d37ba18abbdb..147a181985db 100644 --- a/cmd/example-hook-sidecar/smbios.go +++ b/cmd/example-hook-sidecar/smbios.go @@ -45,7 +45,7 @@ func (s v1alpha1Server) OnDefineDomain(ctx context.Context, params *hooksV1alpha vmJSON := params.GetVm() vmSpec := vmSchema.VirtualMachine{} - err := json.Unmarshal([]byte(vmJSON), &vmSpec) + err := json.Unmarshal(vmJSON, &vmSpec) if err != nil { log.Log.Reason(err).Errorf("Failed to unmarshal given VM spec: %s", vmJSON) panic(err) @@ -54,7 +54,7 @@ func (s v1alpha1Server) OnDefineDomain(ctx context.Context, params *hooksV1alpha annotations := vmSpec.GetAnnotations() if _, found := annotations[baseBoardManufacturerAnnotation]; !found { - log.Log.Info("SMBIOS hook sidecar was requested, but no attributes provided. Returning original domain spec") + log.Log.Info("SM BIOS hook sidecar was requested, but no attributes provided. Returning original domain spec") return &hooksV1alpha1.OnDefineDomainResult{ DomainXML: params.GetDomainXML(), }, nil @@ -62,22 +62,18 @@ func (s v1alpha1Server) OnDefineDomain(ctx context.Context, params *hooksV1alpha domainXML := params.GetDomainXML() domainSpec := domainSchema.DomainSpec{} - err = xml.Unmarshal([]byte(domainXML), &domainSpec) + err = xml.Unmarshal(domainXML, &domainSpec) if err != nil { log.Log.Reason(err).Errorf("Failed to unmarshal given domain spec: %s", domainXML) panic(err) } - if domainSpec.OS.SMBios == nil { - domainSpec.OS.SMBios = &domainSchema.SMBios{Mode: "sysinfo"} - } else { - domainSpec.OS.SMBios.Mode = "sysinfo" - } + domainSpec.OS.SMBios = &domainSchema.SMBios{Mode: "sysinfo"} if domainSpec.SysInfo == nil { domainSpec.SysInfo = &domainSchema.SysInfo{} } - domainSpec.Type = "smbios" + domainSpec.SysInfo.Type = "smbios" if baseBoardManufacturer, found := annotations[baseBoardManufacturerAnnotation]; found { domainSpec.SysInfo.BaseBoard = append(domainSpec.SysInfo.BaseBoard, domainSchema.Entry{ Name: "manufacturer", @@ -85,14 +81,12 @@ func (s v1alpha1Server) OnDefineDomain(ctx context.Context, params *hooksV1alpha }) } - newDomainXMLRaw, err := xml.Marshal(domainSpec) + newDomainXML, err := xml.Marshal(domainSpec) if err != nil { log.Log.Reason(err).Errorf("Failed to marshal updated domain spec: %s", domainSpec) panic(err) } - newDomainXML := string(newDomainXMLRaw[:]) - log.Log.Info("Successfully updated original domain spec with requested SMBIOS attributes") return &hooksV1alpha1.OnDefineDomainResult{ diff --git a/pkg/hooks/info/api.pb.go b/pkg/hooks/info/api.pb.go index 2edd5f1c85fb..7c2b870d919c 100644 --- a/pkg/hooks/info/api.pb.go +++ b/pkg/hooks/info/api.pb.go @@ -81,7 +81,7 @@ type HookPoint struct { // name represents name of the subscribed hook point Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` // priority is used to sort hooks prior to their execution (second key is the name) - Priority uint32 `protobuf:"varint,2,opt,name=priority" json:"priority,omitempty"` + Priority int32 `protobuf:"varint,2,opt,name=priority" json:"priority,omitempty"` } func (m *HookPoint) Reset() { *m = HookPoint{} } @@ -96,7 +96,7 @@ func (m *HookPoint) GetName() string { return "" } -func (m *HookPoint) GetPriority() uint32 { +func (m *HookPoint) GetPriority() int32 { if m != nil { return m.Priority } @@ -194,8 +194,8 @@ var fileDescriptor0 = []byte{ 0x66, 0x05, 0x66, 0x0d, 0x6e, 0x23, 0x39, 0x3d, 0x2c, 0x26, 0xeb, 0x79, 0xc0, 0x94, 0x05, 0x21, 0xe9, 0x10, 0x92, 0xe2, 0xe2, 0x28, 0x4b, 0x2d, 0x2a, 0xce, 0xcc, 0xcf, 0x2b, 0x96, 0x60, 0x51, 0x60, 0xd6, 0xe0, 0x0c, 0x82, 0xf3, 0x95, 0xac, 0xb9, 0x38, 0xe1, 0x9a, 0xb0, 0x5a, 0x2e, 0xc5, - 0xc5, 0x51, 0x50, 0x94, 0x99, 0x5f, 0x94, 0x59, 0x52, 0x29, 0xc1, 0xa4, 0xc0, 0xa8, 0xc1, 0x1b, + 0xc5, 0x51, 0x50, 0x94, 0x99, 0x5f, 0x94, 0x59, 0x52, 0x29, 0xc1, 0xa4, 0xc0, 0xa8, 0xc1, 0x1a, 0x04, 0xe7, 0x1b, 0x05, 0x70, 0xb1, 0x80, 0x9c, 0x2e, 0xe4, 0x01, 0xa5, 0xe5, 0xb1, 0x3a, 0x0a, 0xe1, 0x57, 0x29, 0xdc, 0x0a, 0x20, 0xde, 0x4f, 0x62, 0x03, 0x07, 0x9b, 0x31, 0x20, 0x00, 0x00, - 0xff, 0xff, 0xea, 0x1a, 0x70, 0x71, 0x43, 0x01, 0x00, 0x00, + 0xff, 0xff, 0x94, 0x8e, 0x0d, 0xdc, 0x43, 0x01, 0x00, 0x00, } diff --git a/pkg/hooks/manager.go b/pkg/hooks/manager.go index 350b92c13c21..ff7c2cebfb63 100644 --- a/pkg/hooks/manager.go +++ b/pkg/hooks/manager.go @@ -2,6 +2,8 @@ package hooks import ( "context" + "encoding/json" + "encoding/xml" "fmt" "io/ioutil" "net" @@ -12,16 +14,18 @@ import ( "google.golang.org/grpc" + "kubevirt.io/kubevirt/pkg/api/v1" hooksInfo "kubevirt.io/kubevirt/pkg/hooks/info" hooksV1alpha1 "kubevirt.io/kubevirt/pkg/hooks/v1alpha1" "kubevirt.io/kubevirt/pkg/log" + domainSchema "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api" + virtwrapApi "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api" ) -type hookClient struct { - client interface{} - version string - name string - hookPoints []*hooksInfo.HookPoint +type callackClient struct { + SocketPath string + Version string + subsribedHookPoints []*hooksInfo.HookPoint } var manager *Manager @@ -29,7 +33,7 @@ var once sync.Once type Manager struct { collected bool - callbacksPerHookPoint map[string][]*hookClient + callbacksPerHookPoint map[string][]*callackClient } func GetManager() *Manager { @@ -55,8 +59,8 @@ func (m *Manager) Collect(numberOfRequestedHookSidecars uint) error { return nil } -func collectSideCarSockets(numberOfRequestedHookSidecars uint) (map[string][]*hookClient, error) { - callbacksPerHookPoint := make(map[string][]*hookClient) +func collectSideCarSockets(numberOfRequestedHookSidecars uint) (map[string][]*callackClient, error) { + callbacksPerHookPoint := make(map[string][]*callackClient) processedSockets := make(map[string]bool) for uint(len(processedSockets)) < numberOfRequestedHookSidecars { @@ -70,7 +74,7 @@ func collectSideCarSockets(numberOfRequestedHookSidecars uint) (map[string][]*ho continue } - hookClient, notReady, err := processSideCarSocket(HookSocketsSharedDirectory + "/" + socket.Name()) + callackClient, notReady, err := processSideCarSocket(HookSocketsSharedDirectory + "/" + socket.Name()) if notReady { log.Log.Info("Sidecar server might not be ready yet, retrying in the next iteration") continue @@ -79,8 +83,8 @@ func collectSideCarSockets(numberOfRequestedHookSidecars uint) (map[string][]*ho return nil, err } - for _, hookPoint := range hookClient.hookPoints { - callbacksPerHookPoint[hookPoint.GetName()] = append(callbacksPerHookPoint[hookPoint.GetName()], hookClient) + for _, subsribedHookPoint := range callackClient.subsribedHookPoints { + callbacksPerHookPoint[subsribedHookPoint.GetName()] = append(callbacksPerHookPoint[subsribedHookPoint.GetName()], callackClient) } processedSockets[socket.Name()] = true @@ -92,14 +96,8 @@ func collectSideCarSockets(numberOfRequestedHookSidecars uint) (map[string][]*ho return callbacksPerHookPoint, nil } -func processSideCarSocket(socketPath string) (*hookClient, bool, error) { - conn, err := grpc.Dial( - socketPath, - grpc.WithInsecure(), - grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) { - return net.DialTimeout("unix", addr, timeout) - }), - ) +func processSideCarSocket(socketPath string) (*callackClient, bool, error) { + conn, err := dialSocket(socketPath) if err != nil { log.Log.Reason(err).Infof("Failed to Dial hook socket: %s", socketPath) return nil, true, nil @@ -118,27 +116,87 @@ func processSideCarSocket(socketPath string) (*hookClient, bool, error) { } if _, found := versionsSet[hooksV1alpha1.Version]; found { - return &hookClient{ - client: hooksV1alpha1.NewCallbacksClient(conn), - name: info.GetName(), - version: hooksV1alpha1.Version, - hookPoints: info.GetHookPoints(), + return &callackClient{ + SocketPath: socketPath, + Version: hooksV1alpha1.Version, + subsribedHookPoints: info.GetHookPoints(), }, false, nil } else { return nil, false, fmt.Errorf("Hook sidecar does not expose a supported version. Exposed versions: %v, supported versions: %s", versionsSet, hooksV1alpha1.Version) } } -func sortCallbacksPerHookPoint(callbacksPerHookPoint map[string][]*hookClient) { +func sortCallbacksPerHookPoint(callbacksPerHookPoint map[string][]*callackClient) { for _, callbacks := range callbacksPerHookPoint { for _, callback := range callbacks { sort.Slice(callbacks, func(i, j int) bool { - if callback.hookPoints[i].Priority == callback.hookPoints[j].Priority { - return strings.Compare(callback.hookPoints[i].Name, callback.hookPoints[j].Name) < 0 + if callback.subsribedHookPoints[i].Priority == callback.subsribedHookPoints[j].Priority { + return strings.Compare(callback.subsribedHookPoints[i].Name, callback.subsribedHookPoints[j].Name) < 0 } else { - return callback.hookPoints[i].Priority > callback.hookPoints[j].Priority + return callback.subsribedHookPoints[i].Priority > callback.subsribedHookPoints[j].Priority } }) } } } + +func (m *Manager) OnDefineDomain(domainSpec *virtwrapApi.DomainSpec, vm *v1.VirtualMachine) (*virtwrapApi.DomainSpec, error) { + if !m.collected { + return nil, fmt.Errorf("Hook sidecars have not been collected yet") + } + + if callbacks, found := m.callbacksPerHookPoint[hooksInfo.OnDefineDomainHookPointName]; found { + for _, callback := range callbacks { + if callback.Version == hooksV1alpha1.Version { + domainSpecXML, err := xml.Marshal(domainSpec) + if err != nil { + return nil, fmt.Errorf("Failed to marshal domain spec: %v", domainSpec) + } + vmJSON, err := json.Marshal(vm) + if err != nil { + return nil, fmt.Errorf("Failed to marshal VM spec: %v", vm) + } + + conn, err := dialSocket(callback.SocketPath) + if err != nil { + log.Log.Reason(err).Infof("Failed to Dial hook socket: %s", callback.SocketPath) + return nil, err + } + defer conn.Close() + + client := hooksV1alpha1.NewCallbacksClient(conn) + + result, err := client.OnDefineDomain(context.Background(), &hooksV1alpha1.OnDefineDomainParams{ + DomainXML: domainSpecXML, + Vm: vmJSON, + }) + if err != nil { + return nil, err + } + + newDomainSpecXML := result.GetDomainXML() + newDomainSpec := domainSchema.DomainSpec{} + err = xml.Unmarshal(newDomainSpecXML, &newDomainSpec) + if err != nil { + return nil, fmt.Errorf("Failed to unmarshal given domain spec: %s", newDomainSpecXML) + } + + domainSpec = &newDomainSpec + } else { + panic("Should never happen, version compatibility check is done during Info call") + } + } + } + + return domainSpec, nil +} + +func dialSocket(socketPath string) (*grpc.ClientConn, error) { + return grpc.Dial( + socketPath, + grpc.WithInsecure(), + grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) { + return net.DialTimeout("unix", addr, timeout) + }), + ) +} diff --git a/pkg/hooks/v1alpha1/api.pb.go b/pkg/hooks/v1alpha1/api.pb.go index 383552556fe1..748e3ae4eb15 100644 --- a/pkg/hooks/v1alpha1/api.pb.go +++ b/pkg/hooks/v1alpha1/api.pb.go @@ -34,10 +34,10 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type OnDefineDomainParams struct { - // domainXML is original libvirt domain specification, it is XML passed as string - DomainXML string `protobuf:"bytes,1,opt,name=domainXML" json:"domainXML,omitempty"` - // vm is VirtualMachine is object of virtual machine currently processed by virt-launcher, it is encoded as JSON and passed as string - Vm string `protobuf:"bytes,2,opt,name=vm" json:"vm,omitempty"` + // domainXML is original libvirt domain specification + DomainXML []byte `protobuf:"bytes,1,opt,name=domainXML,proto3" json:"domainXML,omitempty"` + // vm is VirtualMachine is object of virtual machine currently processed by virt-launcher, it is encoded as JSON + Vm []byte `protobuf:"bytes,2,opt,name=vm,proto3" json:"vm,omitempty"` } func (m *OnDefineDomainParams) Reset() { *m = OnDefineDomainParams{} } @@ -45,23 +45,23 @@ func (m *OnDefineDomainParams) String() string { return proto.Compact func (*OnDefineDomainParams) ProtoMessage() {} func (*OnDefineDomainParams) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } -func (m *OnDefineDomainParams) GetDomainXML() string { +func (m *OnDefineDomainParams) GetDomainXML() []byte { if m != nil { return m.DomainXML } - return "" + return nil } -func (m *OnDefineDomainParams) GetVm() string { +func (m *OnDefineDomainParams) GetVm() []byte { if m != nil { return m.Vm } - return "" + return nil } type OnDefineDomainResult struct { - // domainXML is processed libvirt domain specification, it is XML passed as string - DomainXML string `protobuf:"bytes,1,opt,name=domainXML" json:"domainXML,omitempty"` + // domainXML is processed libvirt domain specification + DomainXML []byte `protobuf:"bytes,1,opt,name=domainXML,proto3" json:"domainXML,omitempty"` } func (m *OnDefineDomainResult) Reset() { *m = OnDefineDomainResult{} } @@ -69,11 +69,11 @@ func (m *OnDefineDomainResult) String() string { return proto.Compact func (*OnDefineDomainResult) ProtoMessage() {} func (*OnDefineDomainResult) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } -func (m *OnDefineDomainResult) GetDomainXML() string { +func (m *OnDefineDomainResult) GetDomainXML() []byte { if m != nil { return m.DomainXML } - return "" + return nil } func init() { @@ -162,10 +162,10 @@ var fileDescriptor0 = []byte{ 0xcb, 0xc8, 0xcf, 0xcf, 0x2e, 0xd6, 0x2b, 0x33, 0x4c, 0xcc, 0x29, 0xc8, 0x48, 0x34, 0x54, 0x72, 0xe1, 0x12, 0xf1, 0xcf, 0x73, 0x49, 0x4d, 0xcb, 0xcc, 0x4b, 0x75, 0xc9, 0xcf, 0x4d, 0xcc, 0xcc, 0x0b, 0x48, 0x2c, 0x4a, 0xcc, 0x2d, 0x16, 0x92, 0xe1, 0xe2, 0x4c, 0x01, 0xf3, 0x23, 0x7c, 0x7d, - 0x24, 0x18, 0x15, 0x18, 0x35, 0x38, 0x83, 0x10, 0x02, 0x42, 0x7c, 0x5c, 0x4c, 0x65, 0xb9, 0x12, + 0x24, 0x18, 0x15, 0x18, 0x35, 0x78, 0x82, 0x10, 0x02, 0x42, 0x7c, 0x5c, 0x4c, 0x65, 0xb9, 0x12, 0x4c, 0x60, 0x61, 0xa6, 0xb2, 0x5c, 0x25, 0x13, 0x74, 0x53, 0x82, 0x52, 0x8b, 0x4b, 0x73, 0x4a, 0xf0, 0x9b, 0x62, 0x54, 0xcd, 0xc5, 0xe9, 0x9c, 0x98, 0x93, 0x93, 0x94, 0x98, 0x9c, 0x5d, 0x2c, 0x94, 0xc7, 0xc5, 0x87, 0x6a, 0x84, 0x90, 0xae, 0x1e, 0x0e, 0x47, 0xeb, 0x61, 0x73, 0xb1, 0x14, - 0xb1, 0xca, 0x21, 0x4e, 0x4b, 0x62, 0x03, 0x07, 0x8c, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x31, - 0x71, 0xed, 0xd5, 0x25, 0x01, 0x00, 0x00, + 0xb1, 0xca, 0x21, 0x4e, 0x4b, 0x62, 0x03, 0x07, 0x8c, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x64, + 0x3e, 0xd4, 0x0e, 0x25, 0x01, 0x00, 0x00, } diff --git a/pkg/hooks/v1alpha1/api.proto b/pkg/hooks/v1alpha1/api.proto index 0b0a395aea73..e7bdfde1b207 100644 --- a/pkg/hooks/v1alpha1/api.proto +++ b/pkg/hooks/v1alpha1/api.proto @@ -7,13 +7,13 @@ service Callbacks { } message OnDefineDomainParams { - // domainXML is original libvirt domain specification, it is XML passed as string - string domainXML = 1; - // vm is VirtualMachine is object of virtual machine currently processed by virt-launcher, it is encoded as JSON and passed as string - string vm = 2; + // domainXML is original libvirt domain specification + bytes domainXML = 1; + // vm is VirtualMachine is object of virtual machine currently processed by virt-launcher, it is encoded as JSON + bytes vm = 2; } message OnDefineDomainResult { - // domainXML is processed libvirt domain specification, it is XML passed as string - string domainXML = 1; + // domainXML is processed libvirt domain specification + bytes domainXML = 1; } \ No newline at end of file diff --git a/pkg/hooks/v1alpha1/v1alpha1.go b/pkg/hooks/v1alpha1/v1alpha1.go index 224bd3a933ff..ecf8170e8ecb 100644 --- a/pkg/hooks/v1alpha1/v1alpha1.go +++ b/pkg/hooks/v1alpha1/v1alpha1.go @@ -1,7 +1,3 @@ package v1alpha const Version = "v1alpha" - -const ( - OnDefineDomainHookPointName = "OnDefineDomain" -) diff --git a/pkg/virt-launcher/virtwrap/manager.go b/pkg/virt-launcher/virtwrap/manager.go index e17cff8e072d..effe1b52f261 100644 --- a/pkg/virt-launcher/virtwrap/manager.go +++ b/pkg/virt-launcher/virtwrap/manager.go @@ -37,6 +37,7 @@ import ( cloudinit "kubevirt.io/kubevirt/pkg/cloud-init" "kubevirt.io/kubevirt/pkg/emptydisk" "kubevirt.io/kubevirt/pkg/ephemeral-disk" + "kubevirt.io/kubevirt/pkg/hooks" "kubevirt.io/kubevirt/pkg/log" "kubevirt.io/kubevirt/pkg/registry-disk" "kubevirt.io/kubevirt/pkg/util/net/dns" @@ -110,6 +111,13 @@ func (l *LibvirtDomainManager) preStartHook(vmi *v1.VirtualMachineInstance, doma return domain, fmt.Errorf("creating empty disks failed: %v", err) } + hooksManager := hooks.GetManager() + domainSpec, err := hooksManager.OnDefineDomain(&domain.Spec, vm) + if err != nil { + return domain, err + } + domain.Spec = *domainSpec + return domain, err }