From 2eb5a132e26666df8fac07c078215f11acc6e2f5 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Mon, 22 Jun 2026 19:22:02 +0200 Subject: [PATCH 01/39] feat(vm): add GPU DRA resource claim support Signed-off-by: Daniil Antoshin --- .../pkg/common/annotations/annotations.go | 2 + .../pkg/controller/kvbuilder/gpu.go | 73 +++++++++ .../pkg/controller/kvbuilder/gpu_test.go | 53 +++++++ .../pkg/controller/kvbuilder/kvvm_utils.go | 2 + .../vm/internal/gpu_resourceclaim_handler.go | 146 ++++++++++++++++++ .../gpu_resourceclaim_handler_test.go | 81 ++++++++++ .../pkg/controller/vm/internal/sync_kvvm.go | 31 ++++ .../controller/vm/internal/sync_kvvm_test.go | 27 ++++ .../watcher/resourceclaimtemplate_watcher.go | 53 +++++++ .../pkg/controller/vm/vm_controller.go | 1 + .../pkg/controller/vm/vm_reconciler.go | 1 + 11 files changed, 470 insertions(+) create mode 100644 images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go create mode 100644 images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go create mode 100644 images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go create mode 100644 images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go create mode 100644 images/virtualization-artifact/pkg/controller/vm/internal/watcher/resourceclaimtemplate_watcher.go diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index 75e06031e7..9b63c8b408 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -256,6 +256,8 @@ const ( // AnnVMFilesystemRequest is an annotation on a virtual machine that indicates a request to freeze or unfreeze the filesystem has been sent. AnnVMFilesystemRequest = AnnAPIGroupV + "/virtual-machine-filesystem-request" + // AnnVMGPUID is an annotation on a virtual machine that selects a GPU UUID for DRA passthrough. + AnnVMGPUID = AnnAPIGroupV + "/gpu-id" // AnnDVCRDeploymentSwitchToGarbageCollectionMode is an annotation on secret/dvcr-garbage-collection to indicate that deployment/dcvr should be switched to garbage-collection mode. AnnDVCRDeploymentSwitchToGarbageCollectionMode = AnnAPIGroupV + "/dvcr-deployment-switch-to-garbage-collection-mode" diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go new file mode 100644 index 0000000000..a47eb6fd51 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -0,0 +1,73 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kvbuilder + +import ( + "slices" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" +) + +const ( + GPUName = "gpu" + GPUResourceClaimTemplateNameSuffix = "-gpu-template" + GPUResourceClaimRequestName = "req-gpu" + AppliedGPUAnnotation = "internal.virtualization.deckhouse.io/applied-gpu-id" +) + +func GPUResourceClaimTemplateName(vmName string) string { + return vmName + GPUResourceClaimTemplateNameSuffix +} + +func (b *KVVM) SetGPU(vmName, gpuID string) { + b.Resource.Spec.Template.Spec.ResourceClaims = slices.DeleteFunc( + b.Resource.Spec.Template.Spec.ResourceClaims, + func(claim virtv1.ResourceClaim) bool { return claim.Name == GPUName }, + ) + b.Resource.Spec.Template.Spec.Domain.Devices.GPUs = slices.DeleteFunc( + b.Resource.Spec.Template.Spec.Domain.Devices.GPUs, + func(gpu virtv1.GPU) bool { return gpu.Name == GPUName }, + ) + + if gpuID == "" { + if b.Resource.Annotations != nil { + delete(b.Resource.Annotations, AppliedGPUAnnotation) + } + return + } + + b.Resource.Spec.Template.Spec.ResourceClaims = append(b.Resource.Spec.Template.Spec.ResourceClaims, virtv1.ResourceClaim{ + PodResourceClaim: corev1.PodResourceClaim{ + Name: GPUName, + ResourceClaimTemplateName: ptr.To(GPUResourceClaimTemplateName(vmName)), + }, + }) + b.Resource.Spec.Template.Spec.Domain.Devices.GPUs = append(b.Resource.Spec.Template.Spec.Domain.Devices.GPUs, virtv1.GPU{ + Name: GPUName, + ClaimRequest: &virtv1.ClaimRequest{ + ClaimName: ptr.To(GPUName), + RequestName: ptr.To(GPUResourceClaimRequestName), + }, + }) + + if b.Resource.Annotations == nil { + b.Resource.Annotations = make(map[string]string, 1) + } + b.Resource.Annotations[AppliedGPUAnnotation] = gpuID +} diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go new file mode 100644 index 0000000000..809f7582c1 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kvbuilder + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("GPU", func() { + It("should render DRA GPU resource claim", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + + kvvm.SetGPU("vm-a", "GPU-test") + res := kvvm.GetResource() + + Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal(GPUName)) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu-template")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal(GPUName)) + Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal(GPUName)) + Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal(GPUResourceClaimRequestName)) + Expect(res.Annotations).To(HaveKeyWithValue(AppliedGPUAnnotation, "GPU-test")) + }) + + It("should remove rendered DRA GPU resource claim", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + kvvm.SetGPU("vm-a", "GPU-test") + + kvvm.SetGPU("vm-a", "") + res := kvvm.GetResource() + + Expect(res.Spec.Template.Spec.ResourceClaims).To(BeEmpty()) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(BeEmpty()) + Expect(res.Annotations).NotTo(HaveKey(AppliedGPUAnnotation)) + }) +}) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go index c35a11bce9..aa703677ad 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go @@ -247,6 +247,8 @@ func ApplyVirtualMachineSpec( return err } + kvvm.SetGPU(vm.Name, vm.Annotations[annotations.AnnVMGPUID]) + if err := kvvm.SetProvisioning(vm.Spec.Provisioning); err != nil { return err } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go new file mode 100644 index 0000000000..cb36d8348f --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -0,0 +1,146 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "fmt" + "reflect" + "strconv" + + resourcev1 "k8s.io/api/resource/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization-controller/pkg/controller/service" + "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" + "github.com/deckhouse/virtualization-controller/pkg/logger" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +const ( + nameGPUResourceClaimHandler = "GPUResourceClaimHandler" + gpuDeviceClassName = "gpu.deckhouse.io" +) + +func NewGPUResourceClaimHandler(client client.Client) *GPUResourceClaimHandler { + return &GPUResourceClaimHandler{client: client} +} + +type GPUResourceClaimHandler struct { + client client.Client +} + +func (h *GPUResourceClaimHandler) Name() string { + return nameGPUResourceClaimHandler +} + +func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) { + if s.VirtualMachine().IsEmpty() { + return reconcile.Result{}, nil + } + + vm := s.VirtualMachine().Current() + gpuID := vm.Annotations[annotations.AnnVMGPUID] + templateName := kvbuilder.GPUResourceClaimTemplateName(vm.Name) + template := &resourcev1.ResourceClaimTemplate{} + key := types.NamespacedName{Name: templateName, Namespace: vm.Namespace} + + if gpuID == "" { + if err := h.client.Get(ctx, key, template); err != nil { + if apierrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, fmt.Errorf("failed to get GPU ResourceClaimTemplate: %w", err) + } + if metav1.IsControlledBy(template, vm) { + if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to delete GPU ResourceClaimTemplate: %w", err) + } + } + return reconcile.Result{}, nil + } + + desiredSpec := buildGPUResourceClaimTemplateSpec(gpuID) + log := logger.FromContext(ctx).With(logger.SlogHandler(nameGPUResourceClaimHandler)) + err := h.client.Get(ctx, key, template) + if err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to get GPU ResourceClaimTemplate: %w", err) + } + + if apierrors.IsNotFound(err) { + template = buildGPUResourceClaimTemplate(vm, templateName, desiredSpec) + if err := h.client.Create(ctx, template); err != nil && !apierrors.IsAlreadyExists(err) { + return reconcile.Result{}, fmt.Errorf("failed to create GPU ResourceClaimTemplate: %w", err) + } + log.Info("created GPU ResourceClaimTemplate", "template", templateName) + return reconcile.Result{}, nil + } + + if reflect.DeepEqual(template.Spec, desiredSpec) { + return reconcile.Result{}, nil + } + if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to delete outdated GPU ResourceClaimTemplate: %w", err) + } + template = buildGPUResourceClaimTemplate(vm, templateName, desiredSpec) + if err := h.client.Create(ctx, template); err != nil && !apierrors.IsAlreadyExists(err) { + return reconcile.Result{}, fmt.Errorf("failed to recreate GPU ResourceClaimTemplate: %w", err) + } + log.Info("recreated GPU ResourceClaimTemplate", "template", templateName) + return reconcile.Result{}, nil +} + +func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spec resourcev1.ResourceClaimTemplateSpec) *resourcev1.ResourceClaimTemplate { + return &resourcev1.ResourceClaimTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: vm.Namespace, + OwnerReferences: []metav1.OwnerReference{service.MakeControllerOwnerReference(vm)}, + }, + Spec: spec, + } +} + +func buildGPUResourceClaimTemplateSpec(gpuID string) resourcev1.ResourceClaimTemplateSpec { + selector := fmt.Sprintf( + `device.attributes["gpu.deckhouse.io"].gpuUUID == %s && device.attributes["gpu.deckhouse.io"].deviceType == "physical" && !has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`, + strconv.Quote(gpuID), + ) + return resourcev1.ResourceClaimTemplateSpec{ + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{{ + Name: kvbuilder.GPUResourceClaimRequestName, + Exactly: &resourcev1.ExactDeviceRequest{ + DeviceClassName: gpuDeviceClassName, + AllocationMode: resourcev1.DeviceAllocationModeExactCount, + Count: 1, + Selectors: []resourcev1.DeviceSelector{{ + CEL: &resourcev1.CELDeviceSelector{Expression: selector}, + }}, + }, + }}, + }, + }, + } +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go new file mode 100644 index 0000000000..946db3027d --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + context "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + resourcev1 "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +var _ = Describe("GPUResourceClaimHandler", func() { + const ( + vmName = "vm-a" + namespace = "default" + gpuID = "GPU-test" + ) + + newVM := func(id string) *v1alpha2.VirtualMachine { + vm := &v1alpha2.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: vmName, Namespace: namespace, Annotations: map[string]string{}}, + } + if id != "" { + vm.Annotations[annotations.AnnVMGPUID] = id + } + return vm + } + + It("should create GPU ResourceClaimTemplate", func() { + fakeClient, _, vmState := setupEnvironment(newVM(gpuID)) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + template := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName), Namespace: namespace}, template)).To(Succeed()) + Expect(template.Spec.Spec.Devices.Requests).To(HaveLen(1)) + request := template.Spec.Spec.Devices.Requests[0] + Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimRequestName)) + Expect(request.Exactly.DeviceClassName).To(Equal(gpuDeviceClassName)) + Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`gpuUUID == "GPU-test"`)) + Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`deviceType == "physical"`)) + Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`!has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`)) + }) + + It("should delete owned GPU ResourceClaimTemplate when annotation is removed", func() { + vm := newVM("") + template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName), buildGPUResourceClaimTemplateSpec(gpuID)) + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + err = fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName), Namespace: namespace}, stored) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index b8a72e409d..0768d2590a 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -157,6 +157,8 @@ func (h *SyncKvvmHandler) Handle(ctx context.Context, s state.VirtualMachineStat lastAppliedSpec = h.loadLastAppliedSpec(current, kvvm) lastClassAppliedSpec := h.loadClassLastAppliedSpec(class, kvvm) changes = h.detectSpecChanges(ctx, kvvm, ¤t.Spec, lastAppliedSpec) + gpuAnnotationChanges := detectGPUAnnotationChanges(kvvm, current) + changes.Add(gpuAnnotationChanges.GetAll()...) if !changes.IsEmpty() { kvvmi, kvvmiErr := s.KVVMI(ctx) if kvvmiErr == nil { @@ -732,6 +734,35 @@ func (h *SyncKvvmHandler) detectSpecChanges( return specChanges } +func detectGPUAnnotationChanges(kvvm *virtv1.VirtualMachine, vm *v1alpha2.VirtualMachine) vmchange.SpecChanges { + var changes vmchange.SpecChanges + if kvvm == nil || vm == nil { + return changes + } + + currentValue := kvvm.Annotations[kvbuilder.AppliedGPUAnnotation] + desiredValue := vm.Annotations[annotations.AnnVMGPUID] + if currentValue == desiredValue { + return changes + } + + operation := vmchange.ChangeReplace + if currentValue == "" { + operation = vmchange.ChangeAdd + } + if desiredValue == "" { + operation = vmchange.ChangeRemove + } + changes.Add(vmchange.FieldChange{ + Operation: operation, + Path: "metadata.annotations." + annotations.AnnVMGPUID, + CurrentValue: currentValue, + DesiredValue: desiredValue, + ActionRequired: vmchange.ActionRestart, + }) + return changes +} + func (h *SyncKvvmHandler) detectClassSpecChanges(ctx context.Context, currentClassSpec, lastClassSpec *v1alpha2.VirtualMachineClassSpec) vmchange.SpecChanges { log := logger.FromContext(ctx) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 62d4186486..868d10f566 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -421,6 +421,33 @@ var _ = Describe("SyncKvvmHandler", func() { Entry("Pending phase without changes, shouldn't have condition", v1alpha2.MachinePending, false, metav1.ConditionUnknown, false), ) + It("should require restart when GPU annotation changes on a running VM", func() { + ip := makeVMIP() + vmClass := makeVMClass() + + vm := makeVM(v1alpha2.MachineRunning) + vm.Annotations = map[string]string{annotations.AnnVMGPUID: "GPU-new"} + kvvm := makeKVVM(vm) + kvvm.Annotations[kvbuilder.AppliedGPUAnnotation] = "GPU-old" + kvvmi := makeKVVMI() + + fakeClient, reconcileObj, vmState = setupEnvironment(vm, kvvm, kvvmi, ip, vmClass) + + reconcile() + + newVM := &v1alpha2.VirtualMachine{} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), newVM)).To(Succeed()) + awaitCond, awaitExists := conditions.GetCondition(vmcondition.TypeAwaitingRestartToApplyConfiguration, newVM.Status.Conditions) + Expect(awaitExists).To(BeTrue()) + Expect(awaitCond.Status).To(Equal(metav1.ConditionTrue)) + Expect(newVM.Status.RestartAwaitingChanges).NotTo(BeEmpty()) + + updatedKVVM := &virtv1.VirtualMachine{} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(kvvm), updatedKVVM)).To(Succeed()) + Expect(updatedKVVM.Annotations).To(HaveKeyWithValue(kvbuilder.AppliedGPUAnnotation, "GPU-old")) + Expect(updatedKVVM.Spec.Template.Spec.Domain.Devices.GPUs).To(BeEmpty()) + }) + DescribeTable("AwaitingRestart Condition for NonMigratable VM", func(phase v1alpha2.MachinePhase, featureGate featuregate.FeatureGate, mutateFn func(fakeClient client.WithWatch, vm *v1alpha2.VirtualMachine, kvvm *virtv1.VirtualMachine), expectedStatus metav1.ConditionStatus, expectedExistence bool) { ip := makeVMIP() diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/watcher/resourceclaimtemplate_watcher.go b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/resourceclaimtemplate_watcher.go new file mode 100644 index 0000000000..14dabd46d1 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/resourceclaimtemplate_watcher.go @@ -0,0 +1,53 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "fmt" + + resourcev1 "k8s.io/api/resource/v1" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func NewResourceClaimTemplateWatcher() *ResourceClaimTemplateWatcher { + return &ResourceClaimTemplateWatcher{} +} + +type ResourceClaimTemplateWatcher struct{} + +func (w *ResourceClaimTemplateWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + if err := ctr.Watch( + source.Kind( + mgr.GetCache(), + &resourcev1.ResourceClaimTemplate{}, + handler.TypedEnqueueRequestForOwner[*resourcev1.ResourceClaimTemplate]( + mgr.GetScheme(), + mgr.GetRESTMapper(), + &v1alpha2.VirtualMachine{}, + handler.OnlyControllerOwner(), + ), + ), + ); err != nil { + return fmt.Errorf("error setting watch on ResourceClaimTemplate: %w", err) + } + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go index 4f131f9e01..6b5c52b218 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go @@ -71,6 +71,7 @@ func SetupController( internal.NewBlockDeviceHandler(client, blockDeviceService), internal.NewUSBDeviceDetachHandler(client, virtClient), internal.NewUSBDeviceAttachHandler(client, virtClient), + internal.NewGPUResourceClaimHandler(client), internal.NewProvisioningHandler(client), internal.NewAgentHandler(), internal.NewFilesystemHandler(), diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_reconciler.go b/images/virtualization-artifact/pkg/controller/vm/vm_reconciler.go index 300b65acab..2d49683161 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_reconciler.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_reconciler.go @@ -71,6 +71,7 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr watcher.NewClusterVirtualImageWatcher(mgr.GetClient()), watcher.NewVirtualDiskWatcher(mgr.GetClient()), watcher.NewUSBDeviceWatcher(mgr.GetClient()), + watcher.NewResourceClaimTemplateWatcher(), watcher.NewVMIPWatcher(), watcher.NewVirtualMachineClassWatcher(), watcher.NewVirtualMachineSnapshotWatcher(), From 065808f8add5b229352cbb357d16b7df6f41ba7a Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Mon, 22 Jun 2026 19:27:02 +0200 Subject: [PATCH 02/39] chore(module): checkout to kubevirt PR Signed-off-by: Daniil Antoshin --- build/components/versions.yml | 3 ++- images/virt-artifact/werf.inc.yaml | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build/components/versions.yml b/build/components/versions.yml index 675a7435fd..942e074f77 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,8 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.59 + 3p-kubevirt: feat/gpu/add-deckhouse-dra-support + 3p-containerized-data-importer: v1.60.3-v12n.21 distribution: 3.1.1 package: acl: v2.3.1 diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index 2ddeb2960d..754825132d 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -13,8 +13,10 @@ secrets: - id: SOURCE_REPO value: {{ $.SOURCE_REPO }} shell: + installCacheVersion: "{{ now | date "Mon Jan 2 15:04:05 MST 2006" }}" install: - | + echo "$date" echo "Git clone {{ $gitRepoName }} repository..." git clone --depth=1 $(cat /run/secrets/SOURCE_REPO)/{{ $gitRepoUrl }} --branch {{ $tag }} /src/kubevirt From 59187c376e38011686825d1dcfde23d602fab90b Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Tue, 23 Jun 2026 13:01:48 +0200 Subject: [PATCH 03/39] feat(api): add model-based VM GPU devices Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/virtual_machine.go | 20 ++++ api/core/v1alpha2/zz_generated.deepcopy.go | 21 ++++ crds/doc-ru-virtualmachines.yaml | 12 +++ crds/virtualmachines.yaml | 29 ++++++ .../pkg/common/annotations/annotations.go | 2 - .../pkg/controller/kvbuilder/gpu.go | 99 ++++++++++++++----- .../pkg/controller/kvbuilder/gpu_test.go | 26 ++--- .../pkg/controller/kvbuilder/kvvm_utils.go | 2 +- .../vm/internal/gpu_resourceclaim_handler.go | 95 ++++++++++-------- .../gpu_resourceclaim_handler_test.go | 28 +++--- .../pkg/controller/vm/internal/sync_kvvm.go | 31 ------ .../controller/vm/internal/sync_kvvm_test.go | 28 +++++- .../pkg/controller/vmchange/compare.go | 1 + .../pkg/controller/vmchange/gpu_change.go | 36 +++++++ 14 files changed, 297 insertions(+), 133 deletions(-) create mode 100644 images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index af8ea3d05e..8945f2b221 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -123,6 +123,12 @@ type VirtualMachineSpec struct { // Devices are referenced by name of USBDevice resource in the same namespace. // +kubebuilder:validation:MaxItems:=8 USBDevices []USBDeviceSpecRef `json:"usbDevices,omitempty"` + // List of GPU devices to attach to the virtual machine. + // Devices are requested by GPU model. + // +kubebuilder:validation:MaxItems:=8 + // +listType=map + // +listMapKey=name + GPUDevices []GPUDeviceSpec `json:"gpuDevices,omitempty"` } func (s *VirtualMachineSpec) IsParavirtualizationEnabled() bool { @@ -505,6 +511,20 @@ type USBDeviceSpecRef struct { Name string `json:"name"` } +// GPUDeviceSpec requests a GPU device by model. +type GPUDeviceSpec struct { + // A unique GPU device name inside the virtual machine spec. + // +kubebuilder:validation:MinLength:=1 + // +kubebuilder:validation:MaxLength:=63 + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + Name string `json:"name"` + // GPU model identifier, for example h100-sxm5-96gb. + // +kubebuilder:validation:MinLength:=1 + // +kubebuilder:validation:MaxLength:=63 + // +kubebuilder:validation:Pattern:=`^[A-Za-z0-9]([A-Za-z0-9_.-]*[A-Za-z0-9])?$` + Model string `json:"model"` +} + // USBDeviceStatusRef represents the status of a USB device attached to the virtual machine. type USBDeviceStatusRef struct { // The name of USBDevice resource. diff --git a/api/core/v1alpha2/zz_generated.deepcopy.go b/api/core/v1alpha2/zz_generated.deepcopy.go index 8ee5d37f0a..926d8f4474 100644 --- a/api/core/v1alpha2/zz_generated.deepcopy.go +++ b/api/core/v1alpha2/zz_generated.deepcopy.go @@ -473,6 +473,22 @@ func (in *Disruptions) DeepCopy() *Disruptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUDeviceSpec) DeepCopyInto(out *GPUDeviceSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceSpec. +func (in *GPUDeviceSpec) DeepCopy() *GPUDeviceSpec { + if in == nil { + return nil + } + out := new(GPUDeviceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImagePullSecret) DeepCopyInto(out *ImagePullSecret) { *out = *in @@ -3516,6 +3532,11 @@ func (in *VirtualMachineSpec) DeepCopyInto(out *VirtualMachineSpec) { *out = make([]USBDeviceSpecRef, len(*in)) copy(*out, *in) } + if in.GPUDevices != nil { + in, out := &in.GPUDevices, &out.GPUDevices + *out = make([]GPUDeviceSpec, len(*in)) + copy(*out, *in) + } return } diff --git a/crds/doc-ru-virtualmachines.yaml b/crds/doc-ru-virtualmachines.yaml index 8a4697f05f..1e503eda11 100644 --- a/crds/doc-ru-virtualmachines.yaml +++ b/crds/doc-ru-virtualmachines.yaml @@ -588,6 +588,18 @@ spec: name: description: | Имя ресурса `USBDevice` в том же пространстве имен. + gpuDevices: + description: | + Список GPU-устройств для подключения к виртуальной машине. + Устройства запрашиваются по модели GPU. + items: + properties: + model: + description: | + Идентификатор модели GPU, например `h100-sxm5-96gb`. + name: + description: | + Уникальное имя GPU-устройства внутри спецификации виртуальной машины. status: properties: blockDeviceRefs: diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index 380e4bac38..a535d50a73 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1058,6 +1058,35 @@ spec: type: string description: | The name of USBDevice resource in the same namespace. + gpuDevices: + type: array + maxItems: 8 + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + description: | + List of GPU devices to attach to the virtual machine. + Devices are requested by GPU model. + items: + type: object + required: + - model + - name + properties: + model: + minLength: 1 + maxLength: 63 + pattern: ^[A-Za-z0-9]([A-Za-z0-9_.-]*[A-Za-z0-9])?$ + type: string + description: | + GPU model identifier, for example h100-sxm5-96gb. + name: + minLength: 1 + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + description: | + A unique GPU device name inside the virtual machine spec. status: type: object properties: diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index 9b63c8b408..75e06031e7 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -256,8 +256,6 @@ const ( // AnnVMFilesystemRequest is an annotation on a virtual machine that indicates a request to freeze or unfreeze the filesystem has been sent. AnnVMFilesystemRequest = AnnAPIGroupV + "/virtual-machine-filesystem-request" - // AnnVMGPUID is an annotation on a virtual machine that selects a GPU UUID for DRA passthrough. - AnnVMGPUID = AnnAPIGroupV + "/gpu-id" // AnnDVCRDeploymentSwitchToGarbageCollectionMode is an annotation on secret/dvcr-garbage-collection to indicate that deployment/dcvr should be switched to garbage-collection mode. AnnDVCRDeploymentSwitchToGarbageCollectionMode = AnnAPIGroupV + "/dvcr-deployment-switch-to-garbage-collection-mode" diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index a47eb6fd51..a678a08303 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -17,57 +17,106 @@ limitations under the License. package kvbuilder import ( + "encoding/json" + "fmt" + "strings" + "slices" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" virtv1 "kubevirt.io/api/core/v1" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" ) const ( - GPUName = "gpu" - GPUResourceClaimTemplateNameSuffix = "-gpu-template" - GPUResourceClaimRequestName = "req-gpu" - AppliedGPUAnnotation = "internal.virtualization.deckhouse.io/applied-gpu-id" + GPUNamePrefix = "gpu-" + GPUResourceClaimTemplateNameSuffixFormat = "-gpu-%s-template" + GPUResourceClaimRequestNamePrefix = "req-gpu-" + AppliedGPUDevicesAnnotation = "internal.virtualization.deckhouse.io/applied-gpu-devices" ) -func GPUResourceClaimTemplateName(vmName string) string { - return vmName + GPUResourceClaimTemplateNameSuffix +func GPUResourceClaimName(deviceName string) string { + return GPUNamePrefix + deviceName +} + +func GPUResourceClaimTemplateName(vmName, deviceName string) string { + return vmName + fmt.Sprintf(GPUResourceClaimTemplateNameSuffixFormat, deviceName) +} + +func IsGPUResourceClaimTemplateName(vmName, templateName string) bool { + return templateName == vmName+"-gpu-template" || strings.HasPrefix(templateName, vmName+"-gpu-") && strings.HasSuffix(templateName, "-template") +} + +func GPUResourceClaimRequestName(deviceName string) string { + return GPUResourceClaimRequestNamePrefix + deviceName +} + +func EncodeGPUDevices(devices []v1alpha2.GPUDeviceSpec) string { + if len(devices) == 0 { + return "" + } + data, err := json.Marshal(sortGPUDevices(devices)) + if err != nil { + return "" + } + return string(data) } -func (b *KVVM) SetGPU(vmName, gpuID string) { +func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { + devices = sortGPUDevices(devices) + b.Resource.Spec.Template.Spec.ResourceClaims = slices.DeleteFunc( b.Resource.Spec.Template.Spec.ResourceClaims, - func(claim virtv1.ResourceClaim) bool { return claim.Name == GPUName }, + func(claim virtv1.ResourceClaim) bool { + return strings.HasPrefix(claim.Name, GPUNamePrefix) + }, ) b.Resource.Spec.Template.Spec.Domain.Devices.GPUs = slices.DeleteFunc( b.Resource.Spec.Template.Spec.Domain.Devices.GPUs, - func(gpu virtv1.GPU) bool { return gpu.Name == GPUName }, + func(gpu virtv1.GPU) bool { + return strings.HasPrefix(gpu.Name, GPUNamePrefix) + }, ) - if gpuID == "" { + if len(devices) == 0 { if b.Resource.Annotations != nil { - delete(b.Resource.Annotations, AppliedGPUAnnotation) + delete(b.Resource.Annotations, AppliedGPUDevicesAnnotation) } return } - b.Resource.Spec.Template.Spec.ResourceClaims = append(b.Resource.Spec.Template.Spec.ResourceClaims, virtv1.ResourceClaim{ - PodResourceClaim: corev1.PodResourceClaim{ - Name: GPUName, - ResourceClaimTemplateName: ptr.To(GPUResourceClaimTemplateName(vmName)), - }, - }) - b.Resource.Spec.Template.Spec.Domain.Devices.GPUs = append(b.Resource.Spec.Template.Spec.Domain.Devices.GPUs, virtv1.GPU{ - Name: GPUName, - ClaimRequest: &virtv1.ClaimRequest{ - ClaimName: ptr.To(GPUName), - RequestName: ptr.To(GPUResourceClaimRequestName), - }, - }) + for _, device := range devices { + claimName := GPUResourceClaimName(device.Name) + b.Resource.Spec.Template.Spec.ResourceClaims = append(b.Resource.Spec.Template.Spec.ResourceClaims, virtv1.ResourceClaim{ + PodResourceClaim: corev1.PodResourceClaim{ + Name: claimName, + ResourceClaimTemplateName: ptr.To(GPUResourceClaimTemplateName(vmName, device.Name)), + }, + }) + b.Resource.Spec.Template.Spec.Domain.Devices.GPUs = append(b.Resource.Spec.Template.Spec.Domain.Devices.GPUs, virtv1.GPU{ + Name: claimName, + ClaimRequest: &virtv1.ClaimRequest{ + ClaimName: ptr.To(claimName), + RequestName: ptr.To(GPUResourceClaimRequestName(device.Name)), + }, + }) + } if b.Resource.Annotations == nil { b.Resource.Annotations = make(map[string]string, 1) } - b.Resource.Annotations[AppliedGPUAnnotation] = gpuID + b.Resource.Annotations[AppliedGPUDevicesAnnotation] = EncodeGPUDevices(devices) +} + +func sortGPUDevices(devices []v1alpha2.GPUDeviceSpec) []v1alpha2.GPUDeviceSpec { + if len(devices) == 0 { + return nil + } + sorted := slices.Clone(devices) + slices.SortFunc(sorted, func(a, b v1alpha2.GPUDeviceSpec) int { + return strings.Compare(a.Name, b.Name) + }) + return sorted } diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index 809f7582c1..882bfa2c98 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -20,34 +20,36 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/types" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" ) var _ = Describe("GPU", func() { - It("should render DRA GPU resource claim", func() { + It("should render DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPU("vm-a", "GPU-test") + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "h100-sxm5-96gb"}}) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) - Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal(GPUName)) - Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu-template")) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu0")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu-gpu0-template")) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) - Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal(GPUName)) - Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal(GPUName)) - Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal(GPUResourceClaimRequestName)) - Expect(res.Annotations).To(HaveKeyWithValue(AppliedGPUAnnotation, "GPU-test")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu0")) + Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal("gpu-gpu0")) + Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal("req-gpu-gpu0")) + Expect(res.Annotations).To(HaveKeyWithValue(AppliedGPUDevicesAnnotation, `[{"name":"gpu0","model":"h100-sxm5-96gb"}]`)) }) - It("should remove rendered DRA GPU resource claim", func() { + It("should remove rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPU("vm-a", "GPU-test") + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "h100-sxm5-96gb"}}) - kvvm.SetGPU("vm-a", "") + kvvm.SetGPUDevices("vm-a", nil) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(BeEmpty()) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(BeEmpty()) - Expect(res.Annotations).NotTo(HaveKey(AppliedGPUAnnotation)) + Expect(res.Annotations).NotTo(HaveKey(AppliedGPUDevicesAnnotation)) }) }) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go index aa703677ad..23ce868dfa 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go @@ -247,7 +247,7 @@ func ApplyVirtualMachineSpec( return err } - kvvm.SetGPU(vm.Name, vm.Annotations[annotations.AnnVMGPUID]) + kvvm.SetGPUDevices(vm.Name, vm.Spec.GPUDevices) if err := kvvm.SetProvisioning(vm.Spec.Provisioning); err != nil { return err diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index cb36d8348f..16443a9bf7 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -29,7 +29,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" @@ -60,53 +59,46 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac } vm := s.VirtualMachine().Current() - gpuID := vm.Annotations[annotations.AnnVMGPUID] - templateName := kvbuilder.GPUResourceClaimTemplateName(vm.Name) - template := &resourcev1.ResourceClaimTemplate{} - key := types.NamespacedName{Name: templateName, Namespace: vm.Namespace} - - if gpuID == "" { - if err := h.client.Get(ctx, key, template); err != nil { - if apierrors.IsNotFound(err) { - return reconcile.Result{}, nil - } + log := logger.FromContext(ctx).With(logger.SlogHandler(nameGPUResourceClaimHandler)) + desiredTemplateNames := make(map[string]struct{}, len(vm.Spec.GPUDevices)) + + for _, device := range vm.Spec.GPUDevices { + templateName := kvbuilder.GPUResourceClaimTemplateName(vm.Name, device.Name) + desiredTemplateNames[templateName] = struct{}{} + desiredSpec := buildGPUResourceClaimTemplateSpec(device) + template := &resourcev1.ResourceClaimTemplate{} + key := types.NamespacedName{Name: templateName, Namespace: vm.Namespace} + + err := h.client.Get(ctx, key, template) + if err != nil && !apierrors.IsNotFound(err) { return reconcile.Result{}, fmt.Errorf("failed to get GPU ResourceClaimTemplate: %w", err) } - if metav1.IsControlledBy(template, vm) { - if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("failed to delete GPU ResourceClaimTemplate: %w", err) + + if apierrors.IsNotFound(err) { + template = buildGPUResourceClaimTemplate(vm, templateName, desiredSpec) + if err := h.client.Create(ctx, template); err != nil && !apierrors.IsAlreadyExists(err) { + return reconcile.Result{}, fmt.Errorf("failed to create GPU ResourceClaimTemplate: %w", err) } + log.Info("created GPU ResourceClaimTemplate", "template", templateName) + continue } - return reconcile.Result{}, nil - } - - desiredSpec := buildGPUResourceClaimTemplateSpec(gpuID) - log := logger.FromContext(ctx).With(logger.SlogHandler(nameGPUResourceClaimHandler)) - err := h.client.Get(ctx, key, template) - if err != nil && !apierrors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("failed to get GPU ResourceClaimTemplate: %w", err) - } - if apierrors.IsNotFound(err) { + if reflect.DeepEqual(template.Spec, desiredSpec) { + continue + } + if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to delete outdated GPU ResourceClaimTemplate: %w", err) + } template = buildGPUResourceClaimTemplate(vm, templateName, desiredSpec) if err := h.client.Create(ctx, template); err != nil && !apierrors.IsAlreadyExists(err) { - return reconcile.Result{}, fmt.Errorf("failed to create GPU ResourceClaimTemplate: %w", err) + return reconcile.Result{}, fmt.Errorf("failed to recreate GPU ResourceClaimTemplate: %w", err) } - log.Info("created GPU ResourceClaimTemplate", "template", templateName) - return reconcile.Result{}, nil + log.Info("recreated GPU ResourceClaimTemplate", "template", templateName) } - if reflect.DeepEqual(template.Spec, desiredSpec) { - return reconcile.Result{}, nil - } - if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("failed to delete outdated GPU ResourceClaimTemplate: %w", err) + if err := h.deleteOrphanedTemplates(ctx, vm, desiredTemplateNames); err != nil { + return reconcile.Result{}, err } - template = buildGPUResourceClaimTemplate(vm, templateName, desiredSpec) - if err := h.client.Create(ctx, template); err != nil && !apierrors.IsAlreadyExists(err) { - return reconcile.Result{}, fmt.Errorf("failed to recreate GPU ResourceClaimTemplate: %w", err) - } - log.Info("recreated GPU ResourceClaimTemplate", "template", templateName) return reconcile.Result{}, nil } @@ -121,16 +113,16 @@ func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spe } } -func buildGPUResourceClaimTemplateSpec(gpuID string) resourcev1.ResourceClaimTemplateSpec { +func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1.ResourceClaimTemplateSpec { selector := fmt.Sprintf( - `device.attributes["gpu.deckhouse.io"].gpuUUID == %s && device.attributes["gpu.deckhouse.io"].deviceType == "physical" && !has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`, - strconv.Quote(gpuID), + `device.attributes["gpu.deckhouse.io"].device == %s && device.attributes["gpu.deckhouse.io"].deviceType == "physical" && !has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`, + strconv.Quote(device.Model), ) return resourcev1.ResourceClaimTemplateSpec{ Spec: resourcev1.ResourceClaimSpec{ Devices: resourcev1.DeviceClaim{ Requests: []resourcev1.DeviceRequest{{ - Name: kvbuilder.GPUResourceClaimRequestName, + Name: kvbuilder.GPUResourceClaimRequestName(device.Name), Exactly: &resourcev1.ExactDeviceRequest{ DeviceClassName: gpuDeviceClassName, AllocationMode: resourcev1.DeviceAllocationModeExactCount, @@ -144,3 +136,24 @@ func buildGPUResourceClaimTemplateSpec(gpuID string) resourcev1.ResourceClaimTem }, } } + +func (h *GPUResourceClaimHandler) deleteOrphanedTemplates(ctx context.Context, vm *v1alpha2.VirtualMachine, desiredTemplateNames map[string]struct{}) error { + var templates resourcev1.ResourceClaimTemplateList + if err := h.client.List(ctx, &templates, client.InNamespace(vm.Namespace)); err != nil { + return fmt.Errorf("failed to list GPU ResourceClaimTemplates: %w", err) + } + + for i := range templates.Items { + template := &templates.Items[i] + if !metav1.IsControlledBy(template, vm) || !kvbuilder.IsGPUResourceClaimTemplateName(vm.Name, template.Name) { + continue + } + if _, ok := desiredTemplateNames[template.Name]; ok { + continue + } + if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete GPU ResourceClaimTemplate: %w", err) + } + } + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index 946db3027d..bb6b9cf05e 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -25,7 +25,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -34,40 +33,37 @@ var _ = Describe("GPUResourceClaimHandler", func() { const ( vmName = "vm-a" namespace = "default" - gpuID = "GPU-test" + gpuModel = "h100-sxm5-96gb" ) - newVM := func(id string) *v1alpha2.VirtualMachine { - vm := &v1alpha2.VirtualMachine{ - ObjectMeta: metav1.ObjectMeta{Name: vmName, Namespace: namespace, Annotations: map[string]string{}}, + newVM := func(devices ...v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { + return &v1alpha2.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: vmName, Namespace: namespace}, + Spec: v1alpha2.VirtualMachineSpec{GPUDevices: devices}, } - if id != "" { - vm.Annotations[annotations.AnnVMGPUID] = id - } - return vm } It("should create GPU ResourceClaimTemplate", func() { - fakeClient, _, vmState := setupEnvironment(newVM(gpuID)) + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", Model: gpuModel})) handler := NewGPUResourceClaimHandler(fakeClient) _, err := handler.Handle(context.Background(), vmState) Expect(err).NotTo(HaveOccurred()) template := &resourcev1.ResourceClaimTemplate{} - Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName), Namespace: namespace}, template)).To(Succeed()) + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, template)).To(Succeed()) Expect(template.Spec.Spec.Devices.Requests).To(HaveLen(1)) request := template.Spec.Spec.Devices.Requests[0] - Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimRequestName)) + Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimRequestName("gpu0"))) Expect(request.Exactly.DeviceClassName).To(Equal(gpuDeviceClassName)) - Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`gpuUUID == "GPU-test"`)) + Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`device == "h100-sxm5-96gb"`)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`deviceType == "physical"`)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`!has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`)) }) It("should delete owned GPU ResourceClaimTemplate when annotation is removed", func() { - vm := newVM("") - template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName), buildGPUResourceClaimTemplateSpec(gpuID)) + vm := newVM() + template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", Model: gpuModel})) fakeClient, _, vmState := setupEnvironment(vm, template) handler := NewGPUResourceClaimHandler(fakeClient) @@ -75,7 +71,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(err).NotTo(HaveOccurred()) stored := &resourcev1.ResourceClaimTemplate{} - err = fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName), Namespace: namespace}, stored) + err = fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, stored) Expect(err).To(HaveOccurred()) }) }) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index 0768d2590a..b8a72e409d 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -157,8 +157,6 @@ func (h *SyncKvvmHandler) Handle(ctx context.Context, s state.VirtualMachineStat lastAppliedSpec = h.loadLastAppliedSpec(current, kvvm) lastClassAppliedSpec := h.loadClassLastAppliedSpec(class, kvvm) changes = h.detectSpecChanges(ctx, kvvm, ¤t.Spec, lastAppliedSpec) - gpuAnnotationChanges := detectGPUAnnotationChanges(kvvm, current) - changes.Add(gpuAnnotationChanges.GetAll()...) if !changes.IsEmpty() { kvvmi, kvvmiErr := s.KVVMI(ctx) if kvvmiErr == nil { @@ -734,35 +732,6 @@ func (h *SyncKvvmHandler) detectSpecChanges( return specChanges } -func detectGPUAnnotationChanges(kvvm *virtv1.VirtualMachine, vm *v1alpha2.VirtualMachine) vmchange.SpecChanges { - var changes vmchange.SpecChanges - if kvvm == nil || vm == nil { - return changes - } - - currentValue := kvvm.Annotations[kvbuilder.AppliedGPUAnnotation] - desiredValue := vm.Annotations[annotations.AnnVMGPUID] - if currentValue == desiredValue { - return changes - } - - operation := vmchange.ChangeReplace - if currentValue == "" { - operation = vmchange.ChangeAdd - } - if desiredValue == "" { - operation = vmchange.ChangeRemove - } - changes.Add(vmchange.FieldChange{ - Operation: operation, - Path: "metadata.annotations." + annotations.AnnVMGPUID, - CurrentValue: currentValue, - DesiredValue: desiredValue, - ActionRequired: vmchange.ActionRestart, - }) - return changes -} - func (h *SyncKvvmHandler) detectClassSpecChanges(ctx context.Context, currentClassSpec, lastClassSpec *v1alpha2.VirtualMachineClassSpec) vmchange.SpecChanges { log := logger.FromContext(ctx) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 868d10f566..b3230d99ad 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -33,7 +33,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" - "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/common/network" "github.com/deckhouse/virtualization-controller/pkg/common/testutil" "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" @@ -421,14 +420,33 @@ var _ = Describe("SyncKvvmHandler", func() { Entry("Pending phase without changes, shouldn't have condition", v1alpha2.MachinePending, false, metav1.ConditionUnknown, false), ) - It("should require restart when GPU annotation changes on a running VM", func() { + It("should require restart when GPU devices change on a running VM", func() { ip := makeVMIP() vmClass := makeVMClass() vm := makeVM(v1alpha2.MachineRunning) - vm.Annotations = map[string]string{annotations.AnnVMGPUID: "GPU-new"} + vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "h100-sxm5-96gb"}} kvvm := makeKVVM(vm) - kvvm.Annotations[kvbuilder.AppliedGPUAnnotation] = "GPU-old" + Expect(kvbuilder.SetLastAppliedSpec(kvvm, &v1alpha2.VirtualMachine{ + Spec: v1alpha2.VirtualMachineSpec{ + CPU: v1alpha2.CPUSpec{ + Cores: vm.Spec.CPU.Cores, + }, + Memory: v1alpha2.MemorySpec{ + Size: vm.Spec.Memory.Size, + }, + VirtualMachineIPAddress: vm.Spec.VirtualMachineIPAddress, + RunPolicy: vm.Spec.RunPolicy, + OsType: vm.Spec.OsType, + VirtualMachineClassName: vm.Spec.VirtualMachineClassName, + Disruptions: &v1alpha2.Disruptions{ + RestartApprovalMode: vm.Spec.Disruptions.RestartApprovalMode, + }, + GPUDevices: []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "a100-sxm4-40gb"}}, + }, + })).To(Succeed()) + kvvm.Annotations[kvbuilder.AppliedGPUDevicesAnnotation] = `[{"name":"gpu0","model":"a100-sxm4-40gb"}]` + kvvm.SetGroupVersionKind(virtv1.VirtualMachineGroupVersionKind) kvvmi := makeKVVMI() fakeClient, reconcileObj, vmState = setupEnvironment(vm, kvvm, kvvmi, ip, vmClass) @@ -444,7 +462,7 @@ var _ = Describe("SyncKvvmHandler", func() { updatedKVVM := &virtv1.VirtualMachine{} Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(kvvm), updatedKVVM)).To(Succeed()) - Expect(updatedKVVM.Annotations).To(HaveKeyWithValue(kvbuilder.AppliedGPUAnnotation, "GPU-old")) + Expect(updatedKVVM.Annotations).To(HaveKeyWithValue(kvbuilder.AppliedGPUDevicesAnnotation, `[{"name":"gpu0","model":"a100-sxm4-40gb"}]`)) Expect(updatedKVVM.Spec.Template.Spec.Domain.Devices.GPUs).To(BeEmpty()) }) diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare.go b/images/virtualization-artifact/pkg/controller/vmchange/compare.go index d734167761..b350187440 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare.go @@ -81,6 +81,7 @@ func (v *VMSpecComparator) comparators() []VMSpecFieldComparator { vmSpecFieldComparator(compareProvisioning), vmSpecFieldComparator(compareNetworks), vmSpecFieldComparator(compareUSBDevices), + vmSpecFieldComparator(compareGPUDevices), } } diff --git a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go new file mode 100644 index 0000000000..45cf30098e --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go @@ -0,0 +1,36 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmchange + +import ( + "reflect" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func compareGPUDevices(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { + currentValue := NewValue(current.GPUDevices, current.GPUDevices == nil, false) + desiredValue := NewValue(desired.GPUDevices, desired.GPUDevices == nil, false) + + return compareValues( + "gpuDevices", + currentValue, + desiredValue, + reflect.DeepEqual(current.GPUDevices, desired.GPUDevices), + ActionRestart, + ) +} From 46804099d7b851d980cde5063461543826c581a5 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Tue, 23 Jun 2026 13:10:15 +0200 Subject: [PATCH 04/39] feat(api): add VM GPU devices builder option Signed-off-by: Daniil Antoshin --- images/virtualization-artifact/pkg/builder/vm/option.go | 6 ++++++ .../virtualization-artifact/pkg/controller/kvbuilder/gpu.go | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/images/virtualization-artifact/pkg/builder/vm/option.go b/images/virtualization-artifact/pkg/builder/vm/option.go index 5e3c3dcac3..085214e233 100644 --- a/images/virtualization-artifact/pkg/builder/vm/option.go +++ b/images/virtualization-artifact/pkg/builder/vm/option.go @@ -168,6 +168,12 @@ func WithUSBDevices(usbDevices []v1alpha2.USBDeviceSpecRef) Option { } } +func WithGPUDevices(gpuDevices []v1alpha2.GPUDeviceSpec) Option { + return func(vm *v1alpha2.VirtualMachine) { + vm.Spec.GPUDevices = gpuDevices + } +} + func WithIpAddress(ipAddress string) Option { return func(vm *v1alpha2.VirtualMachine) { vm.Spec.VirtualMachineIPAddress = ipAddress diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index a678a08303..c92b4c1775 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -19,9 +19,8 @@ package kvbuilder import ( "encoding/json" "fmt" - "strings" - "slices" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" From 63102dc1f2d1f0f58c85c5f100381b98aa5d0701 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Tue, 23 Jun 2026 13:26:33 +0200 Subject: [PATCH 05/39] refactor(api): remove GPU annotations from KVVM rendering Signed-off-by: Daniil Antoshin --- .../pkg/controller/kvbuilder/gpu.go | 21 ----------- .../pkg/controller/kvbuilder/gpu_test.go | 35 +++++++++++++++++-- .../controller/vm/internal/sync_kvvm_test.go | 2 -- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index c92b4c1775..c273765319 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -17,7 +17,6 @@ limitations under the License. package kvbuilder import ( - "encoding/json" "fmt" "slices" "strings" @@ -33,7 +32,6 @@ const ( GPUNamePrefix = "gpu-" GPUResourceClaimTemplateNameSuffixFormat = "-gpu-%s-template" GPUResourceClaimRequestNamePrefix = "req-gpu-" - AppliedGPUDevicesAnnotation = "internal.virtualization.deckhouse.io/applied-gpu-devices" ) func GPUResourceClaimName(deviceName string) string { @@ -52,17 +50,6 @@ func GPUResourceClaimRequestName(deviceName string) string { return GPUResourceClaimRequestNamePrefix + deviceName } -func EncodeGPUDevices(devices []v1alpha2.GPUDeviceSpec) string { - if len(devices) == 0 { - return "" - } - data, err := json.Marshal(sortGPUDevices(devices)) - if err != nil { - return "" - } - return string(data) -} - func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { devices = sortGPUDevices(devices) @@ -80,9 +67,6 @@ func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { ) if len(devices) == 0 { - if b.Resource.Annotations != nil { - delete(b.Resource.Annotations, AppliedGPUDevicesAnnotation) - } return } @@ -102,11 +86,6 @@ func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { }, }) } - - if b.Resource.Annotations == nil { - b.Resource.Annotations = make(map[string]string, 1) - } - b.Resource.Annotations[AppliedGPUDevicesAnnotation] = EncodeGPUDevices(devices) } func sortGPUDevices(devices []v1alpha2.GPUDeviceSpec) []v1alpha2.GPUDeviceSpec { diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index 882bfa2c98..35a9ba4089 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -38,7 +38,38 @@ var _ = Describe("GPU", func() { Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu0")) Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal("gpu-gpu0")) Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal("req-gpu-gpu0")) - Expect(res.Annotations).To(HaveKeyWithValue(AppliedGPUDevicesAnnotation, `[{"name":"gpu0","model":"h100-sxm5-96gb"}]`)) + Expect(res.Annotations).To(BeEmpty()) + }) + + It("should render DRA GPU resource claims in stable order", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{ + {Name: "gpu1", Model: "h100-sxm5-96gb"}, + {Name: "gpu0", Model: "a100-sxm4-40gb"}, + }) + res := kvvm.GetResource() + + Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(2)) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu0")) + Expect(res.Spec.Template.Spec.ResourceClaims[1].Name).To(Equal("gpu-gpu1")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(2)) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu0")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[1].Name).To(Equal("gpu-gpu1")) + }) + + It("should replace rendered DRA GPU resource claims", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "h100-sxm5-96gb"}}) + + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu1", Model: "a100-sxm4-40gb"}}) + res := kvvm.GetResource() + + Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu1")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu-gpu1-template")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu1")) }) It("should remove rendered DRA GPU resource claims", func() { @@ -50,6 +81,6 @@ var _ = Describe("GPU", func() { Expect(res.Spec.Template.Spec.ResourceClaims).To(BeEmpty()) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(BeEmpty()) - Expect(res.Annotations).NotTo(HaveKey(AppliedGPUDevicesAnnotation)) + Expect(res.Annotations).To(BeEmpty()) }) }) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index b3230d99ad..955058558e 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -445,7 +445,6 @@ var _ = Describe("SyncKvvmHandler", func() { GPUDevices: []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "a100-sxm4-40gb"}}, }, })).To(Succeed()) - kvvm.Annotations[kvbuilder.AppliedGPUDevicesAnnotation] = `[{"name":"gpu0","model":"a100-sxm4-40gb"}]` kvvm.SetGroupVersionKind(virtv1.VirtualMachineGroupVersionKind) kvvmi := makeKVVMI() @@ -462,7 +461,6 @@ var _ = Describe("SyncKvvmHandler", func() { updatedKVVM := &virtv1.VirtualMachine{} Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(kvvm), updatedKVVM)).To(Succeed()) - Expect(updatedKVVM.Annotations).To(HaveKeyWithValue(kvbuilder.AppliedGPUDevicesAnnotation, `[{"name":"gpu0","model":"a100-sxm4-40gb"}]`)) Expect(updatedKVVM.Spec.Template.Spec.Domain.Devices.GPUs).To(BeEmpty()) }) From 305e4f9011564b39a87b698ef6ae7714d833d124 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Tue, 23 Jun 2026 13:47:29 +0200 Subject: [PATCH 06/39] fix(api): select VM GPUs by product name Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/virtual_machine.go | 5 ++--- crds/doc-ru-virtualmachines.yaml | 2 +- crds/virtualmachines.yaml | 5 ++--- .../pkg/controller/kvbuilder/gpu_test.go | 12 ++++++------ .../vm/internal/gpu_resourceclaim_handler.go | 2 +- .../vm/internal/gpu_resourceclaim_handler_test.go | 4 ++-- .../pkg/controller/vm/internal/sync_kvvm_test.go | 4 ++-- 7 files changed, 16 insertions(+), 18 deletions(-) diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index 8945f2b221..d430814de3 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -518,10 +518,9 @@ type GPUDeviceSpec struct { // +kubebuilder:validation:MaxLength:=63 // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` Name string `json:"name"` - // GPU model identifier, for example h100-sxm5-96gb. + // GPU product name, for example NVIDIA H100. // +kubebuilder:validation:MinLength:=1 - // +kubebuilder:validation:MaxLength:=63 - // +kubebuilder:validation:Pattern:=`^[A-Za-z0-9]([A-Za-z0-9_.-]*[A-Za-z0-9])?$` + // +kubebuilder:validation:MaxLength:=128 Model string `json:"model"` } diff --git a/crds/doc-ru-virtualmachines.yaml b/crds/doc-ru-virtualmachines.yaml index 1e503eda11..ca3262cef6 100644 --- a/crds/doc-ru-virtualmachines.yaml +++ b/crds/doc-ru-virtualmachines.yaml @@ -596,7 +596,7 @@ spec: properties: model: description: | - Идентификатор модели GPU, например `h100-sxm5-96gb`. + Название продукта GPU, например `NVIDIA H100`. name: description: | Уникальное имя GPU-устройства внутри спецификации виртуальной машины. diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index a535d50a73..2fa35d85eb 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1075,11 +1075,10 @@ spec: properties: model: minLength: 1 - maxLength: 63 - pattern: ^[A-Za-z0-9]([A-Za-z0-9_.-]*[A-Za-z0-9])?$ + maxLength: 128 type: string description: | - GPU model identifier, for example h100-sxm5-96gb. + GPU product name, for example NVIDIA H100. name: minLength: 1 maxLength: 63 diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index 35a9ba4089..4ca3b6f77f 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -28,7 +28,7 @@ var _ = Describe("GPU", func() { It("should render DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "h100-sxm5-96gb"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}}) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) @@ -45,8 +45,8 @@ var _ = Describe("GPU", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{ - {Name: "gpu1", Model: "h100-sxm5-96gb"}, - {Name: "gpu0", Model: "a100-sxm4-40gb"}, + {Name: "gpu1", Model: "NVIDIA H100"}, + {Name: "gpu0", Model: "NVIDIA A100-SXM4-40GB"}, }) res := kvvm.GetResource() @@ -60,9 +60,9 @@ var _ = Describe("GPU", func() { It("should replace rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "h100-sxm5-96gb"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu1", Model: "a100-sxm4-40gb"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu1", Model: "NVIDIA A100-SXM4-40GB"}}) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) @@ -74,7 +74,7 @@ var _ = Describe("GPU", func() { It("should remove rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "h100-sxm5-96gb"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}}) kvvm.SetGPUDevices("vm-a", nil) res := kvvm.GetResource() diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index 16443a9bf7..ce72992002 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -115,7 +115,7 @@ func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spe func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1.ResourceClaimTemplateSpec { selector := fmt.Sprintf( - `device.attributes["gpu.deckhouse.io"].device == %s && device.attributes["gpu.deckhouse.io"].deviceType == "physical" && !has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`, + `device.attributes["gpu.deckhouse.io"].productName == %s && device.attributes["gpu.deckhouse.io"].deviceType == "physical" && !has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`, strconv.Quote(device.Model), ) return resourcev1.ResourceClaimTemplateSpec{ diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index bb6b9cf05e..36348d809e 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -33,7 +33,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { const ( vmName = "vm-a" namespace = "default" - gpuModel = "h100-sxm5-96gb" + gpuModel = "NVIDIA H100" ) newVM := func(devices ...v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { @@ -56,7 +56,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { request := template.Spec.Spec.Devices.Requests[0] Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimRequestName("gpu0"))) Expect(request.Exactly.DeviceClassName).To(Equal(gpuDeviceClassName)) - Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`device == "h100-sxm5-96gb"`)) + Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`productName == "NVIDIA H100"`)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`deviceType == "physical"`)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`!has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`)) }) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 955058558e..8a38223103 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -425,7 +425,7 @@ var _ = Describe("SyncKvvmHandler", func() { vmClass := makeVMClass() vm := makeVM(v1alpha2.MachineRunning) - vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "h100-sxm5-96gb"}} + vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}} kvvm := makeKVVM(vm) Expect(kvbuilder.SetLastAppliedSpec(kvvm, &v1alpha2.VirtualMachine{ Spec: v1alpha2.VirtualMachineSpec{ @@ -442,7 +442,7 @@ var _ = Describe("SyncKvvmHandler", func() { Disruptions: &v1alpha2.Disruptions{ RestartApprovalMode: vm.Spec.Disruptions.RestartApprovalMode, }, - GPUDevices: []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "a100-sxm4-40gb"}}, + GPUDevices: []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA A100-SXM4-40GB"}}, }, })).To(Succeed()) kvvm.SetGroupVersionKind(virtv1.VirtualMachineGroupVersionKind) From e69aadaa66b3e418f567393ab535a2e0d61c7b56 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Tue, 23 Jun 2026 14:57:54 +0200 Subject: [PATCH 07/39] fix(api): allow up to 16 VM GPU devices Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/virtual_machine.go | 2 +- crds/virtualmachines.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index d430814de3..7045477421 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -125,7 +125,7 @@ type VirtualMachineSpec struct { USBDevices []USBDeviceSpecRef `json:"usbDevices,omitempty"` // List of GPU devices to attach to the virtual machine. // Devices are requested by GPU model. - // +kubebuilder:validation:MaxItems:=8 + // +kubebuilder:validation:MaxItems:=16 // +listType=map // +listMapKey=name GPUDevices []GPUDeviceSpec `json:"gpuDevices,omitempty"` diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index 2fa35d85eb..ab7a750641 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1060,7 +1060,7 @@ spec: The name of USBDevice resource in the same namespace. gpuDevices: type: array - maxItems: 8 + maxItems: 16 x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map From 237ca5f9bf1e3064961fd1ce353b5db2c8f4bf01 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Tue, 23 Jun 2026 16:38:52 +0200 Subject: [PATCH 08/39] fix(vm): validate GPU DRA device configuration Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/virtual_machine.go | 4 +- crds/doc-ru-virtualmachines.yaml | 2 + crds/virtualmachines.yaml | 4 +- docs/USER_GUIDE.md | 46 +++++++ docs/USER_GUIDE.ru.md | 46 +++++++ .../pkg/controller/kvbuilder/gpu.go | 1 + .../vm/internal/gpu_resourceclaim_handler.go | 11 +- .../gpu_resourceclaim_handler_test.go | 18 ++- .../validators/gpu_devices_validator.go | 69 ++++++++++ .../validators/gpu_devices_validator_test.go | 122 ++++++++++++++++++ .../pkg/controller/vm/vm_webhook.go | 1 + .../pkg/controller/vmchange/compare_test.go | 44 +++++++ .../pkg/controller/vmchange/gpu_change.go | 22 +++- .../pkg/featuregates/featuregate.go | 20 ++- openapi/config-values.yaml | 4 + openapi/doc-ru-config-values.yaml | 2 + .../rbac-for-us.yaml | 8 ++ 17 files changed, 406 insertions(+), 18 deletions(-) create mode 100644 images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go create mode 100644 images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index 7045477421..f6ab812fbd 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -125,6 +125,7 @@ type VirtualMachineSpec struct { USBDevices []USBDeviceSpecRef `json:"usbDevices,omitempty"` // List of GPU devices to attach to the virtual machine. // Devices are requested by GPU model. + // This feature requires the GPU feature gate and the gpu.deckhouse.io DeviceClass. // +kubebuilder:validation:MaxItems:=16 // +listType=map // +listMapKey=name @@ -514,8 +515,9 @@ type USBDeviceSpecRef struct { // GPUDeviceSpec requests a GPU device by model. type GPUDeviceSpec struct { // A unique GPU device name inside the virtual machine spec. + // The value is used to generate DRA claim and request names. // +kubebuilder:validation:MinLength:=1 - // +kubebuilder:validation:MaxLength:=63 + // +kubebuilder:validation:MaxLength:=55 // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` Name string `json:"name"` // GPU product name, for example NVIDIA H100. diff --git a/crds/doc-ru-virtualmachines.yaml b/crds/doc-ru-virtualmachines.yaml index ca3262cef6..9be5b1005e 100644 --- a/crds/doc-ru-virtualmachines.yaml +++ b/crds/doc-ru-virtualmachines.yaml @@ -592,6 +592,7 @@ spec: description: | Список GPU-устройств для подключения к виртуальной машине. Устройства запрашиваются по модели GPU. + Для использования требуется feature gate `GPU` и DeviceClass `gpu.deckhouse.io`. items: properties: model: @@ -600,6 +601,7 @@ spec: name: description: | Уникальное имя GPU-устройства внутри спецификации виртуальной машины. + Значение используется для генерации имён DRA claim и request. status: properties: blockDeviceRefs: diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index ab7a750641..c954c0ce1c 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1067,6 +1067,7 @@ spec: description: | List of GPU devices to attach to the virtual machine. Devices are requested by GPU model. + This feature requires the GPU feature gate and the gpu.deckhouse.io DeviceClass. items: type: object required: @@ -1081,11 +1082,12 @@ spec: GPU product name, for example NVIDIA H100. name: minLength: 1 - maxLength: 63 + maxLength: 55 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string description: | A unique GPU device name inside the virtual machine spec. + The value is used to generate DRA claim and request names. status: type: object properties: diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 48eadac4d5..0534dd8bd0 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -4015,6 +4015,52 @@ spec: As a result, a VM named `clone-database-prod` and a disk named `clone-database-root-prod` will be created. +## GPU Devices + +{{< alert level="warning" >}} +GPU device passthrough is an experimental feature. It requires the Enterprise Edition (EE), Kubernetes DRA support, and an external GPU DRA provider that creates the `gpu.deckhouse.io` `DeviceClass`. +{{< /alert >}} + +The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by product model through the `.spec.gpuDevices` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. + +GPU device passthrough requires: + +- Kubernetes version 1.34 or higher with DRA feature gates required by the cluster configuration. +- The `GPU` feature gate enabled in the `virtualization` module settings. +- A GPU DRA provider installed in the cluster. +- The `gpu.deckhouse.io` [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) created by the GPU DRA provider. + +To enable the module feature gate: + +```yaml +apiVersion: deckhouse.io/v1alpha1 +kind: ModuleConfig +metadata: + name: virtualization +spec: + settings: + featureGates: + - GPU +``` + +To request a GPU device, add `.spec.gpuDevices` to the VM specification: + +```yaml +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualMachine +metadata: + name: linux-vm +spec: + # ... other VM settings ... + gpuDevices: + - name: gpu0 + model: NVIDIA H100 +``` + +The `name` field must be unique within `.spec.gpuDevices` and can contain up to 55 DNS-label characters. The `model` field must match the GPU product name exposed by the GPU DRA provider in the `device.attributes["gpu.deckhouse.io"].productName` device attribute. + +Changing `.spec.gpuDevices` requires restarting the virtual machine to apply the new configuration. + ## USB Devices {{< alert level="warning" >}} diff --git a/docs/USER_GUIDE.ru.md b/docs/USER_GUIDE.ru.md index 6946c25786..719b701d00 100644 --- a/docs/USER_GUIDE.ru.md +++ b/docs/USER_GUIDE.ru.md @@ -4046,6 +4046,52 @@ spec: В результате будет создана ВМ с именем `clone-database-prod` и диск с именем `clone-database-root-prod`. +## GPU-устройства + +{{< alert level="warning" >}} +Проброс GPU-устройств — экспериментальная возможность. Для работы требуются Enterprise Edition (EE), поддержка Kubernetes DRA и внешний GPU DRA-провайдер, создающий `DeviceClass` с именем `gpu.deckhouse.io`. +{{< /alert >}} + +Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по модели продукта через поле `.spec.gpuDevices` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). + +Для проброса GPU требуются: + +- Kubernetes версии 1.34 или выше с DRA feature gates, необходимыми для конфигурации кластера. +- Feature gate `GPU`, включённый в настройках модуля `virtualization`. +- Установленный в кластере GPU DRA-провайдер. +- [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) `gpu.deckhouse.io`, созданный GPU DRA-провайдером. + +Чтобы включить feature gate модуля: + +```yaml +apiVersion: deckhouse.io/v1alpha1 +kind: ModuleConfig +metadata: + name: virtualization +spec: + settings: + featureGates: + - GPU +``` + +Чтобы запросить GPU-устройство, добавьте `.spec.gpuDevices` в спецификацию ВМ: + +```yaml +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualMachine +metadata: + name: linux-vm +spec: + # ... другие настройки ВМ ... + gpuDevices: + - name: gpu0 + model: NVIDIA H100 +``` + +Поле `name` должно быть уникальным внутри `.spec.gpuDevices` и может содержать до 55 символов DNS label. Поле `model` должно совпадать с названием продукта GPU, которое GPU DRA-провайдер публикует в атрибуте устройства `device.attributes["gpu.deckhouse.io"].productName`. + +Изменение `.spec.gpuDevices` требует перезапуска виртуальной машины для применения новой конфигурации. + ## USB-устройства {{< alert level="warning">}} diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index c273765319..838de2427e 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -30,6 +30,7 @@ import ( const ( GPUNamePrefix = "gpu-" + GPUDeviceClassName = "gpu.deckhouse.io" GPUResourceClaimTemplateNameSuffixFormat = "-gpu-%s-template" GPUResourceClaimRequestNamePrefix = "req-gpu-" ) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index ce72992002..c3e0f5e73f 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -36,10 +36,7 @@ import ( "github.com/deckhouse/virtualization/api/core/v1alpha2" ) -const ( - nameGPUResourceClaimHandler = "GPUResourceClaimHandler" - gpuDeviceClassName = "gpu.deckhouse.io" -) +const nameGPUResourceClaimHandler = "GPUResourceClaimHandler" func NewGPUResourceClaimHandler(client client.Client) *GPUResourceClaimHandler { return &GPUResourceClaimHandler{client: client} @@ -83,6 +80,10 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac continue } + if !metav1.IsControlledBy(template, vm) { + return reconcile.Result{}, fmt.Errorf("GPU ResourceClaimTemplate %s/%s is not controlled by VirtualMachine %s/%s", template.Namespace, template.Name, vm.Namespace, vm.Name) + } + if reflect.DeepEqual(template.Spec, desiredSpec) { continue } @@ -124,7 +125,7 @@ func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1 Requests: []resourcev1.DeviceRequest{{ Name: kvbuilder.GPUResourceClaimRequestName(device.Name), Exactly: &resourcev1.ExactDeviceRequest{ - DeviceClassName: gpuDeviceClassName, + DeviceClassName: kvbuilder.GPUDeviceClassName, AllocationMode: resourcev1.DeviceAllocationModeExactCount, Count: 1, Selectors: []resourcev1.DeviceSelector{{ diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index 36348d809e..4ee24c30f3 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -55,7 +55,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(template.Spec.Spec.Devices.Requests).To(HaveLen(1)) request := template.Spec.Spec.Devices.Requests[0] Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimRequestName("gpu0"))) - Expect(request.Exactly.DeviceClassName).To(Equal(gpuDeviceClassName)) + Expect(request.Exactly.DeviceClassName).To(Equal(kvbuilder.GPUDeviceClassName)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`productName == "NVIDIA H100"`)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`deviceType == "physical"`)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`!has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`)) @@ -74,4 +74,20 @@ var _ = Describe("GPUResourceClaimHandler", func() { err = fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, stored) Expect(err).To(HaveOccurred()) }) + + It("should not replace GPU ResourceClaimTemplate owned by another controller", func() { + vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", Model: gpuModel}) + template := &resourcev1.ResourceClaimTemplate{ + ObjectMeta: metav1.ObjectMeta{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, + } + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).To(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: template.Name, Namespace: namespace}, stored)).To(Succeed()) + Expect(stored.OwnerReferences).To(BeEmpty()) + }) }) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go new file mode 100644 index 0000000000..e5a92fe51c --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -0,0 +1,69 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validators + +import ( + "context" + "fmt" + + resourcev1 "k8s.io/api/resource/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/component-base/featuregate" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +type GPUDevicesValidator struct { + client client.Client + featureGate featuregate.FeatureGate +} + +func NewGPUDevicesValidator(client client.Client, featureGate featuregate.FeatureGate) *GPUDevicesValidator { + return &GPUDevicesValidator{client: client, featureGate: featureGate} +} + +func (v *GPUDevicesValidator) ValidateCreate(ctx context.Context, vm *v1alpha2.VirtualMachine) (admission.Warnings, error) { + return nil, v.validateGPUDevices(ctx, vm) +} + +func (v *GPUDevicesValidator) ValidateUpdate(ctx context.Context, _, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) { + return nil, v.validateGPUDevices(ctx, newVM) +} + +func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alpha2.VirtualMachine) error { + if len(vm.Spec.GPUDevices) == 0 { + return nil + } + + if !v.featureGate.Enabled(featuregates.GPU) { + return fmt.Errorf("GPU device attachment requires the GPU feature gate") + } + + deviceClass := &resourcev1.DeviceClass{} + err := v.client.Get(ctx, client.ObjectKey{Name: kvbuilder.GPUDeviceClassName}, deviceClass) + if err == nil { + return nil + } + if apierrors.IsNotFound(err) { + return fmt.Errorf("GPU device attachment requires DeviceClass %q", kvbuilder.GPUDeviceClassName) + } + return fmt.Errorf("failed to get GPU DeviceClass %q: %w", kvbuilder.GPUDeviceClassName, err) +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go new file mode 100644 index 0000000000..133b622aa6 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -0,0 +1,122 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validators + +import ( + "strings" + "testing" + + resourcev1 "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/component-base/featuregate" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func TestGPUDevicesValidatorValidateCreate(t *testing.T) { + tests := []struct { + name string + featureEnabled bool + objects []client.Object + wantErrorPart string + }{ + { + name: "should reject GPU devices when feature is disabled", + featureEnabled: false, + objects: []client.Object{newGPUDeviceClass()}, + wantErrorPart: "GPU feature gate", + }, + { + name: "should reject GPU devices when DeviceClass is missing", + featureEnabled: true, + wantErrorPart: "DeviceClass", + }, + { + name: "should accept GPU devices when feature and DeviceClass are available", + featureEnabled: true, + objects: []client.Object{newGPUDeviceClass()}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}}) + validator := NewGPUDevicesValidator(newFakeClientWithResourceObjects(t, tt.objects...), newGPUFeatureGate(t, tt.featureEnabled)) + + _, err := validator.ValidateCreate(t.Context(), vm) + + if tt.wantErrorPart == "" { + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + return + } + + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), tt.wantErrorPart) { + t.Fatalf("expected error containing %q, got %v", tt.wantErrorPart, err) + } + }) + } +} + +func newVirtualMachineWithGPU(name string, gpuDevices []v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { + return &v1alpha2.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Spec: v1alpha2.VirtualMachineSpec{GPUDevices: gpuDevices}, + } +} + +func newGPUDeviceClass() *resourcev1.DeviceClass { + return &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: kvbuilder.GPUDeviceClassName}} +} + +func newFakeClientWithResourceObjects(t *testing.T, objects ...client.Object) client.Client { + t.Helper() + + scheme := runtime.NewScheme() + if err := v1alpha2.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add virtualization API scheme: %v", err) + } + if err := resourcev1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add resource API scheme: %v", err) + } + + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() +} + +func newGPUFeatureGate(t *testing.T, enabled bool) featuregate.FeatureGate { + t.Helper() + + gate, setFromMap, err := featuregates.NewUnlocked() + if err != nil { + t.Fatalf("failed to create feature gate: %v", err) + } + + if err = setFromMap(map[string]bool{string(featuregates.GPU): enabled}); err != nil { + t.Fatalf("failed to set GPU feature gate: %v", err) + } + + return gate +} diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go index c7e68d3a8b..cab4cedc87 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go @@ -56,6 +56,7 @@ func NewValidator(client client.Client, blockDeviceService *service.BlockDeviceS validators.NewNetworksValidator(client, featureGate), validators.NewFirstDiskValidator(client), validators.NewUSBDevicesValidator(client, featureGate), + validators.NewGPUDevicesValidator(client, featureGate), validators.NewVMBDAConflictValidator(client), validators.NewPVNodeAffinityValidator(client, attachmentService), }, diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go index e95e193d82..b46e2faf68 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go @@ -684,6 +684,43 @@ networks: requirePathOperation("networks", ChangeReplace), ), }, + { + "no restart when gpu devices only change order", + ` +gpuDevices: +- name: gpu1 + model: NVIDIA H100 +- name: gpu0 + model: NVIDIA A100-SXM4-40GB +`, + ` +gpuDevices: +- name: gpu0 + model: NVIDIA A100-SXM4-40GB +- name: gpu1 + model: NVIDIA H100 +`, + nil, + assertNoChanges(), + }, + { + "restart when gpu device model changes", + ` +gpuDevices: +- name: gpu0 + model: NVIDIA A100-SXM4-40GB +`, + ` +gpuDevices: +- name: gpu0 + model: NVIDIA H100 +`, + nil, + assertChanges( + actionRequired(ActionRestart), + requirePathOperation("gpuDevices", ChangeReplace), + ), + }, } for _, tt := range tests { @@ -769,6 +806,13 @@ func assertChanges(asserts ...func(t *testing.T, changes SpecChanges)) func(t *t } } +func assertNoChanges() func(t *testing.T, changes SpecChanges) { + return func(t *testing.T, changes SpecChanges) { + t.Helper() + require.True(t, changes.IsEmpty(), "expected no changes, got %+v", changes.GetAll()) + } +} + func actionRequired(actionType ActionType) func(t *testing.T, changes SpecChanges) { return func(t *testing.T, changes SpecChanges) { t.Helper() diff --git a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go index 45cf30098e..988e0c2590 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go @@ -18,19 +18,35 @@ package vmchange import ( "reflect" + "slices" + "strings" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) func compareGPUDevices(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { - currentValue := NewValue(current.GPUDevices, current.GPUDevices == nil, false) - desiredValue := NewValue(desired.GPUDevices, desired.GPUDevices == nil, false) + currentGPUDevices := sortedGPUDevicesForCompare(current.GPUDevices) + desiredGPUDevices := sortedGPUDevicesForCompare(desired.GPUDevices) + currentValue := NewValue(currentGPUDevices, current.GPUDevices == nil, false) + desiredValue := NewValue(desiredGPUDevices, desired.GPUDevices == nil, false) return compareValues( "gpuDevices", currentValue, desiredValue, - reflect.DeepEqual(current.GPUDevices, desired.GPUDevices), + reflect.DeepEqual(currentGPUDevices, desiredGPUDevices), ActionRestart, ) } + +func sortedGPUDevicesForCompare(devices []v1alpha2.GPUDeviceSpec) []v1alpha2.GPUDeviceSpec { + if devices == nil { + return nil + } + + sorted := slices.Clone(devices) + slices.SortFunc(sorted, func(a, b v1alpha2.GPUDeviceSpec) int { + return strings.Compare(a.Name, b.Name) + }) + return sorted +} diff --git a/images/virtualization-artifact/pkg/featuregates/featuregate.go b/images/virtualization-artifact/pkg/featuregates/featuregate.go index 633c3d9931..dc51cfd523 100644 --- a/images/virtualization-artifact/pkg/featuregates/featuregate.go +++ b/images/virtualization-artifact/pkg/featuregates/featuregate.go @@ -25,13 +25,14 @@ import ( ) const ( - SDN featuregate.Feature = "SDN" - AutoMigrationIfNodePlacementChanged featuregate.Feature = "AutoMigrationIfNodePlacementChanged" - VolumeMigration featuregate.Feature = "VolumeMigration" - TargetMigration featuregate.Feature = "TargetMigration" - USB featuregate.Feature = "USB" - HotplugCPUWithLiveMigration featuregate.Feature = "HotplugCPUWithLiveMigration" - HotplugMemoryWithLiveMigration featuregate.Feature = "HotplugMemoryWithLiveMigration" + SDN featuregate.Feature = "SDN" + AutoMigrationIfNodePlacementChanged featuregate.Feature = "AutoMigrationIfNodePlacementChanged" + VolumeMigration featuregate.Feature = "VolumeMigration" + TargetMigration featuregate.Feature = "TargetMigration" + USB featuregate.Feature = "USB" + GPU featuregate.Feature = "GPU" + HotplugCPUWithLiveMigration featuregate.Feature = "HotplugCPUWithLiveMigration" + HotplugMemoryWithLiveMigration featuregate.Feature = "HotplugMemoryWithLiveMigration" HotplugCPUAndMemoryWithInPlaceResize featuregate.Feature = "HotplugCPUAndMemoryWithInPlaceResize" VirtualMachinePool featuregate.Feature = "VirtualMachinePool" ) @@ -61,6 +62,11 @@ var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ LockToDefault: true, PreRelease: featuregate.Alpha, }, + GPU: { + Default: false, + LockToDefault: version.GetEdition() == version.EditionCE, + PreRelease: featuregate.Alpha, + }, HotplugCPUWithLiveMigration: { Default: false, LockToDefault: version.GetEdition() == version.EditionCE, diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 503a76325e..f8b27a9a68 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -267,9 +267,13 @@ properties: - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number via LiveMigration. (Not available in CE); - `HotplugMemoryWithLiveMigration` — enable live changing of memory size via LiveMigration. (Not available in CE); - `HotplugCPUAndMemoryWithInPlaceResize` - enable live changing of cpu cores number or memory size via InPlaceResize. (Not available in CE); + - `VirtualMachinePool` — enable the VirtualMachinePool resource for group management of virtual machines. (Not available in CE); + - `GPU` — enable attaching GPU devices to virtual machines via DRA by product model. (Not available in CE); items: type: string enum: - "HotplugCPUWithLiveMigration" - "HotplugMemoryWithLiveMigration" - "HotplugCPUAndMemoryWithInPlaceResize" + - "VirtualMachinePool" + - "GPU" diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index e1b17f65af..0ccff0ed73 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -179,5 +179,7 @@ properties: - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки через живую миграцию. (Не доступно в CE); - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки через живую миграцию. (Не доступно в CE); - `HotplugCPUAndMemoryWithInPlaceResize` - включить изменение количества ядер процессора или размера памяти без перезагрузки через InPlaceResize (Не доступно в CE) + - `VirtualMachinePool` — включить ресурс VirtualMachinePool для группового управления виртуальными машинами. (Не доступно в CE) + - `GPU` — включить подключение GPU-устройств к виртуальным машинам через DRA по модели продукта. (Не доступно в CE); items: type: string diff --git a/templates/virtualization-controller/rbac-for-us.yaml b/templates/virtualization-controller/rbac-for-us.yaml index 84eb2fef25..e895370513 100644 --- a/templates/virtualization-controller/rbac-for-us.yaml +++ b/templates/virtualization-controller/rbac-for-us.yaml @@ -344,6 +344,14 @@ rules: - update - patch - delete +- apiGroups: + - resource.k8s.io + resources: + - deviceclasses + verbs: + - get + - list + - watch - apiGroups: - apiextensions.k8s.io resources: From d13a675afbd0d6bd90a7ff69ef949d12c0fcd3fc Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Tue, 23 Jun 2026 16:48:25 +0200 Subject: [PATCH 09/39] refactor(api): drop dead branch in GPU claim template name matcher Signed-off-by: Daniil Antoshin --- images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index 838de2427e..5058bf6bde 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -44,7 +44,8 @@ func GPUResourceClaimTemplateName(vmName, deviceName string) string { } func IsGPUResourceClaimTemplateName(vmName, templateName string) bool { - return templateName == vmName+"-gpu-template" || strings.HasPrefix(templateName, vmName+"-gpu-") && strings.HasSuffix(templateName, "-template") + prefix := vmName + "-gpu-" + return strings.HasPrefix(templateName, prefix) && strings.HasSuffix(templateName, "-template") } func GPUResourceClaimRequestName(deviceName string) string { From 8cab5128b34f55339f9e9f71c4821c281f4bd0b7 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Tue, 23 Jun 2026 17:41:05 +0200 Subject: [PATCH 10/39] refactor(api): shorten GPU DRA claim/request/template names Drop the req- and -template suffixes from generated DRA names and align the device request name with the resource claim name (gpu-). The template name becomes -. This raises the user-facing gpuDevices[].name MaxLength from 55 to 59: the previous 55-char limit was dictated by the req-gpu- prefix (8 chars) that left no headroom against the 63-char DNS label limit. Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/virtual_machine.go | 2 +- crds/virtualmachines.yaml | 2 +- docs/USER_GUIDE.md | 2 +- docs/USER_GUIDE.ru.md | 2 +- .../pkg/controller/kvbuilder/gpu.go | 20 ++++++++----------- .../pkg/controller/kvbuilder/gpu_test.go | 6 +++--- 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index f6ab812fbd..38cf252211 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -517,7 +517,7 @@ type GPUDeviceSpec struct { // A unique GPU device name inside the virtual machine spec. // The value is used to generate DRA claim and request names. // +kubebuilder:validation:MinLength:=1 - // +kubebuilder:validation:MaxLength:=55 + // +kubebuilder:validation:MaxLength:=59 // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` Name string `json:"name"` // GPU product name, for example NVIDIA H100. diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index c954c0ce1c..877b06709b 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1082,7 +1082,7 @@ spec: GPU product name, for example NVIDIA H100. name: minLength: 1 - maxLength: 55 + maxLength: 59 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string description: | diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 0534dd8bd0..87cea08a88 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -4057,7 +4057,7 @@ spec: model: NVIDIA H100 ``` -The `name` field must be unique within `.spec.gpuDevices` and can contain up to 55 DNS-label characters. The `model` field must match the GPU product name exposed by the GPU DRA provider in the `device.attributes["gpu.deckhouse.io"].productName` device attribute. +The `name` field must be unique within `.spec.gpuDevices` and can contain up to 59 DNS-label characters. The `model` field must match the GPU product name exposed by the GPU DRA provider in the `device.attributes["gpu.deckhouse.io"].productName` device attribute. Changing `.spec.gpuDevices` requires restarting the virtual machine to apply the new configuration. diff --git a/docs/USER_GUIDE.ru.md b/docs/USER_GUIDE.ru.md index 719b701d00..d1c8413794 100644 --- a/docs/USER_GUIDE.ru.md +++ b/docs/USER_GUIDE.ru.md @@ -4088,7 +4088,7 @@ spec: model: NVIDIA H100 ``` -Поле `name` должно быть уникальным внутри `.spec.gpuDevices` и может содержать до 55 символов DNS label. Поле `model` должно совпадать с названием продукта GPU, которое GPU DRA-провайдер публикует в атрибуте устройства `device.attributes["gpu.deckhouse.io"].productName`. +Поле `name` должно быть уникальным внутри `.spec.gpuDevices` и может содержать до 59 символов DNS label. Поле `model` должно совпадать с названием продукта GPU, которое GPU DRA-провайдер публикует в атрибуте устройства `device.attributes["gpu.deckhouse.io"].productName`. Изменение `.spec.gpuDevices` требует перезапуска виртуальной машины для применения новой конфигурации. diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index 5058bf6bde..fd7ef61071 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -17,7 +17,6 @@ limitations under the License. package kvbuilder import ( - "fmt" "slices" "strings" @@ -29,27 +28,24 @@ import ( ) const ( - GPUNamePrefix = "gpu-" - GPUDeviceClassName = "gpu.deckhouse.io" - GPUResourceClaimTemplateNameSuffixFormat = "-gpu-%s-template" - GPUResourceClaimRequestNamePrefix = "req-gpu-" + GPUNamePrefix = "gpu-" + GPUDeviceClassName = "gpu.deckhouse.io" ) func GPUResourceClaimName(deviceName string) string { return GPUNamePrefix + deviceName } -func GPUResourceClaimTemplateName(vmName, deviceName string) string { - return vmName + fmt.Sprintf(GPUResourceClaimTemplateNameSuffixFormat, deviceName) +func GPUResourceClaimRequestName(deviceName string) string { + return GPUNamePrefix + deviceName } -func IsGPUResourceClaimTemplateName(vmName, templateName string) bool { - prefix := vmName + "-gpu-" - return strings.HasPrefix(templateName, prefix) && strings.HasSuffix(templateName, "-template") +func GPUResourceClaimTemplateName(vmName, deviceName string) string { + return vmName + "-" + deviceName } -func GPUResourceClaimRequestName(deviceName string) string { - return GPUResourceClaimRequestNamePrefix + deviceName +func IsGPUResourceClaimTemplateName(vmName, templateName string) bool { + return strings.HasPrefix(templateName, vmName+"-") } func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index 4ca3b6f77f..1babc5c931 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -33,11 +33,11 @@ var _ = Describe("GPU", func() { Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu0")) - Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu-gpu0-template")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu0")) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu0")) Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal("gpu-gpu0")) - Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal("req-gpu-gpu0")) + Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal("gpu-gpu0")) Expect(res.Annotations).To(BeEmpty()) }) @@ -67,7 +67,7 @@ var _ = Describe("GPU", func() { Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu1")) - Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu-gpu1-template")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu1")) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu1")) }) From b98ca92776a248ed30c613c5d9cb0a8460e82013 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Fri, 26 Jun 2026 17:45:15 +0200 Subject: [PATCH 11/39] =?UTF-8?q?=1B[0;38;2;204;204;204mrefactor(vm,=20gpu?= =?UTF-8?q?):=20deduplicate=20GPU=20sort=20and=20claim=20request=20name=20?= =?UTF-8?q?functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove GPUResourceClaimRequestName (identical to GPUResourceClaimName) and sortedGPUDevicesForCompare (duplicate of sortGPUDevices). Export SortGPUDevices so vmchange can reuse it. Signed-off-by: Daniil Antoshin --- .../pkg/controller/kvbuilder/gpu.go | 10 +++------- .../vm/internal/gpu_resourceclaim_handler.go | 2 +- .../gpu_resourceclaim_handler_test.go | 2 +- .../pkg/controller/vmchange/gpu_change.go | 19 +++---------------- 4 files changed, 8 insertions(+), 25 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index fd7ef61071..de191eb84e 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -36,10 +36,6 @@ func GPUResourceClaimName(deviceName string) string { return GPUNamePrefix + deviceName } -func GPUResourceClaimRequestName(deviceName string) string { - return GPUNamePrefix + deviceName -} - func GPUResourceClaimTemplateName(vmName, deviceName string) string { return vmName + "-" + deviceName } @@ -49,7 +45,7 @@ func IsGPUResourceClaimTemplateName(vmName, templateName string) bool { } func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { - devices = sortGPUDevices(devices) + devices = SortGPUDevices(devices) b.Resource.Spec.Template.Spec.ResourceClaims = slices.DeleteFunc( b.Resource.Spec.Template.Spec.ResourceClaims, @@ -80,13 +76,13 @@ func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { Name: claimName, ClaimRequest: &virtv1.ClaimRequest{ ClaimName: ptr.To(claimName), - RequestName: ptr.To(GPUResourceClaimRequestName(device.Name)), + RequestName: ptr.To(GPUResourceClaimName(device.Name)), }, }) } } -func sortGPUDevices(devices []v1alpha2.GPUDeviceSpec) []v1alpha2.GPUDeviceSpec { +func SortGPUDevices(devices []v1alpha2.GPUDeviceSpec) []v1alpha2.GPUDeviceSpec { if len(devices) == 0 { return nil } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index c3e0f5e73f..0f9a3dfbd8 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -123,7 +123,7 @@ func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1 Spec: resourcev1.ResourceClaimSpec{ Devices: resourcev1.DeviceClaim{ Requests: []resourcev1.DeviceRequest{{ - Name: kvbuilder.GPUResourceClaimRequestName(device.Name), + Name: kvbuilder.GPUResourceClaimName(device.Name), Exactly: &resourcev1.ExactDeviceRequest{ DeviceClassName: kvbuilder.GPUDeviceClassName, AllocationMode: resourcev1.DeviceAllocationModeExactCount, diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index 4ee24c30f3..cfd48dbf02 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -54,7 +54,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, template)).To(Succeed()) Expect(template.Spec.Spec.Devices.Requests).To(HaveLen(1)) request := template.Spec.Spec.Devices.Requests[0] - Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimRequestName("gpu0"))) + Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimName("gpu0"))) Expect(request.Exactly.DeviceClassName).To(Equal(kvbuilder.GPUDeviceClassName)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`productName == "NVIDIA H100"`)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`deviceType == "physical"`)) diff --git a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go index 988e0c2590..a7be7c057d 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go @@ -18,15 +18,14 @@ package vmchange import ( "reflect" - "slices" - "strings" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) func compareGPUDevices(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { - currentGPUDevices := sortedGPUDevicesForCompare(current.GPUDevices) - desiredGPUDevices := sortedGPUDevicesForCompare(desired.GPUDevices) + currentGPUDevices := kvbuilder.SortGPUDevices(current.GPUDevices) + desiredGPUDevices := kvbuilder.SortGPUDevices(desired.GPUDevices) currentValue := NewValue(currentGPUDevices, current.GPUDevices == nil, false) desiredValue := NewValue(desiredGPUDevices, desired.GPUDevices == nil, false) @@ -38,15 +37,3 @@ func compareGPUDevices(current, desired *v1alpha2.VirtualMachineSpec) []FieldCha ActionRestart, ) } - -func sortedGPUDevicesForCompare(devices []v1alpha2.GPUDeviceSpec) []v1alpha2.GPUDeviceSpec { - if devices == nil { - return nil - } - - sorted := slices.Clone(devices) - slices.SortFunc(sorted, func(a, b v1alpha2.GPUDeviceSpec) int { - return strings.Compare(a.Name, b.Name) - }) - return sorted -} From ebd7cf423fba302924ed2a9abc38849fb4bf9a44 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Sat, 27 Jun 2026 10:54:33 +0200 Subject: [PATCH 12/39] fix(vm): prevent GPU ResourceClaimTemplate name collision between VMs The template name was built as "-", which two VMs in the same namespace could collide on (e.g. VM "a" with device "b-c" and VM "a-b" with device "c" both yield "a-b-c"). The losing VM hit a permanent not-controlled-by reconcile error. Append a vmName hash suffix to make the name unique per VM. Signed-off-by: Daniil Antoshin --- .../virtualization-artifact/pkg/controller/kvbuilder/gpu.go | 6 +++++- .../pkg/controller/kvbuilder/gpu_test.go | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index de191eb84e..40d481e5c0 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -37,7 +37,11 @@ func GPUResourceClaimName(deviceName string) string { } func GPUResourceClaimTemplateName(vmName, deviceName string) string { - return vmName + "-" + deviceName + // The vmName hash suffix keeps the template name unique per VM: two VMs in the + // same namespace whose "-" prefixes collide (e.g. VM "a" with + // device "b-c" and VM "a-b" with device "c") would otherwise fight over one name + // and deadlock the losing VM's reconciliation on a not-controlled-by error. + return vmName + "-" + deviceName + "-" + GenerateSerial(vmName)[:8] } func IsGPUResourceClaimTemplateName(vmName, templateName string) bool { diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index 1babc5c931..736426ea59 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -33,7 +33,7 @@ var _ = Describe("GPU", func() { Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu0")) - Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu0")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal(GPUResourceClaimTemplateName("vm-a", "gpu0"))) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu0")) Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal("gpu-gpu0")) @@ -67,7 +67,7 @@ var _ = Describe("GPU", func() { Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu1")) - Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal("vm-a-gpu1")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal(GPUResourceClaimTemplateName("vm-a", "gpu1"))) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu1")) }) From 8b43fbb2e6322af26157e1a14cd8e2d33fe89805 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Sat, 27 Jun 2026 10:54:38 +0200 Subject: [PATCH 13/39] fix(vm): avoid spurious restart when gpuDevices toggles between empty list and unset isEmpty was derived from "GPUDevices == nil" while the compared values are sorted slices that collapse both nil and []{} to nil. A non-nil empty list versus unset produced a ChangeRemove with ActionRestart despite no real change. Derive isEmpty from the sorted slice length. Signed-off-by: Daniil Antoshin --- .../pkg/controller/vmchange/compare_test.go | 9 +++++++++ .../pkg/controller/vmchange/gpu_change.go | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go index b46e2faf68..0f5e046e5a 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go @@ -721,6 +721,15 @@ gpuDevices: requirePathOperation("gpuDevices", ChangeReplace), ), }, + { + "no restart when gpu devices change between empty list and unset", + ` +gpuDevices: [] +`, + ``, + nil, + assertNoChanges(), + }, } for _, tt := range tests { diff --git a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go index a7be7c057d..8a636a3988 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go @@ -26,8 +26,8 @@ import ( func compareGPUDevices(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { currentGPUDevices := kvbuilder.SortGPUDevices(current.GPUDevices) desiredGPUDevices := kvbuilder.SortGPUDevices(desired.GPUDevices) - currentValue := NewValue(currentGPUDevices, current.GPUDevices == nil, false) - desiredValue := NewValue(desiredGPUDevices, desired.GPUDevices == nil, false) + currentValue := NewValue(currentGPUDevices, len(currentGPUDevices) == 0, false) + desiredValue := NewValue(desiredGPUDevices, len(desiredGPUDevices) == 0, false) return compareValues( "gpuDevices", From eb5a631efed7c2dd93244ad243b846b394b1b7cb Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Sat, 27 Jun 2026 11:13:43 +0200 Subject: [PATCH 14/39] refactor(vm, module): ship gpu.deckhouse.io DeviceClass with the module The GPU devices validator no longer checks for an externally created DeviceClass; the module now ships the gpu.deckhouse.io DeviceClass itself (gated by the GPU feature gate and Kubernetes >= 1.34). The validator keeps only the feature gate check, and the now-unused deviceclasses RBAC grant is dropped. Signed-off-by: Daniil Antoshin --- docs/USER_GUIDE.md | 5 ++- docs/USER_GUIDE.ru.md | 5 ++- .../validators/gpu_devices_validator.go | 29 +++++---------- .../validators/gpu_devices_validator_test.go | 35 ++----------------- .../pkg/controller/vm/vm_webhook.go | 2 +- .../deviceclass-gpu.yaml | 11 ++++++ .../rbac-for-us.yaml | 8 ----- 7 files changed, 26 insertions(+), 69 deletions(-) create mode 100644 templates/virtualization-controller/deviceclass-gpu.yaml diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 87cea08a88..113f0426e6 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -4018,7 +4018,7 @@ As a result, a VM named `clone-database-prod` and a disk named `clone-database-r ## GPU Devices {{< alert level="warning" >}} -GPU device passthrough is an experimental feature. It requires the Enterprise Edition (EE), Kubernetes DRA support, and an external GPU DRA provider that creates the `gpu.deckhouse.io` `DeviceClass`. +GPU device passthrough is an experimental feature. It requires the Enterprise Edition (EE), Kubernetes DRA support, and an external GPU DRA provider that publishes GPUs as DRA resources with `gpu.deckhouse.io` device attributes. {{< /alert >}} The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by product model through the `.spec.gpuDevices` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. @@ -4027,8 +4027,7 @@ GPU device passthrough requires: - Kubernetes version 1.34 or higher with DRA feature gates required by the cluster configuration. - The `GPU` feature gate enabled in the `virtualization` module settings. -- A GPU DRA provider installed in the cluster. -- The `gpu.deckhouse.io` [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) created by the GPU DRA provider. +- A GPU DRA provider installed in the cluster that publishes GPUs with `gpu.deckhouse.io` device attributes. The `gpu.deckhouse.io` [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) is created by the module. To enable the module feature gate: diff --git a/docs/USER_GUIDE.ru.md b/docs/USER_GUIDE.ru.md index d1c8413794..b0e62cc28d 100644 --- a/docs/USER_GUIDE.ru.md +++ b/docs/USER_GUIDE.ru.md @@ -4049,7 +4049,7 @@ spec: ## GPU-устройства {{< alert level="warning" >}} -Проброс GPU-устройств — экспериментальная возможность. Для работы требуются Enterprise Edition (EE), поддержка Kubernetes DRA и внешний GPU DRA-провайдер, создающий `DeviceClass` с именем `gpu.deckhouse.io`. +Проброс GPU-устройств — экспериментальная возможность. Для работы требуются Enterprise Edition (EE), поддержка Kubernetes DRA и внешний GPU DRA-провайдер, публикующий GPU как DRA-ресурсы с атрибутами устройства `gpu.deckhouse.io`. {{< /alert >}} Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по модели продукта через поле `.spec.gpuDevices` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). @@ -4058,8 +4058,7 @@ spec: - Kubernetes версии 1.34 или выше с DRA feature gates, необходимыми для конфигурации кластера. - Feature gate `GPU`, включённый в настройках модуля `virtualization`. -- Установленный в кластере GPU DRA-провайдер. -- [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) `gpu.deckhouse.io`, созданный GPU DRA-провайдером. +- Установленный в кластере GPU DRA-провайдер, публикующий GPU с атрибутами устройства `gpu.deckhouse.io`. [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) `gpu.deckhouse.io` создаётся модулем. Чтобы включить feature gate модуля: diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index e5a92fe51c..cfada74216 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -20,35 +20,30 @@ import ( "context" "fmt" - resourcev1 "k8s.io/api/resource/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/component-base/featuregate" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) type GPUDevicesValidator struct { - client client.Client featureGate featuregate.FeatureGate } -func NewGPUDevicesValidator(client client.Client, featureGate featuregate.FeatureGate) *GPUDevicesValidator { - return &GPUDevicesValidator{client: client, featureGate: featureGate} +func NewGPUDevicesValidator(featureGate featuregate.FeatureGate) *GPUDevicesValidator { + return &GPUDevicesValidator{featureGate: featureGate} } -func (v *GPUDevicesValidator) ValidateCreate(ctx context.Context, vm *v1alpha2.VirtualMachine) (admission.Warnings, error) { - return nil, v.validateGPUDevices(ctx, vm) +func (v *GPUDevicesValidator) ValidateCreate(_ context.Context, vm *v1alpha2.VirtualMachine) (admission.Warnings, error) { + return nil, v.validateGPUDevices(vm) } -func (v *GPUDevicesValidator) ValidateUpdate(ctx context.Context, _, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) { - return nil, v.validateGPUDevices(ctx, newVM) +func (v *GPUDevicesValidator) ValidateUpdate(_ context.Context, _, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) { + return nil, v.validateGPUDevices(newVM) } -func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alpha2.VirtualMachine) error { +func (v *GPUDevicesValidator) validateGPUDevices(vm *v1alpha2.VirtualMachine) error { if len(vm.Spec.GPUDevices) == 0 { return nil } @@ -57,13 +52,5 @@ func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alph return fmt.Errorf("GPU device attachment requires the GPU feature gate") } - deviceClass := &resourcev1.DeviceClass{} - err := v.client.Get(ctx, client.ObjectKey{Name: kvbuilder.GPUDeviceClassName}, deviceClass) - if err == nil { - return nil - } - if apierrors.IsNotFound(err) { - return fmt.Errorf("GPU device attachment requires DeviceClass %q", kvbuilder.GPUDeviceClassName) - } - return fmt.Errorf("failed to get GPU DeviceClass %q: %w", kvbuilder.GPUDeviceClassName, err) + return nil } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index 133b622aa6..b0b3485f11 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -20,14 +20,9 @@ import ( "strings" "testing" - resourcev1 "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/component-base/featuregate" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -36,31 +31,23 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { tests := []struct { name string featureEnabled bool - objects []client.Object wantErrorPart string }{ { name: "should reject GPU devices when feature is disabled", featureEnabled: false, - objects: []client.Object{newGPUDeviceClass()}, wantErrorPart: "GPU feature gate", }, { - name: "should reject GPU devices when DeviceClass is missing", + name: "should accept GPU devices when feature is enabled", featureEnabled: true, - wantErrorPart: "DeviceClass", - }, - { - name: "should accept GPU devices when feature and DeviceClass are available", - featureEnabled: true, - objects: []client.Object{newGPUDeviceClass()}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}}) - validator := NewGPUDevicesValidator(newFakeClientWithResourceObjects(t, tt.objects...), newGPUFeatureGate(t, tt.featureEnabled)) + validator := NewGPUDevicesValidator(newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) @@ -88,24 +75,6 @@ func newVirtualMachineWithGPU(name string, gpuDevices []v1alpha2.GPUDeviceSpec) } } -func newGPUDeviceClass() *resourcev1.DeviceClass { - return &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: kvbuilder.GPUDeviceClassName}} -} - -func newFakeClientWithResourceObjects(t *testing.T, objects ...client.Object) client.Client { - t.Helper() - - scheme := runtime.NewScheme() - if err := v1alpha2.AddToScheme(scheme); err != nil { - t.Fatalf("failed to add virtualization API scheme: %v", err) - } - if err := resourcev1.AddToScheme(scheme); err != nil { - t.Fatalf("failed to add resource API scheme: %v", err) - } - - return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() -} - func newGPUFeatureGate(t *testing.T, enabled bool) featuregate.FeatureGate { t.Helper() diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go index cab4cedc87..d6457844ca 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go @@ -56,7 +56,7 @@ func NewValidator(client client.Client, blockDeviceService *service.BlockDeviceS validators.NewNetworksValidator(client, featureGate), validators.NewFirstDiskValidator(client), validators.NewUSBDevicesValidator(client, featureGate), - validators.NewGPUDevicesValidator(client, featureGate), + validators.NewGPUDevicesValidator(featureGate), validators.NewVMBDAConflictValidator(client), validators.NewPVNodeAffinityValidator(client, attachmentService), }, diff --git a/templates/virtualization-controller/deviceclass-gpu.yaml b/templates/virtualization-controller/deviceclass-gpu.yaml new file mode 100644 index 0000000000..233b931eef --- /dev/null +++ b/templates/virtualization-controller/deviceclass-gpu.yaml @@ -0,0 +1,11 @@ +{{- if and (has "GPU" .Values.virtualization.internal.moduleConfig.featureGates) (semverCompare ">=1.34" .Values.global.discovery.kubernetesVersion) }} +apiVersion: resource.k8s.io/v1 +kind: DeviceClass +metadata: + name: gpu.deckhouse.io + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-controller")) | nindent 2 }} +spec: + selectors: + - cel: + expression: 'has(device.attributes["gpu.deckhouse.io"].deviceType)' +{{- end }} diff --git a/templates/virtualization-controller/rbac-for-us.yaml b/templates/virtualization-controller/rbac-for-us.yaml index e895370513..84eb2fef25 100644 --- a/templates/virtualization-controller/rbac-for-us.yaml +++ b/templates/virtualization-controller/rbac-for-us.yaml @@ -344,14 +344,6 @@ rules: - update - patch - delete -- apiGroups: - - resource.k8s.io - resources: - - deviceclasses - verbs: - - get - - list - - watch - apiGroups: - apiextensions.k8s.io resources: From a3b78c5f484279dae5db542d9046cebcb0754d72 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Wed, 8 Jul 2026 17:16:40 +0200 Subject: [PATCH 15/39] feat(vm): request VFIO passthrough via VfioDeviceConfig in GPU ResourceClaimTemplate The GPU DRA driver binds a card to vfio-pci and exposes /dev/vfio/* only when the claim carries an opaque VfioDeviceConfig for the gpu.deckhouse.io driver. The CEL selector alone picks the device but does not request passthrough, so add the opaque config to the generated template. Signed-off-by: Daniil Antoshin --- .../vm/internal/gpu_resourceclaim_handler.go | 13 ++++++++++++- .../vm/internal/gpu_resourceclaim_handler_test.go | 5 +++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index 0f9a3dfbd8..9ecfe6ce18 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -25,6 +25,7 @@ import ( resourcev1 "k8s.io/api/resource/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -115,6 +116,7 @@ func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spe } func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1.ResourceClaimTemplateSpec { + requestName := kvbuilder.GPUResourceClaimName(device.Name) selector := fmt.Sprintf( `device.attributes["gpu.deckhouse.io"].productName == %s && device.attributes["gpu.deckhouse.io"].deviceType == "physical" && !has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`, strconv.Quote(device.Model), @@ -123,7 +125,7 @@ func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1 Spec: resourcev1.ResourceClaimSpec{ Devices: resourcev1.DeviceClaim{ Requests: []resourcev1.DeviceRequest{{ - Name: kvbuilder.GPUResourceClaimName(device.Name), + Name: requestName, Exactly: &resourcev1.ExactDeviceRequest{ DeviceClassName: kvbuilder.GPUDeviceClassName, AllocationMode: resourcev1.DeviceAllocationModeExactCount, @@ -133,6 +135,15 @@ func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1 }}, }, }}, + Config: []resourcev1.DeviceClaimConfiguration{{ + Requests: []string{requestName}, + DeviceConfiguration: resourcev1.DeviceConfiguration{ + Opaque: &resourcev1.OpaqueDeviceConfiguration{ + Driver: kvbuilder.GPUDeviceClassName, + Parameters: runtime.RawExtension{Raw: []byte(`{"apiVersion":"resource.gpu.deckhouse.io/v1alpha1","kind":"VfioDeviceConfig"}`)}, + }, + }, + }}, }, }, } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index cfd48dbf02..f895c38dad 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -59,6 +59,11 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`productName == "NVIDIA H100"`)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`deviceType == "physical"`)) Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`!has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`)) + Expect(template.Spec.Spec.Devices.Config).To(HaveLen(1)) + config := template.Spec.Spec.Devices.Config[0] + Expect(config.Requests).To(ConsistOf(kvbuilder.GPUResourceClaimName("gpu0"))) + Expect(config.Opaque.Driver).To(Equal(kvbuilder.GPUDeviceClassName)) + Expect(string(config.Opaque.Parameters.Raw)).To(ContainSubstring(`"kind":"VfioDeviceConfig"`)) }) It("should delete owned GPU ResourceClaimTemplate when annotation is removed", func() { From f46bd5767ff0ca381f3150e1bcf7c564b2a89d14 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Wed, 8 Jul 2026 17:33:10 +0200 Subject: [PATCH 16/39] feat(vm): reference DRA DeviceClass by name for GPU devices Replace the model-based GPU request with an explicit DeviceClass reference. spec.gpuDevices[].model becomes deviceClassName, and the generated ResourceClaimTemplate targets that DeviceClass directly instead of selecting devices through a module-owned CEL selector. The module no longer ships the gpu.deckhouse.io DeviceClass; the class is provided out-of-band (manually for now, by the GPU module later). Device selection (physical/exclusive/sharing constraints) now lives in the DeviceClass, and KubeVirt keeps enforcing passthrough safety. The opaque VfioDeviceConfig for the gpu.deckhouse.io driver is retained. Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/virtual_machine.go | 13 ++++++------ crds/doc-ru-virtualmachines.yaml | 8 ++++---- crds/virtualmachines.yaml | 13 ++++++------ docs/USER_GUIDE.md | 9 +++++---- docs/USER_GUIDE.ru.md | 9 +++++---- .../pkg/controller/kvbuilder/gpu.go | 4 ++-- .../pkg/controller/kvbuilder/gpu_test.go | 12 +++++------ .../vm/internal/gpu_resourceclaim_handler.go | 12 ++--------- .../gpu_resourceclaim_handler_test.go | 20 +++++++++---------- .../controller/vm/internal/sync_kvvm_test.go | 4 ++-- .../validators/gpu_devices_validator_test.go | 2 +- .../deviceclass-gpu.yaml | 11 ---------- 12 files changed, 50 insertions(+), 67 deletions(-) delete mode 100644 templates/virtualization-controller/deviceclass-gpu.yaml diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index 38cf252211..a8757c1f93 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -124,8 +124,8 @@ type VirtualMachineSpec struct { // +kubebuilder:validation:MaxItems:=8 USBDevices []USBDeviceSpecRef `json:"usbDevices,omitempty"` // List of GPU devices to attach to the virtual machine. - // Devices are requested by GPU model. - // This feature requires the GPU feature gate and the gpu.deckhouse.io DeviceClass. + // Each device references a DRA DeviceClass by name. + // This feature requires the GPU feature gate. // +kubebuilder:validation:MaxItems:=16 // +listType=map // +listMapKey=name @@ -512,7 +512,7 @@ type USBDeviceSpecRef struct { Name string `json:"name"` } -// GPUDeviceSpec requests a GPU device by model. +// GPUDeviceSpec requests a GPU device by DRA DeviceClass. type GPUDeviceSpec struct { // A unique GPU device name inside the virtual machine spec. // The value is used to generate DRA claim and request names. @@ -520,10 +520,11 @@ type GPUDeviceSpec struct { // +kubebuilder:validation:MaxLength:=59 // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` Name string `json:"name"` - // GPU product name, for example NVIDIA H100. + // Name of the DRA DeviceClass that selects the GPU to attach, for example nvidia-h100. // +kubebuilder:validation:MinLength:=1 - // +kubebuilder:validation:MaxLength:=128 - Model string `json:"model"` + // +kubebuilder:validation:MaxLength:=253 + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + DeviceClassName string `json:"deviceClassName"` } // USBDeviceStatusRef represents the status of a USB device attached to the virtual machine. diff --git a/crds/doc-ru-virtualmachines.yaml b/crds/doc-ru-virtualmachines.yaml index 9be5b1005e..4737a59289 100644 --- a/crds/doc-ru-virtualmachines.yaml +++ b/crds/doc-ru-virtualmachines.yaml @@ -591,13 +591,13 @@ spec: gpuDevices: description: | Список GPU-устройств для подключения к виртуальной машине. - Устройства запрашиваются по модели GPU. - Для использования требуется feature gate `GPU` и DeviceClass `gpu.deckhouse.io`. + Каждое устройство ссылается на DRA DeviceClass по имени. + Для использования требуется feature gate `GPU`. items: properties: - model: + deviceClassName: description: | - Название продукта GPU, например `NVIDIA H100`. + Имя DRA DeviceClass, который выбирает подключаемый GPU, например `nvidia-h100`. name: description: | Уникальное имя GPU-устройства внутри спецификации виртуальной машины. diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index 877b06709b..feef58ff94 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1066,20 +1066,21 @@ spec: x-kubernetes-list-type: map description: | List of GPU devices to attach to the virtual machine. - Devices are requested by GPU model. - This feature requires the GPU feature gate and the gpu.deckhouse.io DeviceClass. + Each device references a DRA DeviceClass by name. + This feature requires the GPU feature gate. items: type: object required: - - model + - deviceClassName - name properties: - model: + deviceClassName: minLength: 1 - maxLength: 128 + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string description: | - GPU product name, for example NVIDIA H100. + Name of the DRA DeviceClass that selects the GPU to attach, for example nvidia-h100. name: minLength: 1 maxLength: 59 diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 113f0426e6..8ea489d9af 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -4021,13 +4021,14 @@ As a result, a VM named `clone-database-prod` and a disk named `clone-database-r GPU device passthrough is an experimental feature. It requires the Enterprise Edition (EE), Kubernetes DRA support, and an external GPU DRA provider that publishes GPUs as DRA resources with `gpu.deckhouse.io` device attributes. {{< /alert >}} -The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by product model through the `.spec.gpuDevices` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. +The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by referencing a DRA [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) through the `.spec.gpuDevices` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. GPU device passthrough requires: - Kubernetes version 1.34 or higher with DRA feature gates required by the cluster configuration. - The `GPU` feature gate enabled in the `virtualization` module settings. -- A GPU DRA provider installed in the cluster that publishes GPUs with `gpu.deckhouse.io` device attributes. The `gpu.deckhouse.io` [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) is created by the module. +- A GPU DRA provider installed in the cluster that publishes GPUs with `gpu.deckhouse.io` device attributes. +- A DRA [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) that selects the GPU, referenced from `.spec.gpuDevices[].deviceClassName`. To enable the module feature gate: @@ -4053,10 +4054,10 @@ spec: # ... other VM settings ... gpuDevices: - name: gpu0 - model: NVIDIA H100 + deviceClassName: nvidia-h100 ``` -The `name` field must be unique within `.spec.gpuDevices` and can contain up to 59 DNS-label characters. The `model` field must match the GPU product name exposed by the GPU DRA provider in the `device.attributes["gpu.deckhouse.io"].productName` device attribute. +The `name` field must be unique within `.spec.gpuDevices` and can contain up to 59 DNS-label characters. The `deviceClassName` field must be the name of an existing DRA `DeviceClass` that selects the GPU to attach. Changing `.spec.gpuDevices` requires restarting the virtual machine to apply the new configuration. diff --git a/docs/USER_GUIDE.ru.md b/docs/USER_GUIDE.ru.md index b0e62cc28d..307be170c1 100644 --- a/docs/USER_GUIDE.ru.md +++ b/docs/USER_GUIDE.ru.md @@ -4052,13 +4052,14 @@ spec: Проброс GPU-устройств — экспериментальная возможность. Для работы требуются Enterprise Edition (EE), поддержка Kubernetes DRA и внешний GPU DRA-провайдер, публикующий GPU как DRA-ресурсы с атрибутами устройства `gpu.deckhouse.io`. {{< /alert >}} -Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по модели продукта через поле `.spec.gpuDevices` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). +Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по ссылке на DRA [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) через поле `.spec.gpuDevices` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). Для проброса GPU требуются: - Kubernetes версии 1.34 или выше с DRA feature gates, необходимыми для конфигурации кластера. - Feature gate `GPU`, включённый в настройках модуля `virtualization`. -- Установленный в кластере GPU DRA-провайдер, публикующий GPU с атрибутами устройства `gpu.deckhouse.io`. [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) `gpu.deckhouse.io` создаётся модулем. +- Установленный в кластере GPU DRA-провайдер, публикующий GPU с атрибутами устройства `gpu.deckhouse.io`. +- DRA [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes), выбирающий GPU, на который ссылается `.spec.gpuDevices[].deviceClassName`. Чтобы включить feature gate модуля: @@ -4084,10 +4085,10 @@ spec: # ... другие настройки ВМ ... gpuDevices: - name: gpu0 - model: NVIDIA H100 + deviceClassName: nvidia-h100 ``` -Поле `name` должно быть уникальным внутри `.spec.gpuDevices` и может содержать до 59 символов DNS label. Поле `model` должно совпадать с названием продукта GPU, которое GPU DRA-провайдер публикует в атрибуте устройства `device.attributes["gpu.deckhouse.io"].productName`. +Поле `name` должно быть уникальным внутри `.spec.gpuDevices` и может содержать до 59 символов DNS label. Поле `deviceClassName` должно быть именем существующего DRA `DeviceClass`, который выбирает подключаемый GPU. Изменение `.spec.gpuDevices` требует перезапуска виртуальной машины для применения новой конфигурации. diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index 40d481e5c0..f5f6e8bb4c 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -28,8 +28,8 @@ import ( ) const ( - GPUNamePrefix = "gpu-" - GPUDeviceClassName = "gpu.deckhouse.io" + GPUNamePrefix = "gpu-" + GPUDRADriverName = "gpu.deckhouse.io" ) func GPUResourceClaimName(deviceName string) string { diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index 736426ea59..9a4784e22d 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -28,7 +28,7 @@ var _ = Describe("GPU", func() { It("should render DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) @@ -45,8 +45,8 @@ var _ = Describe("GPU", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{ - {Name: "gpu1", Model: "NVIDIA H100"}, - {Name: "gpu0", Model: "NVIDIA A100-SXM4-40GB"}, + {Name: "gpu1", DeviceClassName: "nvidia-h100"}, + {Name: "gpu0", DeviceClassName: "nvidia-a100"}, }) res := kvvm.GetResource() @@ -60,9 +60,9 @@ var _ = Describe("GPU", func() { It("should replace rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu1", Model: "NVIDIA A100-SXM4-40GB"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu1", DeviceClassName: "nvidia-a100"}}) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) @@ -74,7 +74,7 @@ var _ = Describe("GPU", func() { It("should remove rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) kvvm.SetGPUDevices("vm-a", nil) res := kvvm.GetResource() diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index 9ecfe6ce18..f4078f6439 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "reflect" - "strconv" resourcev1 "k8s.io/api/resource/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -117,29 +116,22 @@ func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spe func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1.ResourceClaimTemplateSpec { requestName := kvbuilder.GPUResourceClaimName(device.Name) - selector := fmt.Sprintf( - `device.attributes["gpu.deckhouse.io"].productName == %s && device.attributes["gpu.deckhouse.io"].deviceType == "physical" && !has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`, - strconv.Quote(device.Model), - ) return resourcev1.ResourceClaimTemplateSpec{ Spec: resourcev1.ResourceClaimSpec{ Devices: resourcev1.DeviceClaim{ Requests: []resourcev1.DeviceRequest{{ Name: requestName, Exactly: &resourcev1.ExactDeviceRequest{ - DeviceClassName: kvbuilder.GPUDeviceClassName, + DeviceClassName: device.DeviceClassName, AllocationMode: resourcev1.DeviceAllocationModeExactCount, Count: 1, - Selectors: []resourcev1.DeviceSelector{{ - CEL: &resourcev1.CELDeviceSelector{Expression: selector}, - }}, }, }}, Config: []resourcev1.DeviceClaimConfiguration{{ Requests: []string{requestName}, DeviceConfiguration: resourcev1.DeviceConfiguration{ Opaque: &resourcev1.OpaqueDeviceConfiguration{ - Driver: kvbuilder.GPUDeviceClassName, + Driver: kvbuilder.GPUDRADriverName, Parameters: runtime.RawExtension{Raw: []byte(`{"apiVersion":"resource.gpu.deckhouse.io/v1alpha1","kind":"VfioDeviceConfig"}`)}, }, }, diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index f895c38dad..135075b06f 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -31,9 +31,9 @@ import ( var _ = Describe("GPUResourceClaimHandler", func() { const ( - vmName = "vm-a" - namespace = "default" - gpuModel = "NVIDIA H100" + vmName = "vm-a" + namespace = "default" + deviceClass = "nvidia-h100" ) newVM := func(devices ...v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { @@ -44,7 +44,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { } It("should create GPU ResourceClaimTemplate", func() { - fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", Model: gpuModel})) + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) handler := NewGPUResourceClaimHandler(fakeClient) _, err := handler.Handle(context.Background(), vmState) @@ -55,20 +55,18 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(template.Spec.Spec.Devices.Requests).To(HaveLen(1)) request := template.Spec.Spec.Devices.Requests[0] Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimName("gpu0"))) - Expect(request.Exactly.DeviceClassName).To(Equal(kvbuilder.GPUDeviceClassName)) - Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`productName == "NVIDIA H100"`)) - Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`deviceType == "physical"`)) - Expect(request.Exactly.Selectors[0].CEL.Expression).To(ContainSubstring(`!has(device.attributes["gpu.deckhouse.io"].sharingStrategy)`)) + Expect(request.Exactly.DeviceClassName).To(Equal(deviceClass)) + Expect(request.Exactly.Selectors).To(BeEmpty()) Expect(template.Spec.Spec.Devices.Config).To(HaveLen(1)) config := template.Spec.Spec.Devices.Config[0] Expect(config.Requests).To(ConsistOf(kvbuilder.GPUResourceClaimName("gpu0"))) - Expect(config.Opaque.Driver).To(Equal(kvbuilder.GPUDeviceClassName)) + Expect(config.Opaque.Driver).To(Equal(kvbuilder.GPUDRADriverName)) Expect(string(config.Opaque.Parameters.Raw)).To(ContainSubstring(`"kind":"VfioDeviceConfig"`)) }) It("should delete owned GPU ResourceClaimTemplate when annotation is removed", func() { vm := newVM() - template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", Model: gpuModel})) + template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) fakeClient, _, vmState := setupEnvironment(vm, template) handler := NewGPUResourceClaimHandler(fakeClient) @@ -81,7 +79,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { }) It("should not replace GPU ResourceClaimTemplate owned by another controller", func() { - vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", Model: gpuModel}) + vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass}) template := &resourcev1.ResourceClaimTemplate{ ObjectMeta: metav1.ObjectMeta{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 8a38223103..060aa15620 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -425,7 +425,7 @@ var _ = Describe("SyncKvvmHandler", func() { vmClass := makeVMClass() vm := makeVM(v1alpha2.MachineRunning) - vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}} + vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}} kvvm := makeKVVM(vm) Expect(kvbuilder.SetLastAppliedSpec(kvvm, &v1alpha2.VirtualMachine{ Spec: v1alpha2.VirtualMachineSpec{ @@ -442,7 +442,7 @@ var _ = Describe("SyncKvvmHandler", func() { Disruptions: &v1alpha2.Disruptions{ RestartApprovalMode: vm.Spec.Disruptions.RestartApprovalMode, }, - GPUDevices: []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA A100-SXM4-40GB"}}, + GPUDevices: []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-a100"}}, }, })).To(Succeed()) kvvm.SetGroupVersionKind(virtv1.VirtualMachineGroupVersionKind) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index b0b3485f11..5a9d07cb5c 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -46,7 +46,7 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", Model: "NVIDIA H100"}}) + vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) validator := NewGPUDevicesValidator(newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) diff --git a/templates/virtualization-controller/deviceclass-gpu.yaml b/templates/virtualization-controller/deviceclass-gpu.yaml deleted file mode 100644 index 233b931eef..0000000000 --- a/templates/virtualization-controller/deviceclass-gpu.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if and (has "GPU" .Values.virtualization.internal.moduleConfig.featureGates) (semverCompare ">=1.34" .Values.global.discovery.kubernetesVersion) }} -apiVersion: resource.k8s.io/v1 -kind: DeviceClass -metadata: - name: gpu.deckhouse.io - {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-controller")) | nindent 2 }} -spec: - selectors: - - cel: - expression: 'has(device.attributes["gpu.deckhouse.io"].deviceType)' -{{- end }} From cca7bdfa0dafd041963d5b8e59d5b22996be45af Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 12:41:29 +0200 Subject: [PATCH 17/39] fix(vm): enable GPU DRA passthrough in the KubeVirt layer Two paired blockers surfaced when applying a DRA GPU to the KubeVirt VM: - The embedded VirtualMachine CRD kept "deviceName" in gpus.items.required (only "name" is required in the VMI CRD since the 1.6.2 bump). A DRA GPU carries claimName/requestName without deviceName, so the apiserver rejected the KVVM update with "gpus[0].deviceName: Required value". Drop deviceName from required so it matches the embedded VMISpec. - Add the GPUsWithDRA feature gate to the KubeVirt configuration; the DRA admitter rejects DRA GPUs unless it is enabled. Signed-off-by: Daniil Antoshin --- crds/embedded/virtualmachines.yaml | 1 - templates/kubevirt/_kubevirt_helpers.tpl | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/crds/embedded/virtualmachines.yaml b/crds/embedded/virtualmachines.yaml index 45622d1a0c..bdeb54135f 100644 --- a/crds/embedded/virtualmachines.yaml +++ b/crds/embedded/virtualmachines.yaml @@ -2394,7 +2394,6 @@ spec: type: object type: object required: - - deviceName - name type: object type: array diff --git a/templates/kubevirt/_kubevirt_helpers.tpl b/templates/kubevirt/_kubevirt_helpers.tpl index 57a9b5feb5..7fd0bc0b2b 100644 --- a/templates/kubevirt/_kubevirt_helpers.tpl +++ b/templates/kubevirt/_kubevirt_helpers.tpl @@ -68,6 +68,7 @@ because its business logic does not yet have their default behavior implemented. - HostDevicesWithDRA - HostDevices - HotplugHostDevicesWithDRA # custom feature gate - added in our KubeVirt fork, not present in upstream +- GPUsWithDRA {{- if has "HotplugCPUAndMemoryWithInPlaceResize" .Values.virtualization.internal.moduleConfig.featureGates }} - InPlaceResize # custom feature gate - added in our KubeVirt fork, not present in upstream {{- end }} From 4ee197f1d13b03703666d8e123d2ee7f22c1f934 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 13:15:46 +0200 Subject: [PATCH 18/39] fix(vm): restore missing annotations import in sync_kvvm_test The test still references annotations.AnnVMRestorePowerState in the restore-power-state cases, but the import was dropped, breaking test compilation with "undefined: annotations". Signed-off-by: Daniil Antoshin --- .../pkg/controller/vm/internal/sync_kvvm_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 060aa15620..8540669e18 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/common/network" "github.com/deckhouse/virtualization-controller/pkg/common/testutil" "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" From 7ef0d67c6f337661bb730797d81bb06ae5c40256 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 13:36:20 +0200 Subject: [PATCH 19/39] fix(vm): update GPU compare test to deviceClassName and fix featuregate alignment - compare_test.go used the old model field in its gpuDevices YAML; after the field was renamed to deviceClassName the YAML no longer parsed, so the "restart on change" case detected no change. Switch the fixtures to deviceClassName. - Re-align the featuregate const block; the GPU entry left it aligned to two different columns, tripping the formatter. Signed-off-by: Daniil Antoshin --- .../pkg/controller/vmchange/compare_test.go | 14 +++++++------- .../pkg/featuregates/featuregate.go | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go index 0f5e046e5a..6d2cb21bc1 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go @@ -689,31 +689,31 @@ networks: ` gpuDevices: - name: gpu1 - model: NVIDIA H100 + deviceClassName: nvidia-h100 - name: gpu0 - model: NVIDIA A100-SXM4-40GB + deviceClassName: nvidia-a100 `, ` gpuDevices: - name: gpu0 - model: NVIDIA A100-SXM4-40GB + deviceClassName: nvidia-a100 - name: gpu1 - model: NVIDIA H100 + deviceClassName: nvidia-h100 `, nil, assertNoChanges(), }, { - "restart when gpu device model changes", + "restart when gpu device deviceClassName changes", ` gpuDevices: - name: gpu0 - model: NVIDIA A100-SXM4-40GB + deviceClassName: nvidia-a100 `, ` gpuDevices: - name: gpu0 - model: NVIDIA H100 + deviceClassName: nvidia-h100 `, nil, assertChanges( diff --git a/images/virtualization-artifact/pkg/featuregates/featuregate.go b/images/virtualization-artifact/pkg/featuregates/featuregate.go index dc51cfd523..0ebdda2ee0 100644 --- a/images/virtualization-artifact/pkg/featuregates/featuregate.go +++ b/images/virtualization-artifact/pkg/featuregates/featuregate.go @@ -25,14 +25,14 @@ import ( ) const ( - SDN featuregate.Feature = "SDN" - AutoMigrationIfNodePlacementChanged featuregate.Feature = "AutoMigrationIfNodePlacementChanged" - VolumeMigration featuregate.Feature = "VolumeMigration" - TargetMigration featuregate.Feature = "TargetMigration" - USB featuregate.Feature = "USB" - GPU featuregate.Feature = "GPU" - HotplugCPUWithLiveMigration featuregate.Feature = "HotplugCPUWithLiveMigration" - HotplugMemoryWithLiveMigration featuregate.Feature = "HotplugMemoryWithLiveMigration" + SDN featuregate.Feature = "SDN" + AutoMigrationIfNodePlacementChanged featuregate.Feature = "AutoMigrationIfNodePlacementChanged" + VolumeMigration featuregate.Feature = "VolumeMigration" + TargetMigration featuregate.Feature = "TargetMigration" + USB featuregate.Feature = "USB" + GPU featuregate.Feature = "GPU" + HotplugCPUWithLiveMigration featuregate.Feature = "HotplugCPUWithLiveMigration" + HotplugMemoryWithLiveMigration featuregate.Feature = "HotplugMemoryWithLiveMigration" HotplugCPUAndMemoryWithInPlaceResize featuregate.Feature = "HotplugCPUAndMemoryWithInPlaceResize" VirtualMachinePool featuregate.Feature = "VirtualMachinePool" ) From 00230d0e96335ee531d7af1fd1743dcebab6194b Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 14:02:28 +0200 Subject: [PATCH 20/39] fix(vm): require GPU feature gate only when GPU devices change ValidateUpdate rejected any update to a VM that still has gpuDevices when the GPU feature gate is off. A VM created while the gate was enabled would then be unmanageable (even unrelated edits) if the gate is later disabled. Skip the gate check when gpuDevices are unchanged, so the gate is enforced only on introducing or changing GPU devices. Signed-off-by: Daniil Antoshin --- .../validators/gpu_devices_validator.go | 9 ++- .../validators/gpu_devices_validator_test.go | 70 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index cfada74216..0d4a09f4ff 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -19,6 +19,7 @@ package validators import ( "context" "fmt" + "reflect" "k8s.io/component-base/featuregate" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -39,7 +40,13 @@ func (v *GPUDevicesValidator) ValidateCreate(_ context.Context, vm *v1alpha2.Vir return nil, v.validateGPUDevices(vm) } -func (v *GPUDevicesValidator) ValidateUpdate(_ context.Context, _, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) { +func (v *GPUDevicesValidator) ValidateUpdate(_ context.Context, oldVM, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) { + // The feature gate is required only when GPU devices are introduced or changed. + // Unchanged GPU devices must not block unrelated updates (or removal) of a VM + // created while the gate was enabled and later disabled. + if reflect.DeepEqual(oldVM.Spec.GPUDevices, newVM.Spec.GPUDevices) { + return nil, nil + } return nil, v.validateGPUDevices(newVM) } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index 5a9d07cb5c..4d66dea457 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -68,6 +68,76 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { } } +func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { + gpu := func(class string) []v1alpha2.GPUDeviceSpec { + return []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: class}} + } + + tests := []struct { + name string + featureEnabled bool + oldGPU []v1alpha2.GPUDeviceSpec + newGPU []v1alpha2.GPUDeviceSpec + wantErrorPart string + }{ + { + name: "unchanged GPU devices are allowed when feature is disabled", + featureEnabled: false, + oldGPU: gpu("nvidia-h100"), + newGPU: gpu("nvidia-h100"), + }, + { + name: "removing GPU devices is allowed when feature is disabled", + featureEnabled: false, + oldGPU: gpu("nvidia-h100"), + newGPU: nil, + }, + { + name: "adding GPU devices is rejected when feature is disabled", + featureEnabled: false, + oldGPU: nil, + newGPU: gpu("nvidia-h100"), + wantErrorPart: "GPU feature gate", + }, + { + name: "changing GPU devices is rejected when feature is disabled", + featureEnabled: false, + oldGPU: gpu("nvidia-h100"), + newGPU: gpu("nvidia-a100"), + wantErrorPart: "GPU feature gate", + }, + { + name: "changing GPU devices is allowed when feature is enabled", + featureEnabled: true, + oldGPU: gpu("nvidia-h100"), + newGPU: gpu("nvidia-a100"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + oldVM := newVirtualMachineWithGPU("vm-current", tt.oldGPU) + newVM := newVirtualMachineWithGPU("vm-current", tt.newGPU) + validator := NewGPUDevicesValidator(newGPUFeatureGate(t, tt.featureEnabled)) + + _, err := validator.ValidateUpdate(t.Context(), oldVM, newVM) + + if tt.wantErrorPart == "" { + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + return + } + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), tt.wantErrorPart) { + t.Fatalf("expected error containing %q, got %v", tt.wantErrorPart, err) + } + }) + } +} + func newVirtualMachineWithGPU(name string, gpuDevices []v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { return &v1alpha2.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, From 3144334699a9cf2466c56cb255ea1af6e919841b Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 14:35:54 +0200 Subject: [PATCH 21/39] test(vm): orphan cleanup keeps RCT owned by another VM with shared name prefix VM "vm-a" and "vm-a-b" collide on the IsGPUResourceClaimTemplateName prefix check; ensure the IsControlledBy owner guard prevents vm-a from deleting the ResourceClaimTemplate owned by vm-a-b. Signed-off-by: Daniil Antoshin --- .../gpu_resourceclaim_handler_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index 135075b06f..e41f3e46ea 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -93,4 +93,23 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: template.Name, Namespace: namespace}, stored)).To(Succeed()) Expect(stored.OwnerReferences).To(BeEmpty()) }) + + It("should not delete a ResourceClaimTemplate owned by another VM with a shared name prefix", func() { + vm := newVM() + vm.UID = "uid-vm-a" + otherVM := &v1alpha2.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: vmName + "-b", Namespace: namespace, UID: "uid-vm-a-b"}, + } + otherName := kvbuilder.GPUResourceClaimTemplateName(otherVM.Name, "gpu0") + Expect(kvbuilder.IsGPUResourceClaimTemplateName(vmName, otherName)).To(BeTrue()) + template := buildGPUResourceClaimTemplate(otherVM, otherName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: otherName, Namespace: namespace}, stored)).To(Succeed()) + }) }) From 76274c8a24127334e07a918e59f9aec60e8d0ea1 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 14:39:46 +0200 Subject: [PATCH 22/39] feat(vm): reject GPU devices referencing a missing DeviceClass Without this check a VM referencing a non-existent DeviceClass passes admission and its ResourceClaim hangs Pending forever with no clear reason. Validate on admission that each gpuDevices[].deviceClassName resolves to an existing DeviceClass, and fail fast otherwise. The check runs together with the feature-gate check, i.e. only when GPU devices are introduced or changed. Signed-off-by: Daniil Antoshin --- .../validators/gpu_devices_validator.go | 37 ++++++-- .../validators/gpu_devices_validator_test.go | 92 ++++++++++++------- .../pkg/controller/vm/vm_webhook.go | 2 +- 3 files changed, 87 insertions(+), 44 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index 0d4a09f4ff..b9b7d634a1 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -21,7 +21,11 @@ import ( "fmt" "reflect" + resourcev1 "k8s.io/api/resource/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/component-base/featuregate" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/deckhouse/virtualization-controller/pkg/featuregates" @@ -29,28 +33,30 @@ import ( ) type GPUDevicesValidator struct { + client client.Client featureGate featuregate.FeatureGate } -func NewGPUDevicesValidator(featureGate featuregate.FeatureGate) *GPUDevicesValidator { - return &GPUDevicesValidator{featureGate: featureGate} +func NewGPUDevicesValidator(client client.Client, featureGate featuregate.FeatureGate) *GPUDevicesValidator { + return &GPUDevicesValidator{client: client, featureGate: featureGate} } -func (v *GPUDevicesValidator) ValidateCreate(_ context.Context, vm *v1alpha2.VirtualMachine) (admission.Warnings, error) { - return nil, v.validateGPUDevices(vm) +func (v *GPUDevicesValidator) ValidateCreate(ctx context.Context, vm *v1alpha2.VirtualMachine) (admission.Warnings, error) { + return nil, v.validateGPUDevices(ctx, vm) } -func (v *GPUDevicesValidator) ValidateUpdate(_ context.Context, oldVM, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) { - // The feature gate is required only when GPU devices are introduced or changed. - // Unchanged GPU devices must not block unrelated updates (or removal) of a VM - // created while the gate was enabled and later disabled. +func (v *GPUDevicesValidator) ValidateUpdate(ctx context.Context, oldVM, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) { + // The feature gate and DeviceClass existence are checked only when GPU devices + // are introduced or changed. Unchanged GPU devices must not block unrelated + // updates (or removal) of a VM created while the gate was enabled and later + // disabled, or whose DeviceClass was removed out of band. if reflect.DeepEqual(oldVM.Spec.GPUDevices, newVM.Spec.GPUDevices) { return nil, nil } - return nil, v.validateGPUDevices(newVM) + return nil, v.validateGPUDevices(ctx, newVM) } -func (v *GPUDevicesValidator) validateGPUDevices(vm *v1alpha2.VirtualMachine) error { +func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alpha2.VirtualMachine) error { if len(vm.Spec.GPUDevices) == 0 { return nil } @@ -59,5 +65,16 @@ func (v *GPUDevicesValidator) validateGPUDevices(vm *v1alpha2.VirtualMachine) er return fmt.Errorf("GPU device attachment requires the GPU feature gate") } + for _, device := range vm.Spec.GPUDevices { + deviceClass := &resourcev1.DeviceClass{} + err := v.client.Get(ctx, types.NamespacedName{Name: device.DeviceClassName}, deviceClass) + if apierrors.IsNotFound(err) { + return fmt.Errorf("GPU device %q references DeviceClass %q that does not exist", device.Name, device.DeviceClassName) + } + if err != nil { + return fmt.Errorf("failed to get DeviceClass %q for GPU device %q: %w", device.DeviceClassName, device.Name, err) + } + } + return nil } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index 4d66dea457..907c814f12 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -20,9 +20,12 @@ import ( "strings" "testing" + resourcev1 "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-base/featuregate" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/deckhouse/virtualization-controller/pkg/common/testutil" "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -31,39 +34,38 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { tests := []struct { name string featureEnabled bool + deviceClasses []string + gpuClass string wantErrorPart string }{ { name: "should reject GPU devices when feature is disabled", featureEnabled: false, + gpuClass: "nvidia-h100", wantErrorPart: "GPU feature gate", }, { - name: "should accept GPU devices when feature is enabled", + name: "should accept GPU devices when feature is enabled and DeviceClass exists", featureEnabled: true, + deviceClasses: []string{"nvidia-h100"}, + gpuClass: "nvidia-h100", + }, + { + name: "should reject GPU devices when DeviceClass does not exist", + featureEnabled: true, + gpuClass: "nvidia-h100", + wantErrorPart: "does not exist", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) - validator := NewGPUDevicesValidator(newGPUFeatureGate(t, tt.featureEnabled)) + vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: tt.gpuClass}}) + validator := NewGPUDevicesValidator(newValidatorClient(t, tt.deviceClasses...), newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) - if tt.wantErrorPart == "" { - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - return - } - - if err == nil { - t.Fatal("expected error, got nil") - } - if !strings.Contains(err.Error(), tt.wantErrorPart) { - t.Fatalf("expected error containing %q, got %v", tt.wantErrorPart, err) - } + assertValidationError(t, err, tt.wantErrorPart) }) } } @@ -76,6 +78,7 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { tests := []struct { name string featureEnabled bool + deviceClasses []string oldGPU []v1alpha2.GPUDeviceSpec newGPU []v1alpha2.GPUDeviceSpec wantErrorPart string @@ -100,17 +103,19 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { wantErrorPart: "GPU feature gate", }, { - name: "changing GPU devices is rejected when feature is disabled", - featureEnabled: false, + name: "changing to an existing DeviceClass is allowed when feature is enabled", + featureEnabled: true, + deviceClasses: []string{"nvidia-h100", "nvidia-a100"}, oldGPU: gpu("nvidia-h100"), newGPU: gpu("nvidia-a100"), - wantErrorPart: "GPU feature gate", }, { - name: "changing GPU devices is allowed when feature is enabled", + name: "changing to a missing DeviceClass is rejected", featureEnabled: true, + deviceClasses: []string{"nvidia-h100"}, oldGPU: gpu("nvidia-h100"), newGPU: gpu("nvidia-a100"), + wantErrorPart: "does not exist", }, } @@ -118,26 +123,32 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { oldVM := newVirtualMachineWithGPU("vm-current", tt.oldGPU) newVM := newVirtualMachineWithGPU("vm-current", tt.newGPU) - validator := NewGPUDevicesValidator(newGPUFeatureGate(t, tt.featureEnabled)) + validator := NewGPUDevicesValidator(newValidatorClient(t, tt.deviceClasses...), newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateUpdate(t.Context(), oldVM, newVM) - if tt.wantErrorPart == "" { - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - return - } - if err == nil { - t.Fatal("expected error, got nil") - } - if !strings.Contains(err.Error(), tt.wantErrorPart) { - t.Fatalf("expected error containing %q, got %v", tt.wantErrorPart, err) - } + assertValidationError(t, err, tt.wantErrorPart) }) } } +func assertValidationError(t *testing.T, err error, wantErrorPart string) { + t.Helper() + + if wantErrorPart == "" { + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + return + } + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), wantErrorPart) { + t.Fatalf("expected error containing %q, got %v", wantErrorPart, err) + } +} + func newVirtualMachineWithGPU(name string, gpuDevices []v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { return &v1alpha2.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, @@ -145,6 +156,21 @@ func newVirtualMachineWithGPU(name string, gpuDevices []v1alpha2.GPUDeviceSpec) } } +func newValidatorClient(t *testing.T, deviceClasses ...string) client.Client { + t.Helper() + + objs := make([]client.Object, 0, len(deviceClasses)) + for _, name := range deviceClasses { + objs = append(objs, &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: name}}) + } + + fakeClient, err := testutil.NewFakeClientWithObjects(objs...) + if err != nil { + t.Fatalf("failed to create fake client: %v", err) + } + return fakeClient +} + func newGPUFeatureGate(t *testing.T, enabled bool) featuregate.FeatureGate { t.Helper() diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go index d6457844ca..cab4cedc87 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go @@ -56,7 +56,7 @@ func NewValidator(client client.Client, blockDeviceService *service.BlockDeviceS validators.NewNetworksValidator(client, featureGate), validators.NewFirstDiskValidator(client), validators.NewUSBDevicesValidator(client, featureGate), - validators.NewGPUDevicesValidator(featureGate), + validators.NewGPUDevicesValidator(client, featureGate), validators.NewVMBDAConflictValidator(client), validators.NewPVNodeAffinityValidator(client, attachmentService), }, From 044b8530ee39cb642654a9575c736529ea996db3 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 14:43:43 +0200 Subject: [PATCH 23/39] fix(vm): grant controller read access to DeviceClass The GPU devices validator resolves gpuDevices[].deviceClassName through the cached client, which needs get/list/watch on resource.k8s.io deviceclasses. Without it admission fails for every VM referencing a GPU DeviceClass. Add a read-only rule for deviceclasses. Signed-off-by: Daniil Antoshin --- templates/virtualization-controller/rbac-for-us.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/templates/virtualization-controller/rbac-for-us.yaml b/templates/virtualization-controller/rbac-for-us.yaml index 84eb2fef25..e895370513 100644 --- a/templates/virtualization-controller/rbac-for-us.yaml +++ b/templates/virtualization-controller/rbac-for-us.yaml @@ -344,6 +344,14 @@ rules: - update - patch - delete +- apiGroups: + - resource.k8s.io + resources: + - deviceclasses + verbs: + - get + - list + - watch - apiGroups: - apiextensions.k8s.io resources: From 020397b4e5309a374ddf671cd1c4529e1d888e8d Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 14:52:40 +0200 Subject: [PATCH 24/39] fix(vm): only clean up GPU claims and devices owned by this controller SetGPUDevices rebuilt the GPU part of the KVVM by deleting every resourceClaim/GPU whose name starts with the "gpu-" prefix, which could drop a foreign entry that happens to share the prefix. Narrow the match: a resourceClaim must also reference a ResourceClaimTemplate named for this VM, and a GPU must be a DRA entry (ClaimRequest set) rather than a device-plugin GPU. Foreign or device-plugin entries are left intact. Signed-off-by: Daniil Antoshin --- .../pkg/controller/kvbuilder/gpu.go | 6 ++-- .../pkg/controller/kvbuilder/gpu_test.go | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index f5f6e8bb4c..73b0980c27 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -54,13 +54,15 @@ func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { b.Resource.Spec.Template.Spec.ResourceClaims = slices.DeleteFunc( b.Resource.Spec.Template.Spec.ResourceClaims, func(claim virtv1.ResourceClaim) bool { - return strings.HasPrefix(claim.Name, GPUNamePrefix) + return strings.HasPrefix(claim.Name, GPUNamePrefix) && + claim.ResourceClaimTemplateName != nil && + IsGPUResourceClaimTemplateName(vmName, *claim.ResourceClaimTemplateName) }, ) b.Resource.Spec.Template.Spec.Domain.Devices.GPUs = slices.DeleteFunc( b.Resource.Spec.Template.Spec.Domain.Devices.GPUs, func(gpu virtv1.GPU) bool { - return strings.HasPrefix(gpu.Name, GPUNamePrefix) + return strings.HasPrefix(gpu.Name, GPUNamePrefix) && gpu.ClaimRequest != nil }, ) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index 9a4784e22d..61b554374c 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -19,7 +19,10 @@ package kvbuilder import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -72,6 +75,35 @@ var _ = Describe("GPU", func() { Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu1")) }) + It("should not touch resource claims and GPUs it does not own", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + res := kvvm.GetResource() + res.Spec.Template.Spec.ResourceClaims = []virtv1.ResourceClaim{{ + PodResourceClaim: corev1.PodResourceClaim{ + Name: "gpu-foreign", + ResourceClaimTemplateName: ptr.To("foreign-template"), + }, + }} + res.Spec.Template.Spec.Domain.Devices.GPUs = []virtv1.GPU{{ + Name: "gpu-legacy", + DeviceName: "nvidia.com/GH100", + }} + + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) + res = kvvm.GetResource() + + claimNames := make([]string, 0) + for _, c := range res.Spec.Template.Spec.ResourceClaims { + claimNames = append(claimNames, c.Name) + } + gpuNames := make([]string, 0) + for _, g := range res.Spec.Template.Spec.Domain.Devices.GPUs { + gpuNames = append(gpuNames, g.Name) + } + Expect(claimNames).To(ConsistOf("gpu-foreign", "gpu-gpu0")) + Expect(gpuNames).To(ConsistOf("gpu-legacy", "gpu-gpu0")) + }) + It("should remove rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) From fb5b8f06726de8290c9314bbee36cb8c9582a1a4 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 15:04:17 +0200 Subject: [PATCH 25/39] fix(vmpool): validate GPU feature gate in the VM template A VirtualMachinePool template carries a full VirtualMachineSpec, so gpuDevices in the template were only validated when a replica VM was created, turning a bad template into a delayed FailedCreate instead of an up-front rejection. Add the GPU validator to the template-spec validator. It runs in gate-only mode (nil client): the feature gate is enforced at pool creation, while DeviceClass existence stays a per-replica check so a pool can be defined before the GPU provider is installed. Signed-off-by: Daniil Antoshin --- .../validators/gpu_devices_validator.go | 7 +++++ .../validators/gpu_devices_validator_test.go | 31 +++++++++++++++++++ .../pkg/controller/vm/vm_webhook.go | 1 + 3 files changed, 39 insertions(+) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index b9b7d634a1..cce5153ae5 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -65,6 +65,13 @@ func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alph return fmt.Errorf("GPU device attachment requires the GPU feature gate") } + // A nil client means template validation (e.g. a VirtualMachinePool template): + // DeviceClass existence is verified when the actual replica VM is created, + // so that a pool may be defined before the GPU provider and its classes exist. + if v.client == nil { + return nil + } + for _, device := range vm.Spec.GPUDevices { deviceClass := &resourcev1.DeviceClass{} err := v.client.Get(ctx, types.NamespacedName{Name: device.DeviceClassName}, deviceClass) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index 907c814f12..f1f722627f 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -132,6 +132,37 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { } } +func TestGPUDevicesValidatorTemplateMode(t *testing.T) { + // A nil client (template validation) enforces the feature gate but skips + // DeviceClass existence. + tests := []struct { + name string + featureEnabled bool + wantErrorPart string + }{ + { + name: "gate disabled is rejected in template mode", + featureEnabled: false, + wantErrorPart: "GPU feature gate", + }, + { + name: "missing DeviceClass is allowed in template mode", + featureEnabled: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "does-not-exist"}}) + validator := NewGPUDevicesValidator(nil, newGPUFeatureGate(t, tt.featureEnabled)) + + _, err := validator.ValidateCreate(t.Context(), vm) + + assertValidationError(t, err, tt.wantErrorPart) + }) + } +} + func assertValidationError(t *testing.T, err error, wantErrorPart string) { t.Helper() diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go index cab4cedc87..bb493bdbbb 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go @@ -79,6 +79,7 @@ func NewTemplateSpecValidator(client client.Client, featureGate featuregate.Feat validators.NewSizingPolicyValidator(client), validators.NewNetworksValidator(client, featureGate), validators.NewFirstDiskValidator(client), + validators.NewGPUDevicesValidator(nil, featureGate), }, log: log.With("webhook", "vmpool-template-validation"), } From 1b5ffc363719a9ce7171fe69a687277161b071bf Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 15:12:41 +0200 Subject: [PATCH 26/39] refactor(vm): drop redundant empty-devices check in GPU kvbuilder Signed-off-by: Daniil Antoshin --- .../virtualization-artifact/pkg/controller/kvbuilder/gpu.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index 73b0980c27..602e79cc84 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -66,10 +66,6 @@ func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { }, ) - if len(devices) == 0 { - return - } - for _, device := range devices { claimName := GPUResourceClaimName(device.Name) b.Resource.Spec.Template.Spec.ResourceClaims = append(b.Resource.Spec.Template.Spec.ResourceClaims, virtv1.ResourceClaim{ From f87a3d6d499f0098754165cb11e454b68a419f1a Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 15:12:41 +0200 Subject: [PATCH 27/39] test(vm): fix stale GPU test name and drop leftover annotation assertions The annotation-based rendering was replaced earlier, but a test title and two empty-annotations assertions still referred to it. Signed-off-by: Daniil Antoshin --- .../pkg/controller/kvbuilder/gpu_test.go | 2 -- .../controller/vm/internal/gpu_resourceclaim_handler_test.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index 61b554374c..da928b864f 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -41,7 +41,6 @@ var _ = Describe("GPU", func() { Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu0")) Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal("gpu-gpu0")) Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal("gpu-gpu0")) - Expect(res.Annotations).To(BeEmpty()) }) It("should render DRA GPU resource claims in stable order", func() { @@ -113,6 +112,5 @@ var _ = Describe("GPU", func() { Expect(res.Spec.Template.Spec.ResourceClaims).To(BeEmpty()) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(BeEmpty()) - Expect(res.Annotations).To(BeEmpty()) }) }) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index e41f3e46ea..44c0e21a51 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -64,7 +64,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(string(config.Opaque.Parameters.Raw)).To(ContainSubstring(`"kind":"VfioDeviceConfig"`)) }) - It("should delete owned GPU ResourceClaimTemplate when annotation is removed", func() { + It("should delete owned GPU ResourceClaimTemplate when device is removed from spec", func() { vm := newVM() template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) fakeClient, _, vmState := setupEnvironment(vm, template) From f801eb3017e24dec0637c229cb232777408a6ab4 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 15:12:51 +0200 Subject: [PATCH 28/39] fix(vm): ignore GPU device order when revalidating on update The vmchange comparator sorts gpuDevices and treats reordering as no change, while the webhook compared them order-sensitively and re-ran the feature gate and DeviceClass checks on a pure reorder. Sort both sides before comparing so a reorder cannot be rejected after the gate is disabled or a DeviceClass is removed. Signed-off-by: Daniil Antoshin --- .../vm/internal/validators/gpu_devices_validator.go | 6 ++++-- .../validators/gpu_devices_validator_test.go | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index cce5153ae5..e99a565d11 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -49,8 +50,9 @@ func (v *GPUDevicesValidator) ValidateUpdate(ctx context.Context, oldVM, newVM * // The feature gate and DeviceClass existence are checked only when GPU devices // are introduced or changed. Unchanged GPU devices must not block unrelated // updates (or removal) of a VM created while the gate was enabled and later - // disabled, or whose DeviceClass was removed out of band. - if reflect.DeepEqual(oldVM.Spec.GPUDevices, newVM.Spec.GPUDevices) { + // disabled, or whose DeviceClass was removed out of band. Order is ignored + // to match the vmchange comparator, which treats reordering as no change. + if reflect.DeepEqual(kvbuilder.SortGPUDevices(oldVM.Spec.GPUDevices), kvbuilder.SortGPUDevices(newVM.Spec.GPUDevices)) { return nil, nil } return nil, v.validateGPUDevices(ctx, newVM) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index f1f722627f..6fc7e2c95c 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -95,6 +95,18 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { oldGPU: gpu("nvidia-h100"), newGPU: nil, }, + { + name: "reordering GPU devices is allowed when feature is disabled", + featureEnabled: false, + oldGPU: []v1alpha2.GPUDeviceSpec{ + {Name: "gpu0", DeviceClassName: "nvidia-h100"}, + {Name: "gpu1", DeviceClassName: "nvidia-a100"}, + }, + newGPU: []v1alpha2.GPUDeviceSpec{ + {Name: "gpu1", DeviceClassName: "nvidia-a100"}, + {Name: "gpu0", DeviceClassName: "nvidia-h100"}, + }, + }, { name: "adding GPU devices is rejected when feature is disabled", featureEnabled: false, From 9e8c7a0ae8604a5a841bab85ca83fec6df865c29 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 15:12:51 +0200 Subject: [PATCH 29/39] docs(module): describe GPU feature gate as DeviceClass-based The gate description still said GPUs are requested by product model, which was replaced by referencing a DRA DeviceClass. Signed-off-by: Daniil Antoshin --- openapi/config-values.yaml | 2 +- openapi/doc-ru-config-values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index f8b27a9a68..e96c0ab1e7 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -268,7 +268,7 @@ properties: - `HotplugMemoryWithLiveMigration` — enable live changing of memory size via LiveMigration. (Not available in CE); - `HotplugCPUAndMemoryWithInPlaceResize` - enable live changing of cpu cores number or memory size via InPlaceResize. (Not available in CE); - `VirtualMachinePool` — enable the VirtualMachinePool resource for group management of virtual machines. (Not available in CE); - - `GPU` — enable attaching GPU devices to virtual machines via DRA by product model. (Not available in CE); + - `GPU` — enable attaching GPU devices to virtual machines via DRA by referencing a DeviceClass. (Not available in CE); items: type: string enum: diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 0ccff0ed73..d6b3ccc48b 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -180,6 +180,6 @@ properties: - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки через живую миграцию. (Не доступно в CE); - `HotplugCPUAndMemoryWithInPlaceResize` - включить изменение количества ядер процессора или размера памяти без перезагрузки через InPlaceResize (Не доступно в CE) - `VirtualMachinePool` — включить ресурс VirtualMachinePool для группового управления виртуальными машинами. (Не доступно в CE) - - `GPU` — включить подключение GPU-устройств к виртуальным машинам через DRA по модели продукта. (Не доступно в CE); + - `GPU` — включить подключение GPU-устройств к виртуальным машинам через DRA по ссылке на DeviceClass. (Не доступно в CE); items: type: string From 3ed0f598137563639da9442f5f198ecd74784bd9 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 15:19:53 +0200 Subject: [PATCH 30/39] fix(vm): track GPU ResourceClaimTemplate spec via hash annotation Comparing the stored spec with the rendered one via DeepEqual breaks as soon as the API server defaults any field absent from the rendered spec: the comparison never converges and the handler deletes and recreates the template on every reconcile. Stamp the rendered spec hash into an annotation and compare hashes instead, following the tolerations-hash pattern. Signed-off-by: Daniil Antoshin --- .../pkg/common/annotations/annotations.go | 2 ++ .../vm/internal/gpu_resourceclaim_handler.go | 15 +++++++++++++-- .../internal/gpu_resourceclaim_handler_test.go | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index 75e06031e7..c7aa6e1cd3 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -58,6 +58,8 @@ const ( // AnnTolerationsHash provides a const for annotation with hash of applied tolerations. AnnTolerationsHash = AnnAPIGroup + "/tolerations-hash" + // AnnGPUClaimSpecHash provides a const for annotation with hash of the rendered GPU ResourceClaimTemplate spec. + AnnGPUClaimSpecHash = AnnAPIGroup + "/gpu-claim-spec-hash" // AnnTolerationsHashLegacy provides a const for legacy annotation with hash of applied tolerations. AnnTolerationsHashLegacy = AnnAPIGroupLegacy + "/tolerations-hash" diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index f4078f6439..12a53c8211 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -18,8 +18,8 @@ package internal import ( "context" + "encoding/json" "fmt" - "reflect" resourcev1 "k8s.io/api/resource/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" @@ -84,7 +85,9 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac return reconcile.Result{}, fmt.Errorf("GPU ResourceClaimTemplate %s/%s is not controlled by VirtualMachine %s/%s", template.Namespace, template.Name, vm.Namespace, vm.Name) } - if reflect.DeepEqual(template.Spec, desiredSpec) { + // The stored spec is not compared directly: API-server defaulting could make + // it permanently differ from the rendered spec and loop delete/recreate. + if template.Annotations[annotations.AnnGPUClaimSpecHash] == gpuClaimSpecHash(desiredSpec) { continue } if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { @@ -108,12 +111,20 @@ func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spe ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: vm.Namespace, + Annotations: map[string]string{annotations.AnnGPUClaimSpecHash: gpuClaimSpecHash(spec)}, OwnerReferences: []metav1.OwnerReference{service.MakeControllerOwnerReference(vm)}, }, Spec: spec, } } +func gpuClaimSpecHash(spec resourcev1.ResourceClaimTemplateSpec) string { + // Marshalling a plain API struct cannot fail; on the impossible failure both + // sides hash the same empty payload, so the comparison still converges. + raw, _ := json.Marshal(&spec) + return kvbuilder.GenerateSerial(string(raw)) +} + func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1.ResourceClaimTemplateSpec { requestName := kvbuilder.GPUResourceClaimName(device.Name) return resourcev1.ResourceClaimTemplateSpec{ diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index 44c0e21a51..4c9d71af28 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -78,6 +79,22 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(err).To(HaveOccurred()) }) + It("should recreate GPU ResourceClaimTemplate when the desired spec changes", func() { + vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: "nvidia-a100"}) + templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0") + template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: templateName, Namespace: namespace}, stored)).To(Succeed()) + Expect(stored.Spec.Spec.Devices.Requests[0].Exactly.DeviceClassName).To(Equal("nvidia-a100")) + Expect(stored.Annotations).To(HaveKeyWithValue(annotations.AnnGPUClaimSpecHash, gpuClaimSpecHash(stored.Spec))) + }) + It("should not replace GPU ResourceClaimTemplate owned by another controller", func() { vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass}) template := &resourcev1.ResourceClaimTemplate{ From 052194216da3d557ab8521943cd49ec849fde7a8 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 15:19:54 +0200 Subject: [PATCH 31/39] chore(module): drop unrelated VirtualMachinePool feature gate from config values The VirtualMachinePool enum entry is not related to GPU support and does not belong to this change set. Signed-off-by: Daniil Antoshin --- openapi/config-values.yaml | 2 -- openapi/doc-ru-config-values.yaml | 1 - 2 files changed, 3 deletions(-) diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index e96c0ab1e7..57a40b2c59 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -267,7 +267,6 @@ properties: - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number via LiveMigration. (Not available in CE); - `HotplugMemoryWithLiveMigration` — enable live changing of memory size via LiveMigration. (Not available in CE); - `HotplugCPUAndMemoryWithInPlaceResize` - enable live changing of cpu cores number or memory size via InPlaceResize. (Not available in CE); - - `VirtualMachinePool` — enable the VirtualMachinePool resource for group management of virtual machines. (Not available in CE); - `GPU` — enable attaching GPU devices to virtual machines via DRA by referencing a DeviceClass. (Not available in CE); items: type: string @@ -275,5 +274,4 @@ properties: - "HotplugCPUWithLiveMigration" - "HotplugMemoryWithLiveMigration" - "HotplugCPUAndMemoryWithInPlaceResize" - - "VirtualMachinePool" - "GPU" diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index d6b3ccc48b..bb18839c3b 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -179,7 +179,6 @@ properties: - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки через живую миграцию. (Не доступно в CE); - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки через живую миграцию. (Не доступно в CE); - `HotplugCPUAndMemoryWithInPlaceResize` - включить изменение количества ядер процессора или размера памяти без перезагрузки через InPlaceResize (Не доступно в CE) - - `VirtualMachinePool` — включить ресурс VirtualMachinePool для группового управления виртуальными машинами. (Не доступно в CE) - `GPU` — включить подключение GPU-устройств к виртуальным машинам через DRA по ссылке на DeviceClass. (Не доступно в CE); items: type: string From 537f65d5d71fb97cbcef558427c26ae84605a3a6 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Thu, 9 Jul 2026 15:57:32 +0200 Subject: [PATCH 32/39] fix(vm): fall back to spec comparison for GPU claim templates without hash Templates created before the hash annotation existed would otherwise be recreated once on controller upgrade. Compare their stored spec directly instead; they migrate to the hash on their next legitimate recreation. Signed-off-by: Daniil Antoshin --- .../vm/internal/gpu_resourceclaim_handler.go | 18 +++++++++++++++--- .../internal/gpu_resourceclaim_handler_test.go | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index 12a53c8211..73572ed610 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" resourcev1 "k8s.io/api/resource/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -85,9 +86,7 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac return reconcile.Result{}, fmt.Errorf("GPU ResourceClaimTemplate %s/%s is not controlled by VirtualMachine %s/%s", template.Namespace, template.Name, vm.Namespace, vm.Name) } - // The stored spec is not compared directly: API-server defaulting could make - // it permanently differ from the rendered spec and loop delete/recreate. - if template.Annotations[annotations.AnnGPUClaimSpecHash] == gpuClaimSpecHash(desiredSpec) { + if gpuClaimTemplateUpToDate(template, desiredSpec) { continue } if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { @@ -118,6 +117,19 @@ func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spe } } +// gpuClaimTemplateUpToDate prefers the spec-hash annotation over comparing the +// stored spec directly: API-server defaulting could make the stored spec +// permanently differ from the rendered one and loop delete/recreate. +// Templates created before the annotation existed fall back to DeepEqual and +// migrate to the hash on their next legitimate recreation. +func gpuClaimTemplateUpToDate(template *resourcev1.ResourceClaimTemplate, desiredSpec resourcev1.ResourceClaimTemplateSpec) bool { + storedHash, ok := template.Annotations[annotations.AnnGPUClaimSpecHash] + if !ok { + return reflect.DeepEqual(template.Spec, desiredSpec) + } + return storedHash == gpuClaimSpecHash(desiredSpec) +} + func gpuClaimSpecHash(spec resourcev1.ResourceClaimTemplateSpec) string { // Marshalling a plain API struct cannot fail; on the impossible failure both // sides hash the same empty payload, so the comparison still converges. diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index 4c9d71af28..14251a2012 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -95,6 +95,24 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(stored.Annotations).To(HaveKeyWithValue(annotations.AnnGPUClaimSpecHash, gpuClaimSpecHash(stored.Spec))) }) + It("should not recreate GPU ResourceClaimTemplate without hash annotation when spec matches", func() { + vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass}) + templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0") + template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) + template.Annotations = nil + template.Labels = map[string]string{"keep": "me"} + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: templateName, Namespace: namespace}, stored)).To(Succeed()) + Expect(stored.Labels).To(HaveKeyWithValue("keep", "me")) + Expect(stored.Annotations).NotTo(HaveKey(annotations.AnnGPUClaimSpecHash)) + }) + It("should not replace GPU ResourceClaimTemplate owned by another controller", func() { vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass}) template := &resourcev1.ResourceClaimTemplate{ From 86dcc137e2a3a5bcfa95181b20a1c425a1959ee2 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Sat, 11 Jul 2026 02:43:42 +0200 Subject: [PATCH 33/39] feat(vm): select GPU by GPUClass instead of DeviceClass The VM now references a GPUClass in spec.gpuDevices[].gpuClassName (renamed from deviceClassName). The GPU module creates a DRA DeviceClass named exactly after the GPUClass, so the generated ResourceClaimTemplate targets that same-named DeviceClass, and the validator resolves GPUClass readiness by checking that DeviceClass exists. CRD, docs and tests updated. Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/virtual_machine.go | 10 ++++--- crds/doc-ru-virtualmachines.yaml | 7 ++--- crds/virtualmachines.yaml | 9 ++++--- docs/USER_GUIDE.md | 8 +++--- docs/USER_GUIDE.ru.md | 8 +++--- .../pkg/controller/kvbuilder/gpu_test.go | 14 +++++----- .../vm/internal/gpu_resourceclaim_handler.go | 3 ++- .../gpu_resourceclaim_handler_test.go | 16 ++++++------ .../controller/vm/internal/sync_kvvm_test.go | 4 +-- .../validators/gpu_devices_validator.go | 12 +++++---- .../validators/gpu_devices_validator_test.go | 26 +++++++++---------- .../pkg/controller/vmchange/compare_test.go | 14 +++++----- 12 files changed, 69 insertions(+), 62 deletions(-) diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index a8757c1f93..7f07291c1c 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -124,7 +124,7 @@ type VirtualMachineSpec struct { // +kubebuilder:validation:MaxItems:=8 USBDevices []USBDeviceSpecRef `json:"usbDevices,omitempty"` // List of GPU devices to attach to the virtual machine. - // Each device references a DRA DeviceClass by name. + // Each device references a GPUClass by name. // This feature requires the GPU feature gate. // +kubebuilder:validation:MaxItems:=16 // +listType=map @@ -512,7 +512,7 @@ type USBDeviceSpecRef struct { Name string `json:"name"` } -// GPUDeviceSpec requests a GPU device by DRA DeviceClass. +// GPUDeviceSpec requests a GPU device by GPUClass. type GPUDeviceSpec struct { // A unique GPU device name inside the virtual machine spec. // The value is used to generate DRA claim and request names. @@ -520,11 +520,13 @@ type GPUDeviceSpec struct { // +kubebuilder:validation:MaxLength:=59 // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` Name string `json:"name"` - // Name of the DRA DeviceClass that selects the GPU to attach, for example nvidia-h100. + // Name of the GPUClass that selects the GPU to attach, for example nvidia-h100. + // The GPUClass must already exist: a DRA DeviceClass with the same name is + // created by the GPU module and is used to allocate the device. // +kubebuilder:validation:MinLength:=1 // +kubebuilder:validation:MaxLength:=253 // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` - DeviceClassName string `json:"deviceClassName"` + GPUClassName string `json:"gpuClassName"` } // USBDeviceStatusRef represents the status of a USB device attached to the virtual machine. diff --git a/crds/doc-ru-virtualmachines.yaml b/crds/doc-ru-virtualmachines.yaml index 4737a59289..6779593387 100644 --- a/crds/doc-ru-virtualmachines.yaml +++ b/crds/doc-ru-virtualmachines.yaml @@ -591,13 +591,14 @@ spec: gpuDevices: description: | Список GPU-устройств для подключения к виртуальной машине. - Каждое устройство ссылается на DRA DeviceClass по имени. + Каждое устройство ссылается на GPUClass по имени. Для использования требуется feature gate `GPU`. items: properties: - deviceClassName: + gpuClassName: description: | - Имя DRA DeviceClass, который выбирает подключаемый GPU, например `nvidia-h100`. + Имя GPUClass, который выбирает подключаемый GPU, например `nvidia-h100`. + GPUClass должен уже существовать: DRA DeviceClass с тем же именем создаётся модулем GPU и используется для выделения устройства. name: description: | Уникальное имя GPU-устройства внутри спецификации виртуальной машины. diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index feef58ff94..c8d4cffde5 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1066,21 +1066,22 @@ spec: x-kubernetes-list-type: map description: | List of GPU devices to attach to the virtual machine. - Each device references a DRA DeviceClass by name. + Each device references a GPUClass by name. This feature requires the GPU feature gate. items: type: object required: - - deviceClassName + - gpuClassName - name properties: - deviceClassName: + gpuClassName: minLength: 1 maxLength: 253 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string description: | - Name of the DRA DeviceClass that selects the GPU to attach, for example nvidia-h100. + Name of the GPUClass that selects the GPU to attach, for example nvidia-h100. + The GPUClass must already exist: a DRA DeviceClass with the same name is created by the GPU module and is used to allocate the device. name: minLength: 1 maxLength: 59 diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 8ea489d9af..460542110d 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -4021,14 +4021,14 @@ As a result, a VM named `clone-database-prod` and a disk named `clone-database-r GPU device passthrough is an experimental feature. It requires the Enterprise Edition (EE), Kubernetes DRA support, and an external GPU DRA provider that publishes GPUs as DRA resources with `gpu.deckhouse.io` device attributes. {{< /alert >}} -The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by referencing a DRA [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) through the `.spec.gpuDevices` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. +The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by referencing a `GPUClass` through the `.spec.gpuDevices` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. GPU device passthrough requires: - Kubernetes version 1.34 or higher with DRA feature gates required by the cluster configuration. - The `GPU` feature gate enabled in the `virtualization` module settings. - A GPU DRA provider installed in the cluster that publishes GPUs with `gpu.deckhouse.io` device attributes. -- A DRA [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) that selects the GPU, referenced from `.spec.gpuDevices[].deviceClassName`. +- A `GPUClass` that selects the GPU, referenced from `.spec.gpuDevices[].gpuClassName`. The GPU module creates a DRA DeviceClass with the same name, which is used to allocate the device. To enable the module feature gate: @@ -4054,10 +4054,10 @@ spec: # ... other VM settings ... gpuDevices: - name: gpu0 - deviceClassName: nvidia-h100 + gpuClassName: nvidia-h100 ``` -The `name` field must be unique within `.spec.gpuDevices` and can contain up to 59 DNS-label characters. The `deviceClassName` field must be the name of an existing DRA `DeviceClass` that selects the GPU to attach. +The `name` field must be unique within `.spec.gpuDevices` and can contain up to 59 DNS-label characters. The `gpuClassName` field must be the name of an existing `GPUClass` that selects the GPU to attach. Changing `.spec.gpuDevices` requires restarting the virtual machine to apply the new configuration. diff --git a/docs/USER_GUIDE.ru.md b/docs/USER_GUIDE.ru.md index 307be170c1..6014762afe 100644 --- a/docs/USER_GUIDE.ru.md +++ b/docs/USER_GUIDE.ru.md @@ -4052,14 +4052,14 @@ spec: Проброс GPU-устройств — экспериментальная возможность. Для работы требуются Enterprise Edition (EE), поддержка Kubernetes DRA и внешний GPU DRA-провайдер, публикующий GPU как DRA-ресурсы с атрибутами устройства `gpu.deckhouse.io`. {{< /alert >}} -Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по ссылке на DRA [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes) через поле `.spec.gpuDevices` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). +Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по ссылке на `GPUClass` через поле `.spec.gpuDevices` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). Для проброса GPU требуются: - Kubernetes версии 1.34 или выше с DRA feature gates, необходимыми для конфигурации кластера. - Feature gate `GPU`, включённый в настройках модуля `virtualization`. - Установленный в кластере GPU DRA-провайдер, публикующий GPU с атрибутами устройства `gpu.deckhouse.io`. -- DRA [DeviceClass](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/#device-classes), выбирающий GPU, на который ссылается `.spec.gpuDevices[].deviceClassName`. +- `GPUClass`, выбирающий GPU, на который ссылается `.spec.gpuDevices[].gpuClassName`. Модуль GPU создаёт DRA DeviceClass с тем же именем, который используется для выделения устройства. Чтобы включить feature gate модуля: @@ -4085,10 +4085,10 @@ spec: # ... другие настройки ВМ ... gpuDevices: - name: gpu0 - deviceClassName: nvidia-h100 + gpuClassName: nvidia-h100 ``` -Поле `name` должно быть уникальным внутри `.spec.gpuDevices` и может содержать до 59 символов DNS label. Поле `deviceClassName` должно быть именем существующего DRA `DeviceClass`, который выбирает подключаемый GPU. +Поле `name` должно быть уникальным внутри `.spec.gpuDevices` и может содержать до 59 символов DNS label. Поле `gpuClassName` должно быть именем существующего `GPUClass`, который выбирает подключаемый GPU. Изменение `.spec.gpuDevices` требует перезапуска виртуальной машины для применения новой конфигурации. diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index da928b864f..6743bf8414 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -31,7 +31,7 @@ var _ = Describe("GPU", func() { It("should render DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}}) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) @@ -47,8 +47,8 @@ var _ = Describe("GPU", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{ - {Name: "gpu1", DeviceClassName: "nvidia-h100"}, - {Name: "gpu0", DeviceClassName: "nvidia-a100"}, + {Name: "gpu1", GPUClassName: "nvidia-h100"}, + {Name: "gpu0", GPUClassName: "nvidia-a100"}, }) res := kvvm.GetResource() @@ -62,9 +62,9 @@ var _ = Describe("GPU", func() { It("should replace rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu1", DeviceClassName: "nvidia-a100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu1", GPUClassName: "nvidia-a100"}}) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) @@ -88,7 +88,7 @@ var _ = Describe("GPU", func() { DeviceName: "nvidia.com/GH100", }} - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}}) res = kvvm.GetResource() claimNames := make([]string, 0) @@ -105,7 +105,7 @@ var _ = Describe("GPU", func() { It("should remove rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}}) kvvm.SetGPUDevices("vm-a", nil) res := kvvm.GetResource() diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index 73572ed610..e24c181ff6 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -145,7 +145,8 @@ func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1 Requests: []resourcev1.DeviceRequest{{ Name: requestName, Exactly: &resourcev1.ExactDeviceRequest{ - DeviceClassName: device.DeviceClassName, + // The GPU module creates a DeviceClass named exactly after the GPUClass. + DeviceClassName: device.GPUClassName, AllocationMode: resourcev1.DeviceAllocationModeExactCount, Count: 1, }, diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index 14251a2012..d33adb47ae 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -45,7 +45,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { } It("should create GPU ResourceClaimTemplate", func() { - fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) handler := NewGPUResourceClaimHandler(fakeClient) _, err := handler.Handle(context.Background(), vmState) @@ -67,7 +67,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { It("should delete owned GPU ResourceClaimTemplate when device is removed from spec", func() { vm := newVM() - template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) + template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) fakeClient, _, vmState := setupEnvironment(vm, template) handler := NewGPUResourceClaimHandler(fakeClient) @@ -80,9 +80,9 @@ var _ = Describe("GPUResourceClaimHandler", func() { }) It("should recreate GPU ResourceClaimTemplate when the desired spec changes", func() { - vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: "nvidia-a100"}) + vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: "nvidia-a100"}) templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0") - template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) + template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) fakeClient, _, vmState := setupEnvironment(vm, template) handler := NewGPUResourceClaimHandler(fakeClient) @@ -96,9 +96,9 @@ var _ = Describe("GPUResourceClaimHandler", func() { }) It("should not recreate GPU ResourceClaimTemplate without hash annotation when spec matches", func() { - vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass}) + vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass}) templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0") - template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) + template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) template.Annotations = nil template.Labels = map[string]string{"keep": "me"} fakeClient, _, vmState := setupEnvironment(vm, template) @@ -114,7 +114,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { }) It("should not replace GPU ResourceClaimTemplate owned by another controller", func() { - vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass}) + vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass}) template := &resourcev1.ResourceClaimTemplate{ ObjectMeta: metav1.ObjectMeta{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, } @@ -137,7 +137,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { } otherName := kvbuilder.GPUResourceClaimTemplateName(otherVM.Name, "gpu0") Expect(kvbuilder.IsGPUResourceClaimTemplateName(vmName, otherName)).To(BeTrue()) - template := buildGPUResourceClaimTemplate(otherVM, otherName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", DeviceClassName: deviceClass})) + template := buildGPUResourceClaimTemplate(otherVM, otherName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) fakeClient, _, vmState := setupEnvironment(vm, template) handler := NewGPUResourceClaimHandler(fakeClient) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 8540669e18..dce3fdf4f9 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -426,7 +426,7 @@ var _ = Describe("SyncKvvmHandler", func() { vmClass := makeVMClass() vm := makeVM(v1alpha2.MachineRunning) - vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-h100"}} + vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}} kvvm := makeKVVM(vm) Expect(kvbuilder.SetLastAppliedSpec(kvvm, &v1alpha2.VirtualMachine{ Spec: v1alpha2.VirtualMachineSpec{ @@ -443,7 +443,7 @@ var _ = Describe("SyncKvvmHandler", func() { Disruptions: &v1alpha2.Disruptions{ RestartApprovalMode: vm.Spec.Disruptions.RestartApprovalMode, }, - GPUDevices: []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "nvidia-a100"}}, + GPUDevices: []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-a100"}}, }, })).To(Succeed()) kvvm.SetGroupVersionKind(virtv1.VirtualMachineGroupVersionKind) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index e99a565d11..3c89b12208 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -68,20 +68,22 @@ func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alph } // A nil client means template validation (e.g. a VirtualMachinePool template): - // DeviceClass existence is verified when the actual replica VM is created, - // so that a pool may be defined before the GPU provider and its classes exist. + // GPUClass readiness is verified when the actual replica VM is created, so that + // a pool may be defined before the GPU provider and its classes exist. if v.client == nil { return nil } + // The GPU module creates a DeviceClass named exactly after the GPUClass, so an + // existing DeviceClass with that name means the GPUClass is ready for allocation. for _, device := range vm.Spec.GPUDevices { deviceClass := &resourcev1.DeviceClass{} - err := v.client.Get(ctx, types.NamespacedName{Name: device.DeviceClassName}, deviceClass) + err := v.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, deviceClass) if apierrors.IsNotFound(err) { - return fmt.Errorf("GPU device %q references DeviceClass %q that does not exist", device.Name, device.DeviceClassName) + return fmt.Errorf("GPU device %q references GPUClass %q that does not exist or is not ready", device.Name, device.GPUClassName) } if err != nil { - return fmt.Errorf("failed to get DeviceClass %q for GPU device %q: %w", device.DeviceClassName, device.Name, err) + return fmt.Errorf("failed to resolve GPUClass %q for GPU device %q: %w", device.GPUClassName, device.Name, err) } } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index 6fc7e2c95c..e0b77e58f9 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -45,13 +45,13 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { wantErrorPart: "GPU feature gate", }, { - name: "should accept GPU devices when feature is enabled and DeviceClass exists", + name: "should accept GPU devices when feature is enabled and GPUClass is ready", featureEnabled: true, deviceClasses: []string{"nvidia-h100"}, gpuClass: "nvidia-h100", }, { - name: "should reject GPU devices when DeviceClass does not exist", + name: "should reject GPU devices when GPUClass is not ready", featureEnabled: true, gpuClass: "nvidia-h100", wantErrorPart: "does not exist", @@ -60,7 +60,7 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: tt.gpuClass}}) + vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: tt.gpuClass}}) validator := NewGPUDevicesValidator(newValidatorClient(t, tt.deviceClasses...), newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) @@ -72,7 +72,7 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { gpu := func(class string) []v1alpha2.GPUDeviceSpec { - return []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: class}} + return []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: class}} } tests := []struct { @@ -99,12 +99,12 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { name: "reordering GPU devices is allowed when feature is disabled", featureEnabled: false, oldGPU: []v1alpha2.GPUDeviceSpec{ - {Name: "gpu0", DeviceClassName: "nvidia-h100"}, - {Name: "gpu1", DeviceClassName: "nvidia-a100"}, + {Name: "gpu0", GPUClassName: "nvidia-h100"}, + {Name: "gpu1", GPUClassName: "nvidia-a100"}, }, newGPU: []v1alpha2.GPUDeviceSpec{ - {Name: "gpu1", DeviceClassName: "nvidia-a100"}, - {Name: "gpu0", DeviceClassName: "nvidia-h100"}, + {Name: "gpu1", GPUClassName: "nvidia-a100"}, + {Name: "gpu0", GPUClassName: "nvidia-h100"}, }, }, { @@ -115,14 +115,14 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { wantErrorPart: "GPU feature gate", }, { - name: "changing to an existing DeviceClass is allowed when feature is enabled", + name: "changing to a ready GPUClass is allowed when feature is enabled", featureEnabled: true, deviceClasses: []string{"nvidia-h100", "nvidia-a100"}, oldGPU: gpu("nvidia-h100"), newGPU: gpu("nvidia-a100"), }, { - name: "changing to a missing DeviceClass is rejected", + name: "changing to an unready GPUClass is rejected", featureEnabled: true, deviceClasses: []string{"nvidia-h100"}, oldGPU: gpu("nvidia-h100"), @@ -146,7 +146,7 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { func TestGPUDevicesValidatorTemplateMode(t *testing.T) { // A nil client (template validation) enforces the feature gate but skips - // DeviceClass existence. + // GPUClass readiness (its DeviceClass existence). tests := []struct { name string featureEnabled bool @@ -158,14 +158,14 @@ func TestGPUDevicesValidatorTemplateMode(t *testing.T) { wantErrorPart: "GPU feature gate", }, { - name: "missing DeviceClass is allowed in template mode", + name: "unready GPUClass is allowed in template mode", featureEnabled: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", DeviceClassName: "does-not-exist"}}) + vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "does-not-exist"}}) validator := NewGPUDevicesValidator(nil, newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go index 6d2cb21bc1..f8150fc716 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go @@ -689,31 +689,31 @@ networks: ` gpuDevices: - name: gpu1 - deviceClassName: nvidia-h100 + gpuClassName: nvidia-h100 - name: gpu0 - deviceClassName: nvidia-a100 + gpuClassName: nvidia-a100 `, ` gpuDevices: - name: gpu0 - deviceClassName: nvidia-a100 + gpuClassName: nvidia-a100 - name: gpu1 - deviceClassName: nvidia-h100 + gpuClassName: nvidia-h100 `, nil, assertNoChanges(), }, { - "restart when gpu device deviceClassName changes", + "restart when gpu device gpuClassName changes", ` gpuDevices: - name: gpu0 - deviceClassName: nvidia-a100 + gpuClassName: nvidia-a100 `, ` gpuDevices: - name: gpu0 - deviceClassName: nvidia-h100 + gpuClassName: nvidia-h100 `, nil, assertChanges( From 3555a6e20685f8883c78babc63183cfa15f150bc Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Sat, 11 Jul 2026 02:45:43 +0200 Subject: [PATCH 34/39] test(vm): drop unused name param in newVirtualMachineWithGPU golangci unparam: the helper always received "vm-current". Signed-off-by: Daniil Antoshin --- .../validators/gpu_devices_validator_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index e0b77e58f9..eb617d4580 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -60,7 +60,7 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: tt.gpuClass}}) + vm := newVirtualMachineWithGPU([]v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: tt.gpuClass}}) validator := NewGPUDevicesValidator(newValidatorClient(t, tt.deviceClasses...), newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) @@ -133,8 +133,8 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - oldVM := newVirtualMachineWithGPU("vm-current", tt.oldGPU) - newVM := newVirtualMachineWithGPU("vm-current", tt.newGPU) + oldVM := newVirtualMachineWithGPU(tt.oldGPU) + newVM := newVirtualMachineWithGPU(tt.newGPU) validator := NewGPUDevicesValidator(newValidatorClient(t, tt.deviceClasses...), newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateUpdate(t.Context(), oldVM, newVM) @@ -165,7 +165,7 @@ func TestGPUDevicesValidatorTemplateMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vm := newVirtualMachineWithGPU("vm-current", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "does-not-exist"}}) + vm := newVirtualMachineWithGPU([]v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "does-not-exist"}}) validator := NewGPUDevicesValidator(nil, newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) @@ -192,9 +192,9 @@ func assertValidationError(t *testing.T, err error, wantErrorPart string) { } } -func newVirtualMachineWithGPU(name string, gpuDevices []v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { +func newVirtualMachineWithGPU(gpuDevices []v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { return &v1alpha2.VirtualMachine{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + ObjectMeta: metav1.ObjectMeta{Name: "vm-current", Namespace: "default"}, Spec: v1alpha2.VirtualMachineSpec{GPUDevices: gpuDevices}, } } From cead37b22412ac4aaea1c0cdd7c662c940651cd7 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Sat, 11 Jul 2026 02:49:44 +0200 Subject: [PATCH 35/39] docs(module): describe GPU feature gate as GPUClass-based Follow-up to the DeviceClass -> GPUClass rename in the VM spec. Signed-off-by: Daniil Antoshin --- openapi/config-values.yaml | 2 +- openapi/doc-ru-config-values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 57a40b2c59..c1b1ac53cb 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -267,7 +267,7 @@ properties: - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number via LiveMigration. (Not available in CE); - `HotplugMemoryWithLiveMigration` — enable live changing of memory size via LiveMigration. (Not available in CE); - `HotplugCPUAndMemoryWithInPlaceResize` - enable live changing of cpu cores number or memory size via InPlaceResize. (Not available in CE); - - `GPU` — enable attaching GPU devices to virtual machines via DRA by referencing a DeviceClass. (Not available in CE); + - `GPU` — enable attaching GPU devices to virtual machines via DRA by referencing a GPUClass. (Not available in CE); items: type: string enum: diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index bb18839c3b..c21f4017eb 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -179,6 +179,6 @@ properties: - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки через живую миграцию. (Не доступно в CE); - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки через живую миграцию. (Не доступно в CE); - `HotplugCPUAndMemoryWithInPlaceResize` - включить изменение количества ядер процессора или размера памяти без перезагрузки через InPlaceResize (Не доступно в CE) - - `GPU` — включить подключение GPU-устройств к виртуальным машинам через DRA по ссылке на DeviceClass. (Не доступно в CE); + - `GPU` — включить подключение GPU-устройств к виртуальным машинам через DRA по ссылке на GPUClass. (Не доступно в CE); items: type: string From ebf807f24cf720fd1b149963f0a64939d1d08b9a Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Sat, 11 Jul 2026 02:55:41 +0200 Subject: [PATCH 36/39] fix(vm): validate the GPUClass itself, not the same-named DeviceClass The validator resolved a DeviceClass named after the GPUClass, so an unrelated DeviceClass with that name would pass. Look up the GPUClass custom resource (gpu.deckhouse.io/v1alpha1) directly. A missing CRD (GPU module not installed) is reported distinctly from a missing GPUClass. RBAC switches from deviceclasses to gpuclasses read access. Signed-off-by: Daniil Antoshin --- .../validators/gpu_devices_validator.go | 30 +++++++------ .../validators/gpu_devices_validator_test.go | 42 ++++++++++--------- .../rbac-for-us.yaml | 4 +- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index 3c89b12208..3c9fb2fea5 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -21,8 +21,10 @@ import ( "fmt" "reflect" - resourcev1 "k8s.io/api/resource/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/component-base/featuregate" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,6 +35,9 @@ import ( "github.com/deckhouse/virtualization/api/core/v1alpha2" ) +// gpuClassGVK identifies the GPUClass custom resource provided by the GPU module. +var gpuClassGVK = schema.GroupVersionKind{Group: "gpu.deckhouse.io", Version: "v1alpha1", Kind: "GPUClass"} + type GPUDevicesValidator struct { client client.Client featureGate featuregate.FeatureGate @@ -47,10 +52,10 @@ func (v *GPUDevicesValidator) ValidateCreate(ctx context.Context, vm *v1alpha2.V } func (v *GPUDevicesValidator) ValidateUpdate(ctx context.Context, oldVM, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) { - // The feature gate and DeviceClass existence are checked only when GPU devices + // The feature gate and GPUClass existence are checked only when GPU devices // are introduced or changed. Unchanged GPU devices must not block unrelated // updates (or removal) of a VM created while the gate was enabled and later - // disabled, or whose DeviceClass was removed out of band. Order is ignored + // disabled, or whose GPUClass was removed out of band. Order is ignored // to match the vmchange comparator, which treats reordering as no change. if reflect.DeepEqual(kvbuilder.SortGPUDevices(oldVM.Spec.GPUDevices), kvbuilder.SortGPUDevices(newVM.Spec.GPUDevices)) { return nil, nil @@ -68,21 +73,22 @@ func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alph } // A nil client means template validation (e.g. a VirtualMachinePool template): - // GPUClass readiness is verified when the actual replica VM is created, so that + // GPUClass existence is verified when the actual replica VM is created, so that // a pool may be defined before the GPU provider and its classes exist. if v.client == nil { return nil } - // The GPU module creates a DeviceClass named exactly after the GPUClass, so an - // existing DeviceClass with that name means the GPUClass is ready for allocation. for _, device := range vm.Spec.GPUDevices { - deviceClass := &resourcev1.DeviceClass{} - err := v.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, deviceClass) - if apierrors.IsNotFound(err) { - return fmt.Errorf("GPU device %q references GPUClass %q that does not exist or is not ready", device.Name, device.GPUClassName) - } - if err != nil { + gpuClass := &unstructured.Unstructured{} + gpuClass.SetGroupVersionKind(gpuClassGVK) + err := v.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, gpuClass) + switch { + case apierrors.IsNotFound(err): + return fmt.Errorf("GPU device %q references GPUClass %q that does not exist", device.Name, device.GPUClassName) + case meta.IsNoMatchError(err): + return fmt.Errorf("GPU device %q references GPUClass %q, but the GPUClass CRD is not registered (is the GPU module installed?)", device.Name, device.GPUClassName) + case err != nil: return fmt.Errorf("failed to resolve GPUClass %q for GPU device %q: %w", device.GPUClassName, device.Name, err) } } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index eb617d4580..274e1027ec 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -20,12 +20,13 @@ import ( "strings" "testing" - resourcev1 "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/component-base/featuregate" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/deckhouse/virtualization-controller/pkg/common/testutil" "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -34,7 +35,7 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { tests := []struct { name string featureEnabled bool - deviceClasses []string + gpuClasses []string gpuClass string wantErrorPart string }{ @@ -47,7 +48,7 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { { name: "should accept GPU devices when feature is enabled and GPUClass is ready", featureEnabled: true, - deviceClasses: []string{"nvidia-h100"}, + gpuClasses: []string{"nvidia-h100"}, gpuClass: "nvidia-h100", }, { @@ -61,7 +62,7 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { vm := newVirtualMachineWithGPU([]v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: tt.gpuClass}}) - validator := NewGPUDevicesValidator(newValidatorClient(t, tt.deviceClasses...), newGPUFeatureGate(t, tt.featureEnabled)) + validator := NewGPUDevicesValidator(newValidatorClient(t, tt.gpuClasses...), newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) @@ -78,7 +79,7 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { tests := []struct { name string featureEnabled bool - deviceClasses []string + gpuClasses []string oldGPU []v1alpha2.GPUDeviceSpec newGPU []v1alpha2.GPUDeviceSpec wantErrorPart string @@ -117,14 +118,14 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { { name: "changing to a ready GPUClass is allowed when feature is enabled", featureEnabled: true, - deviceClasses: []string{"nvidia-h100", "nvidia-a100"}, + gpuClasses: []string{"nvidia-h100", "nvidia-a100"}, oldGPU: gpu("nvidia-h100"), newGPU: gpu("nvidia-a100"), }, { name: "changing to an unready GPUClass is rejected", featureEnabled: true, - deviceClasses: []string{"nvidia-h100"}, + gpuClasses: []string{"nvidia-h100"}, oldGPU: gpu("nvidia-h100"), newGPU: gpu("nvidia-a100"), wantErrorPart: "does not exist", @@ -135,7 +136,7 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { oldVM := newVirtualMachineWithGPU(tt.oldGPU) newVM := newVirtualMachineWithGPU(tt.newGPU) - validator := NewGPUDevicesValidator(newValidatorClient(t, tt.deviceClasses...), newGPUFeatureGate(t, tt.featureEnabled)) + validator := NewGPUDevicesValidator(newValidatorClient(t, tt.gpuClasses...), newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateUpdate(t.Context(), oldVM, newVM) @@ -146,7 +147,7 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { func TestGPUDevicesValidatorTemplateMode(t *testing.T) { // A nil client (template validation) enforces the feature gate but skips - // GPUClass readiness (its DeviceClass existence). + // GPUClass existence. tests := []struct { name string featureEnabled bool @@ -199,19 +200,22 @@ func newVirtualMachineWithGPU(gpuDevices []v1alpha2.GPUDeviceSpec) *v1alpha2.Vir } } -func newValidatorClient(t *testing.T, deviceClasses ...string) client.Client { +func newValidatorClient(t *testing.T, gpuClasses ...string) client.Client { t.Helper() - objs := make([]client.Object, 0, len(deviceClasses)) - for _, name := range deviceClasses { - objs = append(objs, &resourcev1.DeviceClass{ObjectMeta: metav1.ObjectMeta{Name: name}}) - } + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(gpuClassGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(gpuClassGVK.GroupVersion().WithKind("GPUClassList"), &unstructured.UnstructuredList{}) - fakeClient, err := testutil.NewFakeClientWithObjects(objs...) - if err != nil { - t.Fatalf("failed to create fake client: %v", err) + objs := make([]client.Object, 0, len(gpuClasses)) + for _, name := range gpuClasses { + gpuClass := &unstructured.Unstructured{} + gpuClass.SetGroupVersionKind(gpuClassGVK) + gpuClass.SetName(name) + objs = append(objs, gpuClass) } - return fakeClient + + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() } func newGPUFeatureGate(t *testing.T, enabled bool) featuregate.FeatureGate { diff --git a/templates/virtualization-controller/rbac-for-us.yaml b/templates/virtualization-controller/rbac-for-us.yaml index e895370513..8299039f38 100644 --- a/templates/virtualization-controller/rbac-for-us.yaml +++ b/templates/virtualization-controller/rbac-for-us.yaml @@ -345,9 +345,9 @@ rules: - patch - delete - apiGroups: - - resource.k8s.io + - gpu.deckhouse.io resources: - - deviceclasses + - gpuclasses verbs: - get - list From cdf003f03dee370a3b46e88c023c0c1e6a528cd5 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Sat, 11 Jul 2026 03:06:35 +0200 Subject: [PATCH 37/39] feat(vm): surface GPUClass readiness via a VM condition A GPUClass deleted under a running VM previously left the next claim silently pending. The GPU handler now sets a GPUClassReady condition: False (reason GPUClassNotFound) when any referenced GPUClass is missing or its CRD is absent, True otherwise, and removes it when no GPU devices remain. The GPUClass GVK is centralized in kvbuilder and reused by the validator; testutil registers it for fake clients. Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/vmcondition/condition.go | 14 ++++++ .../pkg/common/testutil/testutil.go | 8 ++++ .../pkg/controller/kvbuilder/gpu.go | 4 ++ .../vm/internal/gpu_resourceclaim_handler.go | 48 ++++++++++++++++++- .../gpu_resourceclaim_handler_test.go | 31 ++++++++++++ .../validators/gpu_devices_validator.go | 6 +-- .../validators/gpu_devices_validator_test.go | 7 +-- 7 files changed, 109 insertions(+), 9 deletions(-) diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index 2320fd5da9..8ab39f1872 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -50,6 +50,9 @@ const ( // TypeMaintenance indicates that the VirtualMachine is in maintenance mode. // During this condition, the VM remains stopped and no changes are allowed. TypeMaintenance Type = "Maintenance" + + // TypeGPUClassReady indicates whether every GPUClass referenced by spec.gpuDevices exists. + TypeGPUClassReady Type = "GPUClassReady" ) type AgentReadyReason string @@ -283,3 +286,14 @@ func (r MaintenanceReason) String() string { const ( ReasonMaintenanceRestore MaintenanceReason = "RestoreInProgress" ) + +type GPUClassReadyReason string + +func (r GPUClassReadyReason) String() string { + return string(r) +} + +const ( + ReasonGPUClassReady GPUClassReadyReason = "GPUClassReady" + ReasonGPUClassNotFound GPUClassReadyReason = "GPUClassNotFound" +) diff --git a/images/virtualization-artifact/pkg/common/testutil/testutil.go b/images/virtualization-artifact/pkg/common/testutil/testutil.go index 99768fa4da..a89b53f7aa 100644 --- a/images/virtualization-artifact/pkg/common/testutil/testutil.go +++ b/images/virtualization-artifact/pkg/common/testutil/testutil.go @@ -34,6 +34,7 @@ import ( storagev1alpha1 "github.com/deckhouse/virtualization-controller/pkg/apis/storage/v1alpha1" commonnetwork "github.com/deckhouse/virtualization-controller/pkg/common/network" "github.com/deckhouse/virtualization-controller/pkg/controller/indexer" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha3" ) @@ -43,6 +44,11 @@ func registerNetworkGVKs(scheme *apiruntime.Scheme) { scheme.AddKnownTypeWithName(commonnetwork.ClusterNetworkGVK, &unstructured.Unstructured{}) } +func registerGPUGVKs(scheme *apiruntime.Scheme) { + scheme.AddKnownTypeWithName(kvbuilder.GPUClassGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(kvbuilder.GPUClassGVK.GroupVersion().WithKind("GPUClassList"), &unstructured.UnstructuredList{}) +} + func NewFakeClientWithObjects(objs ...client.Object) (client.WithWatch, error) { scheme := apiruntime.NewScheme() for _, f := range []func(*apiruntime.Scheme) error{ @@ -58,6 +64,7 @@ func NewFakeClientWithObjects(objs ...client.Object) (client.WithWatch, error) { } } registerNetworkGVKs(scheme) + registerGPUGVKs(scheme) var newObjs []client.Object for _, obj := range objs { if reflect.ValueOf(obj).IsNil() { @@ -88,6 +95,7 @@ func NewFakeClientWithInterceptorWithObjects(interceptor interceptor.Funcs, objs } } registerNetworkGVKs(scheme) + registerGPUGVKs(scheme) var newObjs []client.Object for _, obj := range objs { if reflect.ValueOf(obj).IsNil() { diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index 602e79cc84..0fb922fceb 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -21,6 +21,7 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/ptr" virtv1 "kubevirt.io/api/core/v1" @@ -32,6 +33,9 @@ const ( GPUDRADriverName = "gpu.deckhouse.io" ) +// GPUClassGVK identifies the GPUClass custom resource provided by the GPU module. +var GPUClassGVK = schema.GroupVersionKind{Group: "gpu.deckhouse.io", Version: "v1alpha1", Kind: "GPUClass"} + func GPUResourceClaimName(deviceName string) string { return GPUNamePrefix + deviceName } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index e24c181ff6..6b32833464 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -21,21 +21,26 @@ import ( "encoding/json" "fmt" "reflect" + "strings" resourcev1 "k8s.io/api/resource/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" "github.com/deckhouse/virtualization-controller/pkg/logger" "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" ) const nameGPUResourceClaimHandler = "GPUResourceClaimHandler" @@ -57,7 +62,7 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac return reconcile.Result{}, nil } - vm := s.VirtualMachine().Current() + vm := s.VirtualMachine().Changed() log := logger.FromContext(ctx).With(logger.SlogHandler(nameGPUResourceClaimHandler)) desiredTemplateNames := make(map[string]struct{}, len(vm.Spec.GPUDevices)) @@ -102,9 +107,50 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac if err := h.deleteOrphanedTemplates(ctx, vm, desiredTemplateNames); err != nil { return reconcile.Result{}, err } + + if err := h.reconcileGPUClassReadyCondition(ctx, vm); err != nil { + return reconcile.Result{}, err + } return reconcile.Result{}, nil } +// reconcileGPUClassReadyCondition surfaces whether every referenced GPUClass +// exists, so a GPUClass deleted under a running VM yields a clear condition +// instead of a silently pending claim on the next restart. +func (h *GPUResourceClaimHandler) reconcileGPUClassReadyCondition(ctx context.Context, vm *v1alpha2.VirtualMachine) error { + if len(vm.Spec.GPUDevices) == 0 { + conditions.RemoveCondition(vmcondition.TypeGPUClassReady, &vm.Status.Conditions) + return nil + } + + cb := conditions.NewConditionBuilder(vmcondition.TypeGPUClassReady).Generation(vm.GetGeneration()) + + var missing []string + for _, device := range vm.Spec.GPUDevices { + gpuClass := &unstructured.Unstructured{} + gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) + err := h.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, gpuClass) + switch { + case apierrors.IsNotFound(err), meta.IsNoMatchError(err): + missing = append(missing, device.GPUClassName) + case err != nil: + return fmt.Errorf("failed to resolve GPUClass %q: %w", device.GPUClassName, err) + } + } + + if len(missing) > 0 { + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonGPUClassNotFound). + Message(fmt.Sprintf("GPUClass not found: %s. The GPU cannot be allocated until the GPUClass exists.", strings.Join(missing, ", "))) + } else { + cb.Status(metav1.ConditionTrue). + Reason(vmcondition.ReasonGPUClassReady). + Message("") + } + conditions.SetCondition(cb, &vm.Status.Conditions) + return nil +} + func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spec resourcev1.ResourceClaimTemplateSpec) *resourcev1.ResourceClaimTemplate { return &resourcev1.ResourceClaimTemplate{ ObjectMeta: metav1.ObjectMeta{ diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index d33adb47ae..d46bd6261c 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -23,11 +23,14 @@ import ( . "github.com/onsi/gomega" resourcev1 "k8s.io/api/resource/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" ) var _ = Describe("GPUResourceClaimHandler", func() { @@ -147,4 +150,32 @@ var _ = Describe("GPUResourceClaimHandler", func() { stored := &resourcev1.ResourceClaimTemplate{} Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: otherName, Namespace: namespace}, stored)).To(Succeed()) }) + + It("should set GPUClassReady=False when the GPUClass does not exist", func() { + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + cond, ok := conditions.GetCondition(vmcondition.TypeGPUClassReady, vmState.VirtualMachine().Changed().Status.Conditions) + Expect(ok).To(BeTrue()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(vmcondition.ReasonGPUClassNotFound.String())) + }) + + It("should set GPUClassReady=True when the GPUClass exists", func() { + gpuClass := &unstructured.Unstructured{} + gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) + gpuClass.SetName(deviceClass) + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass}), gpuClass) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + cond, ok := conditions.GetCondition(vmcondition.TypeGPUClassReady, vmState.VirtualMachine().Changed().Status.Conditions) + Expect(ok).To(BeTrue()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + }) }) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index 3c9fb2fea5..72232d3942 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -24,7 +24,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/component-base/featuregate" "sigs.k8s.io/controller-runtime/pkg/client" @@ -35,9 +34,6 @@ import ( "github.com/deckhouse/virtualization/api/core/v1alpha2" ) -// gpuClassGVK identifies the GPUClass custom resource provided by the GPU module. -var gpuClassGVK = schema.GroupVersionKind{Group: "gpu.deckhouse.io", Version: "v1alpha1", Kind: "GPUClass"} - type GPUDevicesValidator struct { client client.Client featureGate featuregate.FeatureGate @@ -81,7 +77,7 @@ func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alph for _, device := range vm.Spec.GPUDevices { gpuClass := &unstructured.Unstructured{} - gpuClass.SetGroupVersionKind(gpuClassGVK) + gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) err := v.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, gpuClass) switch { case apierrors.IsNotFound(err): diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index 274e1027ec..6b9c7ed4d0 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization-controller/pkg/featuregates" "github.com/deckhouse/virtualization/api/core/v1alpha2" ) @@ -204,13 +205,13 @@ func newValidatorClient(t *testing.T, gpuClasses ...string) client.Client { t.Helper() scheme := runtime.NewScheme() - scheme.AddKnownTypeWithName(gpuClassGVK, &unstructured.Unstructured{}) - scheme.AddKnownTypeWithName(gpuClassGVK.GroupVersion().WithKind("GPUClassList"), &unstructured.UnstructuredList{}) + scheme.AddKnownTypeWithName(kvbuilder.GPUClassGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(kvbuilder.GPUClassGVK.GroupVersion().WithKind("GPUClassList"), &unstructured.UnstructuredList{}) objs := make([]client.Object, 0, len(gpuClasses)) for _, name := range gpuClasses { gpuClass := &unstructured.Unstructured{} - gpuClass.SetGroupVersionKind(gpuClassGVK) + gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) gpuClass.SetName(name) objs = append(objs, gpuClass) } From b17ccfc40286fb344c5b16e42febcf12d58f4e09 Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Mon, 13 Jul 2026 09:53:45 +0200 Subject: [PATCH 38/39] feat(vm): drop gpuDevices name, index by GPUClass, gate on GPUClass Ready Remove the per-device name field: gpuDevices entries now carry only gpuClassName. Claim/GPU/template names are derived from the position in the list sorted by GPUClass, so reordering the spec is a no-op and no restart is triggered. The list becomes atomic (order not significant), and duplicate gpuClassName entries request several cards of one class. The GPUClassReady condition now also checks the GPUClass Ready status, not just its existence, so a class that is still initializing (its DeviceClasses not yet reconciled) reports GPUClassNotReady instead of a silently pending claim. Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/virtual_machine.go | 11 +-- api/core/v1alpha2/vmcondition/condition.go | 1 + crds/doc-ru-virtualmachines.yaml | 4 -- crds/virtualmachines.yaml | 15 +--- docs/USER_GUIDE.md | 5 +- docs/USER_GUIDE.ru.md | 5 +- .../pkg/controller/kvbuilder/gpu.go | 29 ++++---- .../pkg/controller/kvbuilder/gpu_test.go | 45 ++++++------ .../vm/internal/gpu_resourceclaim_handler.go | 53 ++++++++++++--- .../gpu_resourceclaim_handler_test.go | 68 +++++++++++++------ .../controller/vm/internal/sync_kvvm_test.go | 4 +- .../validators/gpu_devices_validator.go | 6 +- .../validators/gpu_devices_validator_test.go | 14 ++-- .../pkg/controller/vmchange/compare_test.go | 18 ++--- 14 files changed, 154 insertions(+), 124 deletions(-) diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index 7f07291c1c..4eb45b6c6c 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -124,11 +124,10 @@ type VirtualMachineSpec struct { // +kubebuilder:validation:MaxItems:=8 USBDevices []USBDeviceSpecRef `json:"usbDevices,omitempty"` // List of GPU devices to attach to the virtual machine. - // Each device references a GPUClass by name. + // Each entry references a GPUClass by name; list order is not significant. // This feature requires the GPU feature gate. // +kubebuilder:validation:MaxItems:=16 - // +listType=map - // +listMapKey=name + // +listType=atomic GPUDevices []GPUDeviceSpec `json:"gpuDevices,omitempty"` } @@ -514,12 +513,6 @@ type USBDeviceSpecRef struct { // GPUDeviceSpec requests a GPU device by GPUClass. type GPUDeviceSpec struct { - // A unique GPU device name inside the virtual machine spec. - // The value is used to generate DRA claim and request names. - // +kubebuilder:validation:MinLength:=1 - // +kubebuilder:validation:MaxLength:=59 - // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` - Name string `json:"name"` // Name of the GPUClass that selects the GPU to attach, for example nvidia-h100. // The GPUClass must already exist: a DRA DeviceClass with the same name is // created by the GPU module and is used to allocate the device. diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index 8ab39f1872..1f99d93133 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -296,4 +296,5 @@ func (r GPUClassReadyReason) String() string { const ( ReasonGPUClassReady GPUClassReadyReason = "GPUClassReady" ReasonGPUClassNotFound GPUClassReadyReason = "GPUClassNotFound" + ReasonGPUClassNotReady GPUClassReadyReason = "GPUClassNotReady" ) diff --git a/crds/doc-ru-virtualmachines.yaml b/crds/doc-ru-virtualmachines.yaml index 6779593387..442c62c549 100644 --- a/crds/doc-ru-virtualmachines.yaml +++ b/crds/doc-ru-virtualmachines.yaml @@ -599,10 +599,6 @@ spec: description: | Имя GPUClass, который выбирает подключаемый GPU, например `nvidia-h100`. GPUClass должен уже существовать: DRA DeviceClass с тем же именем создаётся модулем GPU и используется для выделения устройства. - name: - description: | - Уникальное имя GPU-устройства внутри спецификации виртуальной машины. - Значение используется для генерации имён DRA claim и request. status: properties: blockDeviceRefs: diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index c8d4cffde5..5e5188adc5 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1061,18 +1061,15 @@ spec: gpuDevices: type: array maxItems: 16 - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map + x-kubernetes-list-type: atomic description: | List of GPU devices to attach to the virtual machine. - Each device references a GPUClass by name. + Each entry references a GPUClass by name; list order is not significant. This feature requires the GPU feature gate. items: type: object required: - gpuClassName - - name properties: gpuClassName: minLength: 1 @@ -1082,14 +1079,6 @@ spec: description: | Name of the GPUClass that selects the GPU to attach, for example nvidia-h100. The GPUClass must already exist: a DRA DeviceClass with the same name is created by the GPU module and is used to allocate the device. - name: - minLength: 1 - maxLength: 59 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - description: | - A unique GPU device name inside the virtual machine spec. - The value is used to generate DRA claim and request names. status: type: object properties: diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 460542110d..0dc7087dd0 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -4053,11 +4053,10 @@ metadata: spec: # ... other VM settings ... gpuDevices: - - name: gpu0 - gpuClassName: nvidia-h100 + - gpuClassName: nvidia-h100 ``` -The `name` field must be unique within `.spec.gpuDevices` and can contain up to 59 DNS-label characters. The `gpuClassName` field must be the name of an existing `GPUClass` that selects the GPU to attach. +The `gpuClassName` field must be the name of an existing `GPUClass` that selects the GPU to attach. To attach several GPUs, add more entries (list order is not significant). Changing `.spec.gpuDevices` requires restarting the virtual machine to apply the new configuration. diff --git a/docs/USER_GUIDE.ru.md b/docs/USER_GUIDE.ru.md index 6014762afe..1b2294ef4c 100644 --- a/docs/USER_GUIDE.ru.md +++ b/docs/USER_GUIDE.ru.md @@ -4084,11 +4084,10 @@ metadata: spec: # ... другие настройки ВМ ... gpuDevices: - - name: gpu0 - gpuClassName: nvidia-h100 + - gpuClassName: nvidia-h100 ``` -Поле `name` должно быть уникальным внутри `.spec.gpuDevices` и может содержать до 59 символов DNS label. Поле `gpuClassName` должно быть именем существующего `GPUClass`, который выбирает подключаемый GPU. +Поле `gpuClassName` должно быть именем существующего `GPUClass`, который выбирает подключаемый GPU. Чтобы подключить несколько GPU, добавьте больше элементов (порядок в списке не важен). Изменение `.spec.gpuDevices` требует перезапуска виртуальной машины для применения новой конфигурации. diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go index 0fb922fceb..069cd10c9e 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -18,6 +18,7 @@ package kvbuilder import ( "slices" + "strconv" "strings" corev1 "k8s.io/api/core/v1" @@ -36,16 +37,16 @@ const ( // GPUClassGVK identifies the GPUClass custom resource provided by the GPU module. var GPUClassGVK = schema.GroupVersionKind{Group: "gpu.deckhouse.io", Version: "v1alpha1", Kind: "GPUClass"} -func GPUResourceClaimName(deviceName string) string { - return GPUNamePrefix + deviceName +func GPUResourceClaimName(index int) string { + return GPUNamePrefix + strconv.Itoa(index) } -func GPUResourceClaimTemplateName(vmName, deviceName string) string { +func GPUResourceClaimTemplateName(vmName string, index int) string { // The vmName hash suffix keeps the template name unique per VM: two VMs in the - // same namespace whose "-" prefixes collide (e.g. VM "a" with - // device "b-c" and VM "a-b" with device "c") would otherwise fight over one name - // and deadlock the losing VM's reconciliation on a not-controlled-by error. - return vmName + "-" + deviceName + "-" + GenerateSerial(vmName)[:8] + // same namespace whose "-" prefixes collide (e.g. VM "a" and + // VM "a-0") would otherwise fight over one name and deadlock the losing VM's + // reconciliation on a not-controlled-by error. + return vmName + "-" + strconv.Itoa(index) + "-" + GenerateSerial(vmName)[:8] } func IsGPUResourceClaimTemplateName(vmName, templateName string) bool { @@ -70,31 +71,33 @@ func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { }, ) - for _, device := range devices { - claimName := GPUResourceClaimName(device.Name) + for index := range devices { + claimName := GPUResourceClaimName(index) b.Resource.Spec.Template.Spec.ResourceClaims = append(b.Resource.Spec.Template.Spec.ResourceClaims, virtv1.ResourceClaim{ PodResourceClaim: corev1.PodResourceClaim{ Name: claimName, - ResourceClaimTemplateName: ptr.To(GPUResourceClaimTemplateName(vmName, device.Name)), + ResourceClaimTemplateName: ptr.To(GPUResourceClaimTemplateName(vmName, index)), }, }) b.Resource.Spec.Template.Spec.Domain.Devices.GPUs = append(b.Resource.Spec.Template.Spec.Domain.Devices.GPUs, virtv1.GPU{ Name: claimName, ClaimRequest: &virtv1.ClaimRequest{ ClaimName: ptr.To(claimName), - RequestName: ptr.To(GPUResourceClaimName(device.Name)), + RequestName: ptr.To(claimName), }, }) } } +// SortGPUDevices orders devices by GPUClass so that reordering the spec list is a +// no-op: the generated claim indexes stay stable and no restart is triggered. func SortGPUDevices(devices []v1alpha2.GPUDeviceSpec) []v1alpha2.GPUDeviceSpec { if len(devices) == 0 { return nil } sorted := slices.Clone(devices) - slices.SortFunc(sorted, func(a, b v1alpha2.GPUDeviceSpec) int { - return strings.Compare(a.Name, b.Name) + slices.SortStableFunc(sorted, func(a, b v1alpha2.GPUDeviceSpec) int { + return strings.Compare(a.GPUClassName, b.GPUClassName) }) return sorted } diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go index 6743bf8414..a4de26f995 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -31,47 +31,48 @@ var _ = Describe("GPU", func() { It("should render DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}}) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) - Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu0")) - Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal(GPUResourceClaimTemplateName("vm-a", "gpu0"))) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-0")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal(GPUResourceClaimTemplateName("vm-a", 0))) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) - Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu0")) - Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal("gpu-gpu0")) - Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal("gpu-gpu0")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-0")) + Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal("gpu-0")) + Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal("gpu-0")) }) - It("should render DRA GPU resource claims in stable order", func() { + It("should index claims by GPUClass order regardless of spec order", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{ - {Name: "gpu1", GPUClassName: "nvidia-h100"}, - {Name: "gpu0", GPUClassName: "nvidia-a100"}, + {GPUClassName: "nvidia-h100"}, + {GPUClassName: "nvidia-a100"}, }) res := kvvm.GetResource() + // Sorted by GPUClass: nvidia-a100 -> index 0, nvidia-h100 -> index 1. Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(2)) - Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu0")) - Expect(res.Spec.Template.Spec.ResourceClaims[1].Name).To(Equal("gpu-gpu1")) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-0")) + Expect(res.Spec.Template.Spec.ResourceClaims[1].Name).To(Equal("gpu-1")) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(2)) - Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu0")) - Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[1].Name).To(Equal("gpu-gpu1")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-0")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[1].Name).To(Equal("gpu-1")) }) It("should replace rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu1", GPUClassName: "nvidia-a100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-a100"}}) res := kvvm.GetResource() Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) - Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-gpu1")) - Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal(GPUResourceClaimTemplateName("vm-a", "gpu1"))) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-0")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal(GPUResourceClaimTemplateName("vm-a", 0))) Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) - Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-gpu1")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-0")) }) It("should not touch resource claims and GPUs it does not own", func() { @@ -88,7 +89,7 @@ var _ = Describe("GPU", func() { DeviceName: "nvidia.com/GH100", }} - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}}) res = kvvm.GetResource() claimNames := make([]string, 0) @@ -99,13 +100,13 @@ var _ = Describe("GPU", func() { for _, g := range res.Spec.Template.Spec.Domain.Devices.GPUs { gpuNames = append(gpuNames, g.Name) } - Expect(claimNames).To(ConsistOf("gpu-foreign", "gpu-gpu0")) - Expect(gpuNames).To(ConsistOf("gpu-legacy", "gpu-gpu0")) + Expect(claimNames).To(ConsistOf("gpu-foreign", "gpu-0")) + Expect(gpuNames).To(ConsistOf("gpu-legacy", "gpu-0")) }) It("should remove rendered DRA GPU resource claims", func() { kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) - kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}}) kvvm.SetGPUDevices("vm-a", nil) res := kvvm.GetResource() diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index 6b32833464..def65cd683 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -64,12 +64,15 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac vm := s.VirtualMachine().Changed() log := logger.FromContext(ctx).With(logger.SlogHandler(nameGPUResourceClaimHandler)) - desiredTemplateNames := make(map[string]struct{}, len(vm.Spec.GPUDevices)) + // Sort exactly as kvbuilder.SetGPUDevices does, so the claim index used here + // matches the index the KVVM GPU/claim references point at. + devices := kvbuilder.SortGPUDevices(vm.Spec.GPUDevices) + desiredTemplateNames := make(map[string]struct{}, len(devices)) - for _, device := range vm.Spec.GPUDevices { - templateName := kvbuilder.GPUResourceClaimTemplateName(vm.Name, device.Name) + for index, device := range devices { + templateName := kvbuilder.GPUResourceClaimTemplateName(vm.Name, index) desiredTemplateNames[templateName] = struct{}{} - desiredSpec := buildGPUResourceClaimTemplateSpec(device) + desiredSpec := buildGPUResourceClaimTemplateSpec(index, device) template := &resourcev1.ResourceClaimTemplate{} key := types.NamespacedName{Name: templateName, Namespace: vm.Namespace} @@ -115,8 +118,9 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac } // reconcileGPUClassReadyCondition surfaces whether every referenced GPUClass -// exists, so a GPUClass deleted under a running VM yields a clear condition -// instead of a silently pending claim on the next restart. +// exists and is Ready, so a GPUClass that is missing or still initializing +// (its DeviceClasses not yet reconciled) yields a clear condition instead of a +// silently pending claim on the next restart. func (h *GPUResourceClaimHandler) reconcileGPUClassReadyCondition(ctx context.Context, vm *v1alpha2.VirtualMachine) error { if len(vm.Spec.GPUDevices) == 0 { conditions.RemoveCondition(vmcondition.TypeGPUClassReady, &vm.Status.Conditions) @@ -125,7 +129,7 @@ func (h *GPUResourceClaimHandler) reconcileGPUClassReadyCondition(ctx context.Co cb := conditions.NewConditionBuilder(vmcondition.TypeGPUClassReady).Generation(vm.GetGeneration()) - var missing []string + var missing, notReady []string for _, device := range vm.Spec.GPUDevices { gpuClass := &unstructured.Unstructured{} gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) @@ -135,14 +139,23 @@ func (h *GPUResourceClaimHandler) reconcileGPUClassReadyCondition(ctx context.Co missing = append(missing, device.GPUClassName) case err != nil: return fmt.Errorf("failed to resolve GPUClass %q: %w", device.GPUClassName, err) + default: + if !gpuClassReady(gpuClass) { + notReady = append(notReady, device.GPUClassName) + } } } - if len(missing) > 0 { + switch { + case len(missing) > 0: cb.Status(metav1.ConditionFalse). Reason(vmcondition.ReasonGPUClassNotFound). Message(fmt.Sprintf("GPUClass not found: %s. The GPU cannot be allocated until the GPUClass exists.", strings.Join(missing, ", "))) - } else { + case len(notReady) > 0: + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonGPUClassNotReady). + Message(fmt.Sprintf("GPUClass not ready: %s. The GPU cannot be allocated until the GPUClass becomes Ready.", strings.Join(notReady, ", "))) + default: cb.Status(metav1.ConditionTrue). Reason(vmcondition.ReasonGPUClassReady). Message("") @@ -151,6 +164,24 @@ func (h *GPUResourceClaimHandler) reconcileGPUClassReadyCondition(ctx context.Co return nil } +// gpuClassReady reports whether the GPUClass has a Ready condition set to True. +func gpuClassReady(gpuClass *unstructured.Unstructured) bool { + conds, found, err := unstructured.NestedSlice(gpuClass.Object, "status", "conditions") + if err != nil || !found { + return false + } + for _, c := range conds { + cond, ok := c.(map[string]interface{}) + if !ok { + continue + } + if cond["type"] == "Ready" { + return cond["status"] == string(metav1.ConditionTrue) + } + } + return false +} + func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spec resourcev1.ResourceClaimTemplateSpec) *resourcev1.ResourceClaimTemplate { return &resourcev1.ResourceClaimTemplate{ ObjectMeta: metav1.ObjectMeta{ @@ -183,8 +214,8 @@ func gpuClaimSpecHash(spec resourcev1.ResourceClaimTemplateSpec) string { return kvbuilder.GenerateSerial(string(raw)) } -func buildGPUResourceClaimTemplateSpec(device v1alpha2.GPUDeviceSpec) resourcev1.ResourceClaimTemplateSpec { - requestName := kvbuilder.GPUResourceClaimName(device.Name) +func buildGPUResourceClaimTemplateSpec(index int, device v1alpha2.GPUDeviceSpec) resourcev1.ResourceClaimTemplateSpec { + requestName := kvbuilder.GPUResourceClaimName(index) return resourcev1.ResourceClaimTemplateSpec{ Spec: resourcev1.ResourceClaimSpec{ Devices: resourcev1.DeviceClaim{ diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index d46bd6261c..65fc44bce5 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -48,29 +48,29 @@ var _ = Describe("GPUResourceClaimHandler", func() { } It("should create GPU ResourceClaimTemplate", func() { - fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) handler := NewGPUResourceClaimHandler(fakeClient) _, err := handler.Handle(context.Background(), vmState) Expect(err).NotTo(HaveOccurred()) template := &resourcev1.ResourceClaimTemplate{} - Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, template)).To(Succeed()) + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, 0), Namespace: namespace}, template)).To(Succeed()) Expect(template.Spec.Spec.Devices.Requests).To(HaveLen(1)) request := template.Spec.Spec.Devices.Requests[0] - Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimName("gpu0"))) + Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimName(0))) Expect(request.Exactly.DeviceClassName).To(Equal(deviceClass)) Expect(request.Exactly.Selectors).To(BeEmpty()) Expect(template.Spec.Spec.Devices.Config).To(HaveLen(1)) config := template.Spec.Spec.Devices.Config[0] - Expect(config.Requests).To(ConsistOf(kvbuilder.GPUResourceClaimName("gpu0"))) + Expect(config.Requests).To(ConsistOf(kvbuilder.GPUResourceClaimName(0))) Expect(config.Opaque.Driver).To(Equal(kvbuilder.GPUDRADriverName)) Expect(string(config.Opaque.Parameters.Raw)).To(ContainSubstring(`"kind":"VfioDeviceConfig"`)) }) It("should delete owned GPU ResourceClaimTemplate when device is removed from spec", func() { vm := newVM() - template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) + template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName, 0), buildGPUResourceClaimTemplateSpec(0, v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) fakeClient, _, vmState := setupEnvironment(vm, template) handler := NewGPUResourceClaimHandler(fakeClient) @@ -78,14 +78,14 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(err).NotTo(HaveOccurred()) stored := &resourcev1.ResourceClaimTemplate{} - err = fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, stored) + err = fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, 0), Namespace: namespace}, stored) Expect(err).To(HaveOccurred()) }) It("should recreate GPU ResourceClaimTemplate when the desired spec changes", func() { - vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: "nvidia-a100"}) - templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0") - template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) + vm := newVM(v1alpha2.GPUDeviceSpec{GPUClassName: "nvidia-a100"}) + templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, 0) + template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(0, v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) fakeClient, _, vmState := setupEnvironment(vm, template) handler := NewGPUResourceClaimHandler(fakeClient) @@ -99,9 +99,9 @@ var _ = Describe("GPUResourceClaimHandler", func() { }) It("should not recreate GPU ResourceClaimTemplate without hash annotation when spec matches", func() { - vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass}) - templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0") - template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) + vm := newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass}) + templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, 0) + template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(0, v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) template.Annotations = nil template.Labels = map[string]string{"keep": "me"} fakeClient, _, vmState := setupEnvironment(vm, template) @@ -117,9 +117,9 @@ var _ = Describe("GPUResourceClaimHandler", func() { }) It("should not replace GPU ResourceClaimTemplate owned by another controller", func() { - vm := newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass}) + vm := newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass}) template := &resourcev1.ResourceClaimTemplate{ - ObjectMeta: metav1.ObjectMeta{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, "gpu0"), Namespace: namespace}, + ObjectMeta: metav1.ObjectMeta{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, 0), Namespace: namespace}, } fakeClient, _, vmState := setupEnvironment(vm, template) handler := NewGPUResourceClaimHandler(fakeClient) @@ -138,9 +138,9 @@ var _ = Describe("GPUResourceClaimHandler", func() { otherVM := &v1alpha2.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{Name: vmName + "-b", Namespace: namespace, UID: "uid-vm-a-b"}, } - otherName := kvbuilder.GPUResourceClaimTemplateName(otherVM.Name, "gpu0") + otherName := kvbuilder.GPUResourceClaimTemplateName(otherVM.Name, 0) Expect(kvbuilder.IsGPUResourceClaimTemplateName(vmName, otherName)).To(BeTrue()) - template := buildGPUResourceClaimTemplate(otherVM, otherName, buildGPUResourceClaimTemplateSpec(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) + template := buildGPUResourceClaimTemplate(otherVM, otherName, buildGPUResourceClaimTemplateSpec(0, v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) fakeClient, _, vmState := setupEnvironment(vm, template) handler := NewGPUResourceClaimHandler(fakeClient) @@ -152,7 +152,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { }) It("should set GPUClassReady=False when the GPUClass does not exist", func() { - fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass})) + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) handler := NewGPUResourceClaimHandler(fakeClient) _, err := handler.Handle(context.Background(), vmState) @@ -164,11 +164,8 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(cond.Reason).To(Equal(vmcondition.ReasonGPUClassNotFound.String())) }) - It("should set GPUClassReady=True when the GPUClass exists", func() { - gpuClass := &unstructured.Unstructured{} - gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) - gpuClass.SetName(deviceClass) - fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{Name: "gpu0", GPUClassName: deviceClass}), gpuClass) + It("should set GPUClassReady=True when the GPUClass exists and is Ready", func() { + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass}), newGPUClass(deviceClass, true)) handler := NewGPUResourceClaimHandler(fakeClient) _, err := handler.Handle(context.Background(), vmState) @@ -178,4 +175,31 @@ var _ = Describe("GPUResourceClaimHandler", func() { Expect(ok).To(BeTrue()) Expect(cond.Status).To(Equal(metav1.ConditionTrue)) }) + + It("should set GPUClassReady=False when the GPUClass exists but is not Ready", func() { + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass}), newGPUClass(deviceClass, false)) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + cond, ok := conditions.GetCondition(vmcondition.TypeGPUClassReady, vmState.VirtualMachine().Changed().Status.Conditions) + Expect(ok).To(BeTrue()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(vmcondition.ReasonGPUClassNotReady.String())) + }) }) + +func newGPUClass(name string, ready bool) *unstructured.Unstructured { + status := "False" + if ready { + status = "True" + } + gpuClass := &unstructured.Unstructured{} + gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) + gpuClass.SetName(name) + _ = unstructured.SetNestedSlice(gpuClass.Object, []interface{}{ + map[string]interface{}{"type": "Ready", "status": status}, + }, "status", "conditions") + return gpuClass +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index dce3fdf4f9..7d6ca837a6 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -426,7 +426,7 @@ var _ = Describe("SyncKvvmHandler", func() { vmClass := makeVMClass() vm := makeVM(v1alpha2.MachineRunning) - vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-h100"}} + vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}} kvvm := makeKVVM(vm) Expect(kvbuilder.SetLastAppliedSpec(kvvm, &v1alpha2.VirtualMachine{ Spec: v1alpha2.VirtualMachineSpec{ @@ -443,7 +443,7 @@ var _ = Describe("SyncKvvmHandler", func() { Disruptions: &v1alpha2.Disruptions{ RestartApprovalMode: vm.Spec.Disruptions.RestartApprovalMode, }, - GPUDevices: []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "nvidia-a100"}}, + GPUDevices: []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-a100"}}, }, })).To(Succeed()) kvvm.SetGroupVersionKind(virtv1.VirtualMachineGroupVersionKind) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index 72232d3942..48a8f68fd7 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -81,11 +81,11 @@ func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alph err := v.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, gpuClass) switch { case apierrors.IsNotFound(err): - return fmt.Errorf("GPU device %q references GPUClass %q that does not exist", device.Name, device.GPUClassName) + return fmt.Errorf("GPU device references GPUClass %q that does not exist", device.GPUClassName) case meta.IsNoMatchError(err): - return fmt.Errorf("GPU device %q references GPUClass %q, but the GPUClass CRD is not registered (is the GPU module installed?)", device.Name, device.GPUClassName) + return fmt.Errorf("GPU device references GPUClass %q, but the GPUClass CRD is not registered (is the GPU module installed?)", device.GPUClassName) case err != nil: - return fmt.Errorf("failed to resolve GPUClass %q for GPU device %q: %w", device.GPUClassName, device.Name, err) + return fmt.Errorf("failed to resolve GPUClass %q: %w", device.GPUClassName, err) } } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index 6b9c7ed4d0..818f983106 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -62,7 +62,7 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vm := newVirtualMachineWithGPU([]v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: tt.gpuClass}}) + vm := newVirtualMachineWithGPU([]v1alpha2.GPUDeviceSpec{{GPUClassName: tt.gpuClass}}) validator := NewGPUDevicesValidator(newValidatorClient(t, tt.gpuClasses...), newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) @@ -74,7 +74,7 @@ func TestGPUDevicesValidatorValidateCreate(t *testing.T) { func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { gpu := func(class string) []v1alpha2.GPUDeviceSpec { - return []v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: class}} + return []v1alpha2.GPUDeviceSpec{{GPUClassName: class}} } tests := []struct { @@ -101,12 +101,12 @@ func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { name: "reordering GPU devices is allowed when feature is disabled", featureEnabled: false, oldGPU: []v1alpha2.GPUDeviceSpec{ - {Name: "gpu0", GPUClassName: "nvidia-h100"}, - {Name: "gpu1", GPUClassName: "nvidia-a100"}, + {GPUClassName: "nvidia-h100"}, + {GPUClassName: "nvidia-a100"}, }, newGPU: []v1alpha2.GPUDeviceSpec{ - {Name: "gpu1", GPUClassName: "nvidia-a100"}, - {Name: "gpu0", GPUClassName: "nvidia-h100"}, + {GPUClassName: "nvidia-a100"}, + {GPUClassName: "nvidia-h100"}, }, }, { @@ -167,7 +167,7 @@ func TestGPUDevicesValidatorTemplateMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vm := newVirtualMachineWithGPU([]v1alpha2.GPUDeviceSpec{{Name: "gpu0", GPUClassName: "does-not-exist"}}) + vm := newVirtualMachineWithGPU([]v1alpha2.GPUDeviceSpec{{GPUClassName: "does-not-exist"}}) validator := NewGPUDevicesValidator(nil, newGPUFeatureGate(t, tt.featureEnabled)) _, err := validator.ValidateCreate(t.Context(), vm) diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go index f8150fc716..3ce0196e3e 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go @@ -688,17 +688,13 @@ networks: "no restart when gpu devices only change order", ` gpuDevices: -- name: gpu1 - gpuClassName: nvidia-h100 -- name: gpu0 - gpuClassName: nvidia-a100 +- gpuClassName: nvidia-h100 +- gpuClassName: nvidia-a100 `, ` gpuDevices: -- name: gpu0 - gpuClassName: nvidia-a100 -- name: gpu1 - gpuClassName: nvidia-h100 +- gpuClassName: nvidia-a100 +- gpuClassName: nvidia-h100 `, nil, assertNoChanges(), @@ -707,13 +703,11 @@ gpuDevices: "restart when gpu device gpuClassName changes", ` gpuDevices: -- name: gpu0 - gpuClassName: nvidia-a100 +- gpuClassName: nvidia-a100 `, ` gpuDevices: -- name: gpu0 - gpuClassName: nvidia-h100 +- gpuClassName: nvidia-h100 `, nil, assertChanges( From 2f9665b944bf6e34153ff8340b84ee8fc2e99eec Mon Sep 17 00:00:00 2001 From: Daniil Antoshin Date: Mon, 13 Jul 2026 13:24:34 +0200 Subject: [PATCH 39/39] refactor(vm): rename spec.gpuDevices to spec.gpus Shorter, matches the user-facing intent (a list of GPUs). Go field VirtualMachineSpec.GPUDevices -> GPUs, JSON/CRD gpuDevices -> gpus; GPUDeviceSpec type and Set/Sort helper names kept. Signed-off-by: Daniil Antoshin --- api/core/v1alpha2/virtual_machine.go | 2 +- api/core/v1alpha2/vmcondition/condition.go | 2 +- api/core/v1alpha2/zz_generated.deepcopy.go | 4 ++-- crds/doc-ru-virtualmachines.yaml | 2 +- crds/virtualmachines.yaml | 2 +- docs/USER_GUIDE.md | 10 +++++----- docs/USER_GUIDE.ru.md | 10 +++++----- .../virtualization-artifact/pkg/builder/vm/option.go | 4 ++-- .../pkg/controller/kvbuilder/kvvm_utils.go | 2 +- .../vm/internal/gpu_resourceclaim_handler.go | 6 +++--- .../vm/internal/gpu_resourceclaim_handler_test.go | 2 +- .../pkg/controller/vm/internal/sync_kvvm_test.go | 4 ++-- .../vm/internal/validators/gpu_devices_validator.go | 6 +++--- .../validators/gpu_devices_validator_test.go | 4 ++-- .../pkg/controller/vmchange/compare_test.go | 12 ++++++------ .../pkg/controller/vmchange/gpu_change.go | 6 +++--- 16 files changed, 39 insertions(+), 39 deletions(-) diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index 4eb45b6c6c..5848a339e4 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -128,7 +128,7 @@ type VirtualMachineSpec struct { // This feature requires the GPU feature gate. // +kubebuilder:validation:MaxItems:=16 // +listType=atomic - GPUDevices []GPUDeviceSpec `json:"gpuDevices,omitempty"` + GPUs []GPUDeviceSpec `json:"gpus,omitempty"` } func (s *VirtualMachineSpec) IsParavirtualizationEnabled() bool { diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index 1f99d93133..779a6a502b 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -51,7 +51,7 @@ const ( // During this condition, the VM remains stopped and no changes are allowed. TypeMaintenance Type = "Maintenance" - // TypeGPUClassReady indicates whether every GPUClass referenced by spec.gpuDevices exists. + // TypeGPUClassReady indicates whether every GPUClass referenced by spec.gpus exists. TypeGPUClassReady Type = "GPUClassReady" ) diff --git a/api/core/v1alpha2/zz_generated.deepcopy.go b/api/core/v1alpha2/zz_generated.deepcopy.go index 926d8f4474..fd15c1c388 100644 --- a/api/core/v1alpha2/zz_generated.deepcopy.go +++ b/api/core/v1alpha2/zz_generated.deepcopy.go @@ -3532,8 +3532,8 @@ func (in *VirtualMachineSpec) DeepCopyInto(out *VirtualMachineSpec) { *out = make([]USBDeviceSpecRef, len(*in)) copy(*out, *in) } - if in.GPUDevices != nil { - in, out := &in.GPUDevices, &out.GPUDevices + if in.GPUs != nil { + in, out := &in.GPUs, &out.GPUs *out = make([]GPUDeviceSpec, len(*in)) copy(*out, *in) } diff --git a/crds/doc-ru-virtualmachines.yaml b/crds/doc-ru-virtualmachines.yaml index 442c62c549..7217531e0b 100644 --- a/crds/doc-ru-virtualmachines.yaml +++ b/crds/doc-ru-virtualmachines.yaml @@ -588,7 +588,7 @@ spec: name: description: | Имя ресурса `USBDevice` в том же пространстве имен. - gpuDevices: + gpus: description: | Список GPU-устройств для подключения к виртуальной машине. Каждое устройство ссылается на GPUClass по имени. diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index 5e5188adc5..4b5377855d 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1058,7 +1058,7 @@ spec: type: string description: | The name of USBDevice resource in the same namespace. - gpuDevices: + gpus: type: array maxItems: 16 x-kubernetes-list-type: atomic diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 0dc7087dd0..3977236f5a 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -4021,14 +4021,14 @@ As a result, a VM named `clone-database-prod` and a disk named `clone-database-r GPU device passthrough is an experimental feature. It requires the Enterprise Edition (EE), Kubernetes DRA support, and an external GPU DRA provider that publishes GPUs as DRA resources with `gpu.deckhouse.io` device attributes. {{< /alert >}} -The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by referencing a `GPUClass` through the `.spec.gpuDevices` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. +The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by referencing a `GPUClass` through the `.spec.gpus` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. GPU device passthrough requires: - Kubernetes version 1.34 or higher with DRA feature gates required by the cluster configuration. - The `GPU` feature gate enabled in the `virtualization` module settings. - A GPU DRA provider installed in the cluster that publishes GPUs with `gpu.deckhouse.io` device attributes. -- A `GPUClass` that selects the GPU, referenced from `.spec.gpuDevices[].gpuClassName`. The GPU module creates a DRA DeviceClass with the same name, which is used to allocate the device. +- A `GPUClass` that selects the GPU, referenced from `.spec.gpus[].gpuClassName`. The GPU module creates a DRA DeviceClass with the same name, which is used to allocate the device. To enable the module feature gate: @@ -4043,7 +4043,7 @@ spec: - GPU ``` -To request a GPU device, add `.spec.gpuDevices` to the VM specification: +To request a GPU device, add `.spec.gpus` to the VM specification: ```yaml apiVersion: virtualization.deckhouse.io/v1alpha2 @@ -4052,13 +4052,13 @@ metadata: name: linux-vm spec: # ... other VM settings ... - gpuDevices: + gpus: - gpuClassName: nvidia-h100 ``` The `gpuClassName` field must be the name of an existing `GPUClass` that selects the GPU to attach. To attach several GPUs, add more entries (list order is not significant). -Changing `.spec.gpuDevices` requires restarting the virtual machine to apply the new configuration. +Changing `.spec.gpus` requires restarting the virtual machine to apply the new configuration. ## USB Devices diff --git a/docs/USER_GUIDE.ru.md b/docs/USER_GUIDE.ru.md index 1b2294ef4c..eb12c5b314 100644 --- a/docs/USER_GUIDE.ru.md +++ b/docs/USER_GUIDE.ru.md @@ -4052,14 +4052,14 @@ spec: Проброс GPU-устройств — экспериментальная возможность. Для работы требуются Enterprise Edition (EE), поддержка Kubernetes DRA и внешний GPU DRA-провайдер, публикующий GPU как DRA-ресурсы с атрибутами устройства `gpu.deckhouse.io`. {{< /alert >}} -Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по ссылке на `GPUClass` через поле `.spec.gpuDevices` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). +Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по ссылке на `GPUClass` через поле `.spec.gpus` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). Для проброса GPU требуются: - Kubernetes версии 1.34 или выше с DRA feature gates, необходимыми для конфигурации кластера. - Feature gate `GPU`, включённый в настройках модуля `virtualization`. - Установленный в кластере GPU DRA-провайдер, публикующий GPU с атрибутами устройства `gpu.deckhouse.io`. -- `GPUClass`, выбирающий GPU, на который ссылается `.spec.gpuDevices[].gpuClassName`. Модуль GPU создаёт DRA DeviceClass с тем же именем, который используется для выделения устройства. +- `GPUClass`, выбирающий GPU, на который ссылается `.spec.gpus[].gpuClassName`. Модуль GPU создаёт DRA DeviceClass с тем же именем, который используется для выделения устройства. Чтобы включить feature gate модуля: @@ -4074,7 +4074,7 @@ spec: - GPU ``` -Чтобы запросить GPU-устройство, добавьте `.spec.gpuDevices` в спецификацию ВМ: +Чтобы запросить GPU-устройство, добавьте `.spec.gpus` в спецификацию ВМ: ```yaml apiVersion: virtualization.deckhouse.io/v1alpha2 @@ -4083,13 +4083,13 @@ metadata: name: linux-vm spec: # ... другие настройки ВМ ... - gpuDevices: + gpus: - gpuClassName: nvidia-h100 ``` Поле `gpuClassName` должно быть именем существующего `GPUClass`, который выбирает подключаемый GPU. Чтобы подключить несколько GPU, добавьте больше элементов (порядок в списке не важен). -Изменение `.spec.gpuDevices` требует перезапуска виртуальной машины для применения новой конфигурации. +Изменение `.spec.gpus` требует перезапуска виртуальной машины для применения новой конфигурации. ## USB-устройства diff --git a/images/virtualization-artifact/pkg/builder/vm/option.go b/images/virtualization-artifact/pkg/builder/vm/option.go index 085214e233..67c22181e3 100644 --- a/images/virtualization-artifact/pkg/builder/vm/option.go +++ b/images/virtualization-artifact/pkg/builder/vm/option.go @@ -168,9 +168,9 @@ func WithUSBDevices(usbDevices []v1alpha2.USBDeviceSpecRef) Option { } } -func WithGPUDevices(gpuDevices []v1alpha2.GPUDeviceSpec) Option { +func WithGPUDevices(gpus []v1alpha2.GPUDeviceSpec) Option { return func(vm *v1alpha2.VirtualMachine) { - vm.Spec.GPUDevices = gpuDevices + vm.Spec.GPUs = gpus } } diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go index 23ce868dfa..401e01c957 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go @@ -247,7 +247,7 @@ func ApplyVirtualMachineSpec( return err } - kvvm.SetGPUDevices(vm.Name, vm.Spec.GPUDevices) + kvvm.SetGPUDevices(vm.Name, vm.Spec.GPUs) if err := kvvm.SetProvisioning(vm.Spec.Provisioning); err != nil { return err diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go index def65cd683..59ff8d516c 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -66,7 +66,7 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac log := logger.FromContext(ctx).With(logger.SlogHandler(nameGPUResourceClaimHandler)) // Sort exactly as kvbuilder.SetGPUDevices does, so the claim index used here // matches the index the KVVM GPU/claim references point at. - devices := kvbuilder.SortGPUDevices(vm.Spec.GPUDevices) + devices := kvbuilder.SortGPUDevices(vm.Spec.GPUs) desiredTemplateNames := make(map[string]struct{}, len(devices)) for index, device := range devices { @@ -122,7 +122,7 @@ func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMac // (its DeviceClasses not yet reconciled) yields a clear condition instead of a // silently pending claim on the next restart. func (h *GPUResourceClaimHandler) reconcileGPUClassReadyCondition(ctx context.Context, vm *v1alpha2.VirtualMachine) error { - if len(vm.Spec.GPUDevices) == 0 { + if len(vm.Spec.GPUs) == 0 { conditions.RemoveCondition(vmcondition.TypeGPUClassReady, &vm.Status.Conditions) return nil } @@ -130,7 +130,7 @@ func (h *GPUResourceClaimHandler) reconcileGPUClassReadyCondition(ctx context.Co cb := conditions.NewConditionBuilder(vmcondition.TypeGPUClassReady).Generation(vm.GetGeneration()) var missing, notReady []string - for _, device := range vm.Spec.GPUDevices { + for _, device := range vm.Spec.GPUs { gpuClass := &unstructured.Unstructured{} gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) err := h.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, gpuClass) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go index 65fc44bce5..86be55207a 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -43,7 +43,7 @@ var _ = Describe("GPUResourceClaimHandler", func() { newVM := func(devices ...v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { return &v1alpha2.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{Name: vmName, Namespace: namespace}, - Spec: v1alpha2.VirtualMachineSpec{GPUDevices: devices}, + Spec: v1alpha2.VirtualMachineSpec{GPUs: devices}, } } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 7d6ca837a6..c05bd3867f 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -426,7 +426,7 @@ var _ = Describe("SyncKvvmHandler", func() { vmClass := makeVMClass() vm := makeVM(v1alpha2.MachineRunning) - vm.Spec.GPUDevices = []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}} + vm.Spec.GPUs = []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}} kvvm := makeKVVM(vm) Expect(kvbuilder.SetLastAppliedSpec(kvvm, &v1alpha2.VirtualMachine{ Spec: v1alpha2.VirtualMachineSpec{ @@ -443,7 +443,7 @@ var _ = Describe("SyncKvvmHandler", func() { Disruptions: &v1alpha2.Disruptions{ RestartApprovalMode: vm.Spec.Disruptions.RestartApprovalMode, }, - GPUDevices: []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-a100"}}, + GPUs: []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-a100"}}, }, })).To(Succeed()) kvvm.SetGroupVersionKind(virtv1.VirtualMachineGroupVersionKind) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go index 48a8f68fd7..0c6a31f118 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -53,14 +53,14 @@ func (v *GPUDevicesValidator) ValidateUpdate(ctx context.Context, oldVM, newVM * // updates (or removal) of a VM created while the gate was enabled and later // disabled, or whose GPUClass was removed out of band. Order is ignored // to match the vmchange comparator, which treats reordering as no change. - if reflect.DeepEqual(kvbuilder.SortGPUDevices(oldVM.Spec.GPUDevices), kvbuilder.SortGPUDevices(newVM.Spec.GPUDevices)) { + if reflect.DeepEqual(kvbuilder.SortGPUDevices(oldVM.Spec.GPUs), kvbuilder.SortGPUDevices(newVM.Spec.GPUs)) { return nil, nil } return nil, v.validateGPUDevices(ctx, newVM) } func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alpha2.VirtualMachine) error { - if len(vm.Spec.GPUDevices) == 0 { + if len(vm.Spec.GPUs) == 0 { return nil } @@ -75,7 +75,7 @@ func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alph return nil } - for _, device := range vm.Spec.GPUDevices { + for _, device := range vm.Spec.GPUs { gpuClass := &unstructured.Unstructured{} gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) err := v.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, gpuClass) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go index 818f983106..c934d3f05b 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -194,10 +194,10 @@ func assertValidationError(t *testing.T, err error, wantErrorPart string) { } } -func newVirtualMachineWithGPU(gpuDevices []v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { +func newVirtualMachineWithGPU(gpus []v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { return &v1alpha2.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{Name: "vm-current", Namespace: "default"}, - Spec: v1alpha2.VirtualMachineSpec{GPUDevices: gpuDevices}, + Spec: v1alpha2.VirtualMachineSpec{GPUs: gpus}, } } diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go index 3ce0196e3e..ca3721fba1 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go @@ -687,12 +687,12 @@ networks: { "no restart when gpu devices only change order", ` -gpuDevices: +gpus: - gpuClassName: nvidia-h100 - gpuClassName: nvidia-a100 `, ` -gpuDevices: +gpus: - gpuClassName: nvidia-a100 - gpuClassName: nvidia-h100 `, @@ -702,23 +702,23 @@ gpuDevices: { "restart when gpu device gpuClassName changes", ` -gpuDevices: +gpus: - gpuClassName: nvidia-a100 `, ` -gpuDevices: +gpus: - gpuClassName: nvidia-h100 `, nil, assertChanges( actionRequired(ActionRestart), - requirePathOperation("gpuDevices", ChangeReplace), + requirePathOperation("gpus", ChangeReplace), ), }, { "no restart when gpu devices change between empty list and unset", ` -gpuDevices: [] +gpus: [] `, ``, nil, diff --git a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go index 8a636a3988..a548c90821 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go @@ -24,13 +24,13 @@ import ( ) func compareGPUDevices(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { - currentGPUDevices := kvbuilder.SortGPUDevices(current.GPUDevices) - desiredGPUDevices := kvbuilder.SortGPUDevices(desired.GPUDevices) + currentGPUDevices := kvbuilder.SortGPUDevices(current.GPUs) + desiredGPUDevices := kvbuilder.SortGPUDevices(desired.GPUs) currentValue := NewValue(currentGPUDevices, len(currentGPUDevices) == 0, false) desiredValue := NewValue(desiredGPUDevices, len(desiredGPUDevices) == 0, false) return compareValues( - "gpuDevices", + "gpus", currentValue, desiredValue, reflect.DeepEqual(currentGPUDevices, desiredGPUDevices),