diff --git a/api/core/v1alpha2/virtual_machine.go b/api/core/v1alpha2/virtual_machine.go index af8ea3d05e..5848a339e4 100644 --- a/api/core/v1alpha2/virtual_machine.go +++ b/api/core/v1alpha2/virtual_machine.go @@ -123,6 +123,12 @@ type VirtualMachineSpec struct { // Devices are referenced by name of USBDevice resource in the same namespace. // +kubebuilder:validation:MaxItems:=8 USBDevices []USBDeviceSpecRef `json:"usbDevices,omitempty"` + // List of GPU devices to attach to the virtual machine. + // Each entry references a GPUClass by name; list order is not significant. + // This feature requires the GPU feature gate. + // +kubebuilder:validation:MaxItems:=16 + // +listType=atomic + GPUs []GPUDeviceSpec `json:"gpus,omitempty"` } func (s *VirtualMachineSpec) IsParavirtualizationEnabled() bool { @@ -505,6 +511,17 @@ type USBDeviceSpecRef struct { Name string `json:"name"` } +// GPUDeviceSpec requests a GPU device by GPUClass. +type GPUDeviceSpec struct { + // Name of the GPUClass that selects the GPU to attach, for example nvidia-h100. + // The GPUClass must already exist: a DRA DeviceClass with the same name is + // created by the GPU module and is used to allocate the device. + // +kubebuilder:validation:MinLength:=1 + // +kubebuilder:validation:MaxLength:=253 + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + GPUClassName string `json:"gpuClassName"` +} + // USBDeviceStatusRef represents the status of a USB device attached to the virtual machine. type USBDeviceStatusRef struct { // The name of USBDevice resource. diff --git a/api/core/v1alpha2/vmcondition/condition.go b/api/core/v1alpha2/vmcondition/condition.go index 2320fd5da9..779a6a502b 100644 --- a/api/core/v1alpha2/vmcondition/condition.go +++ b/api/core/v1alpha2/vmcondition/condition.go @@ -50,6 +50,9 @@ const ( // TypeMaintenance indicates that the VirtualMachine is in maintenance mode. // During this condition, the VM remains stopped and no changes are allowed. TypeMaintenance Type = "Maintenance" + + // TypeGPUClassReady indicates whether every GPUClass referenced by spec.gpus exists. + TypeGPUClassReady Type = "GPUClassReady" ) type AgentReadyReason string @@ -283,3 +286,15 @@ func (r MaintenanceReason) String() string { const ( ReasonMaintenanceRestore MaintenanceReason = "RestoreInProgress" ) + +type GPUClassReadyReason string + +func (r GPUClassReadyReason) String() string { + return string(r) +} + +const ( + ReasonGPUClassReady GPUClassReadyReason = "GPUClassReady" + ReasonGPUClassNotFound GPUClassReadyReason = "GPUClassNotFound" + ReasonGPUClassNotReady GPUClassReadyReason = "GPUClassNotReady" +) diff --git a/api/core/v1alpha2/zz_generated.deepcopy.go b/api/core/v1alpha2/zz_generated.deepcopy.go index 8ee5d37f0a..fd15c1c388 100644 --- a/api/core/v1alpha2/zz_generated.deepcopy.go +++ b/api/core/v1alpha2/zz_generated.deepcopy.go @@ -473,6 +473,22 @@ func (in *Disruptions) DeepCopy() *Disruptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUDeviceSpec) DeepCopyInto(out *GPUDeviceSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceSpec. +func (in *GPUDeviceSpec) DeepCopy() *GPUDeviceSpec { + if in == nil { + return nil + } + out := new(GPUDeviceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImagePullSecret) DeepCopyInto(out *ImagePullSecret) { *out = *in @@ -3516,6 +3532,11 @@ func (in *VirtualMachineSpec) DeepCopyInto(out *VirtualMachineSpec) { *out = make([]USBDeviceSpecRef, len(*in)) copy(*out, *in) } + if in.GPUs != nil { + in, out := &in.GPUs, &out.GPUs + *out = make([]GPUDeviceSpec, len(*in)) + copy(*out, *in) + } return } diff --git a/build/components/versions.yml b/build/components/versions.yml index 675a7435fd..942e074f77 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,8 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.59 + 3p-kubevirt: feat/gpu/add-deckhouse-dra-support + 3p-containerized-data-importer: v1.60.3-v12n.21 distribution: 3.1.1 package: acl: v2.3.1 diff --git a/crds/doc-ru-virtualmachines.yaml b/crds/doc-ru-virtualmachines.yaml index 8a4697f05f..7217531e0b 100644 --- a/crds/doc-ru-virtualmachines.yaml +++ b/crds/doc-ru-virtualmachines.yaml @@ -588,6 +588,17 @@ spec: name: description: | Имя ресурса `USBDevice` в том же пространстве имен. + gpus: + description: | + Список GPU-устройств для подключения к виртуальной машине. + Каждое устройство ссылается на GPUClass по имени. + Для использования требуется feature gate `GPU`. + items: + properties: + gpuClassName: + description: | + Имя GPUClass, который выбирает подключаемый GPU, например `nvidia-h100`. + GPUClass должен уже существовать: DRA DeviceClass с тем же именем создаётся модулем GPU и используется для выделения устройства. status: properties: blockDeviceRefs: diff --git a/crds/embedded/virtualmachines.yaml b/crds/embedded/virtualmachines.yaml index 45622d1a0c..bdeb54135f 100644 --- a/crds/embedded/virtualmachines.yaml +++ b/crds/embedded/virtualmachines.yaml @@ -2394,7 +2394,6 @@ spec: type: object type: object required: - - deviceName - name type: object type: array diff --git a/crds/virtualmachines.yaml b/crds/virtualmachines.yaml index 380e4bac38..4b5377855d 100644 --- a/crds/virtualmachines.yaml +++ b/crds/virtualmachines.yaml @@ -1058,6 +1058,27 @@ spec: type: string description: | The name of USBDevice resource in the same namespace. + gpus: + type: array + maxItems: 16 + x-kubernetes-list-type: atomic + description: | + List of GPU devices to attach to the virtual machine. + Each entry references a GPUClass by name; list order is not significant. + This feature requires the GPU feature gate. + items: + type: object + required: + - gpuClassName + properties: + gpuClassName: + minLength: 1 + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + description: | + Name of the GPUClass that selects the GPU to attach, for example nvidia-h100. + The GPUClass must already exist: a DRA DeviceClass with the same name is created by the GPU module and is used to allocate the device. status: type: object properties: diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 48eadac4d5..3977236f5a 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -4015,6 +4015,51 @@ spec: As a result, a VM named `clone-database-prod` and a disk named `clone-database-root-prod` will be created. +## GPU Devices + +{{< alert level="warning" >}} +GPU device passthrough is an experimental feature. It requires the Enterprise Edition (EE), Kubernetes DRA support, and an external GPU DRA provider that publishes GPUs as DRA resources with `gpu.deckhouse.io` device attributes. +{{< /alert >}} + +The virtualization module can attach physical GPU devices to virtual machines using DRA (Dynamic Resource Allocation). A GPU is requested by referencing a `GPUClass` through the `.spec.gpus` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. + +GPU device passthrough requires: + +- Kubernetes version 1.34 or higher with DRA feature gates required by the cluster configuration. +- The `GPU` feature gate enabled in the `virtualization` module settings. +- A GPU DRA provider installed in the cluster that publishes GPUs with `gpu.deckhouse.io` device attributes. +- A `GPUClass` that selects the GPU, referenced from `.spec.gpus[].gpuClassName`. The GPU module creates a DRA DeviceClass with the same name, which is used to allocate the device. + +To enable the module feature gate: + +```yaml +apiVersion: deckhouse.io/v1alpha1 +kind: ModuleConfig +metadata: + name: virtualization +spec: + settings: + featureGates: + - GPU +``` + +To request a GPU device, add `.spec.gpus` to the VM specification: + +```yaml +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualMachine +metadata: + name: linux-vm +spec: + # ... other VM settings ... + gpus: + - gpuClassName: nvidia-h100 +``` + +The `gpuClassName` field must be the name of an existing `GPUClass` that selects the GPU to attach. To attach several GPUs, add more entries (list order is not significant). + +Changing `.spec.gpus` requires restarting the virtual machine to apply the new configuration. + ## USB Devices {{< alert level="warning" >}} diff --git a/docs/USER_GUIDE.ru.md b/docs/USER_GUIDE.ru.md index 6946c25786..eb12c5b314 100644 --- a/docs/USER_GUIDE.ru.md +++ b/docs/USER_GUIDE.ru.md @@ -4046,6 +4046,51 @@ spec: В результате будет создана ВМ с именем `clone-database-prod` и диск с именем `clone-database-root-prod`. +## GPU-устройства + +{{< alert level="warning" >}} +Проброс GPU-устройств — экспериментальная возможность. Для работы требуются Enterprise Edition (EE), поддержка Kubernetes DRA и внешний GPU DRA-провайдер, публикующий GPU как DRA-ресурсы с атрибутами устройства `gpu.deckhouse.io`. +{{< /alert >}} + +Модуль виртуализации может подключать физические GPU-устройства к виртуальным машинам с помощью DRA (Dynamic Resource Allocation). GPU запрашивается по ссылке на `GPUClass` через поле `.spec.gpus` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). + +Для проброса GPU требуются: + +- Kubernetes версии 1.34 или выше с DRA feature gates, необходимыми для конфигурации кластера. +- Feature gate `GPU`, включённый в настройках модуля `virtualization`. +- Установленный в кластере GPU DRA-провайдер, публикующий GPU с атрибутами устройства `gpu.deckhouse.io`. +- `GPUClass`, выбирающий GPU, на который ссылается `.spec.gpus[].gpuClassName`. Модуль GPU создаёт DRA DeviceClass с тем же именем, который используется для выделения устройства. + +Чтобы включить feature gate модуля: + +```yaml +apiVersion: deckhouse.io/v1alpha1 +kind: ModuleConfig +metadata: + name: virtualization +spec: + settings: + featureGates: + - GPU +``` + +Чтобы запросить GPU-устройство, добавьте `.spec.gpus` в спецификацию ВМ: + +```yaml +apiVersion: virtualization.deckhouse.io/v1alpha2 +kind: VirtualMachine +metadata: + name: linux-vm +spec: + # ... другие настройки ВМ ... + gpus: + - gpuClassName: nvidia-h100 +``` + +Поле `gpuClassName` должно быть именем существующего `GPUClass`, который выбирает подключаемый GPU. Чтобы подключить несколько GPU, добавьте больше элементов (порядок в списке не важен). + +Изменение `.spec.gpus` требует перезапуска виртуальной машины для применения новой конфигурации. + ## USB-устройства {{< alert level="warning">}} diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index 2ddeb2960d..754825132d 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -13,8 +13,10 @@ secrets: - id: SOURCE_REPO value: {{ $.SOURCE_REPO }} shell: + installCacheVersion: "{{ now | date "Mon Jan 2 15:04:05 MST 2006" }}" install: - | + echo "$date" echo "Git clone {{ $gitRepoName }} repository..." git clone --depth=1 $(cat /run/secrets/SOURCE_REPO)/{{ $gitRepoUrl }} --branch {{ $tag }} /src/kubevirt diff --git a/images/virtualization-artifact/pkg/builder/vm/option.go b/images/virtualization-artifact/pkg/builder/vm/option.go index 5e3c3dcac3..67c22181e3 100644 --- a/images/virtualization-artifact/pkg/builder/vm/option.go +++ b/images/virtualization-artifact/pkg/builder/vm/option.go @@ -168,6 +168,12 @@ func WithUSBDevices(usbDevices []v1alpha2.USBDeviceSpecRef) Option { } } +func WithGPUDevices(gpus []v1alpha2.GPUDeviceSpec) Option { + return func(vm *v1alpha2.VirtualMachine) { + vm.Spec.GPUs = gpus + } +} + func WithIpAddress(ipAddress string) Option { return func(vm *v1alpha2.VirtualMachine) { vm.Spec.VirtualMachineIPAddress = ipAddress diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index 75e06031e7..c7aa6e1cd3 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -58,6 +58,8 @@ const ( // AnnTolerationsHash provides a const for annotation with hash of applied tolerations. AnnTolerationsHash = AnnAPIGroup + "/tolerations-hash" + // AnnGPUClaimSpecHash provides a const for annotation with hash of the rendered GPU ResourceClaimTemplate spec. + AnnGPUClaimSpecHash = AnnAPIGroup + "/gpu-claim-spec-hash" // AnnTolerationsHashLegacy provides a const for legacy annotation with hash of applied tolerations. AnnTolerationsHashLegacy = AnnAPIGroupLegacy + "/tolerations-hash" diff --git a/images/virtualization-artifact/pkg/common/testutil/testutil.go b/images/virtualization-artifact/pkg/common/testutil/testutil.go index 99768fa4da..a89b53f7aa 100644 --- a/images/virtualization-artifact/pkg/common/testutil/testutil.go +++ b/images/virtualization-artifact/pkg/common/testutil/testutil.go @@ -34,6 +34,7 @@ import ( storagev1alpha1 "github.com/deckhouse/virtualization-controller/pkg/apis/storage/v1alpha1" commonnetwork "github.com/deckhouse/virtualization-controller/pkg/common/network" "github.com/deckhouse/virtualization-controller/pkg/controller/indexer" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha3" ) @@ -43,6 +44,11 @@ func registerNetworkGVKs(scheme *apiruntime.Scheme) { scheme.AddKnownTypeWithName(commonnetwork.ClusterNetworkGVK, &unstructured.Unstructured{}) } +func registerGPUGVKs(scheme *apiruntime.Scheme) { + scheme.AddKnownTypeWithName(kvbuilder.GPUClassGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(kvbuilder.GPUClassGVK.GroupVersion().WithKind("GPUClassList"), &unstructured.UnstructuredList{}) +} + func NewFakeClientWithObjects(objs ...client.Object) (client.WithWatch, error) { scheme := apiruntime.NewScheme() for _, f := range []func(*apiruntime.Scheme) error{ @@ -58,6 +64,7 @@ func NewFakeClientWithObjects(objs ...client.Object) (client.WithWatch, error) { } } registerNetworkGVKs(scheme) + registerGPUGVKs(scheme) var newObjs []client.Object for _, obj := range objs { if reflect.ValueOf(obj).IsNil() { @@ -88,6 +95,7 @@ func NewFakeClientWithInterceptorWithObjects(interceptor interceptor.Funcs, objs } } registerNetworkGVKs(scheme) + registerGPUGVKs(scheme) var newObjs []client.Object for _, obj := range objs { if reflect.ValueOf(obj).IsNil() { diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go new file mode 100644 index 0000000000..069cd10c9e --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu.go @@ -0,0 +1,103 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kvbuilder + +import ( + "slices" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +const ( + GPUNamePrefix = "gpu-" + GPUDRADriverName = "gpu.deckhouse.io" +) + +// GPUClassGVK identifies the GPUClass custom resource provided by the GPU module. +var GPUClassGVK = schema.GroupVersionKind{Group: "gpu.deckhouse.io", Version: "v1alpha1", Kind: "GPUClass"} + +func GPUResourceClaimName(index int) string { + return GPUNamePrefix + strconv.Itoa(index) +} + +func GPUResourceClaimTemplateName(vmName string, index int) string { + // The vmName hash suffix keeps the template name unique per VM: two VMs in the + // same namespace whose "-" prefixes collide (e.g. VM "a" and + // VM "a-0") would otherwise fight over one name and deadlock the losing VM's + // reconciliation on a not-controlled-by error. + return vmName + "-" + strconv.Itoa(index) + "-" + GenerateSerial(vmName)[:8] +} + +func IsGPUResourceClaimTemplateName(vmName, templateName string) bool { + return strings.HasPrefix(templateName, vmName+"-") +} + +func (b *KVVM) SetGPUDevices(vmName string, devices []v1alpha2.GPUDeviceSpec) { + devices = SortGPUDevices(devices) + + b.Resource.Spec.Template.Spec.ResourceClaims = slices.DeleteFunc( + b.Resource.Spec.Template.Spec.ResourceClaims, + func(claim virtv1.ResourceClaim) bool { + return strings.HasPrefix(claim.Name, GPUNamePrefix) && + claim.ResourceClaimTemplateName != nil && + IsGPUResourceClaimTemplateName(vmName, *claim.ResourceClaimTemplateName) + }, + ) + b.Resource.Spec.Template.Spec.Domain.Devices.GPUs = slices.DeleteFunc( + b.Resource.Spec.Template.Spec.Domain.Devices.GPUs, + func(gpu virtv1.GPU) bool { + return strings.HasPrefix(gpu.Name, GPUNamePrefix) && gpu.ClaimRequest != nil + }, + ) + + for index := range devices { + claimName := GPUResourceClaimName(index) + b.Resource.Spec.Template.Spec.ResourceClaims = append(b.Resource.Spec.Template.Spec.ResourceClaims, virtv1.ResourceClaim{ + PodResourceClaim: corev1.PodResourceClaim{ + Name: claimName, + ResourceClaimTemplateName: ptr.To(GPUResourceClaimTemplateName(vmName, index)), + }, + }) + b.Resource.Spec.Template.Spec.Domain.Devices.GPUs = append(b.Resource.Spec.Template.Spec.Domain.Devices.GPUs, virtv1.GPU{ + Name: claimName, + ClaimRequest: &virtv1.ClaimRequest{ + ClaimName: ptr.To(claimName), + RequestName: ptr.To(claimName), + }, + }) + } +} + +// SortGPUDevices orders devices by GPUClass so that reordering the spec list is a +// no-op: the generated claim indexes stay stable and no restart is triggered. +func SortGPUDevices(devices []v1alpha2.GPUDeviceSpec) []v1alpha2.GPUDeviceSpec { + if len(devices) == 0 { + return nil + } + sorted := slices.Clone(devices) + slices.SortStableFunc(sorted, func(a, b v1alpha2.GPUDeviceSpec) int { + return strings.Compare(a.GPUClassName, b.GPUClassName) + }) + return sorted +} diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go new file mode 100644 index 0000000000..a4de26f995 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/gpu_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kvbuilder + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +var _ = Describe("GPU", func() { + It("should render DRA GPU resource claims", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}}) + res := kvvm.GetResource() + + Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-0")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal(GPUResourceClaimTemplateName("vm-a", 0))) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-0")) + Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].ClaimName).To(Equal("gpu-0")) + Expect(*res.Spec.Template.Spec.Domain.Devices.GPUs[0].RequestName).To(Equal("gpu-0")) + }) + + It("should index claims by GPUClass order regardless of spec order", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{ + {GPUClassName: "nvidia-h100"}, + {GPUClassName: "nvidia-a100"}, + }) + res := kvvm.GetResource() + + // Sorted by GPUClass: nvidia-a100 -> index 0, nvidia-h100 -> index 1. + Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(2)) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-0")) + Expect(res.Spec.Template.Spec.ResourceClaims[1].Name).To(Equal("gpu-1")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(2)) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-0")) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[1].Name).To(Equal("gpu-1")) + }) + + It("should replace rendered DRA GPU resource claims", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}}) + + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-a100"}}) + res := kvvm.GetResource() + + Expect(res.Spec.Template.Spec.ResourceClaims).To(HaveLen(1)) + Expect(res.Spec.Template.Spec.ResourceClaims[0].Name).To(Equal("gpu-0")) + Expect(*res.Spec.Template.Spec.ResourceClaims[0].ResourceClaimTemplateName).To(Equal(GPUResourceClaimTemplateName("vm-a", 0))) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(HaveLen(1)) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs[0].Name).To(Equal("gpu-0")) + }) + + It("should not touch resource claims and GPUs it does not own", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + res := kvvm.GetResource() + res.Spec.Template.Spec.ResourceClaims = []virtv1.ResourceClaim{{ + PodResourceClaim: corev1.PodResourceClaim{ + Name: "gpu-foreign", + ResourceClaimTemplateName: ptr.To("foreign-template"), + }, + }} + res.Spec.Template.Spec.Domain.Devices.GPUs = []virtv1.GPU{{ + Name: "gpu-legacy", + DeviceName: "nvidia.com/GH100", + }} + + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}}) + res = kvvm.GetResource() + + claimNames := make([]string, 0) + for _, c := range res.Spec.Template.Spec.ResourceClaims { + claimNames = append(claimNames, c.Name) + } + gpuNames := make([]string, 0) + for _, g := range res.Spec.Template.Spec.Domain.Devices.GPUs { + gpuNames = append(gpuNames, g.Name) + } + Expect(claimNames).To(ConsistOf("gpu-foreign", "gpu-0")) + Expect(gpuNames).To(ConsistOf("gpu-legacy", "gpu-0")) + }) + + It("should remove rendered DRA GPU resource claims", func() { + kvvm := NewEmptyKVVM(types.NamespacedName{Name: "vm-a", Namespace: "default"}, KVVMOptions{}) + kvvm.SetGPUDevices("vm-a", []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}}) + + kvvm.SetGPUDevices("vm-a", nil) + res := kvvm.GetResource() + + Expect(res.Spec.Template.Spec.ResourceClaims).To(BeEmpty()) + Expect(res.Spec.Template.Spec.Domain.Devices.GPUs).To(BeEmpty()) + }) +}) diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go index c35a11bce9..401e01c957 100644 --- a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go +++ b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go @@ -247,6 +247,8 @@ func ApplyVirtualMachineSpec( return err } + kvvm.SetGPUDevices(vm.Name, vm.Spec.GPUs) + if err := kvvm.SetProvisioning(vm.Spec.Provisioning); err != nil { return err } diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go new file mode 100644 index 0000000000..59ff8d516c --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler.go @@ -0,0 +1,264 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + + resourcev1 "k8s.io/api/resource/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization-controller/pkg/controller/service" + "github.com/deckhouse/virtualization-controller/pkg/controller/vm/internal/state" + "github.com/deckhouse/virtualization-controller/pkg/logger" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" +) + +const nameGPUResourceClaimHandler = "GPUResourceClaimHandler" + +func NewGPUResourceClaimHandler(client client.Client) *GPUResourceClaimHandler { + return &GPUResourceClaimHandler{client: client} +} + +type GPUResourceClaimHandler struct { + client client.Client +} + +func (h *GPUResourceClaimHandler) Name() string { + return nameGPUResourceClaimHandler +} + +func (h *GPUResourceClaimHandler) Handle(ctx context.Context, s state.VirtualMachineState) (reconcile.Result, error) { + if s.VirtualMachine().IsEmpty() { + return reconcile.Result{}, nil + } + + vm := s.VirtualMachine().Changed() + log := logger.FromContext(ctx).With(logger.SlogHandler(nameGPUResourceClaimHandler)) + // Sort exactly as kvbuilder.SetGPUDevices does, so the claim index used here + // matches the index the KVVM GPU/claim references point at. + devices := kvbuilder.SortGPUDevices(vm.Spec.GPUs) + desiredTemplateNames := make(map[string]struct{}, len(devices)) + + for index, device := range devices { + templateName := kvbuilder.GPUResourceClaimTemplateName(vm.Name, index) + desiredTemplateNames[templateName] = struct{}{} + desiredSpec := buildGPUResourceClaimTemplateSpec(index, device) + template := &resourcev1.ResourceClaimTemplate{} + key := types.NamespacedName{Name: templateName, Namespace: vm.Namespace} + + err := h.client.Get(ctx, key, template) + if err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to get GPU ResourceClaimTemplate: %w", err) + } + + if apierrors.IsNotFound(err) { + template = buildGPUResourceClaimTemplate(vm, templateName, desiredSpec) + if err := h.client.Create(ctx, template); err != nil && !apierrors.IsAlreadyExists(err) { + return reconcile.Result{}, fmt.Errorf("failed to create GPU ResourceClaimTemplate: %w", err) + } + log.Info("created GPU ResourceClaimTemplate", "template", templateName) + continue + } + + if !metav1.IsControlledBy(template, vm) { + return reconcile.Result{}, fmt.Errorf("GPU ResourceClaimTemplate %s/%s is not controlled by VirtualMachine %s/%s", template.Namespace, template.Name, vm.Namespace, vm.Name) + } + + if gpuClaimTemplateUpToDate(template, desiredSpec) { + continue + } + if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to delete outdated GPU ResourceClaimTemplate: %w", err) + } + template = buildGPUResourceClaimTemplate(vm, templateName, desiredSpec) + if err := h.client.Create(ctx, template); err != nil && !apierrors.IsAlreadyExists(err) { + return reconcile.Result{}, fmt.Errorf("failed to recreate GPU ResourceClaimTemplate: %w", err) + } + log.Info("recreated GPU ResourceClaimTemplate", "template", templateName) + } + + if err := h.deleteOrphanedTemplates(ctx, vm, desiredTemplateNames); err != nil { + return reconcile.Result{}, err + } + + if err := h.reconcileGPUClassReadyCondition(ctx, vm); err != nil { + return reconcile.Result{}, err + } + return reconcile.Result{}, nil +} + +// reconcileGPUClassReadyCondition surfaces whether every referenced GPUClass +// exists and is Ready, so a GPUClass that is missing or still initializing +// (its DeviceClasses not yet reconciled) yields a clear condition instead of a +// silently pending claim on the next restart. +func (h *GPUResourceClaimHandler) reconcileGPUClassReadyCondition(ctx context.Context, vm *v1alpha2.VirtualMachine) error { + if len(vm.Spec.GPUs) == 0 { + conditions.RemoveCondition(vmcondition.TypeGPUClassReady, &vm.Status.Conditions) + return nil + } + + cb := conditions.NewConditionBuilder(vmcondition.TypeGPUClassReady).Generation(vm.GetGeneration()) + + var missing, notReady []string + for _, device := range vm.Spec.GPUs { + gpuClass := &unstructured.Unstructured{} + gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) + err := h.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, gpuClass) + switch { + case apierrors.IsNotFound(err), meta.IsNoMatchError(err): + missing = append(missing, device.GPUClassName) + case err != nil: + return fmt.Errorf("failed to resolve GPUClass %q: %w", device.GPUClassName, err) + default: + if !gpuClassReady(gpuClass) { + notReady = append(notReady, device.GPUClassName) + } + } + } + + switch { + case len(missing) > 0: + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonGPUClassNotFound). + Message(fmt.Sprintf("GPUClass not found: %s. The GPU cannot be allocated until the GPUClass exists.", strings.Join(missing, ", "))) + case len(notReady) > 0: + cb.Status(metav1.ConditionFalse). + Reason(vmcondition.ReasonGPUClassNotReady). + Message(fmt.Sprintf("GPUClass not ready: %s. The GPU cannot be allocated until the GPUClass becomes Ready.", strings.Join(notReady, ", "))) + default: + cb.Status(metav1.ConditionTrue). + Reason(vmcondition.ReasonGPUClassReady). + Message("") + } + conditions.SetCondition(cb, &vm.Status.Conditions) + return nil +} + +// gpuClassReady reports whether the GPUClass has a Ready condition set to True. +func gpuClassReady(gpuClass *unstructured.Unstructured) bool { + conds, found, err := unstructured.NestedSlice(gpuClass.Object, "status", "conditions") + if err != nil || !found { + return false + } + for _, c := range conds { + cond, ok := c.(map[string]interface{}) + if !ok { + continue + } + if cond["type"] == "Ready" { + return cond["status"] == string(metav1.ConditionTrue) + } + } + return false +} + +func buildGPUResourceClaimTemplate(vm *v1alpha2.VirtualMachine, name string, spec resourcev1.ResourceClaimTemplateSpec) *resourcev1.ResourceClaimTemplate { + return &resourcev1.ResourceClaimTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: vm.Namespace, + Annotations: map[string]string{annotations.AnnGPUClaimSpecHash: gpuClaimSpecHash(spec)}, + OwnerReferences: []metav1.OwnerReference{service.MakeControllerOwnerReference(vm)}, + }, + Spec: spec, + } +} + +// gpuClaimTemplateUpToDate prefers the spec-hash annotation over comparing the +// stored spec directly: API-server defaulting could make the stored spec +// permanently differ from the rendered one and loop delete/recreate. +// Templates created before the annotation existed fall back to DeepEqual and +// migrate to the hash on their next legitimate recreation. +func gpuClaimTemplateUpToDate(template *resourcev1.ResourceClaimTemplate, desiredSpec resourcev1.ResourceClaimTemplateSpec) bool { + storedHash, ok := template.Annotations[annotations.AnnGPUClaimSpecHash] + if !ok { + return reflect.DeepEqual(template.Spec, desiredSpec) + } + return storedHash == gpuClaimSpecHash(desiredSpec) +} + +func gpuClaimSpecHash(spec resourcev1.ResourceClaimTemplateSpec) string { + // Marshalling a plain API struct cannot fail; on the impossible failure both + // sides hash the same empty payload, so the comparison still converges. + raw, _ := json.Marshal(&spec) + return kvbuilder.GenerateSerial(string(raw)) +} + +func buildGPUResourceClaimTemplateSpec(index int, device v1alpha2.GPUDeviceSpec) resourcev1.ResourceClaimTemplateSpec { + requestName := kvbuilder.GPUResourceClaimName(index) + return resourcev1.ResourceClaimTemplateSpec{ + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{{ + Name: requestName, + Exactly: &resourcev1.ExactDeviceRequest{ + // The GPU module creates a DeviceClass named exactly after the GPUClass. + DeviceClassName: device.GPUClassName, + AllocationMode: resourcev1.DeviceAllocationModeExactCount, + Count: 1, + }, + }}, + Config: []resourcev1.DeviceClaimConfiguration{{ + Requests: []string{requestName}, + DeviceConfiguration: resourcev1.DeviceConfiguration{ + Opaque: &resourcev1.OpaqueDeviceConfiguration{ + Driver: kvbuilder.GPUDRADriverName, + Parameters: runtime.RawExtension{Raw: []byte(`{"apiVersion":"resource.gpu.deckhouse.io/v1alpha1","kind":"VfioDeviceConfig"}`)}, + }, + }, + }}, + }, + }, + } +} + +func (h *GPUResourceClaimHandler) deleteOrphanedTemplates(ctx context.Context, vm *v1alpha2.VirtualMachine, desiredTemplateNames map[string]struct{}) error { + var templates resourcev1.ResourceClaimTemplateList + if err := h.client.List(ctx, &templates, client.InNamespace(vm.Namespace)); err != nil { + return fmt.Errorf("failed to list GPU ResourceClaimTemplates: %w", err) + } + + for i := range templates.Items { + template := &templates.Items[i] + if !metav1.IsControlledBy(template, vm) || !kvbuilder.IsGPUResourceClaimTemplateName(vm.Name, template.Name) { + continue + } + if _, ok := desiredTemplateNames[template.Name]; ok { + continue + } + if err := h.client.Delete(ctx, template); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete GPU ResourceClaimTemplate: %w", err) + } + } + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go new file mode 100644 index 0000000000..86be55207a --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/gpu_resourceclaim_handler_test.go @@ -0,0 +1,205 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + context "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + resourcev1 "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" +) + +var _ = Describe("GPUResourceClaimHandler", func() { + const ( + vmName = "vm-a" + namespace = "default" + deviceClass = "nvidia-h100" + ) + + newVM := func(devices ...v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { + return &v1alpha2.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: vmName, Namespace: namespace}, + Spec: v1alpha2.VirtualMachineSpec{GPUs: devices}, + } + } + + It("should create GPU ResourceClaimTemplate", func() { + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + template := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, 0), Namespace: namespace}, template)).To(Succeed()) + Expect(template.Spec.Spec.Devices.Requests).To(HaveLen(1)) + request := template.Spec.Spec.Devices.Requests[0] + Expect(request.Name).To(Equal(kvbuilder.GPUResourceClaimName(0))) + Expect(request.Exactly.DeviceClassName).To(Equal(deviceClass)) + Expect(request.Exactly.Selectors).To(BeEmpty()) + Expect(template.Spec.Spec.Devices.Config).To(HaveLen(1)) + config := template.Spec.Spec.Devices.Config[0] + Expect(config.Requests).To(ConsistOf(kvbuilder.GPUResourceClaimName(0))) + Expect(config.Opaque.Driver).To(Equal(kvbuilder.GPUDRADriverName)) + Expect(string(config.Opaque.Parameters.Raw)).To(ContainSubstring(`"kind":"VfioDeviceConfig"`)) + }) + + It("should delete owned GPU ResourceClaimTemplate when device is removed from spec", func() { + vm := newVM() + template := buildGPUResourceClaimTemplate(vm, kvbuilder.GPUResourceClaimTemplateName(vmName, 0), buildGPUResourceClaimTemplateSpec(0, v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + err = fakeClient.Get(context.Background(), types.NamespacedName{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, 0), Namespace: namespace}, stored) + Expect(err).To(HaveOccurred()) + }) + + It("should recreate GPU ResourceClaimTemplate when the desired spec changes", func() { + vm := newVM(v1alpha2.GPUDeviceSpec{GPUClassName: "nvidia-a100"}) + templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, 0) + template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(0, v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: templateName, Namespace: namespace}, stored)).To(Succeed()) + Expect(stored.Spec.Spec.Devices.Requests[0].Exactly.DeviceClassName).To(Equal("nvidia-a100")) + Expect(stored.Annotations).To(HaveKeyWithValue(annotations.AnnGPUClaimSpecHash, gpuClaimSpecHash(stored.Spec))) + }) + + It("should not recreate GPU ResourceClaimTemplate without hash annotation when spec matches", func() { + vm := newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass}) + templateName := kvbuilder.GPUResourceClaimTemplateName(vmName, 0) + template := buildGPUResourceClaimTemplate(vm, templateName, buildGPUResourceClaimTemplateSpec(0, v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) + template.Annotations = nil + template.Labels = map[string]string{"keep": "me"} + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: templateName, Namespace: namespace}, stored)).To(Succeed()) + Expect(stored.Labels).To(HaveKeyWithValue("keep", "me")) + Expect(stored.Annotations).NotTo(HaveKey(annotations.AnnGPUClaimSpecHash)) + }) + + It("should not replace GPU ResourceClaimTemplate owned by another controller", func() { + vm := newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass}) + template := &resourcev1.ResourceClaimTemplate{ + ObjectMeta: metav1.ObjectMeta{Name: kvbuilder.GPUResourceClaimTemplateName(vmName, 0), Namespace: namespace}, + } + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).To(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: template.Name, Namespace: namespace}, stored)).To(Succeed()) + Expect(stored.OwnerReferences).To(BeEmpty()) + }) + + It("should not delete a ResourceClaimTemplate owned by another VM with a shared name prefix", func() { + vm := newVM() + vm.UID = "uid-vm-a" + otherVM := &v1alpha2.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: vmName + "-b", Namespace: namespace, UID: "uid-vm-a-b"}, + } + otherName := kvbuilder.GPUResourceClaimTemplateName(otherVM.Name, 0) + Expect(kvbuilder.IsGPUResourceClaimTemplateName(vmName, otherName)).To(BeTrue()) + template := buildGPUResourceClaimTemplate(otherVM, otherName, buildGPUResourceClaimTemplateSpec(0, v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) + fakeClient, _, vmState := setupEnvironment(vm, template) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + stored := &resourcev1.ResourceClaimTemplate{} + Expect(fakeClient.Get(context.Background(), types.NamespacedName{Name: otherName, Namespace: namespace}, stored)).To(Succeed()) + }) + + It("should set GPUClassReady=False when the GPUClass does not exist", func() { + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass})) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + cond, ok := conditions.GetCondition(vmcondition.TypeGPUClassReady, vmState.VirtualMachine().Changed().Status.Conditions) + Expect(ok).To(BeTrue()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(vmcondition.ReasonGPUClassNotFound.String())) + }) + + It("should set GPUClassReady=True when the GPUClass exists and is Ready", func() { + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass}), newGPUClass(deviceClass, true)) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + cond, ok := conditions.GetCondition(vmcondition.TypeGPUClassReady, vmState.VirtualMachine().Changed().Status.Conditions) + Expect(ok).To(BeTrue()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + }) + + It("should set GPUClassReady=False when the GPUClass exists but is not Ready", func() { + fakeClient, _, vmState := setupEnvironment(newVM(v1alpha2.GPUDeviceSpec{GPUClassName: deviceClass}), newGPUClass(deviceClass, false)) + handler := NewGPUResourceClaimHandler(fakeClient) + + _, err := handler.Handle(context.Background(), vmState) + + Expect(err).NotTo(HaveOccurred()) + cond, ok := conditions.GetCondition(vmcondition.TypeGPUClassReady, vmState.VirtualMachine().Changed().Status.Conditions) + Expect(ok).To(BeTrue()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(vmcondition.ReasonGPUClassNotReady.String())) + }) +}) + +func newGPUClass(name string, ready bool) *unstructured.Unstructured { + status := "False" + if ready { + status = "True" + } + gpuClass := &unstructured.Unstructured{} + gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) + gpuClass.SetName(name) + _ = unstructured.SetNestedSlice(gpuClass.Object, []interface{}{ + map[string]interface{}{"type": "Ready", "status": status}, + }, "status", "conditions") + return gpuClass +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go index 62d4186486..c05bd3867f 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm_test.go @@ -421,6 +421,50 @@ var _ = Describe("SyncKvvmHandler", func() { Entry("Pending phase without changes, shouldn't have condition", v1alpha2.MachinePending, false, metav1.ConditionUnknown, false), ) + It("should require restart when GPU devices change on a running VM", func() { + ip := makeVMIP() + vmClass := makeVMClass() + + vm := makeVM(v1alpha2.MachineRunning) + vm.Spec.GPUs = []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-h100"}} + kvvm := makeKVVM(vm) + Expect(kvbuilder.SetLastAppliedSpec(kvvm, &v1alpha2.VirtualMachine{ + Spec: v1alpha2.VirtualMachineSpec{ + CPU: v1alpha2.CPUSpec{ + Cores: vm.Spec.CPU.Cores, + }, + Memory: v1alpha2.MemorySpec{ + Size: vm.Spec.Memory.Size, + }, + VirtualMachineIPAddress: vm.Spec.VirtualMachineIPAddress, + RunPolicy: vm.Spec.RunPolicy, + OsType: vm.Spec.OsType, + VirtualMachineClassName: vm.Spec.VirtualMachineClassName, + Disruptions: &v1alpha2.Disruptions{ + RestartApprovalMode: vm.Spec.Disruptions.RestartApprovalMode, + }, + GPUs: []v1alpha2.GPUDeviceSpec{{GPUClassName: "nvidia-a100"}}, + }, + })).To(Succeed()) + kvvm.SetGroupVersionKind(virtv1.VirtualMachineGroupVersionKind) + kvvmi := makeKVVMI() + + fakeClient, reconcileObj, vmState = setupEnvironment(vm, kvvm, kvvmi, ip, vmClass) + + reconcile() + + newVM := &v1alpha2.VirtualMachine{} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(vm), newVM)).To(Succeed()) + awaitCond, awaitExists := conditions.GetCondition(vmcondition.TypeAwaitingRestartToApplyConfiguration, newVM.Status.Conditions) + Expect(awaitExists).To(BeTrue()) + Expect(awaitCond.Status).To(Equal(metav1.ConditionTrue)) + Expect(newVM.Status.RestartAwaitingChanges).NotTo(BeEmpty()) + + updatedKVVM := &virtv1.VirtualMachine{} + Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(kvvm), updatedKVVM)).To(Succeed()) + Expect(updatedKVVM.Spec.Template.Spec.Domain.Devices.GPUs).To(BeEmpty()) + }) + DescribeTable("AwaitingRestart Condition for NonMigratable VM", func(phase v1alpha2.MachinePhase, featureGate featuregate.FeatureGate, mutateFn func(fakeClient client.WithWatch, vm *v1alpha2.VirtualMachine, kvvm *virtv1.VirtualMachine), expectedStatus metav1.ConditionStatus, expectedExistence bool) { ip := makeVMIP() diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go new file mode 100644 index 0000000000..0c6a31f118 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator.go @@ -0,0 +1,93 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validators + +import ( + "context" + "fmt" + "reflect" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/component-base/featuregate" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +type GPUDevicesValidator struct { + client client.Client + featureGate featuregate.FeatureGate +} + +func NewGPUDevicesValidator(client client.Client, featureGate featuregate.FeatureGate) *GPUDevicesValidator { + return &GPUDevicesValidator{client: client, featureGate: featureGate} +} + +func (v *GPUDevicesValidator) ValidateCreate(ctx context.Context, vm *v1alpha2.VirtualMachine) (admission.Warnings, error) { + return nil, v.validateGPUDevices(ctx, vm) +} + +func (v *GPUDevicesValidator) ValidateUpdate(ctx context.Context, oldVM, newVM *v1alpha2.VirtualMachine) (admission.Warnings, error) { + // The feature gate and GPUClass existence are checked only when GPU devices + // are introduced or changed. Unchanged GPU devices must not block unrelated + // updates (or removal) of a VM created while the gate was enabled and later + // disabled, or whose GPUClass was removed out of band. Order is ignored + // to match the vmchange comparator, which treats reordering as no change. + if reflect.DeepEqual(kvbuilder.SortGPUDevices(oldVM.Spec.GPUs), kvbuilder.SortGPUDevices(newVM.Spec.GPUs)) { + return nil, nil + } + return nil, v.validateGPUDevices(ctx, newVM) +} + +func (v *GPUDevicesValidator) validateGPUDevices(ctx context.Context, vm *v1alpha2.VirtualMachine) error { + if len(vm.Spec.GPUs) == 0 { + return nil + } + + if !v.featureGate.Enabled(featuregates.GPU) { + return fmt.Errorf("GPU device attachment requires the GPU feature gate") + } + + // A nil client means template validation (e.g. a VirtualMachinePool template): + // GPUClass existence is verified when the actual replica VM is created, so that + // a pool may be defined before the GPU provider and its classes exist. + if v.client == nil { + return nil + } + + for _, device := range vm.Spec.GPUs { + gpuClass := &unstructured.Unstructured{} + gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) + err := v.client.Get(ctx, types.NamespacedName{Name: device.GPUClassName}, gpuClass) + switch { + case apierrors.IsNotFound(err): + return fmt.Errorf("GPU device references GPUClass %q that does not exist", device.GPUClassName) + case meta.IsNoMatchError(err): + return fmt.Errorf("GPU device references GPUClass %q, but the GPUClass CRD is not registered (is the GPU module installed?)", device.GPUClassName) + case err != nil: + return fmt.Errorf("failed to resolve GPUClass %q: %w", device.GPUClassName, err) + } + } + + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go new file mode 100644 index 0000000000..c934d3f05b --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/validators/gpu_devices_validator_test.go @@ -0,0 +1,235 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validators + +import ( + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/component-base/featuregate" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func TestGPUDevicesValidatorValidateCreate(t *testing.T) { + tests := []struct { + name string + featureEnabled bool + gpuClasses []string + gpuClass string + wantErrorPart string + }{ + { + name: "should reject GPU devices when feature is disabled", + featureEnabled: false, + gpuClass: "nvidia-h100", + wantErrorPart: "GPU feature gate", + }, + { + name: "should accept GPU devices when feature is enabled and GPUClass is ready", + featureEnabled: true, + gpuClasses: []string{"nvidia-h100"}, + gpuClass: "nvidia-h100", + }, + { + name: "should reject GPU devices when GPUClass is not ready", + featureEnabled: true, + gpuClass: "nvidia-h100", + wantErrorPart: "does not exist", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vm := newVirtualMachineWithGPU([]v1alpha2.GPUDeviceSpec{{GPUClassName: tt.gpuClass}}) + validator := NewGPUDevicesValidator(newValidatorClient(t, tt.gpuClasses...), newGPUFeatureGate(t, tt.featureEnabled)) + + _, err := validator.ValidateCreate(t.Context(), vm) + + assertValidationError(t, err, tt.wantErrorPart) + }) + } +} + +func TestGPUDevicesValidatorValidateUpdate(t *testing.T) { + gpu := func(class string) []v1alpha2.GPUDeviceSpec { + return []v1alpha2.GPUDeviceSpec{{GPUClassName: class}} + } + + tests := []struct { + name string + featureEnabled bool + gpuClasses []string + oldGPU []v1alpha2.GPUDeviceSpec + newGPU []v1alpha2.GPUDeviceSpec + wantErrorPart string + }{ + { + name: "unchanged GPU devices are allowed when feature is disabled", + featureEnabled: false, + oldGPU: gpu("nvidia-h100"), + newGPU: gpu("nvidia-h100"), + }, + { + name: "removing GPU devices is allowed when feature is disabled", + featureEnabled: false, + oldGPU: gpu("nvidia-h100"), + newGPU: nil, + }, + { + name: "reordering GPU devices is allowed when feature is disabled", + featureEnabled: false, + oldGPU: []v1alpha2.GPUDeviceSpec{ + {GPUClassName: "nvidia-h100"}, + {GPUClassName: "nvidia-a100"}, + }, + newGPU: []v1alpha2.GPUDeviceSpec{ + {GPUClassName: "nvidia-a100"}, + {GPUClassName: "nvidia-h100"}, + }, + }, + { + name: "adding GPU devices is rejected when feature is disabled", + featureEnabled: false, + oldGPU: nil, + newGPU: gpu("nvidia-h100"), + wantErrorPart: "GPU feature gate", + }, + { + name: "changing to a ready GPUClass is allowed when feature is enabled", + featureEnabled: true, + gpuClasses: []string{"nvidia-h100", "nvidia-a100"}, + oldGPU: gpu("nvidia-h100"), + newGPU: gpu("nvidia-a100"), + }, + { + name: "changing to an unready GPUClass is rejected", + featureEnabled: true, + gpuClasses: []string{"nvidia-h100"}, + oldGPU: gpu("nvidia-h100"), + newGPU: gpu("nvidia-a100"), + wantErrorPart: "does not exist", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + oldVM := newVirtualMachineWithGPU(tt.oldGPU) + newVM := newVirtualMachineWithGPU(tt.newGPU) + validator := NewGPUDevicesValidator(newValidatorClient(t, tt.gpuClasses...), newGPUFeatureGate(t, tt.featureEnabled)) + + _, err := validator.ValidateUpdate(t.Context(), oldVM, newVM) + + assertValidationError(t, err, tt.wantErrorPart) + }) + } +} + +func TestGPUDevicesValidatorTemplateMode(t *testing.T) { + // A nil client (template validation) enforces the feature gate but skips + // GPUClass existence. + tests := []struct { + name string + featureEnabled bool + wantErrorPart string + }{ + { + name: "gate disabled is rejected in template mode", + featureEnabled: false, + wantErrorPart: "GPU feature gate", + }, + { + name: "unready GPUClass is allowed in template mode", + featureEnabled: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vm := newVirtualMachineWithGPU([]v1alpha2.GPUDeviceSpec{{GPUClassName: "does-not-exist"}}) + validator := NewGPUDevicesValidator(nil, newGPUFeatureGate(t, tt.featureEnabled)) + + _, err := validator.ValidateCreate(t.Context(), vm) + + assertValidationError(t, err, tt.wantErrorPart) + }) + } +} + +func assertValidationError(t *testing.T, err error, wantErrorPart string) { + t.Helper() + + if wantErrorPart == "" { + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + return + } + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), wantErrorPart) { + t.Fatalf("expected error containing %q, got %v", wantErrorPart, err) + } +} + +func newVirtualMachineWithGPU(gpus []v1alpha2.GPUDeviceSpec) *v1alpha2.VirtualMachine { + return &v1alpha2.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: "vm-current", Namespace: "default"}, + Spec: v1alpha2.VirtualMachineSpec{GPUs: gpus}, + } +} + +func newValidatorClient(t *testing.T, gpuClasses ...string) client.Client { + t.Helper() + + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(kvbuilder.GPUClassGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(kvbuilder.GPUClassGVK.GroupVersion().WithKind("GPUClassList"), &unstructured.UnstructuredList{}) + + objs := make([]client.Object, 0, len(gpuClasses)) + for _, name := range gpuClasses { + gpuClass := &unstructured.Unstructured{} + gpuClass.SetGroupVersionKind(kvbuilder.GPUClassGVK) + gpuClass.SetName(name) + objs = append(objs, gpuClass) + } + + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() +} + +func newGPUFeatureGate(t *testing.T, enabled bool) featuregate.FeatureGate { + t.Helper() + + gate, setFromMap, err := featuregates.NewUnlocked() + if err != nil { + t.Fatalf("failed to create feature gate: %v", err) + } + + if err = setFromMap(map[string]bool{string(featuregates.GPU): enabled}); err != nil { + t.Fatalf("failed to set GPU feature gate: %v", err) + } + + return gate +} diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/watcher/resourceclaimtemplate_watcher.go b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/resourceclaimtemplate_watcher.go new file mode 100644 index 0000000000..14dabd46d1 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vm/internal/watcher/resourceclaimtemplate_watcher.go @@ -0,0 +1,53 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "fmt" + + resourcev1 "k8s.io/api/resource/v1" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func NewResourceClaimTemplateWatcher() *ResourceClaimTemplateWatcher { + return &ResourceClaimTemplateWatcher{} +} + +type ResourceClaimTemplateWatcher struct{} + +func (w *ResourceClaimTemplateWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + if err := ctr.Watch( + source.Kind( + mgr.GetCache(), + &resourcev1.ResourceClaimTemplate{}, + handler.TypedEnqueueRequestForOwner[*resourcev1.ResourceClaimTemplate]( + mgr.GetScheme(), + mgr.GetRESTMapper(), + &v1alpha2.VirtualMachine{}, + handler.OnlyControllerOwner(), + ), + ), + ); err != nil { + return fmt.Errorf("error setting watch on ResourceClaimTemplate: %w", err) + } + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go index 4f131f9e01..6b5c52b218 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_controller.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_controller.go @@ -71,6 +71,7 @@ func SetupController( internal.NewBlockDeviceHandler(client, blockDeviceService), internal.NewUSBDeviceDetachHandler(client, virtClient), internal.NewUSBDeviceAttachHandler(client, virtClient), + internal.NewGPUResourceClaimHandler(client), internal.NewProvisioningHandler(client), internal.NewAgentHandler(), internal.NewFilesystemHandler(), diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_reconciler.go b/images/virtualization-artifact/pkg/controller/vm/vm_reconciler.go index 300b65acab..2d49683161 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_reconciler.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_reconciler.go @@ -71,6 +71,7 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr watcher.NewClusterVirtualImageWatcher(mgr.GetClient()), watcher.NewVirtualDiskWatcher(mgr.GetClient()), watcher.NewUSBDeviceWatcher(mgr.GetClient()), + watcher.NewResourceClaimTemplateWatcher(), watcher.NewVMIPWatcher(), watcher.NewVirtualMachineClassWatcher(), watcher.NewVirtualMachineSnapshotWatcher(), diff --git a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go index c7e68d3a8b..bb493bdbbb 100644 --- a/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go +++ b/images/virtualization-artifact/pkg/controller/vm/vm_webhook.go @@ -56,6 +56,7 @@ func NewValidator(client client.Client, blockDeviceService *service.BlockDeviceS validators.NewNetworksValidator(client, featureGate), validators.NewFirstDiskValidator(client), validators.NewUSBDevicesValidator(client, featureGate), + validators.NewGPUDevicesValidator(client, featureGate), validators.NewVMBDAConflictValidator(client), validators.NewPVNodeAffinityValidator(client, attachmentService), }, @@ -78,6 +79,7 @@ func NewTemplateSpecValidator(client client.Client, featureGate featuregate.Feat validators.NewSizingPolicyValidator(client), validators.NewNetworksValidator(client, featureGate), validators.NewFirstDiskValidator(client), + validators.NewGPUDevicesValidator(nil, featureGate), }, log: log.With("webhook", "vmpool-template-validation"), } diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare.go b/images/virtualization-artifact/pkg/controller/vmchange/compare.go index d734167761..b350187440 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare.go @@ -81,6 +81,7 @@ func (v *VMSpecComparator) comparators() []VMSpecFieldComparator { vmSpecFieldComparator(compareProvisioning), vmSpecFieldComparator(compareNetworks), vmSpecFieldComparator(compareUSBDevices), + vmSpecFieldComparator(compareGPUDevices), } } diff --git a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go index e95e193d82..ca3721fba1 100644 --- a/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go +++ b/images/virtualization-artifact/pkg/controller/vmchange/compare_test.go @@ -684,6 +684,46 @@ networks: requirePathOperation("networks", ChangeReplace), ), }, + { + "no restart when gpu devices only change order", + ` +gpus: +- gpuClassName: nvidia-h100 +- gpuClassName: nvidia-a100 +`, + ` +gpus: +- gpuClassName: nvidia-a100 +- gpuClassName: nvidia-h100 +`, + nil, + assertNoChanges(), + }, + { + "restart when gpu device gpuClassName changes", + ` +gpus: +- gpuClassName: nvidia-a100 +`, + ` +gpus: +- gpuClassName: nvidia-h100 +`, + nil, + assertChanges( + actionRequired(ActionRestart), + requirePathOperation("gpus", ChangeReplace), + ), + }, + { + "no restart when gpu devices change between empty list and unset", + ` +gpus: [] +`, + ``, + nil, + assertNoChanges(), + }, } for _, tt := range tests { @@ -769,6 +809,13 @@ func assertChanges(asserts ...func(t *testing.T, changes SpecChanges)) func(t *t } } +func assertNoChanges() func(t *testing.T, changes SpecChanges) { + return func(t *testing.T, changes SpecChanges) { + t.Helper() + require.True(t, changes.IsEmpty(), "expected no changes, got %+v", changes.GetAll()) + } +} + func actionRequired(actionType ActionType) func(t *testing.T, changes SpecChanges) { return func(t *testing.T, changes SpecChanges) { t.Helper() diff --git a/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go new file mode 100644 index 0000000000..a548c90821 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vmchange/gpu_change.go @@ -0,0 +1,39 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmchange + +import ( + "reflect" + + "github.com/deckhouse/virtualization-controller/pkg/controller/kvbuilder" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func compareGPUDevices(current, desired *v1alpha2.VirtualMachineSpec) []FieldChange { + currentGPUDevices := kvbuilder.SortGPUDevices(current.GPUs) + desiredGPUDevices := kvbuilder.SortGPUDevices(desired.GPUs) + currentValue := NewValue(currentGPUDevices, len(currentGPUDevices) == 0, false) + desiredValue := NewValue(desiredGPUDevices, len(desiredGPUDevices) == 0, false) + + return compareValues( + "gpus", + currentValue, + desiredValue, + reflect.DeepEqual(currentGPUDevices, desiredGPUDevices), + ActionRestart, + ) +} diff --git a/images/virtualization-artifact/pkg/featuregates/featuregate.go b/images/virtualization-artifact/pkg/featuregates/featuregate.go index 633c3d9931..0ebdda2ee0 100644 --- a/images/virtualization-artifact/pkg/featuregates/featuregate.go +++ b/images/virtualization-artifact/pkg/featuregates/featuregate.go @@ -30,6 +30,7 @@ const ( VolumeMigration featuregate.Feature = "VolumeMigration" TargetMigration featuregate.Feature = "TargetMigration" USB featuregate.Feature = "USB" + GPU featuregate.Feature = "GPU" HotplugCPUWithLiveMigration featuregate.Feature = "HotplugCPUWithLiveMigration" HotplugMemoryWithLiveMigration featuregate.Feature = "HotplugMemoryWithLiveMigration" HotplugCPUAndMemoryWithInPlaceResize featuregate.Feature = "HotplugCPUAndMemoryWithInPlaceResize" @@ -61,6 +62,11 @@ var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ LockToDefault: true, PreRelease: featuregate.Alpha, }, + GPU: { + Default: false, + LockToDefault: version.GetEdition() == version.EditionCE, + PreRelease: featuregate.Alpha, + }, HotplugCPUWithLiveMigration: { Default: false, LockToDefault: version.GetEdition() == version.EditionCE, diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 503a76325e..c1b1ac53cb 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -267,9 +267,11 @@ properties: - `HotplugCPUWithLiveMigration` — enable live changing of cpu cores number via LiveMigration. (Not available in CE); - `HotplugMemoryWithLiveMigration` — enable live changing of memory size via LiveMigration. (Not available in CE); - `HotplugCPUAndMemoryWithInPlaceResize` - enable live changing of cpu cores number or memory size via InPlaceResize. (Not available in CE); + - `GPU` — enable attaching GPU devices to virtual machines via DRA by referencing a GPUClass. (Not available in CE); items: type: string enum: - "HotplugCPUWithLiveMigration" - "HotplugMemoryWithLiveMigration" - "HotplugCPUAndMemoryWithInPlaceResize" + - "GPU" diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index e1b17f65af..c21f4017eb 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -179,5 +179,6 @@ properties: - `HotplugCPUWithLiveMigration` — включить изменение количества ядер процессора без перезагрузки через живую миграцию. (Не доступно в CE); - `HotplugMemoryWithLiveMigration` — включить изменение размера памяти без перезагрузки через живую миграцию. (Не доступно в CE); - `HotplugCPUAndMemoryWithInPlaceResize` - включить изменение количества ядер процессора или размера памяти без перезагрузки через InPlaceResize (Не доступно в CE) + - `GPU` — включить подключение GPU-устройств к виртуальным машинам через DRA по ссылке на GPUClass. (Не доступно в CE); items: type: string diff --git a/templates/kubevirt/_kubevirt_helpers.tpl b/templates/kubevirt/_kubevirt_helpers.tpl index 57a9b5feb5..7fd0bc0b2b 100644 --- a/templates/kubevirt/_kubevirt_helpers.tpl +++ b/templates/kubevirt/_kubevirt_helpers.tpl @@ -68,6 +68,7 @@ because its business logic does not yet have their default behavior implemented. - HostDevicesWithDRA - HostDevices - HotplugHostDevicesWithDRA # custom feature gate - added in our KubeVirt fork, not present in upstream +- GPUsWithDRA {{- if has "HotplugCPUAndMemoryWithInPlaceResize" .Values.virtualization.internal.moduleConfig.featureGates }} - InPlaceResize # custom feature gate - added in our KubeVirt fork, not present in upstream {{- end }} diff --git a/templates/virtualization-controller/rbac-for-us.yaml b/templates/virtualization-controller/rbac-for-us.yaml index 84eb2fef25..8299039f38 100644 --- a/templates/virtualization-controller/rbac-for-us.yaml +++ b/templates/virtualization-controller/rbac-for-us.yaml @@ -344,6 +344,14 @@ rules: - update - patch - delete +- apiGroups: + - gpu.deckhouse.io + resources: + - gpuclasses + verbs: + - get + - list + - watch - apiGroups: - apiextensions.k8s.io resources: