From 74b4bb588dacebef4e7e3a239092c7ddd6954ae4 Mon Sep 17 00:00:00 2001 From: Daniil Loktev Date: Wed, 1 Jul 2026 18:57:42 +0300 Subject: [PATCH 1/5] wip Signed-off-by: Daniil Loktev --- api/core/v1alpha2/vmbdacondition/condition.go | 2 + .../controller/vmbda/internal/life_cycle.go | 23 +++++++ .../vmbda/internal/life_cycle_test.go | 63 +++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go diff --git a/api/core/v1alpha2/vmbdacondition/condition.go b/api/core/v1alpha2/vmbdacondition/condition.go index 19d30e18cd..4e664c79ea 100644 --- a/api/core/v1alpha2/vmbdacondition/condition.go +++ b/api/core/v1alpha2/vmbdacondition/condition.go @@ -65,6 +65,8 @@ const ( Conflict AttachedReason = "Conflict" // DeviceNotAvailableOnNode indicates that the block device's PersistentVolume is not available on the node where the virtual machine is running. DeviceNotAvailableOnNode AttachedReason = "DeviceNotAvailableOnNode" + // BlockedByMigration indicates that a ReadWriteOnce block device cannot be hot-plugged while the virtual machine is migrating. + BlockedByMigration AttachedReason = "BlockedByMigration" // CapacityAvailable signifies that the capacity not reached and attaching available. CapacityAvailable DiskAttachmentCapacityAvailableReason = "CapacityAvailable" diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index 6dfce1a459..dde49cdcb5 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -20,8 +20,10 @@ import ( "context" "errors" "fmt" + "slices" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -31,6 +33,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/logger" "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/deckhouse/virtualization/api/core/v1alpha2/vmbdacondition" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" ) type LifeCycleHandler struct { @@ -253,6 +256,17 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac } if pvc != nil { + if isVirtualMachineMigrating(vm) && isReadWriteOnce(pvc) { + log.Info("Cannot hot-plug a ReadWriteOnce disk while the virtual machine is migrating") + + vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhasePending + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockedByMigration). + Message("Cannot hot-plug a ReadWriteOnce disk while the virtual machine is migrating.") + return reconcile.Result{}, nil + } + available, err := h.attacher.IsPVAvailableOnVMNode(ctx, pvc, kvvmi) if err != nil { return reconcile.Result{}, err @@ -301,3 +315,12 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac return reconcile.Result{}, canErr } } + +func isVirtualMachineMigrating(vm *v1alpha2.VirtualMachine) bool { + migrating, _ := conditions.GetCondition(vmcondition.TypeMigrating, vm.Status.Conditions) + return migrating.Status == metav1.ConditionTrue +} + +func isReadWriteOnce(pvc *corev1.PersistentVolumeClaim) bool { + return !slices.Contains(pvc.Spec.AccessModes, corev1.ReadWriteMany) +} diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go new file mode 100644 index 0000000000..685e23b674 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go @@ -0,0 +1,63 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" +) + +var _ = Describe("LifeCycleHandler helpers", func() { + DescribeTable("isVirtualMachineMigrating", func(status metav1.ConditionStatus, hasCondition, expected bool) { + vm := &v1alpha2.VirtualMachine{} + if hasCondition { + vm.Status.Conditions = []metav1.Condition{ + { + Type: vmcondition.TypeMigrating.String(), + Status: status, + }, + } + } + Expect(isVirtualMachineMigrating(vm)).To(Equal(expected)) + }, + Entry("migrating condition is true", metav1.ConditionTrue, true, true), + Entry("migrating condition is false", metav1.ConditionFalse, true, false), + Entry("migrating condition is unknown", metav1.ConditionUnknown, true, false), + Entry("migrating condition is absent", metav1.ConditionTrue, false, false), + ) + + DescribeTable("isReadWriteOnce", func(accessModes []corev1.PersistentVolumeAccessMode, expected bool) { + pvc := &corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: accessModes, + }, + } + Expect(isReadWriteOnce(pvc)).To(Equal(expected)) + }, + Entry("ReadWriteOnce", []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, true), + Entry("ReadWriteOncePod", []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOncePod}, true), + Entry("ReadOnlyMany", []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, true), + Entry("no access modes", []corev1.PersistentVolumeAccessMode{}, true), + Entry("ReadWriteMany", []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, false), + Entry("ReadWriteMany among others", []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany}, false), + ) +}) From 83fa9470ce01520dca2d2147d1550815a71b3638 Mon Sep 17 00:00:00 2001 From: Daniil Loktev Date: Thu, 2 Jul 2026 11:01:06 +0300 Subject: [PATCH 2/5] wip Signed-off-by: Daniil Loktev --- .../pkg/controller/vmbda/internal/life_cycle.go | 8 ++++---- .../pkg/controller/vmbda/internal/life_cycle_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index dde49cdcb5..3e2f830a38 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -257,13 +257,13 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac if pvc != nil { if isVirtualMachineMigrating(vm) && isReadWriteOnce(pvc) { - log.Info("Cannot hot-plug a ReadWriteOnce disk while the virtual machine is migrating") + log.Info("Cannot hotplug a ReadWriteOnce disk while the virtual machine is migrating") vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhasePending cb. Status(metav1.ConditionFalse). Reason(vmbdacondition.BlockedByMigration). - Message("Cannot hot-plug a ReadWriteOnce disk while the virtual machine is migrating.") + Message("Cannot hotplug a ReadWriteOnce disk while the virtual machine is migrating.") return reconcile.Result{}, nil } @@ -317,8 +317,8 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac } func isVirtualMachineMigrating(vm *v1alpha2.VirtualMachine) bool { - migrating, _ := conditions.GetCondition(vmcondition.TypeMigrating, vm.Status.Conditions) - return migrating.Status == metav1.ConditionTrue + _, migrating := conditions.GetCondition(vmcondition.TypeMigrating, vm.Status.Conditions) + return migrating } func isReadWriteOnce(pvc *corev1.PersistentVolumeClaim) bool { diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go index 685e23b674..e599b7c522 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go @@ -40,8 +40,8 @@ var _ = Describe("LifeCycleHandler helpers", func() { Expect(isVirtualMachineMigrating(vm)).To(Equal(expected)) }, Entry("migrating condition is true", metav1.ConditionTrue, true, true), - Entry("migrating condition is false", metav1.ConditionFalse, true, false), - Entry("migrating condition is unknown", metav1.ConditionUnknown, true, false), + Entry("migrating condition is false", metav1.ConditionFalse, true, true), + Entry("migrating condition is unknown", metav1.ConditionUnknown, true, true), Entry("migrating condition is absent", metav1.ConditionTrue, false, false), ) From dee4a53981d97881c1e1eb57da5390407f31b821 Mon Sep 17 00:00:00 2001 From: Daniil Loktev Date: Thu, 2 Jul 2026 11:22:46 +0300 Subject: [PATCH 3/5] wip Signed-off-by: Daniil Loktev --- .../pkg/controller/vmbda/internal/watcher/vm_watcher.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vm_watcher.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vm_watcher.go index efca5f6e84..a08c556e8b 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vm_watcher.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/watcher/vm_watcher.go @@ -61,6 +61,12 @@ func (w VirtualMachineWatcher) Watch(mgr manager.Manager, ctr controller.Control return true } + _, oldMigrating := conditions.GetCondition(vmcondition.TypeMigrating, e.ObjectOld.Status.Conditions) + _, newMigrating := conditions.GetCondition(vmcondition.TypeMigrating, e.ObjectNew.Status.Conditions) + if oldMigrating != newMigrating { + return true + } + return w.hasBlockDeviceAttachmentChanges(e.ObjectOld, e.ObjectNew) }, }, From ba27620d6f611e63a7ed8a5c75dc0f50a6b2a8ed Mon Sep 17 00:00:00 2001 From: Daniil Loktev Date: Mon, 6 Jul 2026 17:37:16 +0300 Subject: [PATCH 4/5] forbid rwx as well Signed-off-by: Daniil Loktev --- api/core/v1alpha2/vmbdacondition/condition.go | 2 +- .../controller/vmbda/internal/life_cycle.go | 28 ++++++++----------- .../vmbda/internal/life_cycle_test.go | 17 ----------- 3 files changed, 12 insertions(+), 35 deletions(-) diff --git a/api/core/v1alpha2/vmbdacondition/condition.go b/api/core/v1alpha2/vmbdacondition/condition.go index 4e664c79ea..23500d3c5c 100644 --- a/api/core/v1alpha2/vmbdacondition/condition.go +++ b/api/core/v1alpha2/vmbdacondition/condition.go @@ -65,7 +65,7 @@ const ( Conflict AttachedReason = "Conflict" // DeviceNotAvailableOnNode indicates that the block device's PersistentVolume is not available on the node where the virtual machine is running. DeviceNotAvailableOnNode AttachedReason = "DeviceNotAvailableOnNode" - // BlockedByMigration indicates that a ReadWriteOnce block device cannot be hot-plugged while the virtual machine is migrating. + // BlockedByMigration indicates that a block device cannot be hot-plugged while the virtual machine is migrating. BlockedByMigration AttachedReason = "BlockedByMigration" // CapacityAvailable signifies that the capacity not reached and attaching available. diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go index 3e2f830a38..105aa02728 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle.go @@ -20,10 +20,8 @@ import ( "context" "errors" "fmt" - "slices" "time" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -250,23 +248,23 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmbda *v1alpha2.VirtualMac } if ad.PVCName != "" { + if isVirtualMachineMigrating(vm) { + log.Info("Cannot hotplug a disk while the virtual machine is migrating") + + vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhasePending + cb. + Status(metav1.ConditionFalse). + Reason(vmbdacondition.BlockedByMigration). + Message("Cannot hotplug a disk while the virtual machine is migrating.") + return reconcile.Result{}, nil + } + pvc, err := h.attacher.GetPersistentVolumeClaim(ctx, ad) if err != nil { return reconcile.Result{}, err } if pvc != nil { - if isVirtualMachineMigrating(vm) && isReadWriteOnce(pvc) { - log.Info("Cannot hotplug a ReadWriteOnce disk while the virtual machine is migrating") - - vmbda.Status.Phase = v1alpha2.BlockDeviceAttachmentPhasePending - cb. - Status(metav1.ConditionFalse). - Reason(vmbdacondition.BlockedByMigration). - Message("Cannot hotplug a ReadWriteOnce disk while the virtual machine is migrating.") - return reconcile.Result{}, nil - } - available, err := h.attacher.IsPVAvailableOnVMNode(ctx, pvc, kvvmi) if err != nil { return reconcile.Result{}, err @@ -320,7 +318,3 @@ func isVirtualMachineMigrating(vm *v1alpha2.VirtualMachine) bool { _, migrating := conditions.GetCondition(vmcondition.TypeMigrating, vm.Status.Conditions) return migrating } - -func isReadWriteOnce(pvc *corev1.PersistentVolumeClaim) bool { - return !slices.Contains(pvc.Spec.AccessModes, corev1.ReadWriteMany) -} diff --git a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go index e599b7c522..5a3c381da0 100644 --- a/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go +++ b/images/virtualization-artifact/pkg/controller/vmbda/internal/life_cycle_test.go @@ -19,7 +19,6 @@ package internal import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/deckhouse/virtualization/api/core/v1alpha2" @@ -44,20 +43,4 @@ var _ = Describe("LifeCycleHandler helpers", func() { Entry("migrating condition is unknown", metav1.ConditionUnknown, true, true), Entry("migrating condition is absent", metav1.ConditionTrue, false, false), ) - - DescribeTable("isReadWriteOnce", func(accessModes []corev1.PersistentVolumeAccessMode, expected bool) { - pvc := &corev1.PersistentVolumeClaim{ - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: accessModes, - }, - } - Expect(isReadWriteOnce(pvc)).To(Equal(expected)) - }, - Entry("ReadWriteOnce", []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, true), - Entry("ReadWriteOncePod", []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOncePod}, true), - Entry("ReadOnlyMany", []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, true), - Entry("no access modes", []corev1.PersistentVolumeAccessMode{}, true), - Entry("ReadWriteMany", []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, false), - Entry("ReadWriteMany among others", []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany}, false), - ) }) From 8ff50898dadef76d1b27e7a3ae1634a2210b6205 Mon Sep 17 00:00:00 2001 From: Vladislav Panfilov Date: Fri, 10 Jul 2026 18:18:44 +0400 Subject: [PATCH 5/5] docs: add info about disk hotplug during migration Signed-off-by: Vladislav Panfilov --- docs/USER_GUIDE.md | 4 ++++ docs/USER_GUIDE.ru.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index ae584d9d62..36b9cf0cfe 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -2245,6 +2245,8 @@ For VMs with `enableParavirtualization: false`, the following applies: Hot-plugging (hotplug) is only possible if the storage is available on the cluster node where the virtual machine runs. When you create or update a VM, or create a `vmbda`, placement rules (`nodeSelector`, `affinity`, `tolerations`) for the volume, the VM, and the VM class are taken into account: there must be at least one valid combined placement. If the VM is already running on a specific node, a new disk must be available on that node. +Hot-plugging a disk is not possible while the virtual machine is undergoing live migration. In this case, the VirtualMachineBlockDeviceAttachment resource remains in the `Pending` phase, and the `Attached` condition reports the `BlockedByMigration` reason. Attachment resumes after the migration completes. + #### Attaching via the VM specification The list of block devices is defined in the `.spec.blockDeviceRefs` field of the [VirtualMachine](/modules/virtualization/cr.html#virtualmachine) resource. @@ -2729,6 +2731,8 @@ For successful live migration, certain requirements must be met. Failure to meet - Disk availability: All disks attached to the VM must be accessible on the target node, otherwise migration will be impossible. For network storage (NFS, Ceph, etc.), this requirement is usually met automatically, as disks are accessible on all cluster nodes. For local storage, the situation is different: the storage system must be available on the target node to create a new local volume. If local storage exists only on the source node, migration cannot be performed. +- Disk hot-plugging: During live migration, attaching disks to a VM via the VirtualMachineBlockDeviceAttachment resource is unavailable. The resource remains in the `Pending` phase with the `BlockedByMigration` reason in the `Attached` condition until the migration completes. + - Network bandwidth: Network speed is critical for live migration. With low bandwidth, the number of memory synchronization iterations increases, VM downtime during the final stage of migration increases, and in the worst case, migration may not complete due to a timeout. To manage the migration process, configure the live migration policy [`.spec.liveMigrationPolicy`](#configuring-migration-policy) in the virtual machine settings. For network problems, use the AutoConverge mechanism (see the [Migration with insufficient network bandwidth](#migration-with-insufficient-network-bandwidth) section). - Kernel versions on nodes: For stable live migration operation, all cluster nodes must use the same Linux kernel version. Differences in kernel versions can lead to incompatible interfaces, system calls, and resource handling features, which can disrupt the virtual machine migration process. diff --git a/docs/USER_GUIDE.ru.md b/docs/USER_GUIDE.ru.md index 42df93d945..c2442d8449 100644 --- a/docs/USER_GUIDE.ru.md +++ b/docs/USER_GUIDE.ru.md @@ -2262,6 +2262,8 @@ d8 k get nodes -o custom-columns=NAME:.metadata.name,ZONE:.metadata.labels.topol Горячее подключение (hotplug) возможно только если хранилище доступно на том узле кластера, где выполняется виртуальная машина. При создании и обновлении ВМ, а также при создании `vmbda`, учитываются правила размещения (`nodeSelector`, `affinity`, `tolerations`) тома, виртуальной машины и класса ВМ: должно быть хотя бы одно общее допустимое размещение. Если ВМ уже запущена и работает на конкретном узле, новый диск должен быть доступен на этом узле. +Горячее подключение диска невозможно, пока виртуальная машина находится в процессе живой миграции. В этом случае ресурс VirtualMachineBlockDeviceAttachment остаётся в фазе `Pending`, а в условии `Attached` указывается причина `BlockedByMigration`. Подключение продолжится после завершения миграции. + #### Подключение через спецификацию ВМ Список блочных устройств задаётся в поле `.spec.blockDeviceRefs` ресурса [VirtualMachine](/modules/virtualization/cr.html#virtualmachine). @@ -2746,6 +2748,8 @@ EOF - Доступность дисков: Все подключённые к ВМ диски должны быть доступны на целевом узле, иначе миграция будет невозможна. Для сетевых хранилищ (NFS, Ceph и т.д.) это требование обычно выполняется автоматически, так как диски доступны на всех узлах кластера. Для локальных хранилищ ситуация иная: хранилище должно быть доступно для создания нового локального тома на целевом узле. Если локальное хранилище есть только на исходном узле, миграция не может быть выполнена. +- Горячее подключение дисков: Во время живой миграции подключение дисков к ВМ через ресурс VirtualMachineBlockDeviceAttachment недоступно. Ресурс остаётся в фазе `Pending` с причиной `BlockedByMigration` в условии `Attached` до завершения миграции. + - Пропускная способность сети: Скорость сети критически важна для живой миграции. При низкой пропускной способности увеличивается количество итераций синхронизации памяти, возрастает время простоя ВМ на финальном этапе миграции, а в худшем случае миграция может не завершиться из-за превышения таймаута. Для управления процессом миграции настройте политику живой миграции [`.spec.liveMigrationPolicy`](#настройка-политики-миграции) в настройках ВМ. При проблемах с сетью используйте механизм AutoConverge (см. раздел [Миграции при недостаточной пропускной способности сети](#миграции-при-недостаточной-пропускной-способности-сети)). - Версии ядер на узлах: Для стабильной работы живой миграции на всех узлах кластера должна использоваться идентичная версия ядра Linux. Различия в версиях ядра могут привести к несовместимости интерфейсов, системных вызовов и особенностей работы с ресурсами, что нарушает процесс миграции виртуальных машин.