diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index 196973e1e..d6e8f9f81 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -109,6 +109,16 @@ spec: requests without headroom, add `params: [{key: enforce, boolValue: true}]` to this step. The shadow default is intentional so that newly-rolled-out releases never silently start rejecting requests. + - name: filter_cr_migration_slot + description: | + During live migrations of VMs that occupy a committed-resource reservation + slot, this filter restricts candidates to hosts that have a ready CR + reservation with sufficient remaining capacity for the full slot size (not + just the VM flavor size). This ensures the reservation slot is migrated + alongside the VM. + If no candidate has a matching slot, all candidates are returned unchanged + so the VM can still migrate using regular (non-slot) capacity. + Only activates for live_migrate requests. All other intents pass through. weighers: - name: kvm_prefer_smaller_hosts params: @@ -261,6 +271,16 @@ spec: requests without headroom, add `params: [{key: enforce, boolValue: true}]` to this step. The shadow default is intentional so that newly-rolled-out releases never silently start rejecting requests. + - name: filter_cr_migration_slot + description: | + During live migrations of VMs that occupy a committed-resource reservation + slot, this filter restricts candidates to hosts that have a ready CR + reservation with sufficient remaining capacity for the full slot size (not + just the VM flavor size). This ensures the reservation slot is migrated + alongside the VM. + If no candidate has a matching slot, all candidates are returned unchanged + so the VM can still migrate using regular (non-slot) capacity. + Only activates for live_migrate requests. All other intents pass through. weighers: - name: kvm_prefer_smaller_hosts params: @@ -714,6 +734,16 @@ spec: requests without headroom, add `params: [{key: enforce, boolValue: true}]` to this step. The shadow default is intentional so that newly-rolled-out releases never silently start rejecting requests. + - name: filter_cr_migration_slot + description: | + During live migrations of VMs that occupy a committed-resource reservation + slot, this filter restricts candidates to hosts that have a ready CR + reservation with sufficient remaining capacity for the full slot size (not + just the VM flavor size). This ensures the reservation slot is migrated + alongside the VM. + If no candidate has a matching slot, all candidates are returned unchanged + so the VM can still migrate using regular (non-slot) capacity. + Only activates for live_migrate requests. All other intents pass through. weighers: - name: kvm_prefer_smaller_hosts params: @@ -866,6 +896,16 @@ spec: requests without headroom, add `params: [{key: enforce, boolValue: true}]` to this step. The shadow default is intentional so that newly-rolled-out releases never silently start rejecting requests. + - name: filter_cr_migration_slot + description: | + During live migrations of VMs that occupy a committed-resource reservation + slot, this filter restricts candidates to hosts that have a ready CR + reservation with sufficient remaining capacity for the full slot size (not + just the VM flavor size). This ensures the reservation slot is migrated + alongside the VM. + If no candidate has a matching slot, all candidates are returned unchanged + so the VM can still migrate using regular (non-slot) capacity. + Only activates for live_migrate requests. All other intents pass through. weighers: - name: kvm_prefer_smaller_hosts params: diff --git a/internal/scheduling/nova/crs/evaluator.go b/internal/scheduling/nova/crs/evaluator.go index bfd5f9607..039d31543 100644 --- a/internal/scheduling/nova/crs/evaluator.go +++ b/internal/scheduling/nova/crs/evaluator.go @@ -24,6 +24,19 @@ type SlotEvaluator struct { // BuildSlotEvaluator lists HV CRDs and CR Reservation CRDs once and returns an evaluator // that can answer slot-usability queries without further K8s reads. func BuildSlotEvaluator(ctx context.Context, c client.Client) (*SlotEvaluator, error) { + var resList v1alpha1.ReservationList + if err := c.List(ctx, &resList, + client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, + ); err != nil { + return nil, err + } + return BuildSlotEvaluatorFromReservations(ctx, c, resList.Items) +} + +// BuildSlotEvaluatorFromReservations builds a SlotEvaluator from an already-fetched +// reservation slice. Use this when the caller has already listed reservations to avoid +// a redundant K8s read. +func BuildSlotEvaluatorFromReservations(ctx context.Context, c client.Client, reservations []v1alpha1.Reservation) (*SlotEvaluator, error) { eval := &SlotEvaluator{ hvFreeMemory: make(map[string]int64), reservationsByHost: make(map[string][]v1alpha1.Reservation), @@ -45,13 +58,7 @@ func BuildSlotEvaluator(ctx context.Context, c client.Client) (*SlotEvaluator, e eval.hvFreeMemory[hv.Name] = max(effectiveMemQ.Value()-allocMemQ.Value(), 0) } - var resList v1alpha1.ReservationList - if err := c.List(ctx, &resList, - client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, - ); err != nil { - return nil, err - } - for _, res := range resList.Items { + for _, res := range reservations { if !res.IsReady() { continue } @@ -104,6 +111,20 @@ func (e *SlotEvaluator) HasUsableSlot(hostName, projectID, flavorGroup string, v return false } +// HasSlotWithCapacity reports whether hostName has at least one ready CR slot +// matching projectID + flavorGroup whose remaining memory is >= requiredBytes. +// Unlike HasUsableSlot, this does not apply the overfill model — it is used +// during migration slot filtering where the full slot size must fit within a +// single reservation on the target host. +func (e *SlotEvaluator) HasSlotWithCapacity(hostName, projectID, flavorGroup string, requiredBytes int64) bool { + for _, slot := range e.SlotsForHost(hostName, projectID, flavorGroup) { + if ReservationRemainingMemory(slot) >= requiredBytes { + return true + } + } + return false +} + // ReservationRemainingMemory returns how many bytes of memory remain // unallocated in a reservation slot. Returns 0 if the slot is full or nil. func ReservationRemainingMemory(res v1alpha1.Reservation) int64 { diff --git a/internal/scheduling/nova/plugins/filters/filter_cr_migration_slot.go b/internal/scheduling/nova/plugins/filters/filter_cr_migration_slot.go new file mode 100644 index 000000000..2d88be2f4 --- /dev/null +++ b/internal/scheduling/nova/plugins/filters/filter_cr_migration_slot.go @@ -0,0 +1,140 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "context" + "log/slog" + + "sigs.k8s.io/controller-runtime/pkg/client" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" +) + +// FilterCRMigrationSlotStep filters live-migration candidates by committed-resource +// slot size rather than VM flavor size. +// +// When a VM that occupies a CR reservation slot is live-migrated, the target host +// must accommodate the full slot, not just the VM's flavor resources. This filter +// removes candidates that lack a ready CR reservation with sufficient remaining +// capacity for the slot. +// +// Placement order in the pipeline: last filter, after all other filters have run. +// Fallback: if no candidate survives the slot-size check, the original candidate +// set is returned unchanged so that the VM can still migrate using flavor-sized +// capacity on the target host. +// +// Only activates for LiveMigrationIntent. All other intents pass through unchanged. +type FilterCRMigrationSlotStep struct { + lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts] +} + +func (s *FilterCRMigrationSlotStep) Run( + traceLog *slog.Logger, + request api.ExternalSchedulerRequest, +) (*lib.FilterWeigherPipelineStepResult, error) { + + result := s.IncludeAllHostsFromRequest(request) + + intent, err := request.GetIntent() + if err != nil || intent != api.LiveMigrationIntent { + traceLog.Info("not a live migration, skipping CR slot filter") + return result, nil //nolint:nilerr + } + + instanceUUID := request.Spec.Data.InstanceUUID + projectID := request.Spec.Data.ProjectID + + // List all CR reservations once. We reuse this list for both finding the + // source slot and building the slot evaluator for target hosts, avoiding + // a second K8s read inside BuildSlotEvaluator. + var allReservations v1alpha1.ReservationList + if err := s.Client.List(context.Background(), &allReservations, + client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, + ); err != nil { + return nil, err + } + + // Find the source reservation that currently holds this VM UUID (confirmed). + var sourceSlot *v1alpha1.Reservation + for i := range allReservations.Items { + res := &allReservations.Items[i] + if res.Status.CommittedResourceReservation == nil { + continue + } + if _, confirmed := res.Status.CommittedResourceReservation.Allocations[instanceUUID]; confirmed { + sourceSlot = res + break + } + } + + if sourceSlot == nil { + traceLog.Info("migrating VM has no confirmed CR reservation slot, skipping slot filter", + "instanceUUID", instanceUUID) + return result, nil + } + + slotMemoryBytes := sourceSlot.Spec.Resources[hv1.ResourceMemory] + if slotMemoryBytes.IsZero() { + traceLog.Info("source CR slot has no memory resource, skipping slot filter", + "instanceUUID", instanceUUID, + "reservation", sourceSlot.Name) + return result, nil + } + + resourceGroup := sourceSlot.Spec.CommittedResourceReservation.ResourceGroup + + traceLog.Info("found source CR reservation slot for migrating VM", + "instanceUUID", instanceUUID, + "reservation", sourceSlot.Name, + "slotMemoryBytes", slotMemoryBytes.Value(), + "resourceGroup", resourceGroup, + ) + + // Build the slot evaluator from the already-fetched reservation list so we + // don't issue a second List call. HVs are still fetched once inside the evaluator. + evaluator, err := crs.BuildSlotEvaluatorFromReservations(context.Background(), s.Client, allReservations.Items) + if err != nil { + return nil, err + } + + // Filter candidates to those with a ready CR slot that has at least slotMemoryBytes + // remaining. This is a strict check — no overfill model — because the slot must + // fully migrate with the VM. + filtered := make(map[string]float64, len(result.Activations)) + for host := range result.Activations { + if evaluator.HasSlotWithCapacity(host, projectID, resourceGroup, slotMemoryBytes.Value()) { + filtered[host] = result.Activations[host] + traceLog.Info("host has usable CR slot for migration", + "host", host, "slotMemoryBytes", slotMemoryBytes.Value()) + } else { + traceLog.Info("host has no usable CR slot for migration slot size, excluding", + "host", host, "slotMemoryBytes", slotMemoryBytes.Value()) + } + } + + // Fallback: if no host has a matching slot, return all candidates so the VM + // can still migrate using regular (non-slot) capacity. + if len(filtered) == 0 { + traceLog.Info("no hosts with matching CR slot found, falling back to all candidates", + "instanceUUID", instanceUUID, + "slotMemoryBytes", slotMemoryBytes.Value(), + "candidateCount", len(result.Activations), + ) + return result, nil + } + + result.Activations = filtered + return result, nil +} + +func init() { + Index["filter_cr_migration_slot"] = func() NovaFilter { + return &FilterCRMigrationSlotStep{} + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_cr_migration_slot_test.go b/internal/scheduling/nova/plugins/filters/filter_cr_migration_slot_test.go new file mode 100644 index 000000000..468c3cf3c --- /dev/null +++ b/internal/scheduling/nova/plugins/filters/filter_cr_migration_slot_test.go @@ -0,0 +1,267 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "log/slog" + "testing" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// newCRMigrationSlotFilter builds a FilterCRMigrationSlotStep backed by a fake client +// seeded with the given objects. +func newCRMigrationSlotFilter(t *testing.T, objs ...client.Object) *FilterCRMigrationSlotStep { + t.Helper() + scheme := buildTestScheme(t) + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + return &FilterCRMigrationSlotStep{ + BaseFilter: lib.BaseFilter[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + BaseFilterWeigherPipelineStep: lib.BaseFilterWeigherPipelineStep[api.ExternalSchedulerRequest, lib.EmptyFilterWeigherPipelineStepOpts]{ + Client: c, + }, + }, + } +} + +// liveMigrateRequest builds a minimal live-migration request for "vm-migrating". +func liveMigrateRequest(hosts ...string) api.ExternalSchedulerRequest { + hostList := make([]api.ExternalSchedulerHost, len(hosts)) + for i, h := range hosts { + hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + } + return api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + InstanceUUID: "vm-migrating", + ProjectID: "proj-1", + SchedulerHints: map[string]any{ + "_nova_check_type": "live_migrate", + }, + }, + }, + Hosts: hostList, + } +} + +// sourceSlotFor builds a ready 16Gi CR reservation slot on host-src with instanceUUID confirmed. +func sourceSlotFor(instanceUUID string) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slot-src", + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-src", + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse("16Gi"), + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + ProjectID: "proj-1", + ResourceGroup: "hana-v2", + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: "host-src", + Conditions: []metav1.Condition{ + {Type: v1alpha1.ReservationConditionReady, Status: metav1.ConditionTrue, Reason: "ReservationActive"}, + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{ + Allocations: map[string]string{instanceUUID: "host-src"}, + }, + }, + } +} + +// emptyReservation builds a ready CR reservation slot with no VM allocations. +func emptyReservation(name, host, resourceGroup, slotMemory string) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: host, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse(slotMemory), + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + ProjectID: "proj-1", + ResourceGroup: resourceGroup, + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: host, + Conditions: []metav1.Condition{ + {Type: v1alpha1.ReservationConditionReady, Status: metav1.ConditionTrue, Reason: "ReservationActive"}, + }, + }, + } +} + +// hvWithFreeMemory builds a Hypervisor with 32Gi effective capacity and zero allocation. +func hvWithFreeMemory(name string) *hv1.Hypervisor { + return &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse("32Gi"), + }, + Allocation: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse("0"), + }, + }, + } +} + +func TestFilterCRMigrationSlot(t *testing.T) { + const instanceUUID = "vm-migrating" + + // zeroMemorySourceSlot is a source slot with no memory resource — used to test + // the zero-slot-memory guard. + zeroMemorySourceSlot := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slot-src", + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: "host-src", + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + ProjectID: "proj-1", + ResourceGroup: "hana-v2", + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: "host-src", + Conditions: []metav1.Condition{ + {Type: v1alpha1.ReservationConditionReady, Status: metav1.ConditionTrue, Reason: "ReservationActive"}, + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{ + Allocations: map[string]string{instanceUUID: "host-src"}, + }, + }, + } + + tests := []struct { + name string + objects []client.Object + request api.ExternalSchedulerRequest + wantHosts []string // hosts that must appear in Activations + wantFiltered []string // hosts that must NOT appear in Activations + wantHostCount int // total expected Activations size + }{ + { + name: "non-migration intent: all hosts pass through unchanged", + objects: nil, + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + InstanceUUID: instanceUUID, + ProjectID: "proj-1", + // no _nova_check_type → CreateIntent + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host-1"}, {ComputeHost: "host-2"}}, + }, + wantHosts: []string{"host-1", "host-2"}, + wantHostCount: 2, + }, + { + name: "no source slot: all candidates pass through (fallback)", + objects: []client.Object{}, + request: liveMigrateRequest("host-1", "host-2"), + wantHosts: []string{"host-1", "host-2"}, + wantHostCount: 2, + }, + { + name: "slot size filtering: only host with matching 16Gi slot passes", + objects: []client.Object{ + sourceSlotFor(instanceUUID), + emptyReservation("slot-a", "host-a", "hana-v2", "16Gi"), + emptyReservation("slot-b", "host-b", "hana-v2", "8Gi"), + hvWithFreeMemory("host-src"), + hvWithFreeMemory("host-a"), + hvWithFreeMemory("host-b"), + hvWithFreeMemory("host-c"), + }, + request: liveMigrateRequest("host-a", "host-b", "host-c"), + wantHosts: []string{"host-a"}, + wantFiltered: []string{"host-b", "host-c"}, + wantHostCount: 1, + }, + { + name: "no slot on any candidate: fallback returns all candidates", + objects: []client.Object{ + sourceSlotFor(instanceUUID), + hvWithFreeMemory("host-src"), + hvWithFreeMemory("host-a"), + hvWithFreeMemory("host-b"), + }, + request: liveMigrateRequest("host-a", "host-b"), + wantHosts: []string{"host-a", "host-b"}, + wantHostCount: 2, + }, + { + name: "wrong resource group on target: slot does not match, fallback", + objects: []client.Object{ + sourceSlotFor(instanceUUID), + emptyReservation("slot-a", "host-a", "general-v3", "16Gi"), + hvWithFreeMemory("host-src"), + hvWithFreeMemory("host-a"), + }, + request: liveMigrateRequest("host-a"), + wantHosts: []string{"host-a"}, + wantHostCount: 1, + }, + { + name: "source slot has zero memory: filter skips slot check, all candidates pass", + objects: []client.Object{ + zeroMemorySourceSlot, + hvWithFreeMemory("host-a"), + }, + request: liveMigrateRequest("host-a"), + wantHosts: []string{"host-a"}, + wantHostCount: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filter := newCRMigrationSlotFilter(t, tt.objects...) + result, err := filter.Run(slog.Default(), tt.request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, host := range tt.wantHosts { + if _, ok := result.Activations[host]; !ok { + t.Errorf("expected host %q in activations", host) + } + } + for _, host := range tt.wantFiltered { + if _, ok := result.Activations[host]; ok { + t.Errorf("expected host %q to be filtered out", host) + } + } + if len(result.Activations) != tt.wantHostCount { + t.Errorf("expected %d hosts, got %d: %v", tt.wantHostCount, len(result.Activations), result.Activations) + } + }) + } +}