diff --git a/api/external/nova/messages.go b/api/external/nova/messages.go index e83d37ced..4779eddeb 100644 --- a/api/external/nova/messages.go +++ b/api/external/nova/messages.go @@ -154,10 +154,15 @@ const ( EvacuateIntent v1alpha1.SchedulingIntent = "evacuate" // CreateIntent indicates that the request is intended for creating a new VM. CreateIntent v1alpha1.SchedulingIntent = "create" - // ReserveForFailoverIntent indicates that the request is for failover reservation scheduling. + // ReserveForFailoverIntent indicates that the request is for creating a new failover reservation slot. ReserveForFailoverIntent v1alpha1.SchedulingIntent = "reserve_for_failover" + // ReuseFailoverReservationIntent indicates that the request is checking whether an existing + // failover reservation slot can be reused by a VM (compatibility check, not a new slot). + ReuseFailoverReservationIntent v1alpha1.SchedulingIntent = "reuse_failover_reservation" // ReserveForCommittedResourceIntent indicates that the request is for CR reservation scheduling. ReserveForCommittedResourceIntent v1alpha1.SchedulingIntent = "reserve_for_committed_resource" + // CapacityProbeIntent indicates that the request is a synthetic capacity probe (not a real VM placement). + CapacityProbeIntent v1alpha1.SchedulingIntent = "capacity_probe" // HintKeyResourceGroup is the scheduler hint key used to pass the resource group // (e.g., flavor group name) for failover reservation scheduling. @@ -188,9 +193,15 @@ func (req ExternalSchedulerRequest) GetIntent() (v1alpha1.SchedulingIntent, erro // Used by cortex failover reservation controller case "reserve_for_failover": return ReserveForFailoverIntent, nil + // Used by cortex failover reservation controller (reuse check) + case "reuse_failover_reservation": + return ReuseFailoverReservationIntent, nil // Used by cortex committed resource reservation controller case "reserve_for_committed_resource": return ReserveForCommittedResourceIntent, nil + // Used by cortex capacity probe controller + case "capacity_probe": + return CapacityProbeIntent, nil default: return CreateIntent, nil } diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 1171f3b01..d931d22f6 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -154,9 +154,9 @@ cortex-scheduling-controllers: # that use committed resources. Requires also enabling of CR controllers and tasks committedResourceTracking: false # Pipeline used for the empty-state capacity probe (ignores allocations and reservations). - capacityTotalPipeline: "kvm-report-capacity" + capacityTotalPipeline: "kvm-general-purpose-load-balancing" # Pipeline used for the current-state capacity probe (considers current VM allocations). - capacityPlaceablePipeline: "kvm-general-purpose-load-balancing-no-history" + capacityPlaceablePipeline: "kvm-general-purpose-load-balancing" # How often the capacity controller re-runs its scheduler probes. capacityReconcileInterval: 5m # If true, the external scheduler API will limit the list of hosts in its @@ -166,11 +166,12 @@ cortex-scheduling-controllers: # Set to 0 or negative to disable shuffling. evacuationShuffleK: 3 committedResourceReservationController: - # Maps flavor group IDs to pipeline names; "*" acts as catch-all fallback + # Pipeline selection for CR reservation scheduling. The catch-all default covers + # general-purpose flavors. For HANA flavor groups, add an explicit entry, e.g.: + # "my-hana-group": "kvm-hana-bin-packing" flavorGroupPipelines: - "*": "kvm-general-purpose-load-balancing-no-history" # Catch-all fallback - # Fallback pipeline when no flavorGroupPipelines entry matches - pipelineDefault: "kvm-general-purpose-load-balancing-no-history" + "*": "kvm-general-purpose-load-balancing" + pipelineDefault: "kvm-general-purpose-load-balancing" # How often to re-verify active Reservation CRDs (healthy state) requeueIntervalActive: "5m" # Back-off interval when knowledge is unavailable diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go index 157a80521..f55e8ab7b 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go @@ -9,6 +9,7 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -21,6 +22,14 @@ type FilterAggregateMetadata struct { // the "filter_tenant_id" metadata key set. func (s *FilterAggregateMetadata) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // Failover and capacity probe calls are not placed on behalf of a tenant project; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } hvs := &hv1.HypervisorList{} if err := s.Client.List(context.Background(), hvs); err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go index d1ff9cd2d..42c096a2b 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go @@ -404,3 +404,46 @@ func TestFilterAggregateMetadata_IndexRegistration(t *testing.T) { t.Errorf("expected factory to return *FilterAggregateMetadata, got %T", filter) } } + +func TestFilterAggregateMetadata_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host1 is in an aggregate restricting to project-x; request is project-y → host1 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host1"}, + Status: hv1.HypervisorStatus{ + Aggregates: []hv1.Aggregate{{ + Name: "restricted", + Metadata: map[string]string{"filter_tenant_id": "project-x"}, + }}, + }, + }, + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, + } + step := &FilterAggregateMetadata{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + ProjectID: "project-y", + SchedulerHints: map[string]any{"_nova_check_type": intent}, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go index a0a486f3d..bd0849cf1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go @@ -9,6 +9,7 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -21,6 +22,14 @@ type FilterAllowedProjectsStep struct { // Note that hosts without specified projects are still accessible. func (s *FilterAllowedProjectsStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // Failover and capacity probe calls are not placed on behalf of a tenant project; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } if request.Spec.Data.ProjectID == "" { traceLog.Info("no project ID in request, skipping filter") return result, nil diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go index 53a4ac958..ccbaf44e5 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go @@ -323,3 +323,41 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }) } } + +func TestFilterAllowedProjectsStep_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host2 restricts to project-x; request is project-y → host2 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: "host2"}, + Spec: hv1.HypervisorSpec{AllowedProjects: []string{"project-x"}}, + }, + } + step := &FilterAllowedProjectsStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + ProjectID: "project-y", + SchedulerHints: map[string]any{"_nova_check_type": intent}, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer.go b/internal/scheduling/nova/plugins/filters/filter_external_customer.go index bcbf74716..b574e4ad1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer.go @@ -11,6 +11,7 @@ import ( "strings" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -36,10 +37,15 @@ type FilterExternalCustomerStep struct { func (s *FilterExternalCustomerStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - // Skip for failover reservation scheduling — domain restrictions don't apply - // since failover reservations are not tied to a specific customer domain. - if intent, err := request.GetIntent(); err == nil && intent == api.ReserveForFailoverIntent { - traceLog.Info("skipping external customer filter for failover reservation intent") + // Failover and capacity probe calls are not associated with a user domain; + // domain-based host restrictions don't apply. CR scheduling does carry a real domain + // and must be checked. + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.CapacityProbeIntent, + }, intent) { + traceLog.Info("skipping external customer filter for non-placement intent", "intent", intent) return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go index 290709f05..cb358e0c1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go @@ -545,3 +545,44 @@ func TestFilterExternalCustomerStepOpts_Validate(t *testing.T) { }) } } + +func TestFilterExternalCustomerStep_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // Domain matches external prefix; host1 lacks the exclusive trait → host1 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Traits: []string{"CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE"}}, + }, + } + step := &FilterExternalCustomerStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = FilterExternalCustomerStepOpts{CustomerDomainNamePrefixes: []string{"iaas-"}} + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + SchedulerHints: map[string]any{ + "domain_name": "iaas-customer", + "_nova_check_type": intent, + }, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index 8940c0d86..e2fadf981 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -66,6 +66,10 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa opts := request.GetOptions() result := s.IncludeAllHostsFromRequest(request) + // Merge call-time options with static step config. + ignoreAllocations := s.Options.IgnoreAllocations || opts.AssumeEmptyHosts + ignoredReservationTypes := slices.Concat(s.Options.IgnoredReservationTypes, opts.IgnoredReservationTypes) + // This map holds the free resources per host. freeResourcesByHost := make(map[string]map[hv1.ResourceName]resource.Quantity) @@ -87,7 +91,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa } // Subtract allocated resources (skip when ignoring allocations for empty-datacenter capacity queries). - if !s.Options.IgnoreAllocations { + if !ignoreAllocations { for resourceName, allocated := range hv.Status.Allocation { free, ok := freeResourcesByHost[hv.Name][resourceName] if !ok { @@ -110,7 +114,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa } for _, reservation := range reservations.Items { // Check if this reservation type should be ignored — applies regardless of ready state. - if slices.Contains(s.Options.IgnoredReservationTypes, reservation.Spec.Type) { + if slices.Contains(ignoredReservationTypes, reservation.Spec.Type) { traceLog.Debug("ignoring reservation type", "type", reservation.Spec.Type, "reservation", reservation.Name) continue } @@ -169,14 +173,23 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa // 2. During live migrations or other operations, we don't want to use failover capacity. // Note: we cannot use failover reservations from other VMs, as that can invalidate our HA guarantees. intent, err := request.GetIntent() - if err == nil && intent == api.EvacuateIntent { - if reservation.Status.FailoverReservation != nil { - if _, contained := reservation.Status.FailoverReservation.Allocations[request.Spec.Data.InstanceUUID]; contained { - traceLog.Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", - "reservation", reservation.Name, - "instanceUUID", request.Spec.Data.InstanceUUID) - continue + if err == nil { + switch intent { + case api.EvacuateIntent: + if reservation.Status.FailoverReservation != nil { + if _, contained := reservation.Status.FailoverReservation.Allocations[request.Spec.Data.InstanceUUID]; contained { + traceLog.Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", + "reservation", reservation.Name, + "instanceUUID", request.Spec.Data.InstanceUUID) + continue + } } + case api.ReuseFailoverReservationIntent: + // Reuse check: the reservation already pre-blocks the right capacity for this VM. + // Don't subtract it from free capacity to avoid double-counting. + traceLog.Debug("skipping failover reservation block for reuse compatibility check", + "reservation", reservation.Name) + continue } } traceLog.Debug("processing failover reservation", "reservation", reservation.Name) @@ -209,7 +222,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa // Oversize spec-only: if a pending VM is larger than the remaining slot, block its full size. // // FailoverReservations: block = Spec.Resources (always fully blocked). - resourcesToBlock := resv.UnusedReservationCapacity(&reservation, s.Options.IgnoreAllocations) + resourcesToBlock := resv.UnusedReservationCapacity(&reservation, ignoreAllocations) // Block the calculated resources on each host for host := range hostsToBlock { diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go index 8b55556f8..b1b659e59 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go @@ -248,7 +248,7 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { //nolint:unparam // vcpus varies in real usage +func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { return newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup, vcpus, memory, "", evacuation, hosts) } @@ -961,6 +961,41 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { } } +func TestFilterHasEnoughCapacity_AssumeEmptyHosts(t *testing.T) { + scheme := buildTestScheme(t) + + // host1: 8 CPU total, 6 CPU allocated to running VMs → 2 free; request needs 4 → fails normally. + // With AssumeEmptyHosts: allocations ignored → 8 free → passes. + hypervisors := []*hv1.Hypervisor{ + newHypervisor("host1", "8", "6", "32Gi", "0"), + } + request := newNovaRequest("vm", "proj", "m1.large", "gp", 4, "1Gi", false, []string{"host1"}) + + objects := make([]client.Object, 0, len(hypervisors)) + for _, h := range hypervisors { + objects = append(objects, h.DeepCopy()) + } + + step := &FilterHasEnoughCapacity{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = FilterHasEnoughCapacityOpts{} + + // Without AssumeEmptyHosts: host1 filtered (only 2 CPU free). + resultWithout, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + assertActivations(t, resultWithout.Activations, []string{}, []string{"host1"}) + + // With AssumeEmptyHosts via call-time options: allocations ignored → host1 passes. + request.Options = scheduling.Options{AssumeEmptyHosts: true} + resultWith, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + assertActivations(t, resultWith.Activations, []string{"host1"}, []string{}) +} + func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) { scheme := buildTestScheme(t) @@ -986,7 +1021,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) step := &FilterHasEnoughCapacity{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - // Ignore CR reservations via pipeline-level opts (call-time opts.IgnoredReservationTypes removed in favour of YAML params). + // Ignore CR reservations via YAML step opts (static, configured per pipeline step). step.Options = FilterHasEnoughCapacityOpts{ LockReserved: true, IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, @@ -999,6 +1034,49 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) assertActivations(t, result.Activations, []string{"host1"}, []string{"host2"}) } +func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTimeMerge(t *testing.T) { + scheme := buildTestScheme(t) + + // Same two-host setup: CR on host1, Failover on host2. + // Each blocks 4 CPU, leaving 4 free; request needs 8 CPU so both hosts fail without ignoring. + // Verify that call-time opts.IgnoredReservationTypes is merged with (not instead of) YAML opts. + hypervisors := []*hv1.Hypervisor{ + newHypervisor("host1", "16", "8", "32Gi", "16Gi"), + newHypervisor("host2", "16", "8", "32Gi", "16Gi"), + } + reservations := []*v1alpha1.Reservation{ + newCommittedReservation("cr-res", "host1", "project-X", "m1.large", "gp-1", "4", "8Gi", nil, nil), + newFailoverReservation("failover-res", "host2", "4", "8Gi", map[string]string{"other-vm": "host3"}), + } + request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) + + objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) + for _, h := range hypervisors { + objects = append(objects, h.DeepCopy()) + } + for _, r := range reservations { + objects = append(objects, r.DeepCopy()) + } + + step := &FilterHasEnoughCapacity{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + // YAML opts ignore CR; call-time opts ignore Failover — both must be respected. + step.Options = FilterHasEnoughCapacityOpts{ + LockReserved: true, + IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, + } + request.Options = scheduling.Options{ + IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, + } + + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + // Both CR (host1) and Failover (host2) reservations ignored → both hosts pass. + assertActivations(t, result.Activations, []string{"host1", "host2"}, []string{}) +} + func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) { scheme := buildTestScheme(t) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go index 326864b9d..8c9045e34 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go @@ -8,6 +8,7 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -22,6 +23,15 @@ func (s *FilterInstanceGroupAffinityStep) Run( ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // Instance group hints are only present in user-initiated VM placement requests; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } ig := request.Spec.Data.InstanceGroup if ig == nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go index 7321747e3..e6e6a8803 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go @@ -352,3 +352,25 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }) } } + +func TestFilterInstanceGroupAffinityStep_SkipsForNonPlacementIntent(t *testing.T) { + // Affinity group on host1 only — host2 would normally be filtered. + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := newNovaRequest("vm", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ + Data: api.NovaInstanceGroup{Policy: "affinity", Hosts: []string{"host1"}}, + } + request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": intent} + step := &FilterInstanceGroupAffinityStep{} + + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go index 0dee29d9e..137ddf04f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go @@ -9,6 +9,7 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -25,6 +26,15 @@ func (s *FilterInstanceGroupAntiAffinityStep) Run( ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // Instance group hints are only present in user-initiated VM placement requests; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } ig := request.Spec.Data.InstanceGroup if ig == nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go index 931265b9b..63a9aff4a 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go @@ -547,3 +547,42 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }) } } + +func TestFilterInstanceGroupAntiAffinityStep_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host2 has the group member vm; max_server_per_host=1 → host2 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Instances: []hv1.Instance{{ID: "vm-existing"}}}, + }, + } + step := &FilterInstanceGroupAntiAffinityStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := newNovaRequest("vm-new", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ + Data: api.NovaInstanceGroup{ + Policy: "anti-affinity", + Members: []string{"vm-existing"}, + Rules: map[string]any{"max_server_per_host": 1}, + }, + } + request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": intent} + + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index 4e139f8d8..8bc0feace 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -88,15 +88,18 @@ func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.External traceLog.Info("skipping quota enforcement for non-consuming intent", "intent", intent) QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") return result, nil - case api.ReserveForFailoverIntent: + case api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent: traceLog.Info("skipping quota enforcement for failover reservation intent") QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") return result, nil case api.ReserveForCommittedResourceIntent: - // TODO: revisit whether committed resource reservation scheduling should also be quota-checked traceLog.Info("skipping quota enforcement for committed resource reservation intent") QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") return result, nil + case api.CapacityProbeIntent: + traceLog.Info("skipping quota enforcement for capacity probe intent") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") + return result, nil } } diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index 32ea16ad4..c96731e33 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -1087,3 +1087,36 @@ func TestQuotaEnforcementMetrics_RecordDecision_NilWarns(t *testing.T) { msg, got, buf.String()) } } + +func TestFilterQuotaEnforcement_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add v1alpha1 to scheme: %v", err) + } + // Enforce mode: quota fully consumed → all hosts would normally be filtered. + objects := []client.Object{ + &v1alpha1.ProjectQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-no-quota-az-1"}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: "project-no-quota", + AvailabilityZone: "az-1", + Quota: map[string]int64{"hw_version_gp_ram": 10, "hw_version_gp_cores": 10, "hw_version_gp_instances": 1}, + }, + Status: v1alpha1.ProjectQuotaStatus{ + PaygUsage: map[string]int64{"hw_version_gp_ram": 10, "hw_version_gp_cores": 10, "hw_version_gp_instances": 1}, + }, + }, + } + request := makeQuotaEnforcementRequest("project-no-quota", "az-1", "gp", 1, 1, map[string]any{"_nova_check_type": "capacity_probe"}) + step := &FilterQuotaEnforcement{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = FilterQuotaEnforcementOpts{Enforce: true} + + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != len(request.Hosts) { + t.Errorf("expected all %d hosts to pass, got %d", len(request.Hosts), len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go index 8922ab8c4..971848dcd 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go @@ -10,6 +10,7 @@ import ( "strings" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -102,6 +103,15 @@ func (s *FilterRequestedDestinationStep) processRequestedHost( // host filtering. func (s *FilterRequestedDestinationStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // The requested_destination hint is only set by Nova for user-directed placement; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } rd := request.Spec.Data.RequestedDestination if rd == nil { traceLog.Info("no requested_destination in request, skipping filter") diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go index 5a752160e..fab4c7aec 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go @@ -782,3 +782,40 @@ func TestFilterRequestedDestinationStep_Run_ClientError(t *testing.T) { t.Errorf("expected error when client fails, got none") } } + +func TestFilterRequestedDestinationStep_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // RequestedDestination forces host1 — host2 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, + } + step := &FilterRequestedDestinationStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ + Data: api.NovaRequestedDestination{Host: "host1"}, + }, + SchedulerHints: map[string]any{"_nova_check_type": intent}, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/reservations/capacity/config.go b/internal/scheduling/reservations/capacity/config.go index 264a0b59d..d4834c53e 100644 --- a/internal/scheduling/reservations/capacity/config.go +++ b/internal/scheduling/reservations/capacity/config.go @@ -46,8 +46,8 @@ func (c *Config) ApplyDefaults() { func DefaultConfig() Config { return Config{ ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, - TotalPipeline: "kvm-report-capacity", - PlaceablePipeline: "kvm-general-purpose-load-balancing-no-history", + TotalPipeline: "kvm-general-purpose-load-balancing", + PlaceablePipeline: "kvm-general-purpose-load-balancing", SchedulerURL: "http://localhost:8080/scheduler/nova/external", } } diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 1e69b22cb..257afe79e 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -529,6 +529,16 @@ func (c *Controller) probeScheduler( eligibleHosts = append(eligibleHosts, schedulerapi.ExternalSchedulerHost{ComputeHost: name}) } + // Total probe ignores all reservation blocks (raw hardware capacity). + // Placeable probe counts reservations as capacity blocks. + var ignoredReservationTypes []v1alpha1.ReservationType + if ignoreAllocations { + ignoredReservationTypes = []v1alpha1.ReservationType{ + v1alpha1.ReservationTypeCommittedResource, + v1alpha1.ReservationTypeFailover, + } + } + resp, err := c.schedulerClient.ScheduleReservation(ctx, reservations.ScheduleReservationRequest{ InstanceUUID: "capacity-" + flavor.Name, ProjectID: "cortex-capacity-probe", @@ -539,8 +549,11 @@ func (c *Controller) probeScheduler( AvailabilityZone: az, Pipeline: pipeline, EligibleHosts: eligibleHosts, + SchedulerHints: map[string]any{"_nova_check_type": string(schedulerapi.CapacityProbeIntent)}, }, scheduling.Options{ ReadOnly: true, + AssumeEmptyHosts: ignoreAllocations, + IgnoredReservationTypes: ignoredReservationTypes, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true, diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index c33d98be4..7d066bee6 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -848,3 +848,36 @@ func TestProbeScheduler_SubtractsReservationBlocksWhenNotIgnored(t *testing.T) { t.Errorf("placeable capacity = %d, want 1 (3 slots − 1 alloc − 1 reservation)", placeableCap) } } + +func TestProbeScheduler_SetsCapacityProbeIntent(t *testing.T) { + scheme := newTestScheme(t) + hv := newHypervisor("host-1", "az-a", 4096*1024*1024) + + var capturedReq schedulerapi.ExternalSchedulerRequest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + json.NewEncoder(w).Encode(schedulerapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}}) //nolint:errcheck + })) + defer srv.Close() + + c := NewController(fake.NewClientBuilder().WithScheme(scheme).Build(), Config{SchedulerURL: srv.URL}, nil) + hvByName := map[string]hv1.Hypervisor{"host-1": *hv} + flavor := compute.FlavorInGroup{Name: "test-flavor", MemoryMB: 4096} + + if _, _, _, err := c.probeScheduler(context.Background(), flavor, "az-a", "test-pipeline", hvByName, true, nil); err != nil { + t.Fatalf("probeScheduler failed: %v", err) + } + hint, err := capturedReq.Spec.Data.GetSchedulerHintStr("_nova_check_type") + if err != nil { + t.Fatalf("failed to get _nova_check_type hint: %v", err) + } + if hint != string(schedulerapi.CapacityProbeIntent) { + t.Errorf("capacity probe must set _nova_check_type=%q, got %q", schedulerapi.CapacityProbeIntent, hint) + } + if capturedReq.Spec.Data.ProjectID != "cortex-capacity-probe" { + t.Errorf("capacity probe must send ProjectID cortex-capacity-probe, got %q", capturedReq.Spec.Data.ProjectID) + } +} diff --git a/internal/scheduling/reservations/commitments/integration_test.go b/internal/scheduling/reservations/commitments/integration_test.go index 92fcc6246..ce968b4f4 100644 --- a/internal/scheduling/reservations/commitments/integration_test.go +++ b/internal/scheduling/reservations/commitments/integration_test.go @@ -1193,3 +1193,41 @@ func TestCRLifecycle(t *testing.T) { } }) } + +func TestCRScheduling_SetsReserveForCommittedResourceIntent(t *testing.T) { + // CR slot scheduling has a real project ID and must run tenant-context filters + // (filter_allowed_projects, filter_aggregate_metadata, etc.). Verify the intent is + // reserve_for_committed_resource so filters can identify the call type correctly. + var capturedReq schedulerdelegationapi.ExternalSchedulerRequest + var schedulerCalled bool + captureScheduler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + schedulerCalled = true + if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} + json.NewEncoder(w).Encode(resp) //nolint:errcheck + }) + + env := newIntgEnv(t, []client.Object{newTestFlavorKnowledge(), intgHypervisor("host-1")}, captureScheduler, nil) + defer env.close() + + cr := intgCR("test-cr", "commit-uuid-1", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + env.reconcileCR(t, cr.Name) + env.reconcileChildReservations(t, cr.Name) + + if !schedulerCalled { + t.Fatal("scheduler was never called — test did not exercise the scheduling path") + } + hint, err := capturedReq.Spec.Data.GetSchedulerHintStr("_nova_check_type") + if err != nil { + t.Fatalf("failed to get _nova_check_type hint: %v", err) + } + if hint != string(schedulerdelegationapi.ReserveForCommittedResourceIntent) { + t.Errorf("CR slot scheduling must set _nova_check_type=%q, got %q", schedulerdelegationapi.ReserveForCommittedResourceIntent, hint) + } +} diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index a145ec676..b7ad34d5f 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -435,6 +435,22 @@ func TestIntegration(t *testing.T) { ExpectedMinRes: 1, // Both HANA VMs can share reservation on host3 UseTraitsFilter: true, }, + { + Name: "HANA VM uses kvm-hana-bin-packing pipeline (trait:CUSTOM_HANA_EXCLUSIVE_HOST=required)", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-hana-exclusive-1", Name: "vm-hana-exclusive-1", Active: true}}, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}), + newHypervisor("host2", 16, 32, 0, 0, nil, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}), + }, + VMs: []reservations.VM{ + newVMWithExtraSpecs("vm-hana-exclusive-1", "m1.hana", "project-A", "host1", 8192, 4, + map[string]string{"trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required"}), + }, + FlavorRequirements: map[string]int{"m1.hana": 1}, + ExpectedMinRes: 1, + ExpectedMaxRes: 1, + VerifyVMReservation: []string{"vm-hana-exclusive-1"}, + UseTraitsFilter: true, + }, } for _, tc := range testCases { @@ -1153,7 +1169,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h }, { ObjectMeta: metav1.ObjectMeta{ - Name: PipelineReuseFailoverReservation, + Name: PipelineNewFailoverReservation, }, Spec: v1alpha1.PipelineSpec{ Type: v1alpha1.PipelineTypeFilterWeigher, @@ -1181,6 +1197,17 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h }, }, } + pipelines = append(pipelines, v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{Name: "kvm-hana-bin-packing"}, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{{Name: "kvm_failover_evacuation"}}, + }, + }) ctx := context.Background() for _, pipeline := range pipelines { @@ -1343,7 +1370,7 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, }, { ObjectMeta: metav1.ObjectMeta{ - Name: PipelineReuseFailoverReservation, + Name: PipelineNewFailoverReservation, }, Spec: v1alpha1.PipelineSpec{ Type: v1alpha1.PipelineTypeFilterWeigher, @@ -1371,6 +1398,18 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, }, }, } + pipelines = append(pipelines, v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{Name: "kvm-hana-bin-packing"}, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_has_requested_traits"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{{Name: "kvm_failover_evacuation"}}, + }, + }) ctx := context.Background() for _, pipeline := range pipelines { diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 5f5903de9..efd3876f3 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -8,6 +8,7 @@ import ( "fmt" "slices" "sort" + "strings" api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" @@ -17,22 +18,29 @@ import ( // Pipeline names for failover reservation scheduling const ( - // PipelineReuseFailoverReservation is used to check if a VM can reuse an existing reservation. - // It validates host compatibility without checking capacity (since reservation already has capacity). - PipelineReuseFailoverReservation = "kvm-valid-host-reuse-failover-reservation" - // PipelineNewFailoverReservation is used to find a host for creating a new reservation. - // It validates host compatibility AND checks capacity. // Uses the general-purpose pipeline; LockReservations and SkipHistory are set via Options. PipelineNewFailoverReservation = "kvm-general-purpose-load-balancing" - - // PipelineAcknowledgeFailoverReservation is used to validate that a failover reservation - // is still valid for all its allocated VMs. It sends an evacuation-style scheduling request - // for each VM with only the reservation's host as the eligible target. - PipelineAcknowledgeFailoverReservation = "kvm-acknowledge-failover-reservation" ) -func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx context.Context, vm reservations.VM, allHypervisors []string, pipeline string, resSpec resolvedReservationSpec, opts scheduling.Options) ([]string, error) { +// DefaultFailoverOptions is the base scheduling.Options for all failover scheduling calls. +// Per-call overrides (ReadOnly, LockReservations) are applied on top. +var DefaultFailoverOptions = scheduling.Options{ + SkipHistory: true, + SkipInflight: true, + SkipCommittedResourceTracking: true, +} + +// inferFailoverPipeline returns the standard pipeline for a failover scheduling call based on +// the VM's flavor extra specs — the same HANA vs general-purpose split used by Nova placement. +func inferFailoverPipeline(extraSpecs map[string]string) string { + if strings.ToLower(extraSpecs["trait:CUSTOM_HANA_EXCLUSIVE_HOST"]) == "required" { + return "kvm-hana-bin-packing" + } + return "kvm-general-purpose-load-balancing" +} + +func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx context.Context, vm reservations.VM, allHypervisors []string, pipeline string, resSpec resolvedReservationSpec, intent v1alpha1.SchedulingIntent, opts scheduling.Options) ([]string, error) { logger := LoggerFromContext(ctx) // Build list of eligible hypervisors (excluding VM's current hypervisor) @@ -82,7 +90,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex Pipeline: pipeline, AvailabilityZone: vm.AvailabilityZone, SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForFailoverIntent), + "_nova_check_type": string(intent), api.HintKeyResourceGroup: resSpec.ResourceGroup(vm.FlavorName), }, } @@ -123,7 +131,9 @@ func (c *FailoverReservationController) tryReuseExistingReservation( logger := LoggerFromContext(ctx) - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineReuseFailoverReservation, resSpec, scheduling.Options{ReadOnly: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + reuseOpts := DefaultFailoverOptions + reuseOpts.ReadOnly = true + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, inferFailoverPipeline(vm.FlavorExtraSpecs), resSpec, api.ReuseFailoverReservationIntent, reuseOpts) if err != nil { logger.Error(err, "failed to get potential hypervisors for VM", "vmUUID", vm.UUID) return nil @@ -213,7 +223,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( VCPUs: vcpus, EligibleHosts: []api.ExternalSchedulerHost{{ComputeHost: reservationHost}}, IgnoreHosts: []string{vm.CurrentHypervisor}, - Pipeline: PipelineAcknowledgeFailoverReservation, + Pipeline: inferFailoverPipeline(flavorExtraSpecs), AvailabilityZone: vm.AvailabilityZone, SchedulerHints: map[string]any{"_nova_check_type": string(api.EvacuateIntent)}, } @@ -222,9 +232,12 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( "vmUUID", vm.UUID, "reservationHost", reservationHost, "vmCurrentHost", vm.CurrentHypervisor, - "pipeline", PipelineAcknowledgeFailoverReservation) + "pipeline", scheduleReq.Pipeline) - resp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, scheduling.Options{ReadOnly: true, LockReservations: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + evacuateOpts := DefaultFailoverOptions + evacuateOpts.ReadOnly = true + evacuateOpts.LockReservations = true + resp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, evacuateOpts) if err != nil { logger.Error(err, "failed to validate VM for reservation host", "vmUUID", vm.UUID, "reservationHost", reservationHost) return false, fmt.Errorf("failed to validate VM for reservation host: %w", err) @@ -266,7 +279,9 @@ func (c *FailoverReservationController) scheduleAndBuildNewFailoverReservation( // Get potential hypervisors from scheduler using the reservation spec resources // (which may be sized to the LargestFlavor from the flavor group) - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineNewFailoverReservation, resSpec, scheduling.Options{LockReservations: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + newResOpts := DefaultFailoverOptions + newResOpts.LockReservations = true + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineNewFailoverReservation, resSpec, api.ReserveForFailoverIntent, newResOpts) if err != nil { return nil, fmt.Errorf("failed to get potential hypervisors for VM: %w", err) } diff --git a/internal/scheduling/reservations/failover/reservation_scheduling_test.go b/internal/scheduling/reservations/failover/reservation_scheduling_test.go index c80ef4bf9..6ae2ab820 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling_test.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling_test.go @@ -5,14 +5,20 @@ package failover import ( "context" + "encoding/json" + "net/http" + "net/http/httptest" "testing" + novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) // ============================================================================ @@ -466,3 +472,103 @@ func buildSchedulingTestVMWithResources(uuid, hypervisor string, memoryMB, vcpus }, } } + +// captureSchedulerRequest spins up a test HTTP server that captures one scheduler request +// and returns a single host. The captured request is written into *out. +func captureSchedulerRequest(t *testing.T, host string, out *novaapi.ExternalSchedulerRequest) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(out); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resp := novaapi.ExternalSchedulerResponse{Hosts: []string{host}} + _ = json.NewEncoder(w).Encode(resp) //nolint:errcheck + })) +} + +// buildMinimalController returns a FailoverReservationController wired to a scheduler at the given URL. +func buildMinimalController(t *testing.T, schedulerURL string) *FailoverReservationController { + t.Helper() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add v1alpha1 to scheme: %v", err) + } + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("add hv1 to scheme: %v", err) + } + k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() + schedulerClient := reservations.NewSchedulerClient(schedulerURL + "/scheduler/nova/external") + config := FailoverConfig{} + config.ApplyDefaults() + return NewFailoverReservationController(k8sClient, nil, config, schedulerClient, nil) +} + +// ============================================================================ +// Test: scheduling options sent to the scheduler +// ============================================================================ + +func TestFailoverSchedulerOptions(t *testing.T) { + vm := buildSchedulingTestVM("vm-1", "host-1") + vm.AvailabilityZone = "az1" + resolved := resolveVMSpecForScheduling(context.Background(), vm, false, nil) + reservation := buildSchedulingTestReservation("res-1", "host-2", nil) + reservation.Status.FailoverReservation = &v1alpha1.FailoverReservationStatus{Allocations: map[string]string{}} + + tests := []struct { + name string + call func(c *FailoverReservationController, ctx context.Context) + wantReadOnly bool + wantLockReservations bool + }{ + { + // Read-only compatibility check — must not write scheduling state. + // Tenant-context filters must run so only hosts the VM can actually reach are returned. + name: "tryReuseExistingReservation", + call: func(c *FailoverReservationController, ctx context.Context) { + _ = c.tryReuseExistingReservation(ctx, vm, []v1alpha1.Reservation{reservation}, []string{"host-1", "host-2"}, resolved) + }, + wantReadOnly: true, + wantLockReservations: false, + }, + { + // Validates whether a VM can land on the reservation host during evacuation. + // LockReservations ensures failover capacity is not unlocked for other VMs. + // Tenant-context filters (aggregate metadata, allowed projects, instance group) must run. + name: "validateVMViaSchedulerEvacuation", + call: func(c *FailoverReservationController, ctx context.Context) { + _, _ = c.validateVMViaSchedulerEvacuation(ctx, vm, "host-2") //nolint:errcheck + }, + wantReadOnly: true, + wantLockReservations: true, + }, + { + // Finds a host for a new reservation — writes state, so not read-only. + // LockReservations ensures existing reservations are treated as unavailable. + // Tenant-context filters must run. + name: "scheduleAndBuildNewFailoverReservation", + call: func(c *FailoverReservationController, ctx context.Context) { + _, _ = c.scheduleAndBuildNewFailoverReservation(ctx, vm, []string{"host-1", "host-2"}, nil, nil, resolved) //nolint:errcheck + }, + wantReadOnly: false, + wantLockReservations: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var capturedReq novaapi.ExternalSchedulerRequest + server := captureSchedulerRequest(t, "host-2", &capturedReq) + defer server.Close() + + tt.call(buildMinimalController(t, server.URL), context.Background()) + + if capturedReq.Options.ReadOnly != tt.wantReadOnly { + t.Errorf("ReadOnly = %v, want %v", capturedReq.Options.ReadOnly, tt.wantReadOnly) + } + if capturedReq.Options.LockReservations != tt.wantLockReservations { + t.Errorf("LockReservations = %v, want %v", capturedReq.Options.LockReservations, tt.wantLockReservations) + } + }) + } +}