Implement in-flight reservations controller

cmd/manager/main.go

@@ -63,6 +63,7 @@ import (
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments"
 	commitmentsapi "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments/api"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover"
+	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/inflight"
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/quota"
 	"github.com/cobaltcore-dev/cortex/pkg/conf"
 	"github.com/cobaltcore-dev/cortex/pkg/monitoring"
@@ -475,6 +476,18 @@ func main() {
 			os.Exit(1)
 		}
 	}
+	if slices.Contains(mainConfig.EnabledControllers, "inflight-reservation-controller") {
+		setupLog.Info("enabling controller",
+			"controller", "inflight-reservation-controller")
+		config := conf.GetConfigOrDie[inflight.NovaVMClientConfig]()
+		vmClient := inflight.NewNovaVMClient(config)
+		controller := &inflight.Controller{Client: multiclusterClient, VMClient: vmClient}
+		if err := controller.SetupWithManager(ctx, mgr); err != nil {
+			setupLog.Error(err, "unable to create controller",
+				"controller", "inflight-reservation-controller")
+			os.Exit(1)
+		}
+	}
 	if slices.Contains(mainConfig.EnabledControllers, "nova-deschedulings-executor") {
 		setupLog.Info("enabling controller", "controller", "nova-deschedulings-executor")
 		executorConfig := conf.GetConfigOrDie[nova.DeschedulingsExecutorConfig]()

helm/bundles/cortex-nova/templates/alerts.yaml

@@ -303,6 +303,43 @@ spec:
           configuration. It is recommended to investigate the
           pipeline status and logs for more details.
 
+    - alert: CortexNovaInFlightReservationsBacklogGrowing
+      # Alert when the number of in-flight reservations stuck with
+      # reason=InstanceNotFound is trending upwards over a long window,
+      # rather than staying roughly the same or draining down. A healthy
+      # scheduler-hypervisor loop should quickly resolve in-flight
+      # reservations once the VM shows up on any hypervisor; a sustained
+      # positive derivative indicates the backlog is not being worked off.
+      #
+      # deriv() over 1h returns the per-second slope of the gauge; multiplied
+      # by 3600 it becomes "expected growth per hour". We require this to be
+      # meaningfully positive AND the current count to be non-trivial, so we
+      # don't page on a single lingering reservation.
+      expr: |
+        deriv(cortex_reservation_state{domain="nova",type="InFlightReservation",state="unknown",reason="InstanceNotFound"}[1h]) * 3600 > 5
+        and
+        cortex_reservation_state{domain="nova",type="InFlightReservation",state="unknown",reason="InstanceNotFound"} > 10
+      for: 2h
+      labels:
+        context: reservations
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/unready
+      annotations:
+        summary: "In-flight reservation backlog with reason=InstanceNotFound keeps growing"
+        description: >
+          The number of in-flight reservations that have not yet observed
+          their VM on any hypervisor (reason=InstanceNotFound) has been
+          increasing steadily for at least two hours. Normally these
+          reservations resolve once the hypervisor operator reports the
+          instance; a sustained positive trend suggests either VMs are not
+          landing on hypervisors, or the hypervisor status feed is stalled,
+          or the in-flight reservation controller is not pruning cleanly.
+          Investigate the hypervisor operator, the nova build pipeline, and
+          the in-flight reservation controller logs.
+
     {{- if .Values.kvm.enabled }}
     - alert: CortexNovaDoesntFindValidKVMHosts
       expr: sum by (az, hvtype) (increase(cortex_vm_faults{hvtype=~"CH|QEMU",faultmsg=~".*No valid host was found.*",faultmsg!~".*No such host.*"}[5m])) > 0

helm/bundles/cortex-nova/templates/kpis.yaml

@@ -146,6 +146,21 @@ spec:
 ---
 apiVersion: cortex.cloud/v1alpha1
 kind: KPI
+metadata:
+  name: cortex-nova-reservation-state
+spec:
+  schedulingDomain: nova
+  impl: reservation_state_kpi
+  opts:
+    reservationSchedulingDomain: nova
+  description: |
+    This KPI tracks the state of reservation resources managed by cortex,
+    labelled by reservation type (e.g. InFlightReservation) and the reason
+    on the Ready condition. It is used to alert on sustained increases of
+    unready in-flight reservations (e.g. reason=InstanceNotFound).
+---
+apiVersion: cortex.cloud/v1alpha1
+kind: KPI
 metadata:
   name: vmware-project-utilization
 spec:

helm/bundles/cortex-nova/values.yaml

@@ -138,6 +138,7 @@ cortex-scheduling-controllers:
         component: nova-scheduling
     enabledControllers:
       - nova-pipeline-controllers
+      - inflight-reservation-controller
       - nova-deschedulings-executor
       - hypervisor-overcommit-controller
       - committed-resource-reservations-controller

internal/knowledge/kpis/plugins/deployment/reservation_state.go

@@ -0,0 +1,117 @@
+// Copyright SAP SE
+// SPDX-License-Identifier: Apache-2.0
+
+package deployment
+
+import (
+	"context"
+	"time"
+
+	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
+	"github.com/cobaltcore-dev/cortex/internal/knowledge/db"
+	"github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins"
+	"github.com/cobaltcore-dev/cortex/pkg/conf"
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var reservationStateKPILogger = ctrl.Log.WithName("reservation-state-kpi")
+
+type ReservationStateKPIOpts struct {
+	// The scheduling domain to filter reservations by.
+	ReservationSchedulingDomain v1alpha1.SchedulingDomain `json:"reservationSchedulingDomain"`
+}
+
+// KPI observing the state of reservation resources managed by cortex.
+// Metrics are labeled by the reservation type (e.g. InFlightReservation,
+// CommittedResourceReservation, FailoverReservation) so that operators can
+// alert on specific reservation kinds. The state label mirrors the
+// v1alpha1.ReservationConditionReady condition status and the reason label
+// carries the condition's Reason so that alerts can target specific failure
+// modes (e.g. reason="InstanceNotFound" for in-flight reservations whose VM
+// has not spawned on any hypervisor yet).
+type ReservationStateKPI struct {
+	// Common base for all KPIs that provides standard functionality.
+	plugins.BaseKPI[ReservationStateKPIOpts]
+
+	// Prometheus descriptor for the reservation state metric.
+	counter *prometheus.Desc
+}
+
+func (ReservationStateKPI) GetName() string { return "reservation_state_kpi" }
+
+// Initialize the KPI.
+func (k *ReservationStateKPI) Init(db *db.DB, client client.Client, opts conf.RawOpts) error {
+	if err := k.BaseKPI.Init(db, client, opts); err != nil {
+		return err
+	}
+	k.counter = prometheus.NewDesc(
+		"cortex_reservation_state",
+		"State of cortex managed reservations",
+		[]string{"domain", "type", "state", "reason"},
+		nil,
+	)
+	return nil
+}
+
+// Conform to the prometheus collector interface by providing the descriptor.
+func (k *ReservationStateKPI) Describe(ch chan<- *prometheus.Desc) { ch <- k.counter }
+
+// Collect the reservation state metrics.
+func (k *ReservationStateKPI) Collect(ch chan<- prometheus.Metric) {
+	// Bound the list call so a slow API server can't hang the Prometheus
+	// scrape indefinitely; if it fails we log so the disappearance of the
+	// reservation metric is not silent.
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	// Get all reservations. The scheduling domain filter is applied per item
+	// since a Reservation is cluster-scoped and may cover multiple domains.
+	reservationList := &v1alpha1.ReservationList{}
+	if err := k.Client.List(ctx, reservationList); err != nil {
+		reservationStateKPILogger.Error(err, "Failed to list reservations")
+		return
+	}
+	// Aggregate counts by (type, state, reason) so that we emit one time
+	// series per bucket rather than one per reservation. This keeps metric
+	// cardinality bounded regardless of how many reservations exist.
+	type bucket struct {
+		reservationType string
+		state           string
+		reason          string
+	}
+	counts := map[bucket]float64{}
+	for _, r := range reservationList.Items {
+		if r.Spec.SchedulingDomain != k.Options.ReservationSchedulingDomain {
+			continue
+		}
+		state := "unknown"
+		reason := ""
+		cond := meta.FindStatusCondition(r.Status.Conditions, v1alpha1.ReservationConditionReady)
+		if cond != nil {
+			reason = cond.Reason
+			switch cond.Status {
+			case metav1.ConditionTrue:
+				state = "ready"
+			case metav1.ConditionFalse:
+				state = "error"
+			default:
+				state = "unknown"
+			}
+		}
+		counts[bucket{
+			reservationType: string(r.Spec.Type),
+			state:           state,
+			reason:          reason,
+		}]++
+	}
+	for b, v := range counts {
+		ch <- prometheus.MustNewConstMetric(
+			k.counter, prometheus.GaugeValue, v,
+			string(k.Options.ReservationSchedulingDomain),
+			b.reservationType, b.state, b.reason,
+		)
+	}
+}