diff --git a/cmd/scheduling-quality-exporter/main.go b/cmd/scheduling-quality-exporter/main.go new file mode 100644 index 000000000..204a9fe63 --- /dev/null +++ b/cmd/scheduling-quality-exporter/main.go @@ -0,0 +1,111 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/cobaltcore-dev/cortex/internal/exporters/schedulingquality/collector" + "github.com/cobaltcore-dev/cortex/internal/exporters/schedulingquality/nova" +) + +func main() { + var ( + listenAddr = flag.String("listen-address", ":9199", "address to listen on for metrics") + metricsPath = flag.String("metrics-path", "/metrics", "path under which to expose metrics") + novaEndpoint = flag.String("nova-endpoint", "", "OpenStack Nova API endpoint (optional; reads OS_AUTH_URL env if unset)") + scrapeInterval = flag.Duration("scrape-interval", 30*time.Second, "minimum interval between Nova API calls") + kubeconfig = flag.String("kubeconfig", "", "path to kubeconfig (uses in-cluster config if unset)") + isNovaDisabled = flag.Bool("disable-nova", false, "disable Nova flavor lookups; use fixed minimum placeable unit") + minCPU = flag.Int64("min-cpu", 1, "minimum placeable CPU cores (used when Nova is disabled)") + minMemoryMB = flag.Int64("min-memory-mb", 512, "minimum placeable memory in MB (used when Nova is disabled)") + logLevel = flag.String("log-level", "info", "log level (debug, info, warn, error)") + ) + flag.Parse() + + level := slog.LevelInfo + switch *logLevel { + case "debug": + level = slog.LevelDebug + case "warn": + level = slog.LevelWarn + case "error": + level = slog.LevelError + } + logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: level})) + slog.SetDefault(logger) + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + var novaClient *nova.Client + if !*isNovaDisabled { + endpoint := *novaEndpoint + if endpoint == "" { + endpoint = os.Getenv("OS_AUTH_URL") + } + if endpoint != "" { + var err error + novaClient, err = nova.NewClientFromEnv() + handleError(err, "failed to create nova client") + } + } + + k8sClient, err := collector.NewK8sClient(*kubeconfig) + handleError(err, "failed to create kubernetes client") + + c := collector.New(collector.Options{ + ScrapeInterval: *scrapeInterval, + NovaClient: novaClient, + K8sClient: k8sClient, + MinCPU: *minCPU, + MinMemoryBytes: *minMemoryMB * 1024 * 1024, + }) + prometheus.MustRegister(c) + + mux := http.NewServeMux() + mux.Handle(*metricsPath, promhttp.Handler()) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, "ok") + }) + + srv := &http.Server{ + Addr: *listenAddr, + Handler: mux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 30 * time.Second, + } + + go func() { + slog.Info("starting cortex-scheduling-quality-exporter", "address", *listenAddr, "path", *metricsPath) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + handleError(err, "server error") + } + }() + + <-ctx.Done() + slog.Info("shutting down") + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + _ = srv.Shutdown(shutdownCtx) +} + +func handleError(err error, message string) { + if err != nil { + slog.Error(message, "error", err) + os.Exit(1) + } +} diff --git a/helm/library/cortex-scheduling-quality-exporter/Chart.yaml b/helm/library/cortex-scheduling-quality-exporter/Chart.yaml new file mode 100644 index 000000000..992c155e2 --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: cortex-scheduling-quality-exporter +description: Prometheus exporter for Cortex scheduling quality and resource efficiency metrics +type: application +version: 0.1.0 +appVersion: "sha-9f115a57" +keywords: + - prometheus + - exporter + - openstack + - cortex + - scheduling +sources: + - https://github.com/cobaltcore-dev/cortex +maintainers: + - name: cobaltcore-dev +dependencies: [] diff --git a/helm/library/cortex-scheduling-quality-exporter/templates/_helpers.tpl b/helm/library/cortex-scheduling-quality-exporter/templates/_helpers.tpl new file mode 100644 index 000000000..82ca6e7bc --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/templates/_helpers.tpl @@ -0,0 +1,72 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cortex-scheduling-quality-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "cortex-scheduling-quality-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart label. +*/}} +{{- define "cortex-scheduling-quality-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels. +*/}} +{{- define "cortex-scheduling-quality-exporter.labels" -}} +helm.sh/chart: {{ include "cortex-scheduling-quality-exporter.chart" . }} +{{ include "cortex-scheduling-quality-exporter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels. +*/}} +{{- define "cortex-scheduling-quality-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cortex-scheduling-quality-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +ServiceAccount name. +*/}} +{{- define "cortex-scheduling-quality-exporter.serviceAccountName" -}} +{{- default (include "cortex-scheduling-quality-exporter.fullname" .) .Values.serviceAccount.name }} +{{- end }} + +{{/* +OpenStack credentials Secret name. +*/}} +{{- define "cortex-scheduling-quality-exporter.osSecretName" -}} +{{- include "cortex-scheduling-quality-exporter.fullname" . }}-openstack +{{- end }} + +{{/* +Returns true if Nova integration is enabled (all required openstack fields are set). +*/}} +{{- define "cortex-scheduling-quality-exporter.novaEnabled" -}} +{{- if and .Values.openstack.authURL .Values.openstack.username .Values.openstack.password .Values.openstack.projectName -}} +true +{{- end }} +{{- end }} diff --git a/helm/library/cortex-scheduling-quality-exporter/templates/clusterrole.yaml b/helm/library/cortex-scheduling-quality-exporter/templates/clusterrole.yaml new file mode 100644 index 000000000..ee06c3b7e --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/templates/clusterrole.yaml @@ -0,0 +1,30 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "cortex-scheduling-quality-exporter.fullname" . }} + labels: + {{- include "cortex-scheduling-quality-exporter.labels" . | nindent 4 }} +rules: + - apiGroups: + - openstack.sapcloud.io + resources: + - hypervisors + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "cortex-scheduling-quality-exporter.fullname" . }} + labels: + {{- include "cortex-scheduling-quality-exporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "cortex-scheduling-quality-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "cortex-scheduling-quality-exporter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/helm/library/cortex-scheduling-quality-exporter/templates/deployment.yaml b/helm/library/cortex-scheduling-quality-exporter/templates/deployment.yaml new file mode 100644 index 000000000..3bd50907f --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/templates/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "cortex-scheduling-quality-exporter.fullname" . }} + labels: + {{- include "cortex-scheduling-quality-exporter.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "cortex-scheduling-quality-exporter.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.port | quote }} + prometheus.io/path: {{ .Values.config.metricsPath | quote }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "cortex-scheduling-quality-exporter.labels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "cortex-scheduling-quality-exporter.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: exporter + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --listen-address=:{{ .Values.port }} + - --metrics-path={{ .Values.config.metricsPath }} + - --scrape-interval={{ .Values.config.scrapeInterval }} + - --log-level={{ .Values.config.logLevel }} + {{- if not (include "cortex-scheduling-quality-exporter.novaEnabled" .) }} + - --disable-nova + {{- end }} + {{- if include "cortex-scheduling-quality-exporter.novaEnabled" . }} + envFrom: + - secretRef: + name: {{ include "cortex-scheduling-quality-exporter.osSecretName" . }} + {{- end }} + ports: + - name: metrics + containerPort: {{ .Values.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 5 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 5 + periodSeconds: 15 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/library/cortex-scheduling-quality-exporter/templates/secret.yaml b/helm/library/cortex-scheduling-quality-exporter/templates/secret.yaml new file mode 100644 index 000000000..d85b08664 --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if include "cortex-scheduling-quality-exporter.novaEnabled" . }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "cortex-scheduling-quality-exporter.osSecretName" . }} + labels: + {{- include "cortex-scheduling-quality-exporter.labels" . | nindent 4 }} +type: Opaque +stringData: + OS_AUTH_URL: {{ .Values.openstack.authURL | quote }} + OS_USERNAME: {{ .Values.openstack.username | quote }} + OS_PASSWORD: {{ .Values.openstack.password | quote }} + OS_PROJECT_NAME: {{ .Values.openstack.projectName | quote }} + OS_USER_DOMAIN_NAME: {{ .Values.openstack.userDomainName | quote }} + OS_PROJECT_DOMAIN_NAME: {{ .Values.openstack.projectDomainName | quote }} +{{- end }} diff --git a/helm/library/cortex-scheduling-quality-exporter/templates/service.yaml b/helm/library/cortex-scheduling-quality-exporter/templates/service.yaml new file mode 100644 index 000000000..911a6b773 --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "cortex-scheduling-quality-exporter.fullname" . }} + labels: + {{- include "cortex-scheduling-quality-exporter.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.port }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "cortex-scheduling-quality-exporter.selectorLabels" . | nindent 4 }} diff --git a/helm/library/cortex-scheduling-quality-exporter/templates/serviceaccount.yaml b/helm/library/cortex-scheduling-quality-exporter/templates/serviceaccount.yaml new file mode 100644 index 000000000..94e60de2f --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "cortex-scheduling-quality-exporter.serviceAccountName" . }} + labels: + {{- include "cortex-scheduling-quality-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} diff --git a/helm/library/cortex-scheduling-quality-exporter/templates/servicemonitor.yaml b/helm/library/cortex-scheduling-quality-exporter/templates/servicemonitor.yaml new file mode 100644 index 000000000..bee598d7e --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/templates/servicemonitor.yaml @@ -0,0 +1,21 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "cortex-scheduling-quality-exporter.fullname" . }} + namespace: {{ .Values.serviceMonitor.namespace | default .Release.Namespace }} + labels: + {{- include "cortex-scheduling-quality-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "cortex-scheduling-quality-exporter.selectorLabels" . | nindent 6 }} + endpoints: + - port: metrics + path: {{ .Values.config.metricsPath }} + interval: {{ .Values.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} +{{- end }} diff --git a/helm/library/cortex-scheduling-quality-exporter/templates/verticalpodautoscaler.yaml b/helm/library/cortex-scheduling-quality-exporter/templates/verticalpodautoscaler.yaml new file mode 100644 index 000000000..f372036e2 --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/templates/verticalpodautoscaler.yaml @@ -0,0 +1,21 @@ +{{- if .Values.vpa.enabled }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: {{ include "cortex-scheduling-quality-exporter.fullname" . }} + labels: + {{- include "cortex-scheduling-quality-exporter.labels" . | nindent 4 }} +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "cortex-scheduling-quality-exporter.fullname" . }} + updatePolicy: + updateMode: {{ .Values.vpa.updateMode | quote }} + resourcePolicy: + containerPolicies: + - containerName: exporter + controlledResources: + - cpu + - memory +{{- end }} diff --git a/helm/library/cortex-scheduling-quality-exporter/values.yaml b/helm/library/cortex-scheduling-quality-exporter/values.yaml new file mode 100644 index 000000000..4545773f9 --- /dev/null +++ b/helm/library/cortex-scheduling-quality-exporter/values.yaml @@ -0,0 +1,67 @@ +replicaCount: 1 + +image: + repository: ghcr.io/cobaltcore-dev/cortex-scheduling-quality-exporter + pullPolicy: IfNotPresent + tag: "" + +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" + +port: 9199 + +config: + metricsPath: /metrics + scrapeInterval: 30s + logLevel: info + +# OpenStack credentials for Nova flavor lookups. +# If all fields are empty, Nova integration is disabled (--disable-nova is passed). +openstack: + authURL: "" + username: "" + password: "" + projectName: "" + userDomainName: Default + projectDomainName: Default + +serviceAccount: + annotations: {} + name: "" + +podAnnotations: {} + +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + +service: + type: ClusterIP + +serviceMonitor: + enabled: false + namespace: "" + interval: 60s + scrapeTimeout: 30s + labels: {} + +vpa: + enabled: false + updateMode: Auto + +resources: {} + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/internal/exporters/schedulingquality/README.md b/internal/exporters/schedulingquality/README.md new file mode 100644 index 000000000..01b30f72e --- /dev/null +++ b/internal/exporters/schedulingquality/README.md @@ -0,0 +1,160 @@ +# cortex-efficiency-exporter + +Prometheus exporter measuring scheduling quality and resource efficiency for Cortex-managed OpenStack infrastructure. +Reads Hypervisor CRDs from the [openstack-hypervisor-operator](https://github.com/cobaltcore-dev/openstack-hypervisor-operator) and (optionally) Nova flavors to compute per-hypervisor and cluster-wide metrics. + +## Metrics + +### Per-hypervisor + +| Metric | Labels | Description | +|--------|--------|-------------| +| `cortex_hypervisor_capacity` | hypervisor, resource, zone, building_block, group | Raw physical capacity | +| `cortex_hypervisor_effective_capacity` | hypervisor, resource, zone, building_block, group | Capacity after overcommit ratios | +| `cortex_hypervisor_allocation` | hypervisor, resource, zone, building_block, group | Current resource allocation | +| `cortex_hypervisor_free` | hypervisor, resource, zone, building_block, group | Free resources (effective_capacity − allocation) | +| `cortex_hypervisor_utilization_ratio` | hypervisor, resource, zone, building_block, group | allocation / effective_capacity | +| `cortex_hypervisor_balance_score` | hypervisor, zone, building_block, group | 1 − σ(fraction_i); 1.0 = balanced, 0.0 = imbalanced | +| `cortex_hypervisor_stranded` | hypervisor, resource, zone, building_block, group | Free capacity unusable due to other dimension exhausted | +| `cortex_hypervisor_instances` | hypervisor, zone, building_block, group | VM count | +| `cortex_hypervisor_maintenance` | hypervisor, zone, building_block, group, maintenance | 1 if in maintenance | + +### Cluster-wide + +| Metric | Labels | Description | +|--------|--------|-------------| +| `cortex_cluster_capacity` | resource | Total raw capacity | +| `cortex_cluster_effective_capacity` | resource | Total effective capacity | +| `cortex_cluster_allocation` | resource | Total allocation | +| `cortex_cluster_free` | resource | Total free capacity | +| `cortex_cluster_stranded` | resource | Total stranded resources | +| `cortex_cluster_fragmentation_ratio` | resource | stranded / free; 0 = no fragmentation | +| `cortex_cluster_balance_score` | - | Mean balance score across hypervisors | +| `cortex_cluster_hypervisor_count` | - | Number of hypervisors | +| `cortex_cluster_instance_count` | - | Total VM instances | +| `cortex_cluster_mean_utilization_ratio` | resource | Mean utilization across hypervisors | +| `cortex_cluster_mean_tetris_alignment` | - | Mean Tetris alignment score | + +## Formulas + +### Balance score (Kubernetes BalancedAllocation) + +``` +fraction_i = allocation_i / effective_capacity_i +score = 1 − stddev(fraction_1, ..., fraction_d) +``` + +Score of 1.0 means all resource dimensions are consumed proportionally. Score near 0.0 means one dimension is saturated while others are idle. + +### Stranded resources (Borg, EuroSys 2015) + +Resources on a host that cannot be used because other resources on the same host are depleted. Two modes: + +**Exhaustion mode**: If free capacity in dimension _j_ falls below the minimum placeable unit, all free capacity in dimensions _i ≠ j_ is stranded. + +**Proportional mode**: When no dimension is exhausted, stranding is computed via the bottleneck dimension: + +``` +placeable = min(free_cpu / min_cpu, free_mem / min_mem) +stranded_cpu = free_cpu − placeable × min_cpu +stranded_mem = free_mem − placeable × min_mem +``` + +The minimum placeable unit is derived from the smallest Nova flavor (queried live) or configured via `--min-cpu` and `--min-memory-mb`. + +### Fragmentation ratio (Tetris, SIGCOMM 2014; FGD, ATC 2023) + +``` +fragmentation_ratio = total_stranded / total_free +``` + +Per resource dimension across the cluster. 0.0 = no fragmentation; 1.0 = all free capacity is stranded. + +### Tetris alignment score (Tetris, SIGCOMM 2014) + +``` +alignment(host) = normalize(free_vec) · normalize(demand_vec) +``` + +The demand vector is the mean per-instance allocation across the cluster. Score of 1.0 means the host's free capacity shape perfectly matches the workload demand shape + +#### TODO + +1) Hypervisor overcommit ratio. Future: Dynamic +2) Hypervisor blocking density +3) Number of (live-)migrations +4) Number of "no valid host found". +5) Packing density (Protean, OSDI 2020): `allocated / (non_empty_hosts × host_capacity)`. Excludes failover-reserved hosts. Different from utilization_ratio. +6) Failover-reserved host count +7) Placement latency histogram per-VM p50/p99 from Nova + +## Usage + +``` +cortex-efficiency-exporter \ + --listen-address=:9199 \ + --kubeconfig=$HOME/.kube/config \ + --min-cpu=1 \ + --min-memory-mb=512 +``` + +### Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--listen-address` | `:9199` | Metrics endpoint | +| `--metrics-path` | `/metrics` | HTTP path | +| `--kubeconfig` | (in-cluster) | Path to kubeconfig | +| `--disable-nova` | `false` | Skip Nova flavor lookups | +| `--min-cpu` | `1` | Fallback min CPU cores | +| `--min-memory-mb` | `512` | Fallback min memory MB | +| `--scrape-interval` | `30s` | Nova API poll interval | +| `--nova-endpoint` | `$OS_AUTH_URL` | Nova endpoint | +| `--log-level` | `info` | Log verbosity | + +### In-cluster deployment + +The exporter needs RBAC to list Hypervisor CRDs (cluster-scoped): + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cortex-efficiency-exporter +rules: + - apiGroups: ["openstack.sapcloud.io"] + resources: ["hypervisors"] + verbs: ["get", "list", "watch"] +``` + +For Nova integration, mount OpenStack credentials as environment variables (`OS_AUTH_URL`, `OS_USERNAME`, `OS_PASSWORD`, `OS_PROJECT_NAME`, `OS_USER_DOMAIN_NAME`, `OS_PROJECT_DOMAIN_NAME`). + +## Example Grafana queries + +```promql +# Cluster-wide CPU fragmentation ratio +cortex_cluster_fragmentation_ratio{resource="cpu"} + +# Top 10 hypervisors by stranded memory (GB) +topk(10, cortex_hypervisor_stranded{resource="memory"} / 1e9) + +# Hypervisors with balance score below 0.5 (high stranding risk) +cortex_hypervisor_balance_score < 0.5 + +# Mean cluster utilization over time +cortex_cluster_mean_utilization_ratio + +# Total stranded CPU cores across cluster +cortex_cluster_stranded{resource="cpu"} + +# NUMA cell imbalance detection +cortex_cell_balance_score < 0.6 +``` + +## References + +- Verma et al., "Large-scale cluster management at Google with Borg," EuroSys 2015 +- Grandl et al., "Multi-Resource Packing for Cluster Schedulers" (Tetris), SIGCOMM 2014 +- Weng et al., "Beware of Fragmentation: Scheduling GPU-Sharing Workloads with Fragmentation Gradient Descent" (FGD), ATC 2023 +- Ghodsi et al., "Dominant Resource Fairness," NSDI 2011 +- Kubernetes `NodeResourcesBalancedAllocation` scoring plugin diff --git a/internal/exporters/schedulingquality/collector/collector.go b/internal/exporters/schedulingquality/collector/collector.go new file mode 100644 index 000000000..05cfe1da5 --- /dev/null +++ b/internal/exporters/schedulingquality/collector/collector.go @@ -0,0 +1,521 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package collector + +import ( + "context" + "log/slog" + "sync" + "time" + + hvv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/cobaltcore-dev/cortex/internal/exporters/schedulingquality/nova" +) + +type Options struct { + ScrapeInterval time.Duration + NovaClient *nova.Client + K8sClient client.Client + MinCPU int64 // fallback: minimum placeable CPU cores + MinMemoryBytes int64 // fallback: minimum placeable memory bytes +} + +type Collector struct { + opts Options + + mu sync.Mutex + lastScrape time.Time + cachedMin minPlaceableUnit + flavorsDone bool +} + +func New(opts Options) *Collector { + return &Collector{ + opts: opts, + cachedMin: minPlaceableUnit{ + CPU: float64(opts.MinCPU), + Memory: float64(opts.MinMemoryBytes), + }, + } +} + +const ( + ns = "cortex" + + labelHypervisor = "hypervisor" + labelResource = "resource" + labelZone = "zone" + labelBuildingBlock = "building_block" + labelGroup = "group" + labelCellID = "cell_id" + labelMaintenance = "maintenance" +) + +var ( + descCapacity = prometheus.NewDesc( + prometheus.BuildFQName(ns, "hypervisor", "capacity"), + "Raw physical capacity of a hypervisor by resource dimension.", + []string{labelHypervisor, labelResource, labelZone, labelBuildingBlock, labelGroup}, nil, + ) + descEffectiveCapacity = prometheus.NewDesc( + prometheus.BuildFQName(ns, "hypervisor", "effective_capacity"), + "Effective capacity of a hypervisor after overcommit ratios.", + []string{labelHypervisor, labelResource, labelZone, labelBuildingBlock, labelGroup}, nil, + ) + descAllocation = prometheus.NewDesc( + prometheus.BuildFQName(ns, "hypervisor", "allocation"), + "Current resource allocation on a hypervisor.", + []string{labelHypervisor, labelResource, labelZone, labelBuildingBlock, labelGroup}, nil, + ) + descFree = prometheus.NewDesc( + prometheus.BuildFQName(ns, "hypervisor", "free"), + "Free resources on a hypervisor (effective_capacity - allocation).", + []string{labelHypervisor, labelResource, labelZone, labelBuildingBlock, labelGroup}, nil, + ) + descUtilization = prometheus.NewDesc( + prometheus.BuildFQName(ns, "hypervisor", "utilization_ratio"), + "Resource utilization ratio (allocation / effective_capacity).", + []string{labelHypervisor, labelResource, labelZone, labelBuildingBlock, labelGroup}, nil, + ) + descBalanceScore = prometheus.NewDesc( + prometheus.BuildFQName(ns, "hypervisor", "balance_score"), + "Balanced allocation score (1 - stddev of per-dimension utilization fractions). "+ + "1.0 = perfectly balanced; 0.0 = maximally imbalanced. "+ + "Based on the Kubernetes NodeResourcesBalancedAllocation scoring plugin.", + []string{labelHypervisor, labelZone, labelBuildingBlock, labelGroup}, nil, + ) + descStranded = prometheus.NewDesc( + prometheus.BuildFQName(ns, "hypervisor", "stranded"), + "Stranded resources on a hypervisor: free capacity that cannot be consumed because "+ + "another resource dimension is exhausted or insufficient for the minimum placeable unit. "+ + "Borg (EuroSys 2015) defines stranded resources as resources on a machine which cannot be "+ + "used because other resources on the same machine are depleted.", + []string{labelHypervisor, labelResource, labelZone, labelBuildingBlock, labelGroup}, nil, + ) + descInstances = prometheus.NewDesc( + prometheus.BuildFQName(ns, "hypervisor", "instances"), + "Number of VM instances on a hypervisor.", + []string{labelHypervisor, labelZone, labelBuildingBlock, labelGroup}, nil, + ) + descMaintenanceInfo = prometheus.NewDesc( + prometheus.BuildFQName(ns, "hypervisor", "maintenance"), + "1 if the hypervisor is in maintenance mode; 0 otherwise.", + []string{labelHypervisor, labelZone, labelBuildingBlock, labelGroup, labelMaintenance}, nil, + ) + + // --- Cluster-wide metric descriptors --- + + descClusterCapacity = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "capacity"), + "Total raw capacity across all hypervisors.", + []string{labelResource}, nil, + ) + descClusterEffectiveCapacity = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "effective_capacity"), + "Total effective capacity across all hypervisors.", + []string{labelResource}, nil, + ) + descClusterAllocation = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "allocation"), + "Total allocation across all hypervisors.", + []string{labelResource}, nil, + ) + descClusterFree = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "free"), + "Total free capacity across all hypervisors.", + []string{labelResource}, nil, + ) + descClusterStranded = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "stranded"), + "Total stranded resources across all hypervisors.", + []string{labelResource}, nil, + ) + descClusterFragmentationRatio = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "fragmentation_ratio"), + "Cluster-wide fragmentation ratio (total_stranded / total_free) per resource dimension. "+ + "0.0 = no fragmentation; 1.0 = all free capacity is stranded. "+ + "Based on Tetris (SIGCOMM 2014) and FGD (ATC 2023) fragmentation analysis.", + []string{labelResource}, nil, + ) + descClusterBalanceScore = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "balance_score"), + "Cluster-wide balanced allocation score (aggregate across all hypervisors).", + nil, nil, + ) + descClusterHypervisorCount = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "hypervisor_count"), + "Total number of hypervisors.", + nil, nil, + ) + descClusterInstanceCount = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "instance_count"), + "Total number of VM instances across all hypervisors.", + nil, nil, + ) + descClusterMeanUtilization = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "mean_utilization_ratio"), + "Mean utilization ratio across all hypervisors per resource dimension.", + []string{labelResource}, nil, + ) + descClusterTetrisAlignment = prometheus.NewDesc( + prometheus.BuildFQName(ns, "cluster", "mean_tetris_alignment"), + "Mean Tetris alignment score across all hypervisors. "+ + "Measures how well free capacity shape matches the average workload demand shape. "+ + "Based on Tetris (SIGCOMM 2014): alignment(host) = normalized_free · normalized_demand.", + nil, nil, + ) + + // --- Exporter health --- + + descScrapeSuccess = prometheus.NewDesc( + prometheus.BuildFQName(ns, "exporter", "scrape_success"), + "1 if the last scrape succeeded; 0 otherwise.", + nil, nil, + ) + descScrapeDuration = prometheus.NewDesc( + prometheus.BuildFQName(ns, "exporter", "scrape_duration_seconds"), + "Duration of the last scrape in seconds.", + nil, nil, + ) + descMinPlaceableCPU = prometheus.NewDesc( + prometheus.BuildFQName(ns, "exporter", "min_placeable_cpu_cores"), + "Minimum placeable CPU cores used for stranding computation.", + nil, nil, + ) + descMinPlaceableMemory = prometheus.NewDesc( + prometheus.BuildFQName(ns, "exporter", "min_placeable_memory_bytes"), + "Minimum placeable memory bytes used for stranding computation.", + nil, nil, + ) +) + +// Describe implements prometheus.Collector. +func (c *Collector) Describe(ch chan<- *prometheus.Desc) { + // Per-hypervisor. + ch <- descCapacity + ch <- descEffectiveCapacity + ch <- descAllocation + ch <- descFree + ch <- descUtilization + ch <- descBalanceScore + ch <- descStranded + ch <- descInstances + ch <- descMaintenanceInfo + + // Cluster-wide. + ch <- descClusterCapacity + ch <- descClusterEffectiveCapacity + ch <- descClusterAllocation + ch <- descClusterFree + ch <- descClusterStranded + ch <- descClusterFragmentationRatio + ch <- descClusterBalanceScore + ch <- descClusterHypervisorCount + ch <- descClusterInstanceCount + ch <- descClusterMeanUtilization + ch <- descClusterTetrisAlignment + + // Exporter. + ch <- descScrapeSuccess + ch <- descScrapeDuration + ch <- descMinPlaceableCPU + ch <- descMinPlaceableMemory +} + +// Collect implements prometheus.Collector. +func (c *Collector) Collect(ch chan<- prometheus.Metric) { + start := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Refresh minimum placeable unit from Nova flavors. + c.refreshMinPlaceableUnit(ctx) + + // List all Hypervisor CRDs. + hypervisors, err := listHypervisors(ctx, c.opts.K8sClient) + if err != nil { + slog.Error("failed to list hypervisors", "error", err) + ch <- prometheus.MustNewConstMetric(descScrapeSuccess, prometheus.GaugeValue, 0) + ch <- prometheus.MustNewConstMetric(descScrapeDuration, prometheus.GaugeValue, time.Since(start).Seconds()) + return + } + + min := c.getMin() + + // Cluster-wide. + clusterCap := map[string]float64{} + clusterEffCap := map[string]float64{} + clusterAlloc := map[string]float64{} + clusterStranded := map[string]float64{} + clusterUtilSum := map[string]float64{} + + var ( + clusterUtilCount int + clusterBalanceSum float64 + clusterInstanceCount int + clusterAlignmentSum float64 + clusterAlignmentCount int + ) + + demandVec := c.computeAverageDemand(hypervisors) + + for i := range hypervisors { + hv := &hypervisors[i] + name := hv.Name + zone := hv.Labels["topology.kubernetes.io/zone"] + bb := hv.Labels["kubernetes.metal.cloud.sap/bb"] + group := hv.Labels["worker.garden.sapcloud.io/group"] + + rv := buildResourceVec(hv.Status.EffectiveCapacity, hv.Status.Allocation) + + // --- Per-resource metrics --- + for dim := range rv.EffectiveCapacity { + dimStr := dim + + // Raw capacity. + if raw, ok := hv.Status.Capacity[hvv1.ResourceName(dim)]; ok { + ch <- prometheus.MustNewConstMetric( + descCapacity, prometheus.GaugeValue, + quantityToFloat64(raw, dim), + name, dimStr, zone, bb, group, + ) + } + + // Effective capacity. + ch <- prometheus.MustNewConstMetric( + descEffectiveCapacity, prometheus.GaugeValue, + rv.EffectiveCapacity[dim], + name, dimStr, zone, bb, group, + ) + + // Allocation. + ch <- prometheus.MustNewConstMetric( + descAllocation, prometheus.GaugeValue, + rv.Allocation[dim], + name, dimStr, zone, bb, group, + ) + + // Free capa. + free := rv.free() + ch <- prometheus.MustNewConstMetric( + descFree, prometheus.GaugeValue, + free[dim], + name, dimStr, zone, bb, group, + ) + + // Utilization ratio. + fracs := rv.fractions() + ch <- prometheus.MustNewConstMetric( + descUtilization, prometheus.GaugeValue, + fracs[dim], + name, dimStr, zone, bb, group, + ) + + // Accumulate cluster-wide. + clusterCap[dim] += quantityToFloat64OrZero(hv.Status.Capacity, dim) + clusterEffCap[dim] += rv.EffectiveCapacity[dim] + clusterAlloc[dim] += rv.Allocation[dim] + clusterUtilSum[dim] += fracs[dim] + } + clusterUtilCount++ + + // Balance score. + bs := rv.balanceScore() + ch <- prometheus.MustNewConstMetric( + descBalanceScore, prometheus.GaugeValue, bs, + name, zone, bb, group, + ) + clusterBalanceSum += bs + + // Stranded resources. + stranded := strandedResources(rv, min) + for dim, val := range stranded { + ch <- prometheus.MustNewConstMetric( + descStranded, prometheus.GaugeValue, val, + name, dim, zone, bb, group, + ) + clusterStranded[dim] += val + } + + // Instance count. + ch <- prometheus.MustNewConstMetric( + descInstances, prometheus.GaugeValue, + float64(hv.Status.NumInstances), + name, zone, bb, group, + ) + clusterInstanceCount += hv.Status.NumInstances + + // Maintenance mode. + maint := hv.Spec.Maintenance + maintVal := 0.0 + if maint != "" { + maintVal = 1.0 + } + ch <- prometheus.MustNewConstMetric( + descMaintenanceInfo, prometheus.GaugeValue, maintVal, + name, zone, bb, group, maint, + ) + + // Tetris alignment. + free := rv.free() + if len(demandVec) > 0 { + alignment := tetrisAlignmentScore(free, demandVec) + clusterAlignmentSum += alignment + clusterAlignmentCount++ + } + } + + // --- Cluster-wide metrics --- + for _, dim := range []string{"cpu", "memory"} { + ch <- prometheus.MustNewConstMetric(descClusterCapacity, prometheus.GaugeValue, clusterCap[dim], dim) + ch <- prometheus.MustNewConstMetric(descClusterEffectiveCapacity, prometheus.GaugeValue, clusterEffCap[dim], dim) + ch <- prometheus.MustNewConstMetric(descClusterAllocation, prometheus.GaugeValue, clusterAlloc[dim], dim) + + totalFree := clusterEffCap[dim] - clusterAlloc[dim] + if totalFree < 0 { + totalFree = 0 + } + ch <- prometheus.MustNewConstMetric(descClusterFree, prometheus.GaugeValue, totalFree, dim) + ch <- prometheus.MustNewConstMetric(descClusterStranded, prometheus.GaugeValue, clusterStranded[dim], dim) + ch <- prometheus.MustNewConstMetric( + descClusterFragmentationRatio, prometheus.GaugeValue, + fragmentationRatio(clusterStranded[dim], totalFree), dim, + ) + + if clusterUtilCount > 0 { + ch <- prometheus.MustNewConstMetric( + descClusterMeanUtilization, prometheus.GaugeValue, + clusterUtilSum[dim]/float64(clusterUtilCount), dim, + ) + } + } + + if clusterUtilCount > 0 { + ch <- prometheus.MustNewConstMetric( + descClusterBalanceScore, prometheus.GaugeValue, + clusterBalanceSum/float64(clusterUtilCount), + ) + } + ch <- prometheus.MustNewConstMetric(descClusterHypervisorCount, prometheus.GaugeValue, float64(len(hypervisors))) + ch <- prometheus.MustNewConstMetric(descClusterInstanceCount, prometheus.GaugeValue, float64(clusterInstanceCount)) + + if clusterAlignmentCount > 0 { + ch <- prometheus.MustNewConstMetric( + descClusterTetrisAlignment, prometheus.GaugeValue, + clusterAlignmentSum/float64(clusterAlignmentCount), + ) + } + + // Exporter health. + ch <- prometheus.MustNewConstMetric(descScrapeSuccess, prometheus.GaugeValue, 1) + ch <- prometheus.MustNewConstMetric(descScrapeDuration, prometheus.GaugeValue, time.Since(start).Seconds()) + ch <- prometheus.MustNewConstMetric(descMinPlaceableCPU, prometheus.GaugeValue, min.CPU) + ch <- prometheus.MustNewConstMetric(descMinPlaceableMemory, prometheus.GaugeValue, min.Memory) +} + +// refreshMinPlaceableUnit queries Nova for the smallest flavor and caches the +// result. Respects the configured scrape interval to avoid hammering Nova. +func (c *Collector) refreshMinPlaceableUnit(ctx context.Context) { + if c.opts.NovaClient == nil { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.flavorsDone && time.Since(c.lastScrape) < c.opts.ScrapeInterval { + return + } + + minCPU, minMem, err := c.opts.NovaClient.SmallestFlavor(ctx) + if err != nil { + slog.Warn("failed to fetch Nova flavors; using cached or fallback minimum", "error", err) + return + } + + c.cachedMin = minPlaceableUnit{ + CPU: float64(minCPU), + Memory: float64(minMem), + } + c.flavorsDone = true + c.lastScrape = time.Now() + slog.Debug("refreshed minimum placeable unit from Nova flavors", + "min_cpu", minCPU, "min_memory_bytes", minMem) +} + +func (c *Collector) getMin() minPlaceableUnit { + c.mu.Lock() + defer c.mu.Unlock() + return c.cachedMin +} + +// computeAverageDemand computes the average per-instance resource demand across +// all hypervisors. This serves as the workload demand vector for Tetris alignment. +func (c *Collector) computeAverageDemand(hypervisors []hvv1.Hypervisor) map[string]float64 { + totalAlloc := map[string]float64{} + totalInstances := 0 + + for i := range hypervisors { + hv := &hypervisors[i] + if hv.Status.NumInstances == 0 { + continue + } + for rn, q := range hv.Status.Allocation { + dim := string(rn) + totalAlloc[dim] += quantityToFloat64(q, dim) + } + totalInstances += hv.Status.NumInstances + } + + if totalInstances == 0 { + return nil + } + + demand := make(map[string]float64, len(totalAlloc)) + for dim, total := range totalAlloc { + demand[dim] = total / float64(totalInstances) + } + return demand +} + +// buildResourceVec constructs a resourceVec from Hypervisor status fields. +func buildResourceVec( + effCap map[hvv1.ResourceName]resource.Quantity, + alloc map[hvv1.ResourceName]resource.Quantity, +) resourceVec { + rv := resourceVec{ + EffectiveCapacity: make(map[string]float64), + Allocation: make(map[string]float64), + } + + for rn, q := range effCap { + dim := string(rn) + rv.EffectiveCapacity[dim] = quantityToFloat64(q, dim) + } + for rn, q := range alloc { + dim := string(rn) + rv.Allocation[dim] = quantityToFloat64(q, dim) + } + + // Ensure all dimensions present in EffectiveCapacity also exist in Allocation. + for dim := range rv.EffectiveCapacity { + if _, ok := rv.Allocation[dim]; !ok { + rv.Allocation[dim] = 0 + } + } + return rv +} + +// quantityToFloat64OrZero looks up a dimension in a resource map and converts it. +func quantityToFloat64OrZero(m map[hvv1.ResourceName]resource.Quantity, dim string) float64 { + q, ok := m[hvv1.ResourceName(dim)] + if !ok { + return 0 + } + return quantityToFloat64(q, dim) +} diff --git a/internal/exporters/schedulingquality/collector/k8s.go b/internal/exporters/schedulingquality/collector/k8s.go new file mode 100644 index 000000000..0b32753dc --- /dev/null +++ b/internal/exporters/schedulingquality/collector/k8s.go @@ -0,0 +1,64 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package collector + +import ( + "context" + "fmt" + + hvv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(hvv1.AddToScheme(scheme)) +} + +// NewK8sClient builds a controller-runtime client configured for the +// Hypervisor CRD. Resolution order: +// 1. --kubeconfig flag (explicit path) +// 2. In-cluster config (KUBERNETES_SERVICE_HOST / SERVICE_PORT env vars) +// 3. Default kubeconfig loading rules (KUBECONFIG env var, ~/.kube/config) +func NewK8sClient(kubeconfig string) (client.Client, error) { + var cfg *rest.Config + var err error + + if kubeconfig != "" { + cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + } else { + cfg, err = rest.InClusterConfig() + if err != nil { + cfg, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + &clientcmd.ConfigOverrides{}, + ).ClientConfig() + } + } + if err != nil { + return nil, fmt.Errorf("build rest config: %w", err) + } + + c, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + return nil, fmt.Errorf("create client: %w", err) + } + return c, nil +} + +// listHypervisors returns all Hypervisor CRs in the cluster. +func listHypervisors(ctx context.Context, c client.Client) ([]hvv1.Hypervisor, error) { + var list hvv1.HypervisorList + if err := c.List(ctx, &list); err != nil { + return nil, fmt.Errorf("list hypervisors: %w", err) + } + return list.Items, nil +} diff --git a/internal/exporters/schedulingquality/collector/stranding.go b/internal/exporters/schedulingquality/collector/stranding.go new file mode 100644 index 000000000..90ebe808e --- /dev/null +++ b/internal/exporters/schedulingquality/collector/stranding.go @@ -0,0 +1,204 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package collector + +import ( + "math" + + "k8s.io/apimachinery/pkg/api/resource" +) + +// resourceVec holds capacity and allocation for a set of resource dimensions. +type resourceVec struct { + EffectiveCapacity map[string]float64 + Allocation map[string]float64 +} + +// free returns effective_capacity - allocation for each dimension; floored at 0. +func (v resourceVec) free() map[string]float64 { + f := make(map[string]float64, len(v.EffectiveCapacity)) + for dim, cap := range v.EffectiveCapacity { + alloc := v.Allocation[dim] + if cap-alloc > 0 { + f[dim] = cap - alloc + } + } + return f +} + +// fractions returns allocation / effective_capacity per dimension. +// Returns 0 for dimensions with zero capacity. +func (v resourceVec) fractions() map[string]float64 { + f := make(map[string]float64, len(v.EffectiveCapacity)) + for dim, cap := range v.EffectiveCapacity { + if cap > 0 { + f[dim] = v.Allocation[dim] / cap + } + } + return f +} + +// balanceScore computes the Kubernetes balanced-allocation score: +// +// score = 1 - stddev(fraction_i) +// +// Returns 1.0 for perfectly balanced, 0.0 for maximally imbalanced. +// With a single dimension the score is always 1.0. +func (v resourceVec) balanceScore() float64 { + fracs := v.fractions() + n := float64(len(fracs)) + if n <= 1 { + return 1.0 + } + + var sum float64 + for _, f := range fracs { + sum += f + } + mean := sum / n + + var variance float64 + for _, f := range fracs { + d := f - mean + variance += d * d + } + variance /= n + std := math.Sqrt(variance) + + score := 1.0 - std + if score < 0 { + return 0 + } + return score +} + +// minPlaceableUnit defines the smallest VM that can be placed, per dimension. +// Used as the threshold below which remaining capacity is considered stranded. +type minPlaceableUnit struct { + CPU float64 // cores + Memory float64 // bytes +} + +// strandedResources computes stranded capacity per dimension on a single host. +// +// A resource dimension is stranded when the free capacity in that dimension +// cannot be consumed because another dimension's free capacity is below the +// minimum placeable unit. +// +// Returns a map from dimension name to stranded quantity. +func strandedResources(v resourceVec, min minPlaceableUnit) map[string]float64 { + free := v.free() + stranded := make(map[string]float64) + + // Map dimension names to their minimum placeable thresholds. + thresholds := map[string]float64{ + "cpu": min.CPU, + "memory": min.Memory, + } + + // For each dimension, check if it's below the minimum placeable unit. + // If so, all other dimensions' free capacity is stranded. + exhausted := make(map[string]bool) + for dim, threshold := range thresholds { + if free[dim] < threshold { + exhausted[dim] = true + } + } + + if len(exhausted) == 0 { + // No dimension is exhausted; compute proportional stranding. + // Even when all dimensions have free capacity above the minimum, + // imbalanced free capacity creates stranding. The stranded amount + // per dimension equals the free capacity that exceeds what the + // bottleneck dimension can support. + // + // For CPU and memory: how many minimum units can we place? + // placeable = min(free_cpu / min_cpu, free_mem / min_mem) + // stranded_cpu = free_cpu - placeable * min_cpu + // stranded_mem = free_mem - placeable * min_mem + if min.CPU > 0 && min.Memory > 0 { + freeCPU := free["cpu"] + freeMem := free["memory"] + placeableByCPU := math.Floor(freeCPU / min.CPU) + placeableByMem := math.Floor(freeMem / min.Memory) + placeable := math.Min(placeableByCPU, placeableByMem) + + cpuUsable := placeable * min.CPU + memUsable := placeable * min.Memory + + if freeCPU-cpuUsable > 0 { + stranded["cpu"] = freeCPU - cpuUsable + } + if freeMem-memUsable > 0 { + stranded["memory"] = freeMem - memUsable + } + } + return stranded + } + + // At least one dimension is exhausted. + // All free capacity in non-exhausted dimensions is stranded. + for dim, f := range free { + if !exhausted[dim] && f > 0 { + stranded[dim] = f + } + } + return stranded +} + +// fragmentationRatio computes the cluster-wide fragmentation ratio: +// +// fragmentation = total_stranded / total_free +// +// per dimension. Returns 0 if total free is 0. +func fragmentationRatio(totalStranded, totalFree float64) float64 { + if totalFree <= 0 { + return 0 + } + return totalStranded / totalFree +} + +// tetrisAlignmentScore computes the Tetris dot-product alignment score +// between a workload demand vector and the host's available capacity vector. +// Both vectors are normalized to unit length before computing the dot product. +// +// For a host with free capacity, this measures how well the free capacity +// shape matches the expected workload shape. Score range: [0, 1]. +// 1.0 = perfectly aligned (free capacity proportions match demand proportions). +func tetrisAlignmentScore(freeVec, demandVec map[string]float64) float64 { + // Compute norms. + var freeNorm, demandNorm float64 + for dim, f := range freeVec { + d := demandVec[dim] + freeNorm += f * f + demandNorm += d * d + } + freeNorm = math.Sqrt(freeNorm) + demandNorm = math.Sqrt(demandNorm) + + if freeNorm == 0 || demandNorm == 0 { + return 0 + } + + // Dot product of normalized vectors. + var dot float64 + for dim, f := range freeVec { + d := demandVec[dim] + dot += (f / freeNorm) * (d / demandNorm) + } + return dot +} + +// quantityToFloat64 converts a resource.Quantity to float64. +// For CPU: returns cores (e.g., "4" -> 4.0, "500m" -> 0.5). +// For Memory: returns bytes. +func quantityToFloat64(q resource.Quantity, dimName string) float64 { + switch dimName { + case "cpu": + return float64(q.MilliValue()) / 1000.0 + default: + // Memory and other resources: use raw value (bytes). + return float64(q.Value()) + } +} diff --git a/internal/exporters/schedulingquality/collector/stranding_test.go b/internal/exporters/schedulingquality/collector/stranding_test.go new file mode 100644 index 000000000..eb643a6b4 --- /dev/null +++ b/internal/exporters/schedulingquality/collector/stranding_test.go @@ -0,0 +1,212 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package collector + +import ( + "math" + "testing" +) + +func TestBalanceScore(t *testing.T) { + tests := []struct { + name string + effCap map[string]float64 + alloc map[string]float64 + wantMin float64 + wantMax float64 + }{ + { + name: "perfectly balanced 50/50", + effCap: map[string]float64{"cpu": 100, "memory": 1000}, + alloc: map[string]float64{"cpu": 50, "memory": 500}, + wantMin: 0.99, + wantMax: 1.01, + }, + { + name: "perfectly balanced 0/0", + effCap: map[string]float64{"cpu": 100, "memory": 1000}, + alloc: map[string]float64{"cpu": 0, "memory": 0}, + wantMin: 0.99, + wantMax: 1.01, + }, + { + name: "maximally imbalanced: cpu full, memory empty", + effCap: map[string]float64{"cpu": 100, "memory": 1000}, + alloc: map[string]float64{"cpu": 100, "memory": 0}, + wantMin: 0.0, + wantMax: 0.6, // stddev = 0.5, so score = 0.5 + }, + { + name: "moderate imbalance: 90% cpu, 10% memory", + effCap: map[string]float64{"cpu": 100, "memory": 1000}, + alloc: map[string]float64{"cpu": 90, "memory": 100}, + wantMin: 0.55, + wantMax: 0.65, + }, + { + name: "single dimension: always balanced", + effCap: map[string]float64{"cpu": 100}, + alloc: map[string]float64{"cpu": 90}, + wantMin: 0.99, + wantMax: 1.01, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rv := resourceVec{EffectiveCapacity: tt.effCap, Allocation: tt.alloc} + got := rv.balanceScore() + if got < tt.wantMin || got > tt.wantMax { + t.Errorf("balanceScore() = %v, want [%v, %v]", got, tt.wantMin, tt.wantMax) + } + }) + } +} + +func TestStrandedResources_Exhausted(t *testing.T) { + min := minPlaceableUnit{CPU: 1, Memory: 512 * 1024 * 1024} // 1 core, 512 MB + + // CPU exhausted, memory has free capacity. + rv := resourceVec{ + EffectiveCapacity: map[string]float64{"cpu": 64, "memory": 256e9}, + Allocation: map[string]float64{"cpu": 64, "memory": 128e9}, + } + stranded := strandedResources(rv, min) + + // All free memory is stranded because CPU is exhausted. + if stranded["memory"] != 128e9 { + t.Errorf("stranded memory = %v, want %v", stranded["memory"], 128e9) + } + // CPU should have nothing stranded (it's the exhausted dimension, not stranded). + if stranded["cpu"] != 0 { + t.Errorf("stranded cpu = %v, want 0", stranded["cpu"]) + } +} + +func TestStrandedResources_BothExhausted(t *testing.T) { + min := minPlaceableUnit{CPU: 1, Memory: 512 * 1024 * 1024} + + rv := resourceVec{ + EffectiveCapacity: map[string]float64{"cpu": 64, "memory": 256e9}, + Allocation: map[string]float64{"cpu": 64, "memory": 256e9}, + } + stranded := strandedResources(rv, min) + + // Both exhausted: nothing is stranded (nothing free at all). + if len(stranded) != 0 { + t.Errorf("stranded = %v, want empty", stranded) + } +} + +func TestStrandedResources_Proportional(t *testing.T) { + min := minPlaceableUnit{CPU: 2, Memory: 4e9} // 2 cores, 4 GB per unit + + // 10 free CPU, 12 GB free memory. + // Placeable units: min(10/2, 12e9/4e9) = min(5, 3) = 3 + // Usable CPU = 3*2 = 6, stranded CPU = 10 - 6 = 4 + // Usable Mem = 3*4e9 = 12e9, stranded Mem = 0 + rv := resourceVec{ + EffectiveCapacity: map[string]float64{"cpu": 64, "memory": 256e9}, + Allocation: map[string]float64{"cpu": 54, "memory": 244e9}, + } + stranded := strandedResources(rv, min) + + if stranded["cpu"] != 4 { + t.Errorf("stranded cpu = %v, want 4", stranded["cpu"]) + } + if stranded["memory"] != 0 { + t.Errorf("stranded memory = %v, want 0", stranded["memory"]) + } +} + +func TestStrandedResources_NothingStranded(t *testing.T) { + min := minPlaceableUnit{CPU: 1, Memory: 1e9} + + // Perfectly proportional free capacity: 10 CPU, 10 GB. + // Placeable = min(10/1, 10e9/1e9) = 10. + // Usable CPU = 10, Usable Mem = 10e9. Nothing stranded. + rv := resourceVec{ + EffectiveCapacity: map[string]float64{"cpu": 64, "memory": 256e9}, + Allocation: map[string]float64{"cpu": 54, "memory": 246e9}, + } + stranded := strandedResources(rv, min) + + if stranded["cpu"] != 0 { + t.Errorf("stranded cpu = %v, want 0", stranded["cpu"]) + } + if stranded["memory"] != 0 { + t.Errorf("stranded memory = %v, want 0", stranded["memory"]) + } +} + +func TestFragmentationRatio(t *testing.T) { + tests := []struct { + name string + stranded float64 + free float64 + want float64 + }{ + {"no free capacity", 0, 0, 0}, + {"no stranding", 0, 100, 0}, + {"half stranded", 50, 100, 0.5}, + {"all stranded", 100, 100, 1.0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := fragmentationRatio(tt.stranded, tt.free) + if math.Abs(got-tt.want) > 0.001 { + t.Errorf("fragmentationRatio(%v, %v) = %v, want %v", tt.stranded, tt.free, got, tt.want) + } + }) + } +} + +func TestTetrisAlignmentScore(t *testing.T) { + tests := []struct { + name string + free map[string]float64 + demand map[string]float64 + wantMin float64 + wantMax float64 + }{ + { + name: "perfectly aligned", + free: map[string]float64{"cpu": 10, "memory": 20}, + demand: map[string]float64{"cpu": 5, "memory": 10}, + wantMin: 0.99, + wantMax: 1.01, + }, + { + name: "orthogonal: free cpu, demand memory", + free: map[string]float64{"cpu": 10, "memory": 0}, + demand: map[string]float64{"cpu": 0, "memory": 10}, + wantMin: -0.01, + wantMax: 0.01, + }, + { + name: "moderate alignment", + free: map[string]float64{"cpu": 10, "memory": 2}, + demand: map[string]float64{"cpu": 1, "memory": 4}, + wantMin: 0.4, + wantMax: 0.8, + }, + { + name: "zero free", + free: map[string]float64{"cpu": 0, "memory": 0}, + demand: map[string]float64{"cpu": 1, "memory": 4}, + wantMin: -0.01, + wantMax: 0.01, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tetrisAlignmentScore(tt.free, tt.demand) + if got < tt.wantMin || got > tt.wantMax { + t.Errorf("tetrisAlignmentScore() = %v, want [%v, %v]", got, tt.wantMin, tt.wantMax) + } + }) + } +} diff --git a/internal/exporters/schedulingquality/nova/client.go b/internal/exporters/schedulingquality/nova/client.go new file mode 100644 index 000000000..6aa8c4b18 --- /dev/null +++ b/internal/exporters/schedulingquality/nova/client.go @@ -0,0 +1,90 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package nova + +import ( + "context" + "fmt" + "math" + "os" + + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/flavors" +) + +// Client wraps the OpenStack Nova API. +type Client struct { + compute *gophercloud.ServiceClient +} + +// NewClientFromEnv creates a Nova client from standard OpenStack environment +// variables (OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_PROJECT_NAME, etc.). +func NewClientFromEnv() (*Client, error) { + opts, err := openstack.AuthOptionsFromEnv() + if err != nil { + return nil, fmt.Errorf("read auth env: %w", err) + } + + provider, err := openstack.AuthenticatedClient(context.Background(), opts) + if err != nil { + return nil, fmt.Errorf("authenticate: %w", err) + } + + region := os.Getenv("OS_REGION_NAME") + compute, err := openstack.NewComputeV2(provider, gophercloud.EndpointOpts{ + Region: region, + }) + if err != nil { + return nil, fmt.Errorf("create compute client: %w", err) + } + + return &Client{compute: compute}, nil +} + +// SmallestFlavor queries the Nova flavor list and returns the smallest CPU +// count and memory size (in bytes) across all public flavors. These values +// define the minimum placeable unit for stranding computation. +// +// Returns (minCPU, minMemoryBytes, error). +func (c *Client) SmallestFlavor(ctx context.Context) (int64, int64, error) { + listOpts := flavors.ListOpts{ + AccessType: flavors.PublicAccess, + } + + allPages, err := flavors.ListDetail(c.compute, listOpts).AllPages(ctx) + if err != nil { + return 0, 0, fmt.Errorf("list flavors: %w", err) + } + + allFlavors, err := flavors.ExtractFlavors(allPages) + if err != nil { + return 0, 0, fmt.Errorf("extract flavors: %w", err) + } + + if len(allFlavors) == 0 { + return 0, 0, fmt.Errorf("no flavors found") + } + + minCPU := int64(math.MaxInt64) + minMemMB := int64(math.MaxInt64) + + for _, f := range allFlavors { + if int64(f.VCPUs) < minCPU && f.VCPUs > 0 { + minCPU = int64(f.VCPUs) + } + if int64(f.RAM) < minMemMB && f.RAM > 0 { + minMemMB = int64(f.RAM) + } + } + + if minCPU == math.MaxInt64 || minMemMB == math.MaxInt64 { + return 0, 0, fmt.Errorf("no valid flavors found") + } + + // Convert MB to bytes. + minMemBytes := minMemMB * 1024 * 1024 + + return minCPU, minMemBytes, nil +}