feat(nvcf-compute-plane): add kai-scheduler to stack

ai-tooling/user/skills/nvcf-self-managed-prerequisite/SKILL.md

@@ -35,7 +35,7 @@ One operator tool plus two cluster-level components the NVCA operator / compute
 | Prereq | Why it is needed | Detail |
 | ------ | ------------------ | ------ |
 | `nvcf-cli` | The compute-plane stack's `make register-cluster` (and `install`/`apply`/`sync`, which abort without the registration values it writes) shells out to `nvcf-cli`. The shipped stack defaults to building it from a sibling `../cli` checkout that the release does not include, so a green-field repo fails with `ensure-nvcf-cli` / "Registration values not found". | See Step 0b below |
-| KAI Scheduler | `selfManaged.featureGateValues` includes `KAIScheduler`; NVCA polls `Queue` CRs and refuses to become healthy until their quotas are `-1` | [references/kai-scheduler.md](references/kai-scheduler.md) |
+| KAI Scheduler | `selfManaged.featureGateValues` includes `KAIScheduler`; NVCA polls `Queue` CRs and refuses to become healthy until their quotas are `-1`. Skip the standalone helm step below when `addons.kaiScheduler.enabled` is set in `nvcf-compute-plane` (required when enabling Grove or Dynamo). | [references/kai-scheduler.md](references/kai-scheduler.md) |
 | SMB CSI driver (`smb.csi.k8s.io`) | NVCA's `selfManaged.sharedStorage` runs Samba sidecar pods that export file shares; the resulting PVCs need this CSI driver to bind | [references/smb-csi.md](references/smb-csi.md) |
 
 The KAI Scheduler and SMB CSI installs are cloud-neutral helm commands pinned to NVCF-validated versions. These are upstream third-party charts (not NVCF images), so they are not in `manifest.yaml`; the per-component reference docs carry the current pin and link the NVCF docs version table. `nvcf-cli` is an operator workstation tool, not an in-cluster install.
@@ -82,6 +82,8 @@ make install CLUSTER_NAME=<name> HELMFILE_ENV=<env>
 
 ### 1 — KAI Scheduler
 
+Skip this step when `addons.kaiScheduler.enabled` is true in `nvcf-compute-plane`. That stack installs KAI Scheduler (release and namespace `kai-scheduler`). Enable that flag whenever Grove or Dynamo is enabled. Use the standalone install below when you need KAI without the compute-plane add-on (for example the NVCA `KAIScheduler` feature gate alone).
+
 ```bash
 cat > nvca-values.yaml << 'EOF'
 scheduler:

ai-tooling/user/skills/nvcf-self-managed-prerequisite/references/kai-scheduler.md

@@ -14,6 +14,8 @@ At the time of writing, the pinned chart version is `v0.14.0`. See [NVCF KAI Sch
 
 ## Install
 
+When `addons.kaiScheduler.enabled` is true in `nvcf-compute-plane`, that stack installs KAI Scheduler for you. Skip this section in that case.
+
 ```bash
 cat > nvca-values.yaml << 'EOF'
 scheduler:

deploy/stacks/nvcf-compute-plane/Makefile.dist

@@ -35,7 +35,7 @@ REQUIRED_HELMFILE_MINOR := 1
 REQUIRED_HELM_MAJOR := 3
 
 # Namespaces owned by the compute-plane stack (used by destroy target)
-NAMESPACES := nvca-operator grove-system dynamo-system
+NAMESPACES := nvca-operator grove-system dynamo-system kai-scheduler
 
 # Kubeconfig file targeting a specific cluster. Used by helmfile, kubectl,
 # and nvcf-cli. Required in multi-cluster workflows to avoid registering

deploy/stacks/nvcf-compute-plane/README.md

@@ -89,17 +89,28 @@ make install CLUSTER_NAME=...   # downloads helmfile v1.1.9 + helm v3.15.4 on fi
 
 ## Optional Add-ons
 
-Grove (topology-aware scheduling) and Dynamo (inference framework scheduling) are disabled
-by default. Enable per-environment in `environments/<env>.yaml`:
+KAI Scheduler, Grove (topology-aware scheduling), and Dynamo (inference framework
+scheduling) are disabled by default. Grove and Dynamo require KAI Scheduler
+(release and namespace `kai-scheduler`). Enable per-environment in
+`environments/<env>.yaml`:
 
 ```yaml
 addons:
+  kaiScheduler:
+    enabled: true
   groveOperator:
     enabled: true
   dynamoOperator:
     enabled: true
 ```
 
+Override KAI component resources under `addons.kaiScheduler.<component>.resources`
+(for example `addons.kaiScheduler.scheduler.resources.requests.memory`). Defaults
+are set in `helmfile.d/01-dependencies.yaml.gotmpl`.
+
+For a standalone KAI install outside this stack, follow the
+[KAI Scheduler guide](https://docs.nvidia.com/cloud-functions/current/latest/cluster-management/kai-scheduler.html).
+
 ## Multi-Cluster Example
 
 Each cluster is registered and installed independently. Pass `KUBECONFIG_FILE`

deploy/stacks/nvcf-compute-plane/environments/base.yaml

@@ -72,6 +72,12 @@ observability:
 # =============================================================================
 addons:
 
+  # KAI Scheduler (disabled by default). Required when enabling Grove or Dynamo.
+  # Per-component resource overrides: addons.kaiScheduler.<component>.resources
+  # (requests/limits). Defaults live in helmfile.d/01-dependencies.yaml.gotmpl.
+  kaiScheduler:
+    enabled: false
+
   # Grove topology-aware scheduling operator (disabled by default)
   groveOperator:
     enabled: false

deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl

@@ -1,6 +1,7 @@
 # NVCF Compute Plane Releases
 #
-# Two optional releases — installed once per GPU cluster:
+# Optional releases — installed once per GPU cluster:
+#   kai-scheduler    — optional; addons.kaiScheduler.enabled (required by grove/dynamo)
 #   grove-operator   — optional; topology-aware scheduling (addons.groveOperator.enabled)
 #   dynamo-operator  — optional; Dynamo inference framework scheduling (addons.dynamoOperator.enabled)
 #
@@ -16,6 +17,9 @@ environments:
 ---
 
 repositories:
+  - name: kai-scheduler
+    url: ghcr.io/kai-scheduler/kai-scheduler
+    oci: true
   - name: nvidia-ai-dynamo-grove
     url: ghcr.io/ai-dynamo/grove
     oci: true
@@ -33,13 +37,206 @@ helmDefaults:
 
 releases:
 
+  - name: kai-scheduler
+    # Required by grove-operator and dynamo-operator. Release name and namespace
+    # must be "kai-scheduler" per KAI installation requirements.
+    # Enable per-environment: addons.kaiScheduler.enabled: true
+    # Per-component resources can be overridden under addons.kaiScheduler.<component>.resources;
+    # defaults below match the managed SBOM template.
+    condition: addons.kaiScheduler.enabled
+    chart: kai-scheduler/kai-scheduler
+    version: v0.14.0
+    namespace: kai-scheduler
+    values:
+      - ../global.yaml.gotmpl
+      - {{- $kai := dig "kaiScheduler" dict .Values.addons }}
+        {{- $opRes := dig "operator" "resources" dict $kai }}
+        {{- $opReq := dig "requests" dict $opRes }}
+        {{- $opLim := dig "limits" dict $opRes }}
+        {{- $pgRes := dig "podgrouper" "resources" dict $kai }}
+        {{- $pgReq := dig "requests" dict $pgRes }}
+        {{- $pgLim := dig "limits" dict $pgRes }}
+        {{- $pgcRes := dig "podgroupcontroller" "resources" dict $kai }}
+        {{- $pgcReq := dig "requests" dict $pgcRes }}
+        {{- $pgcLim := dig "limits" dict $pgcRes }}
+        {{- $bdRes := dig "binder" "resources" dict $kai }}
+        {{- $bdReq := dig "requests" dict $bdRes }}
+        {{- $bdLim := dig "limits" dict $bdRes }}
+        {{- $schRes := dig "scheduler" "resources" dict $kai }}
+        {{- $schReq := dig "requests" dict $schRes }}
+        {{- $schLim := dig "limits" dict $schRes }}
+        {{- $qcRes := dig "queuecontroller" "resources" dict $kai }}
+        {{- $qcReq := dig "requests" dict $qcRes }}
+        {{- $qcLim := dig "limits" dict $qcRes }}
+        {{- $adRes := dig "admission" "resources" dict $kai }}
+        {{- $adReq := dig "requests" dict $adRes }}
+        {{- $adLim := dig "limits" dict $adRes }}
+        {{- $nsaRes := dig "nodescaleadjuster" "resources" dict $kai }}
+        {{- $nsaReq := dig "requests" dict $nsaRes }}
+        {{- $nsaLim := dig "limits" dict $nsaRes }}
+        {{- $npeRes := dig "numaPlacementExporter" "resources" dict $kai }}
+        {{- $npeReq := dig "requests" dict $npeRes }}
+        {{- $npeLim := dig "limits" dict $npeRes }}
+        global:
+          securityContext:
+            runAsUser: 10000
+            runAsNonRoot: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+          {{- if .Values.global.nodeSelector.enabled }}
+          nodeSelector:
+            {{ .Values.global.nodeSelector.key }}: {{ .Values.global.nodeSelector.value }}
+          {{- end }}
+          {{- if .Values.global.imagePullSecrets }}
+          imagePullSecrets:
+          {{- toYaml .Values.global.imagePullSecrets | nindent 12 }}
+          {{- end }}
+          {{- if not (empty .Values.global.tolerations) }}
+          tolerations:
+          {{- toYaml .Values.global.tolerations | nindent 12 }}
+          {{- end }}
+
+        # Image names stay at chart defaults (ghcr.io/kai-scheduler/kai-scheduler/*).
+        # Managed SBOM uses kai-scheduler-* names with nv-ngc-devops; those do not
+        # resolve against the public chart registry.
+        operator:
+          resources:
+            requests:
+              cpu: {{ dig "cpu" "50m" $opReq }}
+              memory: {{ dig "memory" "256Mi" $opReq }}
+            limits:
+              cpu: {{ dig "cpu" "1000m" $opLim }}
+              memory: {{ dig "memory" "1Gi" $opLim }}
+
+        podgrouper:
+          resources:
+            requests:
+              cpu: {{ dig "cpu" "50m" $pgReq }}
+              memory: {{ dig "memory" "512Mi" $pgReq }}
+            limits:
+              cpu: {{ dig "cpu" "1000m" $pgLim }}
+              memory: {{ dig "memory" "4Gi" $pgLim }}
+
+        podgroupcontroller:
+          resources:
+            requests:
+              cpu: {{ dig "cpu" "50m" $pgcReq }}
+              memory: {{ dig "memory" "256Mi" $pgcReq }}
+            limits:
+              cpu: {{ dig "cpu" "1000m" $pgcLim }}
+              memory: {{ dig "memory" "1Gi" $pgcLim }}
+
+        binder:
+          resources:
+            requests:
+              cpu: {{ dig "cpu" "50m" $bdReq }}
+              memory: {{ dig "memory" "2Gi" $bdReq }}
+            limits:
+              cpu: {{ dig "cpu" "4000m" $bdLim }}
+              memory: {{ dig "memory" "4Gi" $bdLim }}
+
+        scheduler:
+          placementStrategy: binpack
+          plugins:
+            nodeplacement:
+              arguments:
+                gpu: binpack
+                cpu: spread
+          actions:
+            preempt:
+              enabled: false
+            consolidation:
+              enabled: false
+          resources:
+            requests:
+              cpu: {{ dig "cpu" "500m" $schReq }}
+              memory: {{ dig "memory" "3Gi" $schReq }}
+            limits:
+              cpu: {{ dig "cpu" "2000m" $schLim }}
+              memory: {{ dig "memory" "6Gi" $schLim }}
+
+        queuecontroller:
+          resources:
+            requests:
+              cpu: {{ dig "cpu" "50m" $qcReq }}
+              memory: {{ dig "memory" "256Mi" $qcReq }}
+            limits:
+              cpu: {{ dig "cpu" "1000m" $qcLim }}
+              memory: {{ dig "memory" "1Gi" $qcLim }}
+
+        admission:
+          resources:
+            requests:
+              cpu: {{ dig "cpu" "50m" $adReq }}
+              memory: {{ dig "memory" "256Mi" $adReq }}
+            limits:
+              cpu: {{ dig "cpu" "1000m" $adLim }}
+              memory: {{ dig "memory" "1Gi" $adLim }}
+
+        nodescaleadjuster:
+          resources:
+            requests:
+              cpu: {{ dig "cpu" "50m" $nsaReq }}
+              memory: {{ dig "memory" "256Mi" $nsaReq }}
+            limits:
+              cpu: {{ dig "cpu" "1000m" $nsaLim }}
+              memory: {{ dig "memory" "1Gi" $nsaLim }}
+
+        numaPlacementExporter:
+          resources:
+            requests:
+              cpu: {{ dig "cpu" "50m" $npeReq }}
+              memory: {{ dig "memory" "64Mi" $npeReq }}
+            limits:
+              cpu: {{ dig "cpu" "1000m" $npeLim }}
+              memory: {{ dig "memory" "1Gi" $npeLim }}
+
+        defaultQueue:
+          createDefaultQueue: true
+          parentName: default-parent-queue
+          childName: default-queue
+          parentResources:
+            cpu:
+              quota: -1
+              limit: -1
+              overQuotaWeight: 1
+            gpu:
+              quota: -1
+              limit: -1
+              overQuotaWeight: 1
+            memory:
+              quota: -1
+              limit: -1
+              overQuotaWeight: 1
+          childResources:
+            cpu:
+              quota: -1
+              limit: -1
+              overQuotaWeight: 1
+            gpu:
+              quota: -1
+              limit: -1
+              overQuotaWeight: 1
+            memory:
+              quota: -1
+              limit: -1
+              overQuotaWeight: 1
+    labels:
+      release-group: workers
+    wait: true
+    waitForJobs: true
+
   - name: grove-operator
     # Topology-aware scheduling operator (optional; disabled by default).
     # Enable per-environment: addons.groveOperator.enabled: true
     condition: addons.groveOperator.enabled
     chart: nvidia-ai-dynamo-grove/grove-charts
     version: v0.1.0-alpha.9
     namespace: grove-system
+    needs:
+      - kai-scheduler/kai-scheduler
     values:
       - ../global.yaml.gotmpl
       - replicaCount: 1

deploy/stacks/nvcf-compute-plane/testdata/environments/local.yaml

@@ -18,6 +18,8 @@ global:
       natsURL: nats://nats.nats-system.svc.cluster.local:4222
 
 addons:
+  kaiScheduler:
+    enabled: true
   groveOperator:
     enabled: true
   dynamoOperator:

deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-queue.yaml

@@ -0,0 +1,47 @@
+---
+# Source: kai-scheduler/templates/default-queue.yaml
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: scheduling.run.ai/v2
+kind: Queue
+metadata:
+    name: default-parent-queue
+    annotations:
+        helm.sh/resource-policy: keep
+spec:
+    resources:
+        cpu:
+            quota: -1
+            limit: -1
+            overQuotaWeight: 1
+        gpu:
+            quota: -1
+            limit: -1
+            overQuotaWeight: 1
+        memory:
+            quota: -1
+            limit: -1
+            overQuotaWeight: 1
+---
+# Source: kai-scheduler/templates/default-queue.yaml
+apiVersion: scheduling.run.ai/v2
+kind: Queue
+metadata:
+    name: default-queue
+    annotations:
+        helm.sh/resource-policy: keep
+spec:
+    parentQueue: default-parent-queue
+    resources:
+        cpu:
+            quota: -1
+            limit: -1
+            overQuotaWeight: 1
+        gpu:
+            quota: -1
+            limit: -1
+            overQuotaWeight: 1
+        memory:
+            quota: -1
+            limit: -1
+            overQuotaWeight: 1

deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-shard.yaml

@@ -0,0 +1,26 @@
+---
+# Source: kai-scheduler/templates/default-shard.yaml
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kai.scheduler/v1
+kind: SchedulingShard
+metadata:
+  name: default
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  partitionLabelValue: ""
+  placementStrategy:
+    gpu: binpack
+    cpu: binpack
+  plugins:
+    nodeplacement:
+      arguments:
+        cpu: spread
+        gpu: binpack
+  actions:
+    consolidation:
+      enabled: false
+    preempt:
+      enabled: false