feat: implement skip toleration mechanism for batch rolling

config/crd/bases/rollout.kusionstack.io_rolloutruns.yaml

@@ -2264,32 +2264,30 @@ spec:
                           - targets
                         type: object
                       type: array
-                    toleration:
-                      description: Toleration is the toleration policy of the canary strategy
-                      properties:
-                        initialDelaySeconds:
-                          description: Number of seconds after the toleration check has started before the task are initiated.
-                          format: int32
-                          type: integer
-                        taskFailureThreshold:
-                          anyOf:
-                            - type: integer
-                            - type: string
-                          description: |-
-                            FailureThreshold indicates how many failed pods can be tolerated before marking the rollout task as success
-                            If not set, the default value is 0, which means no failed pods can be tolerated
-                            This is a task level threshold.
-                          x-kubernetes-int-or-string: true
-                        workloadTotalFailureThreshold:
-                          anyOf:
-                            - type: integer
-                            - type: string
-                          description: |-
-                            WorkloadFailureThreshold indicates how many failed pods can be tolerated in all upgraded pods of one workload.
-                            The default value is 0, which means no failed pods can be tolerated.
-                            This is a workload level threshold.
-                          x-kubernetes-int-or-string: true
-                      type: object
+                    tolerations:
+                      description: |-
+                        Tolerations records the accumulated toleration from skipped batches per workload.
+                        When a batch is skipped, the gap between expected and actual replicas for each workload
+                        is accumulated into this field, allowing subsequent batches to tolerate the deficit.
+                      items:
+                        description: RolloutRunTolerationTarget records the toleration value accumulated from skipped batches for a specific workload.
+                        properties:
+                          cluster:
+                            description: Cluster defines which cluster the workload is in.
+                            type: string
+                          name:
+                            description: Name is the workload name.
+                            type: string
+                          toleration:
+                            description: |-
+                              Toleration is the accumulated toleration value from skipped batches.
+                              It represents how many replicas the workload is allowed to be short of.
+                            format: int32
+                            type: integer
+                        required:
+                          - toleration
+                        type: object
+                      type: array
                   type: object
                 canary:
                   description: Canary defines the canary strategy

go.mod

@@ -23,7 +23,7 @@ require (
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kubernetes v1.22.2
 	k8s.io/utils v0.0.0-20241210054802-24370beab758
-	kusionstack.io/kube-api v0.7.5-0.20260512114711-9570d38337c2
+	kusionstack.io/kube-api v0.7.5-0.20260629032255-be28009453cb
 	kusionstack.io/kube-utils v0.2.1-0.20251125083928-1134a582b341
 	kusionstack.io/resourceconsist v0.0.4
 	sigs.k8s.io/controller-runtime v0.21.0

go.sum

@@ -1021,8 +1021,8 @@ k8s.io/sample-apiserver v0.22.2/go.mod h1:h+/DIV5EmuNq4vfPr5TSXy9mIBVXXlPAKQMPbj
 k8s.io/system-validators v1.5.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-kusionstack.io/kube-api v0.7.5-0.20260512114711-9570d38337c2 h1:jrUVO6a6/fuwdfF8NnehVXGbiYV57AfIJvqnsYRB3sI=
-kusionstack.io/kube-api v0.7.5-0.20260512114711-9570d38337c2/go.mod h1:e1jtrQH2LK5fD2nTyfIXG6nYrYbU8VXShRxTRwVPaLk=
+kusionstack.io/kube-api v0.7.5-0.20260629032255-be28009453cb h1:7lrtNodnXf7rsm3NzVhGZ4TwrM2ekJ656FN0qT9fHaI=
+kusionstack.io/kube-api v0.7.5-0.20260629032255-be28009453cb/go.mod h1:e1jtrQH2LK5fD2nTyfIXG6nYrYbU8VXShRxTRwVPaLk=
 kusionstack.io/kube-utils v0.2.1-0.20251125083928-1134a582b341 h1:dnMtHJvIpU3338WpqGiNN2qXWZFiXaoiuzR9jwhvWpg=
 kusionstack.io/kube-utils v0.2.1-0.20251125083928-1134a582b341/go.mod h1:Lz5SBYWg9+jw+kP0CAyf/b62D5DeUPf6+jE1d0WC4cI=
 kusionstack.io/resourceconsist v0.0.4 h1:wRqLJuNh8O4TT6p0uOklFpHUKiRdRxcAH71Sw/q9LhE=

pkg/controllers/rollout/rollout_controller.go

@@ -677,21 +677,20 @@ func (r *RolloutReconciler) applyOneTimeStrategy(ctx context.Context, obj *rollo
 	var batch *rolloutv1alpha1.RolloutRunBatchStrategy
 
 	// Check if using BatchV2 (V2 strategy scenario)
-	// Note: BatchStrategyV2 does not support Toleration, only V1 BatchStrategy has it
 	if strategy.BatchV2 != nil {
 		batch = &rolloutv1alpha1.RolloutRunBatchStrategy{
 			Batches: constructRolloutRunBatchesV2(strategy.BatchV2, workloads),
 		}
 	} else {
 		// Use original Batch field (V1 scenario)
 		batch = &rolloutv1alpha1.RolloutRunBatchStrategy{
-			Batches:    constructRolloutRunBatches(&strategy.Batch, workloads),
-			Toleration: strategy.Batch.Toleration,
+			Batches: constructRolloutRunBatches(&strategy.Batch, workloads),
 		}
 	}
 
-	if batch.Toleration == nil && run.Spec.Batch != nil {
-		batch.Toleration = run.Spec.Batch.Toleration
+	// Inherit Tolerations from the existing run spec if not changed
+	if run.Spec.Batch != nil {
+		batch.Tolerations = run.Spec.Batch.Tolerations
 	}
 
 	if equality.Semantic.DeepEqual(batch, run.Spec.Batch) {

pkg/controllers/rollout/utils.go

@@ -92,7 +92,6 @@ func constructRolloutRun(obj *rolloutv1alpha1.Rollout, strategy *rolloutv1alpha1
 	// Determine V1 vs V2 path based on BatchV2 presence
 	if strategy.BatchV2 != nil {
 		// V2 path: BatchV2 + optional CanaryV2
-		// Note: BatchStrategyV2 does not support Toleration, only V1 BatchStrategy has it
 		if strategy.CanaryV2 != nil {
 			run.Spec.Canary = constructRolloutRunCanaryV2(strategy.CanaryV2, workloadWrappers)
 		}
@@ -105,8 +104,7 @@ func constructRolloutRun(obj *rolloutv1alpha1.Rollout, strategy *rolloutv1alpha1
 			run.Spec.Canary = constructRolloutRunCanary(strategy.Canary, workloadWrappers)
 		}
 		run.Spec.Batch = &rolloutv1alpha1.RolloutRunBatchStrategy{
-			Toleration: strategy.Batch.Toleration,
-			Batches:    constructRolloutRunBatches(strategy.Batch, workloadWrappers),
+			Batches: constructRolloutRunBatches(strategy.Batch, workloadWrappers),
 		}
 	}
 

pkg/controllers/rolloutrun/executor/batch.go

@@ -217,7 +217,10 @@ func (e *batchExecutor) doBatchUpgrading(ctx *ExecutorContext) (bool, time.Durat
 
 		currentBatchExpectedReplicas, _ := workload.CalculateUpdatedReplicas(&status.Replicas, item.Replicas)
 
-		ready, reason := info.CheckUpdatedReady(currentBatchExpectedReplicas, isLastBatch)
+		// Find skip toleration for this workload
+		skipToleration := findSkipToleration(rolloutRun.Spec.Batch.Tolerations, item.CrossClusterObjectNameReference)
+
+		ready, reason := info.CheckUpdatedReady(currentBatchExpectedReplicas, isLastBatch, skipToleration)
 		if ready {
 			// if the target is ready, we will not change partition
 			continue
@@ -270,3 +273,13 @@ func (e *batchExecutor) calculateExpectedReplicasBySlidingWindow(status rolloutv
 	expected = min(currentBatchExpectedReplicas, expected)
 	return expected, nil
 }
+
+// findSkipToleration finds the accumulated skip toleration for the given workload
+func findSkipToleration(tolerations []rolloutv1alpha1.RolloutRunTolerationTarget, ref rolloutv1alpha1.CrossClusterObjectNameReference) int32 {
+	for _, t := range tolerations {
+		if t.CrossClusterObjectNameReference == ref {
+			return t.Toleration
+		}
+	}
+	return 0
+}

pkg/controllers/rolloutrun/executor/batch_test.go

@@ -318,6 +318,203 @@ func (s *batchExecutorTestSuite) Test_BatchExecutor_Do() {
 	s.runBatchTestCases(tests)
 }
 
+func (s *batchExecutorTestSuite) Test_BatchExecutor_Do_SkipToleration() {
+	tests := []batchExectorTestCase{
+		{
+			name: "skip toleration allows batch to complete with deficit in middle batch",
+			getObjects: func() (*rolloutv1alpha1.Rollout, *rolloutv1alpha1.RolloutRun) {
+				rollout := s.rollout.DeepCopy()
+				rolloutRun := s.rolloutRun.DeepCopy()
+
+				// 3 batches, currently on batch 2 (index 1), not the last batch
+				rolloutRun.Spec.Batch.Batches = []rolloutv1alpha1.RolloutRunStep{
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(30)),
+					}},
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(60)),
+					}},
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(100)),
+					}},
+				}
+				rolloutRun.Status.Phase = rolloutv1alpha1.RolloutRunPhaseProgressing
+				rolloutRun.Status.BatchStatus = &rolloutv1alpha1.RolloutRunBatchStatus{
+					RolloutBatchStatus: rolloutv1alpha1.RolloutBatchStatus{
+						CurrentBatchIndex: 1,
+						CurrentBatchState: StepRunning,
+					},
+					Records: []rolloutv1alpha1.RolloutRunStepStatus{
+						{Index: ptr.To[int32](0), State: StepSkipped},
+						{Index: ptr.To[int32](1), State: StepRunning, StartTime: ptr.To(metav1.Now())},
+						{Index: ptr.To[int32](2), State: StepNone},
+					},
+				}
+				// Batch 0 was skipped: gap = 30 - 25 = 5
+				rolloutRun.Spec.Batch.Tolerations = []rolloutv1alpha1.RolloutRunTolerationTarget{
+					{CrossClusterObjectNameReference: rolloutv1alpha1.CrossClusterObjectNameReference{Cluster: "cluster-a", Name: "test-a"}, Toleration: 5},
+				}
+				return rollout, rolloutRun
+			},
+			getWorkloads: func() []client.Object {
+				// UpdatedAvailableReplicas = 55, expected = 60, gap = 5 <= toleration(5)
+				return []client.Object{
+					newFakeObject("cluster-a", "default", "test-a", 100, 55, 55),
+				}
+			},
+			assertResult: func(done bool, result reconcile.Result, err error) {
+				s.Require().NoError(err)
+				s.False(done) // not all done, move to next batch
+				s.Equal(reconcile.Result{Requeue: true}, result)
+			},
+			assertStatus: func(status *rolloutv1alpha1.RolloutRunStatus) {
+				s.Equal(StepPostBatchStepHook, status.BatchStatus.CurrentBatchState)
+			},
+		},
+		{
+			name: "skip toleration not enough, batch stays running",
+			getObjects: func() (*rolloutv1alpha1.Rollout, *rolloutv1alpha1.RolloutRun) {
+				rollout := s.rollout.DeepCopy()
+				rolloutRun := s.rolloutRun.DeepCopy()
+
+				rolloutRun.Spec.Batch.Batches = []rolloutv1alpha1.RolloutRunStep{
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(30)),
+					}},
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(60)),
+					}},
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(100)),
+					}},
+				}
+				rolloutRun.Status.Phase = rolloutv1alpha1.RolloutRunPhaseProgressing
+				rolloutRun.Status.BatchStatus = &rolloutv1alpha1.RolloutRunBatchStatus{
+					RolloutBatchStatus: rolloutv1alpha1.RolloutBatchStatus{
+						CurrentBatchIndex: 1,
+						CurrentBatchState: StepRunning,
+					},
+					Records: []rolloutv1alpha1.RolloutRunStepStatus{
+						{Index: ptr.To[int32](0), State: StepSkipped},
+						{Index: ptr.To[int32](1), State: StepRunning, StartTime: ptr.To(metav1.Now())},
+						{Index: ptr.To[int32](2), State: StepNone},
+					},
+				}
+				// gap = 60 - 52 = 8 > toleration(5)
+				rolloutRun.Spec.Batch.Tolerations = []rolloutv1alpha1.RolloutRunTolerationTarget{
+					{CrossClusterObjectNameReference: rolloutv1alpha1.CrossClusterObjectNameReference{Cluster: "cluster-a", Name: "test-a"}, Toleration: 5},
+				}
+				return rollout, rolloutRun
+			},
+			getWorkloads: func() []client.Object {
+				return []client.Object{
+					newFakeObject("cluster-a", "default", "test-a", 100, 52, 52),
+				}
+			},
+			assertResult: func(done bool, result reconcile.Result, err error) {
+				s.Require().NoError(err)
+				s.False(done)
+				s.Equal(reconcile.Result{RequeueAfter: retryDefault}, result)
+			},
+			assertStatus: func(status *rolloutv1alpha1.RolloutRunStatus) {
+				s.Equal(StepRunning, status.BatchStatus.CurrentBatchState)
+			},
+		},
+		{
+			name: "skip toleration does not apply on last batch",
+			getObjects: func() (*rolloutv1alpha1.Rollout, *rolloutv1alpha1.RolloutRun) {
+				rollout := s.rollout.DeepCopy()
+				rolloutRun := s.rolloutRun.DeepCopy()
+
+				rolloutRun.Spec.Batch.Batches = []rolloutv1alpha1.RolloutRunStep{
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(30)),
+					}},
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(60)),
+					}},
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(100)),
+					}},
+				}
+				rolloutRun.Status.Phase = rolloutv1alpha1.RolloutRunPhaseProgressing
+				rolloutRun.Status.BatchStatus = &rolloutv1alpha1.RolloutRunBatchStatus{
+					RolloutBatchStatus: rolloutv1alpha1.RolloutBatchStatus{
+						CurrentBatchIndex: 2,
+						CurrentBatchState: StepRunning,
+					},
+					Records: []rolloutv1alpha1.RolloutRunStepStatus{
+						{Index: ptr.To[int32](0), State: StepSkipped},
+						{Index: ptr.To[int32](1), State: StepSucceeded},
+						{Index: ptr.To[int32](2), State: StepRunning, StartTime: ptr.To(metav1.Now())},
+					},
+				}
+				// Last batch: toleration should NOT apply
+				rolloutRun.Spec.Batch.Tolerations = []rolloutv1alpha1.RolloutRunTolerationTarget{
+					{CrossClusterObjectNameReference: rolloutv1alpha1.CrossClusterObjectNameReference{Cluster: "cluster-a", Name: "test-a"}, Toleration: 5},
+				}
+				return rollout, rolloutRun
+			},
+			getWorkloads: func() []client.Object {
+				// UpdatedAvailableReplicas = 96, expected = 100, gap = 4 <= toleration(5)
+				// but last batch, so toleration does NOT apply, needs strict check
+				return []client.Object{
+					newFakeObject("cluster-a", "default", "test-a", 100, 96, 96),
+				}
+			},
+			assertResult: func(done bool, result reconcile.Result, err error) {
+				s.Require().NoError(err)
+				s.False(done)
+				s.Equal(reconcile.Result{RequeueAfter: retryDefault}, result)
+			},
+			assertStatus: func(status *rolloutv1alpha1.RolloutRunStatus) {
+				s.Equal(StepRunning, status.BatchStatus.CurrentBatchState)
+			},
+		},
+		{
+			name: "no skip toleration, behavior unchanged",
+			getObjects: func() (*rolloutv1alpha1.Rollout, *rolloutv1alpha1.RolloutRun) {
+				rollout := s.rollout.DeepCopy()
+				rolloutRun := s.rolloutRun.DeepCopy()
+
+				rolloutRun.Spec.Batch.Batches = []rolloutv1alpha1.RolloutRunStep{
+					{Targets: []rolloutv1alpha1.RolloutRunStepTarget{
+						newRunStepTarget("cluster-a", "test-a", intstr.FromInt(10)),
+					}},
+				}
+				rolloutRun.Status.Phase = rolloutv1alpha1.RolloutRunPhaseProgressing
+				rolloutRun.Status.BatchStatus = &rolloutv1alpha1.RolloutRunBatchStatus{
+					RolloutBatchStatus: rolloutv1alpha1.RolloutBatchStatus{
+						CurrentBatchIndex: 0,
+						CurrentBatchState: StepRunning,
+					},
+					Records: []rolloutv1alpha1.RolloutRunStepStatus{
+						{Index: ptr.To[int32](0), State: StepRunning, StartTime: ptr.To(metav1.Now())},
+					},
+				}
+				// No skip tolerations
+				return rollout, rolloutRun
+			},
+			getWorkloads: func() []client.Object {
+				// UpdatedAvailableReplicas=8 < expected=10, no toleration -> not ready
+				return []client.Object{
+					newFakeObject("cluster-a", "default", "test-a", 100, 8, 8),
+				}
+			},
+			assertResult: func(done bool, result reconcile.Result, err error) {
+				s.Require().NoError(err)
+				s.False(done)
+				s.Equal(reconcile.Result{RequeueAfter: retryDefault}, result)
+			},
+			assertStatus: func(status *rolloutv1alpha1.RolloutRunStatus) {
+				s.Equal(StepRunning, status.BatchStatus.CurrentBatchState)
+			},
+		},
+	}
+
+	s.runBatchTestCases(tests)
+}
+
 func newRunStepTarget(cluster, name string, replicas intstr.IntOrString) rolloutv1alpha1.RolloutRunStepTarget {
 	return newRunStepTargetWithSlidingWindow(cluster, name, replicas, nil)
 }

pkg/controllers/rolloutrun/executor/canary.go

@@ -224,7 +224,7 @@ func (e *canaryExecutor) doCanary(ctx *ExecutorContext) (bool, time.Duration, er
 
 	// 2.b. waiting canary workload ready
 	for _, info := range canaryWorkloads {
-		if ready, _ := info.CheckUpdatedReady(info.Status.DesiredReplicas, false); !ready {
+		if ready, _ := info.CheckUpdatedReady(info.Status.DesiredReplicas, false, 0); !ready {
 			// ready
 			logger.Info("still waiting for canary target ready",
 				"cluster", info.ClusterName,

pkg/controllers/rolloutrun/executor/default_test.go

@@ -50,8 +50,7 @@ var (
 			},
 			Webhooks: []rolloutv1alpha1.RolloutWebhook{},
 			Batch: &rolloutv1alpha1.RolloutRunBatchStrategy{
-				Toleration: &rolloutv1alpha1.TolerationStrategy{},
-				Batches:    []rolloutv1alpha1.RolloutRunStep{},
+				Batches: []rolloutv1alpha1.RolloutRunStep{},
 			},
 		},
 		Status: rolloutv1alpha1.RolloutRunStatus{
@@ -76,8 +75,7 @@ var (
 				Targets: []rolloutv1alpha1.RolloutRunStepTarget{},
 			},
 			Batch: &rolloutv1alpha1.RolloutRunBatchStrategy{
-				Toleration: &rolloutv1alpha1.TolerationStrategy{},
-				Batches:    []rolloutv1alpha1.RolloutRunStep{},
+				Batches: []rolloutv1alpha1.RolloutRunStep{},
 			},
 		},
 		Status: rolloutv1alpha1.RolloutRunStatus{