config-remote-sync: block-aware YAML indexing for tasks/clusters split across top-level and target blocks

acceptance/bundle/config-remote-sync/job_clusters_split/databricks.yml.tmpl

@@ -0,0 +1,41 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# job_clusters is merged by merge.ElementsByKey (NON-sorted, unlike tasks). The
+# sequence is still split across the top-level block (aaa_cluster, bbb_cluster)
+# and the targets.default override block (ccc_cluster). Merged order is
+# aaa,bbb,ccc, so ccc_cluster's whole-file position is 2 but its target-local
+# index is 0. Editing ccc_cluster must patch the targets block at index 0, not
+# the top-level block.
+resources:
+  jobs:
+    my_job:
+      job_clusters:
+        - job_cluster_key: aaa_cluster
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+        - job_cluster_key: bbb_cluster
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          job_cluster_key: aaa_cluster
+
+targets:
+  default:
+    mode: development
+    resources:
+      jobs:
+        my_job:
+          job_clusters:
+            - job_cluster_key: ccc_cluster
+              new_cluster:
+                spark_version: $DEFAULT_SPARK_VERSION
+                node_type_id: $NODE_TYPE_ID
+                num_workers: 1

acceptance/bundle/config-remote-sync/job_clusters_split/output.txt

@@ -0,0 +1,34 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Set num_workers on ccc_cluster remotely (target-block cluster)
+
+=== Detect and save changes
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  job_clusters[job_cluster_key='ccc_cluster'].new_cluster.num_workers: replace
+
+
+
+=== Configuration changes
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -39,3 +39,3 @@
+                 spark_version: 13.3.x-snapshot-scala2.12
+                 node_type_id: [NODE_TYPE_ID]
+-                num_workers: 1
++                num_workers: 5
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/job_clusters_split/script

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+# ccc_cluster is defined in the targets block (target-local index 0); the
+# top-level block holds aaa_cluster, bbb_cluster. Editing ccc_cluster's
+# num_workers must patch the targets block, proving block-aware indexing works
+# for the non-sorted job_clusters merge, not just tasks.
+title "Set num_workers on ccc_cluster remotely (target-block cluster)"
+echo
+edit_resource.py jobs $job_id <<EOF
+for cluster in r.get("job_clusters", []):
+    if cluster.get("job_cluster_key") == "ccc_cluster":
+        cluster["new_cluster"]["num_workers"] = 5
+EOF
+
+title "Detect and save changes"
+echo
+cp databricks.yml databricks.yml.backup
+$CLI bundle config-remote-sync --save
+
+title "Configuration changes"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/job_clusters_split/test.toml

@@ -0,0 +1,10 @@
+# Local-only: exercises block-aware indexing for a non-sorted merged sequence
+# without needing real cluster creation.
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"]

acceptance/bundle/config-remote-sync/pipeline_clusters_split/databricks.yml.tmpl

@@ -0,0 +1,30 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# Pipeline clusters are merged by merge.ElementsByKey on the "label" key. The
+# sequence is split across the top-level block (label: default) and the
+# targets.default override block (label: maintenance). Merged order is
+# default,maintenance, so the maintenance cluster's whole-file position is 1 but
+# its target-local index is 0. Editing the maintenance cluster must patch the
+# targets block. Pipeline is non-UC so the test runs locally.
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-$UNIQUE_NAME
+      development: false
+      libraries:
+        - notebook:
+            path: /Users/{{workspace_user_name}}/notebook
+      clusters:
+        - label: default
+          num_workers: 1
+
+targets:
+  default:
+    mode: development
+    resources:
+      pipelines:
+        my_pipeline:
+          clusters:
+            - label: maintenance
+              num_workers: 1

acceptance/bundle/config-remote-sync/pipeline_clusters_split/output.txt

@@ -0,0 +1,38 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Set num_workers on maintenance cluster remotely (target-block cluster)
+
+=== Detect and save changes
+Detected changes in 1 resource(s):
+
+Resource: resources.pipelines.my_pipeline
+  clusters[1].num_workers: replace
+
+
+
+=== Configuration changes
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -28,3 +28,3 @@
+           clusters:
+             - label: maintenance
+-              num_workers: 1
++              num_workers: 3
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.pipelines.my_pipeline
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them:
+  delete resources.pipelines.my_pipeline
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/pipeline_clusters_split/script

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+$CLI bundle deploy
+pipeline_id="$(read_id.py my_pipeline)"
+
+# The maintenance cluster is defined in the targets block (target-local index
+# 0); the top-level block holds the default cluster. Editing the maintenance
+# cluster's num_workers must patch the targets block, proving block-aware
+# indexing works for pipeline clusters, a different resource type from jobs.
+title "Set num_workers on maintenance cluster remotely (target-block cluster)"
+echo
+edit_resource.py pipelines $pipeline_id <<EOF
+for cluster in r.get("clusters", []):
+    if cluster.get("label") == "maintenance":
+        cluster["num_workers"] = 3
+EOF
+
+title "Detect and save changes"
+echo
+cp databricks.yml databricks.yml.backup
+$CLI bundle config-remote-sync --save
+
+title "Configuration changes"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/pipeline_clusters_split/test.toml

@@ -0,0 +1,11 @@
+# Local-only: exercises block-aware indexing on a pipeline (different resource
+# type) without needing real pipeline/cluster creation. No UC catalog/schema so
+# it is not gated by RequiresUnityCatalog.
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"]

acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/databricks.yml.tmpl

@@ -0,0 +1,31 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# Top-level clusters [default, maintenance]; the target overrides `default`, so
+# the clusters sequence is split (2 locations) but pipeline clusters diff
+# positionally (no KeyedSlices) -> numeric change paths. Editing the maintenance
+# cluster (top-level block-local index 1) must resolve to the top-level
+# clusters[1], NOT be shifted or misclassified as a target element.
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-$UNIQUE_NAME
+      development: false
+      libraries:
+        - notebook:
+            path: /Users/{{workspace_user_name}}/notebook
+      clusters:
+        - label: default
+          num_workers: 1
+        - label: maintenance
+          num_workers: 1
+
+targets:
+  default:
+    mode: development
+    resources:
+      pipelines:
+        my_pipeline:
+          clusters:
+            - label: default
+              num_workers: 2

acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/output.txt

@@ -0,0 +1,40 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Set num_workers on maintenance cluster (top-level block index 1)
+
+=== Detect and save changes
+Detected changes in 1 resource(s):
+
+Resource: resources.pipelines.my_pipeline
+  clusters[1].num_workers: replace
+
+
+
+=== Configuration changes
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -19,5 +19,5 @@
+           num_workers: 1
+         - label: maintenance
+-          num_workers: 1
++          num_workers: 7
+
+ targets:
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.pipelines.my_pipeline
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them:
+  delete resources.pipelines.my_pipeline
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/script

@@ -0,0 +1,26 @@
+#!/bin/bash
+envsubst < databricks.yml.tmpl > databricks.yml
+cleanup() { trace $CLI bundle destroy --auto-approve; }
+trap cleanup EXIT
+$CLI bundle deploy
+pipeline_id="$(read_id.py my_pipeline)"
+
+# maintenance is the SECOND top-level cluster (block-local index 1). Its numeric
+# change path clusters[1] must patch the top-level maintenance cluster, leaving
+# the target `default` override untouched.
+title "Set num_workers on maintenance cluster (top-level block index 1)"
+echo
+edit_resource.py pipelines $pipeline_id <<PY
+for cluster in r.get("clusters", []):
+    if cluster.get("label") == "maintenance":
+        cluster["num_workers"] = 7
+PY
+
+title "Detect and save changes"
+echo
+cp databricks.yml databricks.yml.backup
+$CLI bundle config-remote-sync --save
+title "Configuration changes"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/test.toml

@@ -0,0 +1,10 @@
+# Local-only: numeric (positional) change path for a TOP-LEVEL pipeline cluster
+# at block-local index >= 1 while the sequence is split by a target block.
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"]