bundle: route config sync edits to the block that defines them

acceptance/bundle/config-remote-sync/split/keyed_edit/databricks.yml.tmpl

@@ -0,0 +1,30 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# The task list of split_job is defined in two physical blocks: the top-level
+# one below and the targets.dev override further down. After loading they merge
+# into one list sorted by task_key, so the merged order is
+#   [alpha (target), mu (top-level), zeta (top-level)]
+# while each physical block keeps its own order. Editing a task must therefore
+# resolve to (file, block, index-within-that-block), not to the merged index.
+resources:
+  jobs:
+    split_job:
+      tasks:
+        - task_key: zeta
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/zeta
+        - task_key: mu
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/mu
+
+targets:
+  dev:
+    mode: development
+    resources:
+      jobs:
+        split_job:
+          tasks:
+            - task_key: alpha
+              notebook_task:
+                notebook_path: /Users/{{workspace_user_name}}/alpha

acceptance/bundle/config-remote-sync/split/keyed_edit/output.txt

@@ -0,0 +1,53 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Edit alpha, defined only in the target block
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.split_job
+  tasks[task_key='alpha'].timeout_seconds: add
+
+
+
+=== Only alpha in the target block gains timeout_seconds: 111
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -29,2 +29,3 @@
+               notebook_task:
+                 notebook_path: /Users/{{workspace_user_name}}/alpha
++              timeout_seconds: 111
+
+=== Edit mu, a top-level task that is not first after the merge sort
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.split_job
+  tasks[task_key='mu'].timeout_seconds: add
+
+
+
+=== Only mu in the top-level block gains timeout_seconds: 222
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -18,4 +18,5 @@
+           notebook_task:
+             notebook_path: /Users/{{workspace_user_name}}/mu
++          timeout_seconds: 222
+
+ targets:
+
+>>> [CLI] bundle destroy --auto-approve -t dev
+The following resources will be deleted:
+  delete resources.jobs.split_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/split/keyed_edit/script

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve -t dev
+}
+trap cleanup EXIT
+
+$CLI bundle deploy -t dev
+job_id="$(read_id.py split_job)"
+
+
+# alpha is the only task in the target block, so it must be written at index 0 of
+# that block. Its merged index is also 0, but the top-level block's index 0 is
+# zeta -- a naive merged-index write lands on zeta instead.
+title "Edit alpha, defined only in the target block"
+edit_resource.py jobs $job_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "alpha":
+        task["timeout_seconds"] = 111
+EOF
+
+title "Sync"
+echo
+cp databricks.yml databricks.yml.backup
+errcode $CLI bundle config-remote-sync -t dev --save
+
+title "Only alpha in the target block gains timeout_seconds: 111"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup
+
+
+# mu is at merged index 1 but physical index 1 of the top-level block, and zeta
+# (merged index 2) is physical index 0. The merged order and the block order
+# disagree, so writing by merged index edits the wrong task.
+title "Edit mu, a top-level task that is not first after the merge sort"
+edit_resource.py jobs $job_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "mu":
+        task["timeout_seconds"] = 222
+EOF
+
+title "Sync"
+echo
+cp databricks.yml databricks.yml.backup
+errcode $CLI bundle config-remote-sync -t dev --save
+
+title "Only mu in the top-level block gains timeout_seconds: 222"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/split/keyed_twoblock/databricks.yml.tmpl

@@ -0,0 +1,39 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# "shared" is defined in BOTH blocks: the top-level one carries max_retries and
+# the targets.dev one carries timeout_seconds. Loading merges them into a single
+# element whose fields come from two different physical places, so each field
+# edit has to be routed to the block that actually defines that field.
+#
+# max_retries is set in BOTH blocks, so the target's value is the one that was
+# deployed and the one an edit has to overwrite.
+#
+# "omega" sits after "shared" in the target block. Because "shared" contributes
+# to both blocks its position is easy to miscount, which is what makes editing
+# the plain single-block sibling "omega" the subtle case.
+resources:
+  jobs:
+    twoblock_job:
+      tasks:
+        - task_key: shared
+          max_retries: 1
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/shared
+        - task_key: nu
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/nu
+
+targets:
+  dev:
+    mode: development
+    resources:
+      jobs:
+        twoblock_job:
+          tasks:
+            - task_key: shared
+              max_retries: 2
+              timeout_seconds: 60
+            - task_key: omega
+              notebook_task:
+                notebook_path: /Users/{{workspace_user_name}}/omega

acceptance/bundle/config-remote-sync/split/keyed_twoblock/output.txt

@@ -0,0 +1,64 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Edit omega, whose sibling is defined in two blocks
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.twoblock_job
+  tasks[task_key='omega'].timeout_seconds: add
+
+
+
+=== Only omega changes; shared is untouched in both blocks
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -38,2 +38,3 @@
+               notebook_task:
+                 notebook_path: /Users/{{workspace_user_name}}/omega
++              timeout_seconds: 303
+
+=== Edit a field per block, a field defined in both, and add a new field
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.twoblock_job
+  tasks[task_key='shared'].max_retries: replace
+  tasks[task_key='shared'].min_retry_interval_millis: add
+  tasks[task_key='shared'].timeout_seconds: replace
+
+
+
+=== Each edit lands on the definition that was deployed
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -21,4 +21,5 @@
+           notebook_task:
+             notebook_path: /Users/{{workspace_user_name}}/shared
++          min_retry_interval_millis: 5000
+         - task_key: nu
+           notebook_task:
+@@ -33,6 +34,6 @@
+           tasks:
+             - task_key: shared
+-              max_retries: 2
+-              timeout_seconds: 60
++              max_retries: 9
++              timeout_seconds: 900
+             - task_key: omega
+               notebook_task:
+
+>>> [CLI] bundle destroy --auto-approve -t dev
+The following resources will be deleted:
+  delete resources.jobs.twoblock_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/split/keyed_twoblock/script

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve -t dev
+}
+trap cleanup EXIT
+
+$CLI bundle deploy -t dev
+job_id="$(read_id.py twoblock_job)"
+
+
+# omega is defined only in the target block, but its sibling "shared" contributes
+# to both blocks. Editing omega must land on omega and must leave the two-block
+# element completely untouched.
+title "Edit omega, whose sibling is defined in two blocks"
+edit_resource.py jobs $job_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "omega":
+        task["timeout_seconds"] = 303
+EOF
+
+title "Sync"
+echo
+cp databricks.yml databricks.yml.backup
+errcode $CLI bundle config-remote-sync -t dev --save
+
+title "Only omega changes; shared is untouched in both blocks"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup
+
+
+# timeout_seconds lives only in the target block, so its edit goes there.
+# max_retries is set in BOTH blocks: the target's value is the deployed one, so
+# that is the definition an edit has to overwrite -- writing the top-level copy
+# instead would leave the effective value unchanged. A brand-new field on the same
+# element must land in a single defined place, not be duplicated across both.
+title "Edit a field per block, a field defined in both, and add a new field"
+edit_resource.py jobs $job_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "shared":
+        task["max_retries"] = 9
+        task["timeout_seconds"] = 900
+        task["min_retry_interval_millis"] = 5000
+EOF
+
+title "Sync"
+echo
+cp databricks.yml databricks.yml.backup
+errcode $CLI bundle config-remote-sync -t dev --save
+
+title "Each edit lands on the definition that was deployed"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/split/multifile/databricks.yml.tmpl

@@ -0,0 +1,17 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# One target's override of a single job is spread across two included files.
+# Resource keys are unique across top-level files, but that is not enforced
+# inside the targets subtree, so a target override may legitimately span
+# several files. An edit must land in the exact file that defines the element.
+include:
+  - overrides/*.yml
+
+resources:
+  jobs:
+    multifile_job:
+      tasks:
+        - task_key: base
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/base

acceptance/bundle/config-remote-sync/split/multifile/output.txt

@@ -0,0 +1,40 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Edit a task defined in the second included file
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.multifile_job
+  tasks[task_key='from_second_file'].timeout_seconds: add
+
+
+
+=== databricks.yml is untouched
+
+>>> diff.py databricks.yml.backup databricks.yml
+
+=== overrides/10-first.yml is untouched
+
+>>> diff.py overrides/10-first.yml.backup overrides/10-first.yml
+
+=== overrides/20-second.yml gains timeout_seconds: 777
+
+>>> diff.py overrides/20-second.yml.backup overrides/20-second.yml
+--- overrides/20-second.yml.backup
++++ overrides/20-second.yml
+@@ -8,2 +8,3 @@
+               notebook_task:
+                 notebook_path: /Users/{{workspace_user_name}}/second
++              timeout_seconds: 777
+
+>>> [CLI] bundle destroy --auto-approve -t dev
+The following resources will be deleted:
+  delete resources.jobs.multifile_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/split/multifile/overrides/10-first.yml.tmpl

@@ -0,0 +1,10 @@
+targets:
+  dev:
+    mode: development
+    resources:
+      jobs:
+        multifile_job:
+          tasks:
+            - task_key: from_first_file
+              notebook_task:
+                notebook_path: /Users/{{workspace_user_name}}/first

acceptance/bundle/config-remote-sync/split/multifile/overrides/20-second.yml.tmpl

@@ -0,0 +1,9 @@
+targets:
+  dev:
+    resources:
+      jobs:
+        multifile_job:
+          tasks:
+            - task_key: from_second_file
+              notebook_task:
+                notebook_path: /Users/{{workspace_user_name}}/second