acc: cover config-remote-sync write-back for split lists

acceptance/bundle/config-remote-sync/split/isolation/databricks.yml.tmpl

@@ -0,0 +1,34 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# job_a's task list is split across both blocks and one task, "shared", is defined
+# in both. job_b is an ordinary single-block job.
+#
+# The point of the fixture: whatever happens to job_a, an unrelated resource's
+# unambiguous change must still be applied in the same run. The sync is
+# unattended, so one hard-to-place change must never stop the rest.
+resources:
+  jobs:
+    job_a:
+      tasks:
+        - task_key: shared
+          max_retries: 1
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/shared
+
+    job_b:
+      max_concurrent_runs: 1
+      tasks:
+        - task_key: simple
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/simple
+
+targets:
+  dev:
+    mode: development
+    resources:
+      jobs:
+        job_a:
+          tasks:
+            - task_key: shared
+              timeout_seconds: 45

acceptance/bundle/config-remote-sync/split/isolation/output.txt

@@ -0,0 +1,56 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Rename the two-block task on job_a, and edit job_b in the same run
+=== Sync
+Detected changes in 2 resource(s):
+
+Resource: resources.jobs.job_a
+  tasks[task_key='shared']: remove
+  tasks[task_key='shared_renamed']: add
+
+Resource: resources.jobs.job_b
+  max_concurrent_runs: replace
+
+
+
+=== job_b is updated, and job_a keeps exactly one copy of the task
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -12,11 +12,11 @@
+     job_a:
+       tasks:
+-        - task_key: shared
+-          max_retries: 1
++        - max_retries: 1
+           notebook_task:
+-            notebook_path: /Users/{{workspace_user_name}}/shared
+-
++            notebook_path: '/Users/{{workspace_user_name}}/shared'
++          task_key: shared_renamed
++          timeout_seconds: 45
+     job_b:
+-      max_concurrent_runs: 1
++      max_concurrent_runs: 6
+       tasks:
+         - task_key: simple
+
+>>> grep -c max_concurrent_runs: 6 databricks.yml
+1
+
+>>> grep -c task_key: shared_renamed databricks.yml
+1
+
+>>> [CLI] bundle destroy --auto-approve -t dev
+The following resources will be deleted:
+  delete resources.jobs.job_a
+  delete resources.jobs.job_b
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/split/isolation/script

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve -t dev
+}
+trap cleanup EXIT
+
+$CLI bundle deploy -t dev
+job_a_id="$(read_id.py job_a)"
+job_b_id="$(read_id.py job_b)"
+
+
+# A rename of the two-block task on job_a, and a plain scalar edit on job_b, in the
+# SAME run. job_b's edit is independent of anything job_a does, so it must be
+# applied whether or not job_a's rename can be placed.
+title "Rename the two-block task on job_a, and edit job_b in the same run"
+edit_resource.py jobs $job_a_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "shared":
+        task["task_key"] = "shared_renamed"
+EOF
+
+edit_resource.py jobs $job_b_id <<EOF
+r["max_concurrent_runs"] = 6
+EOF
+
+title "Sync"
+echo
+cp databricks.yml databricks.yml.backup
+errcode $CLI bundle config-remote-sync -t dev --save
+
+title "job_b is updated, and job_a keeps exactly one copy of the task"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+trace grep -c "max_concurrent_runs: 6" databricks.yml
+trace grep -c "task_key: shared_renamed" databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/split/keyed_edit/databricks.yml.tmpl

@@ -0,0 +1,30 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# The task list of split_job is defined in two physical blocks: the top-level
+# one below and the targets.dev override further down. After loading they merge
+# into one list sorted by task_key, so the merged order is
+#   [alpha (target), mu (top-level), zeta (top-level)]
+# while each physical block keeps its own order. Editing a task must therefore
+# resolve to (file, block, index-within-that-block), not to the merged index.
+resources:
+  jobs:
+    split_job:
+      tasks:
+        - task_key: zeta
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/zeta
+        - task_key: mu
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/mu
+
+targets:
+  dev:
+    mode: development
+    resources:
+      jobs:
+        split_job:
+          tasks:
+            - task_key: alpha
+              notebook_task:
+                notebook_path: /Users/{{workspace_user_name}}/alpha

acceptance/bundle/config-remote-sync/split/keyed_edit/output.txt

@@ -0,0 +1,57 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Edit alpha, defined only in the target block
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.split_job
+  tasks[task_key='alpha'].timeout_seconds: add
+
+
+
+=== Only alpha in the target block gains timeout_seconds: 111
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -18,4 +18,5 @@
+           notebook_task:
+             notebook_path: /Users/{{workspace_user_name}}/mu
++        - timeout_seconds: 111
+
+ targets:
+
+=== Edit mu, a top-level task that is not first after the merge sort
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.split_job
+  tasks[task_key='']: remove
+  tasks[task_key='alpha'].timeout_seconds: add
+  tasks[task_key='mu'].timeout_seconds: add
+
+
+
+=== Only mu in the top-level block gains timeout_seconds: 222
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -18,4 +18,5 @@
+           notebook_task:
+             notebook_path: /Users/{{workspace_user_name}}/mu
++          timeout_seconds: 222
+         - timeout_seconds: 111
+
+
+>>> [CLI] bundle destroy --auto-approve -t dev
+The following resources will be deleted:
+  delete resources.jobs.split_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/split/keyed_edit/script

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve -t dev
+}
+trap cleanup EXIT
+
+$CLI bundle deploy -t dev
+job_id="$(read_id.py split_job)"
+
+
+# alpha is the only task in the target block, so it must be written at index 0 of
+# that block. Its merged index is also 0, but the top-level block's index 0 is
+# zeta -- a naive merged-index write lands on zeta instead.
+title "Edit alpha, defined only in the target block"
+edit_resource.py jobs $job_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "alpha":
+        task["timeout_seconds"] = 111
+EOF
+
+title "Sync"
+echo
+cp databricks.yml databricks.yml.backup
+errcode $CLI bundle config-remote-sync -t dev --save
+
+title "Only alpha in the target block gains timeout_seconds: 111"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup
+
+
+# mu is at merged index 1 but physical index 1 of the top-level block, and zeta
+# (merged index 2) is physical index 0. The merged order and the block order
+# disagree, so writing by merged index edits the wrong task.
+title "Edit mu, a top-level task that is not first after the merge sort"
+edit_resource.py jobs $job_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "mu":
+        task["timeout_seconds"] = 222
+EOF
+
+title "Sync"
+echo
+cp databricks.yml databricks.yml.backup
+errcode $CLI bundle config-remote-sync -t dev --save
+
+title "Only mu in the top-level block gains timeout_seconds: 222"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/split/keyed_remove/databricks.yml.tmpl

@@ -0,0 +1,45 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# remove_job: gamma/beta live top-level, alpha lives in the target block. The
+# merged order is [alpha, beta, gamma], so the merged indices collide with the
+# physical ones -- removing by merged index deletes the wrong tasks.
+#
+# twoblock_remove_job: "both" is defined in the two blocks at once. Removing it
+# cannot be expressed as "drop it from one scope but keep the merged result", so
+# the sync must leave the source untouched rather than half-deleting it.
+resources:
+  jobs:
+    remove_job:
+      tasks:
+        - task_key: gamma
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/gamma
+        - task_key: beta
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/beta
+
+    twoblock_remove_job:
+      tasks:
+        - task_key: both
+          max_retries: 2
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/both
+        - task_key: keep
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/keep
+
+targets:
+  dev:
+    mode: development
+    resources:
+      jobs:
+        remove_job:
+          tasks:
+            - task_key: alpha
+              notebook_task:
+                notebook_path: /Users/{{workspace_user_name}}/alpha
+        twoblock_remove_job:
+          tasks:
+            - task_key: both
+              timeout_seconds: 30

acceptance/bundle/config-remote-sync/split/keyed_remove/output.txt

@@ -0,0 +1,54 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Remove gamma from the top-level block and alpha from the target block
+=== Sync
+Error: failed to generate YAML files: failed to apply change to file [TEST_TMP_DIR]/databricks.yml for a field resources.jobs.remove_job.tasks[2]: failed to apply change: op remove /resources/jobs/remove_job/tasks/2: remove index key out of bounds (idx 2, len 2)
+
+Exit code: 1
+
+=== Exactly gamma and alpha are gone; beta survives
+
+>>> diff.py databricks.yml.backup databricks.yml
+
+=== Remove the task defined in BOTH blocks
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.twoblock_remove_job
+  tasks[task_key='both']: remove
+
+
+
+=== 'both' is gone from both blocks; 'keep' survives
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -22,8 +22,4 @@
+     twoblock_remove_job:
+       tasks:
+-        - task_key: both
+-          max_retries: 2
+-          notebook_task:
+-            notebook_path: /Users/{{workspace_user_name}}/both
+         - task_key: keep
+           notebook_task:
+
+>>> grep -c task_key: both databricks.yml
+1
+
+>>> grep -c task_key: keep databricks.yml
+1
+
+>>> [CLI] bundle destroy --auto-approve -t dev
+The following resources will be deleted:
+  delete resources.jobs.remove_job
+  delete resources.jobs.twoblock_remove_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev
+
+Deleting files...
+Destroy complete!