bundle: write config sync key renames as a key rewrite

acceptance/bundle/config-remote-sync/job_fields/output.txt

@@ -58,18 +58,14 @@ Resource: resources.jobs.my_job
 +            - samples.nyctaxi.trips
        environments:
          - environment_key: default
-@@ -25,14 +31,14 @@
+@@ -25,5 +31,5 @@
                - ./*.whl
        job_clusters:
 -        - job_cluster_key: test_cluster
 +        - job_cluster_key: test_cluster_renamed
            new_cluster:
--            spark_version: [[DEFAULT_SPARK_VERSION]]
-             node_type_id: [NODE_TYPE_ID]
-             num_workers: 1
-+            spark_version: [[DEFAULT_SPARK_VERSION]]
-       tasks:
-         - task_key: main
+             spark_version: [[DEFAULT_SPARK_VERSION]]
+@@ -34,5 +40,5 @@
            notebook_task:
              notebook_path: /Users/{{workspace_user_name}}/notebook
 -          job_cluster_key: test_cluster

acceptance/bundle/config-remote-sync/job_multiple_tasks/output.txt

@@ -97,21 +97,14 @@ Resource: resources.jobs.rename_task_job
 >>> diff.py databricks.yml.backup2 databricks.yml
 --- databricks.yml.backup2
 +++ databricks.yml
-@@ -52,14 +52,14 @@
+@@ -52,5 +52,5 @@
      rename_task_job:
        tasks:
 -        - task_key: b_task
-+        - new_cluster:
-+            node_type_id: [NODE_TYPE_ID]
-+            num_workers: 1
-+            spark_version: 13.3.x-snapshot-scala2.12
++        - task_key: b_task_renamed
            notebook_task:
              notebook_path: /Users/{{workspace_user_name}}/b_task
--          new_cluster:
--            spark_version: 13.3.x-snapshot-scala2.12
--            node_type_id: [NODE_TYPE_ID]
--            num_workers: 1
-+          task_key: b_task_renamed
+@@ -61,5 +61,5 @@
          - task_key: d_task
            depends_on:
 -            - task_key: b_task

acceptance/bundle/config-remote-sync/multiple_files/output.txt

@@ -32,27 +32,13 @@ Resource: resources.jobs.job_two
 >>> diff.py resources/job1.yml.backup resources/job1.yml
 --- resources/job1.yml.backup
 +++ resources/job1.yml
-@@ -4,13 +4,13 @@
+@@ -4,5 +4,5 @@
        max_concurrent_runs: 1
        tasks:
 -        - task_key: c_task
-+        - depends_on:
-+            - task_key: b_task
-+          new_cluster:
-+            node_type_id: [NODE_TYPE_ID]
-+            num_workers: 1
-+            spark_version: 13.3.x-snapshot-scala2.12
++        - task_key: c_task_renamed
            notebook_task:
              notebook_path: /Users/{{workspace_user_name}}/c_task
--          new_cluster:
--            spark_version: 13.3.x-snapshot-scala2.12
--            node_type_id: [NODE_TYPE_ID]
--            num_workers: 1
--          depends_on:
--            - task_key: b_task
-+          task_key: c_task_renamed
-         - task_key: a_task
-           notebook_task:
 @@ -21,3 +21,10 @@
              num_workers: 1
            depends_on:
@@ -71,26 +57,16 @@ Resource: resources.jobs.job_two
 >>> diff.py resources/job2.yml.backup resources/job2.yml
 --- resources/job2.yml.backup
 +++ resources/job2.yml
-@@ -2,13 +2,13 @@
+@@ -2,7 +2,7 @@
    jobs:
      job_two:
 -      max_concurrent_runs: 2
 +      max_concurrent_runs: 10
        tasks:
 -        - task_key: run_pipeline
-+        - new_cluster:
-+            node_type_id: [NODE_TYPE_ID]
-+            num_workers: 1
-+            spark_version: 13.3.x-snapshot-scala2.12
++        - task_key: run_pipeline_renamed
            notebook_task:
              notebook_path: /Users/{{workspace_user_name}}/1
--          new_cluster:
--            spark_version: 13.3.x-snapshot-scala2.12
--            node_type_id: [NODE_TYPE_ID]
--            num_workers: 1
-+          task_key: run_pipeline_renamed
-         - task_key: etl_pipeline
-           notebook_task:
 @@ -18,5 +18,9 @@
              node_type_id: [NODE_TYPE_ID]
              num_workers: 2

acceptance/bundle/config-remote-sync/split/isolation/output.txt

@@ -21,36 +21,32 @@ Resource: resources.jobs.job_b
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -12,11 +12,11 @@
+@@ -12,5 +12,5 @@
      job_a:
        tasks:
 -        - task_key: shared
--          max_retries: 1
-+        - max_retries: 1
++        - task_key: shared_renamed
+           max_retries: 1
            notebook_task:
--            notebook_path: /Users/{{workspace_user_name}}/shared
--
-+            notebook_path: '/Users/{{workspace_user_name}}/shared'
-+          task_key: shared_renamed
-+          timeout_seconds: 45
+@@ -18,5 +18,5 @@
+
      job_b:
 -      max_concurrent_runs: 1
 +      max_concurrent_runs: 6
        tasks:
          - task_key: simple
-@@ -30,5 +30,3 @@
-       jobs:
+@@ -31,4 +31,4 @@
          job_a:
--          tasks:
+           tasks:
 -            - task_key: shared
--              timeout_seconds: 45
-+          tasks: []
++            - task_key: shared_renamed
+               timeout_seconds: 45
 
 >>> grep -c max_concurrent_runs: 6 databricks.yml
 1
 
 >>> grep -c task_key: shared_renamed databricks.yml
-1
+2
 
 >>> [CLI] bundle destroy --auto-approve -t dev
 The following resources will be deleted:

acceptance/bundle/config-remote-sync/split/keyed_rename/databricks.yml.tmpl

@@ -0,0 +1,43 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# "shared" is defined in both blocks: max_retries at the top level and
+# timeout_seconds in the target override. Renaming it must rewrite the key in both
+# places and leave each field in the scope it was written in.
+#
+# "solo" lives only in the target block, and the new keys are chosen so their sort
+# order is the reverse of the block order: "a_shared" sorts before "z_solo", while
+# the top-level block comes before the target block. An implementation that pairs
+# renames by order rather than by identity swaps their contents.
+resources:
+  jobs:
+    rename_job:
+      tasks:
+        - task_key: shared
+          max_retries: 1
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/shared
+
+    # A single block, to check that a removal earlier in the list shifts the
+    # position a later rename has to be written at.
+    shift_job:
+      tasks:
+        - task_key: aaa
+          max_retries: 1
+        - task_key: mmm
+          max_retries: 2
+        - task_key: zzz
+          max_retries: 3
+
+targets:
+  dev:
+    mode: development
+    resources:
+      jobs:
+        rename_job:
+          tasks:
+            - task_key: shared
+              timeout_seconds: 45
+            - task_key: solo
+              notebook_task:
+                notebook_path: /Users/{{workspace_user_name}}/solo

acceptance/bundle/config-remote-sync/split/keyed_rename/output.txt

@@ -0,0 +1,115 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Rename a task defined in both blocks and a target-only task
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.rename_job
+  tasks[task_key='a_shared']: add
+  tasks[task_key='shared']: remove
+  tasks[task_key='solo']: remove
+  tasks[task_key='z_solo']: add
+
+
+
+=== Both blocks get the new key; each field keeps its own scope
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -14,5 +14,5 @@
+     rename_job:
+       tasks:
+-        - task_key: shared
++        - task_key: a_shared
+           max_retries: 1
+           notebook_task:
+@@ -37,7 +37,7 @@
+         rename_job:
+           tasks:
+-            - task_key: shared
++            - task_key: a_shared
+               timeout_seconds: 45
+-            - task_key: solo
++            - task_key: z_solo
+               notebook_task:
+                 notebook_path: /Users/{{workspace_user_name}}/solo
+
+>>> grep -c task_key: a_shared databricks.yml
+2
+
+>>> grep -c task_key: z_solo databricks.yml
+1
+
+=== Remove a task and rename a later one in the same run, single block
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.shift_job
+  tasks[task_key='aaa']: remove
+  tasks[task_key='mmm']: remove
+  tasks[task_key='mmm2']: add
+
+
+
+=== aaa is gone, mmm became mmm2, zzz keeps its name and its max_retries
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -23,7 +23,5 @@
+     shift_job:
+       tasks:
+-        - task_key: aaa
+-          max_retries: 1
+-        - task_key: mmm
++        - task_key: mmm2
+           max_retries: 2
+         - task_key: zzz
+
+>>> grep -c task_key: zzz databricks.yml
+1
+
+=== Rename the two-block task AND edit one of its fields in the same run
+=== Sync
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.rename_job
+  tasks[task_key='a_shared']: remove
+  tasks[task_key='b_shared']: add
+  tasks[task_key='z_solo']: replace
+
+
+
+=== Left unapplied: the split is intact and timeout_seconds stays target-scoped
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -37,5 +37,5 @@
+             - task_key: a_shared
+               timeout_seconds: 45
+-            - task_key: z_solo
+-              notebook_task:
++            - notebook_task:
+                 notebook_path: /Users/{{workspace_user_name}}/solo
++              task_key: z_solo
+
+>>> grep -c task_key: a_shared databricks.yml
+2
+
+>>> grep -c timeout_seconds: 45 databricks.yml
+1
+
+>>> [CLI] bundle destroy --auto-approve -t dev
+The following resources will be deleted:
+  delete resources.jobs.rename_job
+  delete resources.jobs.shift_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev
+
+Deleting files...
+Destroy complete!