From ce7c5f12333814fb6d72759a66193c620f1add99 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 23 Jul 2026 16:25:50 +0000 Subject: [PATCH] config-remote-sync: block-aware YAML indexing for tasks/clusters split across top-level and target blocks --- .../job_clusters_split/databricks.yml.tmpl | 41 +++++ .../job_clusters_split/out.test.toml | 4 + .../job_clusters_split/output.txt | 34 ++++ .../job_clusters_split/script | 33 ++++ .../job_clusters_split/test.toml | 10 ++ .../databricks.yml.tmpl | 30 ++++ .../pipeline_clusters_split/out.test.toml | 4 + .../pipeline_clusters_split/output.txt | 38 +++++ .../pipeline_clusters_split/script | 33 ++++ .../pipeline_clusters_split/test.toml | 11 ++ .../databricks.yml.tmpl | 31 ++++ .../out.test.toml | 4 + .../output.txt | 40 +++++ .../pipeline_clusters_toplevel_index/script | 26 ++++ .../test.toml | 10 ++ .../databricks.yml.tmpl | 37 +++++ .../out.test.toml | 4 + .../task_split_crossblock_removes/output.txt | 46 ++++++ .../task_split_crossblock_removes/script | 21 +++ .../task_split_crossblock_removes/test.toml | 8 + .../databricks.yml.tmpl | 32 ++++ .../out.test.toml | 4 + .../task_split_crossblock_rename/output.txt | 59 +++++++ .../task_split_crossblock_rename/script | 27 ++++ .../task_split_crossblock_rename/test.toml | 10 ++ .../task_split_mixed_ops/databricks.yml.tmpl | 37 +++++ .../task_split_mixed_ops/out.test.toml | 4 + .../task_split_mixed_ops/output.txt | 42 +++++ .../task_split_mixed_ops/script | 36 +++++ .../task_split_mixed_ops/test.toml | 10 ++ .../databricks.yml.tmpl | 39 +++++ .../task_split_multiple_jobs/out.test.toml | 4 + .../task_split_multiple_jobs/output.txt | 49 ++++++ .../task_split_multiple_jobs/script | 41 +++++ .../task_split_multiple_jobs/test.toml | 10 ++ .../databricks.yml.tmpl | 32 ++++ .../task_split_nonzero_index/out.test.toml | 4 + .../task_split_nonzero_index/output.txt | 36 +++++ .../task_split_nonzero_index/script | 33 ++++ .../task_split_nonzero_index/test.toml | 10 ++ .../databricks.yml.tmpl | 34 ++++ .../task_split_target_index/out.test.toml | 4 + .../task_split_target_index/output.txt | 36 +++++ .../task_split_target_index/script | 37 +++++ .../task_split_target_index/test.toml | 10 ++ .../databricks.yml.tmpl | 28 ++++ .../task_split_target_local/out.test.toml | 4 + .../task_split_target_local/output.txt | 36 +++++ .../task_split_target_local/script | 33 ++++ .../task_split_target_local/test.toml | 10 ++ bundle/configsync/resolve.go | 147 ++++++++++++++---- bundle/configsync/resolve_test.go | 86 +++++++--- 52 files changed, 1402 insertions(+), 47 deletions(-) create mode 100644 acceptance/bundle/config-remote-sync/job_clusters_split/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/job_clusters_split/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/job_clusters_split/output.txt create mode 100644 acceptance/bundle/config-remote-sync/job_clusters_split/script create mode 100644 acceptance/bundle/config-remote-sync/job_clusters_split/test.toml create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_split/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_split/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_split/output.txt create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_split/script create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_split/test.toml create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/output.txt create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/script create mode 100644 acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_removes/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_removes/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_removes/output.txt create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_removes/script create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_removes/test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_rename/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_rename/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_rename/output.txt create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_rename/script create mode 100644 acceptance/bundle/config-remote-sync/task_split_crossblock_rename/test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_mixed_ops/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/task_split_mixed_ops/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_mixed_ops/output.txt create mode 100644 acceptance/bundle/config-remote-sync/task_split_mixed_ops/script create mode 100644 acceptance/bundle/config-remote-sync/task_split_mixed_ops/test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_multiple_jobs/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/task_split_multiple_jobs/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_multiple_jobs/output.txt create mode 100644 acceptance/bundle/config-remote-sync/task_split_multiple_jobs/script create mode 100644 acceptance/bundle/config-remote-sync/task_split_multiple_jobs/test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_nonzero_index/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/task_split_nonzero_index/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_nonzero_index/output.txt create mode 100644 acceptance/bundle/config-remote-sync/task_split_nonzero_index/script create mode 100644 acceptance/bundle/config-remote-sync/task_split_nonzero_index/test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_index/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_index/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_index/output.txt create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_index/script create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_index/test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_local/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_local/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_local/output.txt create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_local/script create mode 100644 acceptance/bundle/config-remote-sync/task_split_target_local/test.toml diff --git a/acceptance/bundle/config-remote-sync/job_clusters_split/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/job_clusters_split/databricks.yml.tmpl new file mode 100644 index 00000000000..999085b275f --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_clusters_split/databricks.yml.tmpl @@ -0,0 +1,41 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +# job_clusters is merged by merge.ElementsByKey (NON-sorted, unlike tasks). The +# sequence is still split across the top-level block (aaa_cluster, bbb_cluster) +# and the targets.default override block (ccc_cluster). Merged order is +# aaa,bbb,ccc, so ccc_cluster's whole-file position is 2 but its target-local +# index is 0. Editing ccc_cluster must patch the targets block at index 0, not +# the top-level block. +resources: + jobs: + my_job: + job_clusters: + - job_cluster_key: aaa_cluster + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + - job_cluster_key: bbb_cluster + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + job_cluster_key: aaa_cluster + +targets: + default: + mode: development + resources: + jobs: + my_job: + job_clusters: + - job_cluster_key: ccc_cluster + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 diff --git a/acceptance/bundle/config-remote-sync/job_clusters_split/out.test.toml b/acceptance/bundle/config-remote-sync/job_clusters_split/out.test.toml new file mode 100644 index 00000000000..4b5914daa2c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_clusters_split/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_clusters_split/output.txt b/acceptance/bundle/config-remote-sync/job_clusters_split/output.txt new file mode 100644 index 00000000000..4351476de8c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_clusters_split/output.txt @@ -0,0 +1,34 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Set num_workers on ccc_cluster remotely (target-block cluster) + +=== Detect and save changes +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + job_clusters[job_cluster_key='ccc_cluster'].new_cluster.num_workers: replace + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -39,3 +39,3 @@ + spark_version: 13.3.x-snapshot-scala2.12 + node_type_id: [NODE_TYPE_ID] +- num_workers: 1 ++ num_workers: 5 + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/job_clusters_split/script b/acceptance/bundle/config-remote-sync/job_clusters_split/script new file mode 100644 index 00000000000..f1bee2a9cb1 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/job_clusters_split/script @@ -0,0 +1,33 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +# ccc_cluster is defined in the targets block (target-local index 0); the +# top-level block holds aaa_cluster, bbb_cluster. Editing ccc_cluster's +# num_workers must patch the targets block, proving block-aware indexing works +# for the non-sorted job_clusters merge, not just tasks. +title "Set num_workers on ccc_cluster remotely (target-block cluster)" +echo +edit_resource.py jobs $job_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -28,3 +28,3 @@ + clusters: + - label: maintenance +- num_workers: 1 ++ num_workers: 3 + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/pipeline_clusters_split/script b/acceptance/bundle/config-remote-sync/pipeline_clusters_split/script new file mode 100644 index 00000000000..3be06bc9413 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/pipeline_clusters_split/script @@ -0,0 +1,33 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +pipeline_id="$(read_id.py my_pipeline)" + +# The maintenance cluster is defined in the targets block (target-local index +# 0); the top-level block holds the default cluster. Editing the maintenance +# cluster's num_workers must patch the targets block, proving block-aware +# indexing works for pipeline clusters, a different resource type from jobs. +title "Set num_workers on maintenance cluster remotely (target-block cluster)" +echo +edit_resource.py pipelines $pipeline_id < numeric change paths. Editing the maintenance +# cluster (top-level block-local index 1) must resolve to the top-level +# clusters[1], NOT be shifted or misclassified as a target element. +resources: + pipelines: + my_pipeline: + name: test-pipeline-$UNIQUE_NAME + development: false + libraries: + - notebook: + path: /Users/{{workspace_user_name}}/notebook + clusters: + - label: default + num_workers: 1 + - label: maintenance + num_workers: 1 + +targets: + default: + mode: development + resources: + pipelines: + my_pipeline: + clusters: + - label: default + num_workers: 2 diff --git a/acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/out.test.toml b/acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/out.test.toml new file mode 100644 index 00000000000..4b5914daa2c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/output.txt b/acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/output.txt new file mode 100644 index 00000000000..3e197c85501 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/output.txt @@ -0,0 +1,40 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Set num_workers on maintenance cluster (top-level block index 1) + +=== Detect and save changes +Detected changes in 1 resource(s): + +Resource: resources.pipelines.my_pipeline + clusters[1].num_workers: replace + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -19,5 +19,5 @@ + num_workers: 1 + - label: maintenance +- num_workers: 1 ++ num_workers: 7 + + targets: + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/script b/acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/script new file mode 100644 index 00000000000..01f92541f70 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/pipeline_clusters_toplevel_index/script @@ -0,0 +1,26 @@ +#!/bin/bash +envsubst < databricks.yml.tmpl > databricks.yml +cleanup() { trace $CLI bundle destroy --auto-approve; } +trap cleanup EXIT +$CLI bundle deploy +pipeline_id="$(read_id.py my_pipeline)" + +# maintenance is the SECOND top-level cluster (block-local index 1). Its numeric +# change path clusters[1] must patch the top-level maintenance cluster, leaving +# the target `default` override untouched. +title "Set num_workers on maintenance cluster (top-level block index 1)" +echo +edit_resource.py pipelines $pipeline_id <= 1 while the sequence is split by a target block. +RecordRequests = false +Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"] + +[Env] +DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/databricks.yml.tmpl new file mode 100644 index 00000000000..c05a1035b4c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/databricks.yml.tmpl @@ -0,0 +1,37 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +# Split tasks: top-level [bbb, ccc], target [aaa, eee]. Merged+sorted = +# aaa,bbb,ccc,eee. One sync removes bbb (top block-local idx 0) AND eee (target +# block-local idx 1). Index-shift bookkeeping must be block-scoped: bbb's +# top-level removal must NOT shift eee's target-block index. Otherwise eee's +# index 1 would shift to 0 and, being flagged as a target-block element, patch +# targets...tasks[0] = aaa. Correct result: bbb + eee removed, aaa + ccc kept. +resources: + jobs: + my_job: + tasks: + - task_key: bbb_task + run_if: ALL_SUCCESS + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/bbb + - task_key: ccc_task + run_if: ALL_SUCCESS + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/ccc + +targets: + default: + mode: development + resources: + jobs: + my_job: + tasks: + - task_key: aaa_task + run_if: ALL_SUCCESS + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/aaa + - task_key: eee_task + run_if: ALL_SUCCESS + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/eee diff --git a/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/out.test.toml b/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/out.test.toml new file mode 100644 index 00000000000..4b5914daa2c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/output.txt b/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/output.txt new file mode 100644 index 00000000000..f738fd44652 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/output.txt @@ -0,0 +1,46 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Remove bbb_task (top block) AND eee_task (target block) remotely + +=== Detect and save changes +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + tasks[task_key='bbb_task']: remove + tasks[task_key='eee_task']: remove + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -12,8 +12,4 @@ + my_job: + tasks: +- - task_key: bbb_task +- run_if: ALL_SUCCESS +- notebook_task: +- notebook_path: /Users/{{workspace_user_name}}/bbb + - task_key: ccc_task + run_if: ALL_SUCCESS +@@ -32,6 +28,2 @@ + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/aaa +- - task_key: eee_task +- run_if: ALL_SUCCESS +- notebook_task: +- notebook_path: /Users/{{workspace_user_name}}/eee + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/script b/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/script new file mode 100644 index 00000000000..3abfaa85d21 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_crossblock_removes/script @@ -0,0 +1,21 @@ +#!/bin/bash +envsubst < databricks.yml.tmpl > databricks.yml +cleanup() { trace $CLI bundle destroy --auto-approve; } +trap cleanup EXIT +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +title "Remove bbb_task (top block) AND eee_task (target block) remotely" +echo +edit_resource.py jobs $job_id < ppp (top block) and aaa -> qqq (target block). Each +# rename is a remove+add; the freed index reuse (indicesToReplaceMap) must be +# block-scoped so the top-level add reuses mmm's slot and the target add reuses +# aaa's slot, never crossing blocks. +resources: + jobs: + my_job: + tasks: + - task_key: mmm_task + run_if: ALL_SUCCESS + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/mmm + - task_key: nnn_task + run_if: ALL_SUCCESS + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/nnn + +targets: + default: + mode: development + resources: + jobs: + my_job: + tasks: + - task_key: aaa_task + run_if: ALL_SUCCESS + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/aaa diff --git a/acceptance/bundle/config-remote-sync/task_split_crossblock_rename/out.test.toml b/acceptance/bundle/config-remote-sync/task_split_crossblock_rename/out.test.toml new file mode 100644 index 00000000000..4b5914daa2c --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_crossblock_rename/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/task_split_crossblock_rename/output.txt b/acceptance/bundle/config-remote-sync/task_split_crossblock_rename/output.txt new file mode 100644 index 00000000000..89bdde14d86 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_crossblock_rename/output.txt @@ -0,0 +1,59 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Rename mmm_task->ppp_task (top) and aaa_task->qqq_task (target) + +=== Detect and save changes +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + tasks[task_key='aaa_task']: remove + tasks[task_key='mmm_task']: remove + tasks[task_key='ppp_task']: add + tasks[task_key='qqq_task']: add + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -11,12 +11,14 @@ + my_job: + tasks: +- - task_key: mmm_task +- run_if: ALL_SUCCESS +- notebook_task: ++ - notebook_task: + notebook_path: /Users/{{workspace_user_name}}/mmm ++ task_key: ppp_task + - task_key: nnn_task + run_if: ALL_SUCCESS + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/nnn ++ - notebook_task: ++ notebook_path: /Users/{{workspace_user_name}}/aaa ++ task_key: qqq_task + + targets: +@@ -26,7 +28,3 @@ + jobs: + my_job: +- tasks: +- - task_key: aaa_task +- run_if: ALL_SUCCESS +- notebook_task: +- notebook_path: /Users/{{workspace_user_name}}/aaa ++ tasks: [] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/task_split_crossblock_rename/script b/acceptance/bundle/config-remote-sync/task_split_crossblock_rename/script new file mode 100644 index 00000000000..b555aa3f870 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_crossblock_rename/script @@ -0,0 +1,27 @@ +#!/bin/bash +envsubst < databricks.yml.tmpl > databricks.yml +cleanup() { trace $CLI bundle destroy --auto-approve; } +trap cleanup EXIT +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +# Rename mmm_task (top block) and aaa_task (target block) in one sync. Each is +# seen as remove old + add new. The adds must land back in their own block. +title "Rename mmm_task->ppp_task (top) and aaa_task->qqq_task (target)" +echo +edit_resource.py jobs $job_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -16,10 +16,6 @@ + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/bbb +- - task_key: ccc_task +- run_if: ALL_SUCCESS +- notebook_task: +- notebook_path: /Users/{{workspace_user_name}}/ccc + - task_key: ddd_task +- run_if: ALL_SUCCESS ++ run_if: ALL_DONE + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/ddd + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/task_split_mixed_ops/script b/acceptance/bundle/config-remote-sync/task_split_mixed_ops/script new file mode 100644 index 00000000000..417540fff74 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_mixed_ops/script @@ -0,0 +1,36 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +# Two operations in one sync, both in the top-level block: +# - remove ccc_task (block-local index 1) +# - replace ddd_task.run_if (block-local index 2, shifts to 1 after the remove) +# The remove's index adjustment must compose with block-aware indexing so +# ddd_task is still patched correctly, and aaa_task in the target block must be +# left untouched. +title "Remove ccc_task and edit ddd_task remotely" +echo +edit_resource.py jobs $job_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -29,5 +29,5 @@ + tasks: + - task_key: a_tgt +- run_if: ALL_SUCCESS ++ run_if: ALL_DONE + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/a_tgt +@@ -35,5 +35,5 @@ + tasks: + - task_key: b_tgt +- run_if: ALL_SUCCESS ++ run_if: ALL_DONE + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/b_tgt + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.job_a + delete resources.jobs.job_b + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/task_split_multiple_jobs/script b/acceptance/bundle/config-remote-sync/task_split_multiple_jobs/script new file mode 100644 index 00000000000..9d7d208310b --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_multiple_jobs/script @@ -0,0 +1,41 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_a_id="$(read_id.py job_a)" +job_b_id="$(read_id.py job_b)" + +# Edit the target-block task of BOTH jobs. Each edit must land in its own job's +# targets block. If blockAnchor grouped elements globally instead of per +# sequence, one job's index could bleed into the other. +title "Set run_if on a_tgt (job_a target task)" +echo +edit_resource.py jobs $job_a_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -16,5 +16,5 @@ + notebook_path: /Users/{{workspace_user_name}}/mmm + - task_key: nnn_task +- run_if: ALL_SUCCESS ++ run_if: ALL_DONE + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/nnn + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/task_split_nonzero_index/script b/acceptance/bundle/config-remote-sync/task_split_nonzero_index/script new file mode 100644 index 00000000000..823d0ba1cea --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_nonzero_index/script @@ -0,0 +1,33 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +# Merged+sorted order is aaa,mmm,nnn. nnn_task is the SECOND task of the +# top-level block (block-local index 1). Its remote run_if edit must land on +# nnn_task, not mmm_task, proving yamlFileIndex counts same-block predecessors +# rather than returning 0. +title "Set run_if on nnn_task remotely (top-level block index 1)" +echo +edit_resource.py jobs $job_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -19,5 +19,5 @@ + tasks: + - task_key: bbb_task +- run_if: ALL_SUCCESS ++ run_if: ALL_DONE + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/bbb + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/task_split_target_index/script b/acceptance/bundle/config-remote-sync/task_split_target_index/script new file mode 100644 index 00000000000..8ab908ffde1 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_target_index/script @@ -0,0 +1,37 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +# my_job's tasks are split across two config blocks in the SAME file: the +# top-level resources block (bbb_task, ccc_task) and the targets.default +# override block (aaa_task). After MergeJobTasks sorts by task_key the merged +# order is aaa,bbb,ccc. yamlFileIndex must be block-aware: bbb_task's remote +# run_if edit resolves to index 0 of the top-level block (bbb_task), not to the +# whole-file position 1 (ccc_task). +title "Set run_if on bbb_task remotely (merged task order: aaa,bbb,ccc)" +echo +edit_resource.py jobs $job_id <>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -24,5 +24,5 @@ + tasks: + - task_key: aaa_tgt +- run_if: ALL_SUCCESS ++ run_if: ALL_DONE + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/aaa + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/config-remote-sync/task_split_target_local/script b/acceptance/bundle/config-remote-sync/task_split_target_local/script new file mode 100644 index 00000000000..3946293f135 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/task_split_target_local/script @@ -0,0 +1,33 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +# Merged+sorted task order is aaa_tgt, zzz_top. aaa_tgt is defined in the +# targets block (target-local index 0); zzz_top in the top-level block +# (top-level index 0) and also has run_if. Editing aaa_tgt's run_if must patch +# the targets block, not zzz_top in the top-level block. +title "Set run_if on aaa_tgt remotely (target-block task)" +echo +edit_resource.py jobs $job_id < "resources.jobs.foo.tasks[1].name" // Returns a PatternNode because for Add operations, [*] may be used as a placeholder for new elements. -func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) (*structpath.PatternNode, dyn.Location, error) { +// +// The returned bool reports whether the resolved element was contributed by a +// target override block. Its numeric index is then local to that block, so only +// the targets..-prefixed candidate is valid (see ResolveChanges). +func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) (*structpath.PatternNode, dyn.Location, bool, error) { node, err := structpath.ParsePath(pathStr) if err != nil { - return nil, dyn.Location{}, fmt.Errorf("failed to parse path %s: %w", pathStr, err) + return nil, dyn.Location{}, false, fmt.Errorf("failed to parse path %s: %w", pathStr, err) } nodes := node.AsSlice() var result *structpath.PatternNode + inOverrideBlock := false currentValue := b.Config.Value() for _, n := range nodes { @@ -47,6 +52,23 @@ func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) } if idx, ok := n.Index(); ok { + // A numeric index reaches here when the change path is positional + // rather than keyed — i.e. the resource registered no KeyedSlices + // entry for this sequence (e.g. pipeline `clusters`, which structdiff + // compares by position). When such a sequence is split across a + // top-level and a target override block, idx is the merged-sequence + // position; convert it to the block-local index and flag target-block + // elements, exactly as the keyed selector below does. + if currentValue.IsValid() && currentValue.Kind() == dyn.KindSequence { + seq, _ := currentValue.AsSequence() + seqLocations := currentValue.Locations() + if len(seqLocations) >= 2 && idx >= 0 && idx < len(seq) { + result = structpath.NewPatternIndex(result, yamlFileIndex(seq, idx, seqLocations)) + inOverrideBlock = inOverrideBlock || elementInOverrideBlock(seq[idx].Location(), seqLocations) + currentValue = seq[idx] + continue + } + } result = structpath.NewPatternIndex(result, idx) if currentValue.IsValid() { currentValue, _ = dyn.GetByPath(currentValue, dyn.Path{dyn.Index(idx)}) @@ -57,10 +79,11 @@ func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) // Check for key-value selector: [key='value'] if key, value, ok := n.KeyValue(); ok { if !currentValue.IsValid() || currentValue.Kind() != dyn.KindSequence { - return nil, dyn.Location{}, fmt.Errorf("cannot apply [%s='%s'] selector to non-array value in path %s", key, value, pathStr) + return nil, dyn.Location{}, false, fmt.Errorf("cannot apply [%s='%s'] selector to non-array value in path %s", key, value, pathStr) } seq, _ := currentValue.AsSequence() + seqLocations := currentValue.Locations() foundIndex := -1 for i, elem := range seq { @@ -82,45 +105,92 @@ func resolveSelectors(pathStr string, b *bundle.Bundle, operation OperationType) currentValue = dyn.Value{} continue } - return nil, dyn.Location{}, fmt.Errorf("no array element found with %s='%s' in path %s", key, value, pathStr) + return nil, dyn.Location{}, false, fmt.Errorf("no array element found with %s='%s' in path %s", key, value, pathStr) } // Mutators may reorder sequence elements (e.g., tasks sorted by task_key). // Use location information to determine the original YAML file position. - yamlIndex := yamlFileIndex(seq, foundIndex) + yamlIndex := yamlFileIndex(seq, foundIndex, seqLocations) result = structpath.NewPatternIndex(result, yamlIndex) + // yamlIndex is local to the element's own block; latch so a nested + // single-block selector further down doesn't clear a target match. + inOverrideBlock = inOverrideBlock || elementInOverrideBlock(seq[foundIndex].Location(), seqLocations) currentValue = seq[foundIndex] continue } } - return result, currentValue.Location(), nil + return result, currentValue.Location(), inOverrideBlock, nil +} + +// elementInOverrideBlock reports whether a merged sequence element was contributed +// by a target override block rather than the top-level resources block. Target +// overrides are merged with the top-level block as the merge reference, and merge +// keeps the reference locations first, so seqLocations[0] is always the top-level +// anchor regardless of where each block appears in the file. An element whose +// nearest block anchor is not that top-level anchor came from a target block, so +// its block-local index is only valid under the targets.. prefix. +// +// A resource may only be declared once across all top-level files (a duplicate +// key is rejected at load), so a second sequence location can only come from a +// target override — never from a second top-level file. +func elementInOverrideBlock(elemLoc dyn.Location, seqLocations []dyn.Location) bool { + if len(seqLocations) < 2 || elemLoc.File == "" { + return false + } + topAnchor := seqLocations[0] + return elemLoc.File != topAnchor.File || blockAnchor(elemLoc, seqLocations) != topAnchor.Line } // yamlFileIndex determines the original YAML file position of a sequence element. // Mutators may reorder sequence elements (e.g., tasks sorted by task_key), so the -// in-memory index may not match the position in the YAML file. This function uses -// location information to count how many elements from the same file appear before -// the target element, giving the correct index for YAML patching. -func yamlFileIndex(seq []dyn.Value, sortedIndex int) int { +// in-memory index may not match the position in the YAML file. +// +// A single field can hold sequences from more than one config block — e.g. a job's +// tasks defined partly under top-level `resources.jobs..tasks` and partly under +// `targets..resources.jobs..tasks`. MergeJobTasks concatenates and sorts them +// into one in-memory sequence, but each block is patched independently in the YAML. +// seqLocations holds one location per contributing block (the location of each block's +// sequence node); an element belongs to the block whose anchor is the closest one at or +// before it in the same file. Counting only same-block elements with a smaller line +// gives the index within that block, which is what the YAML patch targets. When the +// sequence comes from a single block this reduces to a plain same-file count. +func yamlFileIndex(seq []dyn.Value, sortedIndex int, seqLocations []dyn.Location) int { matchLocation := seq[sortedIndex].Location() if matchLocation.File == "" { return sortedIndex } + matchAnchor := blockAnchor(matchLocation, seqLocations) + yamlIndex := 0 for i, elem := range seq { if i == sortedIndex { continue } loc := elem.Location() - if loc.File == matchLocation.File && loc.Line < matchLocation.Line { + if loc.File == matchLocation.File && loc.Line < matchLocation.Line && blockAnchor(loc, seqLocations) == matchAnchor { yamlIndex++ } } return yamlIndex } +// blockAnchor returns the config block a sequence element belongs to, identified by the +// line of that block's sequence node. Blocks never interleave within a file, so the +// element's block is the anchor with the greatest line at or before the element in the +// same file. Returns 0 when no anchor matches (single unlocated block), which keeps all +// elements in one group. +func blockAnchor(loc dyn.Location, seqLocations []dyn.Location) int { + anchor := 0 + for _, l := range seqLocations { + if l.File == loc.File && l.Line <= loc.Line && l.Line > anchor { + anchor = l.Line + } + } + return anchor +} + func pathDepth(pathStr string) int { node, err := structpath.ParsePath(pathStr) if err != nil { @@ -129,10 +199,23 @@ func pathDepth(pathStr string) int { return len(node.AsSlice()) } +// blockScopedParent namespaces per-sequence index bookkeeping by block. A target +// override block and the top-level block share the same unprefixed parent path +// but keep block-local indices, so add/remove operations in one block must not +// shift indices resolved in the other. Prefixing the override block's key with +// targets.. keeps the two buckets separate. +func blockScopedParent(parentPathStr, targetName string, inOverrideBlock bool) string { + if inOverrideBlock { + return "targets." + targetName + "." + parentPathStr + } + return parentPathStr +} + // adjustArrayIndex adjusts the index in a PatternNode based on previous operations. // When operations are applied sequentially, removals and additions shift array indices. -// This function adjusts the index to account for those shifts. -func adjustArrayIndex(path *structpath.PatternNode, operations map[string][]struct { +// This function adjusts the index to account for those shifts. parentKey selects the +// block-scoped bucket of operations that apply to this element's sequence block. +func adjustArrayIndex(path *structpath.PatternNode, parentKey string, operations map[string][]struct { index int operation OperationType }, @@ -143,8 +226,7 @@ func adjustArrayIndex(path *structpath.PatternNode, operations map[string][]stru } parentPath := path.Parent() - parentPathStr := parentPath.String() - ops := operations[parentPathStr] + ops := operations[parentKey] adjustment := 0 for _, op := range ops { @@ -215,7 +297,7 @@ func ResolveChanges(ctx context.Context, b *bundle.Bundle, configChanges Changes configChange := resourceChanges[fieldPath] fullPath := resourceKey + "." + fieldPath - resolvedPath, resolvedLocation, err := resolveSelectors(fullPath, b, configChange.Operation) + resolvedPath, resolvedLocation, inOverrideBlock, err := resolveSelectors(fullPath, b, configChange.Operation) if err != nil { return nil, fmt.Errorf("failed to resolve selectors in path %s: %w", fullPath, err) } @@ -225,37 +307,46 @@ func ResolveChanges(ctx context.Context, b *bundle.Bundle, configChanges Changes if configChange.Operation == OperationRemove { freeIndex, ok := resolvedPath.Index() if ok { - parentPath := resolvedPath.Parent().String() - indicesToReplaceMap[parentPath] = append(indicesToReplaceMap[parentPath], freeIndex) + parentKey := blockScopedParent(resolvedPath.Parent().String(), targetName, inOverrideBlock) + indicesToReplaceMap[parentKey] = append(indicesToReplaceMap[parentKey], freeIndex) } } if configChange.Operation == OperationAdd && resolvedPath.BracketStar() { - parentPath := resolvedPath.Parent().String() - indices, ok := indicesToReplaceMap[parentPath] + parentKey := blockScopedParent(resolvedPath.Parent().String(), targetName, inOverrideBlock) + indices, ok := indicesToReplaceMap[parentKey] if ok && len(indices) > 0 { index := indices[0] - indicesToReplaceMap[parentPath] = indices[1:] + indicesToReplaceMap[parentKey] = indices[1:] resolvedPath = structpath.NewPatternIndex(resolvedPath.Parent(), index) } } - resolvedPath = adjustArrayIndex(resolvedPath, indexOperations) + parentKey := blockScopedParent(resolvedPath.Parent().String(), targetName, inOverrideBlock) + resolvedPath = adjustArrayIndex(resolvedPath, parentKey, indexOperations) // Track this operation for future index adjustments (only for array element operations) if originalIndex, ok := resolvedPath.Index(); ok { - parentPath := resolvedPath.Parent().String() - indexOperations[parentPath] = append(indexOperations[parentPath], struct { + indexOperations[parentKey] = append(indexOperations[parentKey], struct { index int operation OperationType }{originalIndex, configChange.Operation}) } resolvedPathStr := resolvedPath.String() - candidates := []string{resolvedPathStr} - if targetName != "" { - targetPrefixedPath := "targets." + targetName + "." + resolvedPathStr - candidates = append(candidates, targetPrefixedPath) + var candidates []string + targetPrefixedPath := "targets." + targetName + "." + resolvedPathStr + switch { + case inOverrideBlock: + // The index is local to the target override block, so only the + // target-prefixed candidate points at the right element. Emitting the + // unprefixed candidate too would let applyChange write the same index + // into the top-level block and silently patch the wrong element. + candidates = []string{targetPrefixedPath} + case targetName != "": + candidates = []string{resolvedPathStr, targetPrefixedPath} + default: + candidates = []string{resolvedPathStr} } filePath := resolvedLocation.File diff --git a/bundle/configsync/resolve_test.go b/bundle/configsync/resolve_test.go index 9264ad7f5dc..e39174abd00 100644 --- a/bundle/configsync/resolve_test.go +++ b/bundle/configsync/resolve_test.go @@ -32,7 +32,7 @@ func TestResolveSelectors_NoSelectors(t *testing.T) { mutator.DefaultMutators(ctx, b) - result, _, err := resolveSelectors("resources.jobs.test_job.name", b, OperationReplace) + result, _, _, err := resolveSelectors("resources.jobs.test_job.name", b, OperationReplace) require.NoError(t, err) assert.Equal(t, "resources.jobs.test_job.name", result.String()) } @@ -57,11 +57,11 @@ func TestResolveSelectors_NumericIndices(t *testing.T) { mutator.DefaultMutators(ctx, b) - result, _, err := resolveSelectors("resources.jobs.test_job.tasks[0].task_key", b, OperationReplace) + result, _, _, err := resolveSelectors("resources.jobs.test_job.tasks[0].task_key", b, OperationReplace) require.NoError(t, err) assert.Equal(t, "resources.jobs.test_job.tasks[0].task_key", result.String()) - result, _, err = resolveSelectors("resources.jobs.test_job.tasks[1].task_key", b, OperationReplace) + result, _, _, err = resolveSelectors("resources.jobs.test_job.tasks[1].task_key", b, OperationReplace) require.NoError(t, err) assert.Equal(t, "resources.jobs.test_job.tasks[1].task_key", result.String()) } @@ -90,11 +90,11 @@ func TestResolveSelectors_KeyValueSelector(t *testing.T) { mutator.DefaultMutators(ctx, b) - result, _, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].notebook_task.notebook_path", b, OperationReplace) + result, _, _, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].notebook_task.notebook_path", b, OperationReplace) require.NoError(t, err) assert.Equal(t, "resources.jobs.test_job.tasks[1].notebook_task.notebook_path", result.String()) - result, _, err = resolveSelectors("resources.jobs.test_job.tasks[task_key='setup'].notebook_task.notebook_path", b, OperationReplace) + result, _, _, err = resolveSelectors("resources.jobs.test_job.tasks[task_key='setup'].notebook_task.notebook_path", b, OperationReplace) require.NoError(t, err) assert.Equal(t, "resources.jobs.test_job.tasks[0].notebook_task.notebook_path", result.String()) } @@ -120,7 +120,7 @@ func TestResolveSelectors_SelectorNotFound(t *testing.T) { mutator.DefaultMutators(ctx, b) - _, _, err = resolveSelectors("resources.jobs.test_job.tasks[task_key='nonexistent'].notebook_task.notebook_path", b, OperationReplace) + _, _, _, err = resolveSelectors("resources.jobs.test_job.tasks[task_key='nonexistent'].notebook_task.notebook_path", b, OperationReplace) require.Error(t, err) assert.Contains(t, err.Error(), "no array element found with task_key='nonexistent'") } @@ -143,7 +143,7 @@ func TestResolveSelectors_SelectorOnNonArray(t *testing.T) { mutator.DefaultMutators(ctx, b) - _, _, err = resolveSelectors("resources.jobs.test_job[task_key='main'].name", b, OperationReplace) + _, _, _, err = resolveSelectors("resources.jobs.test_job[task_key='main'].name", b, OperationReplace) require.Error(t, err) assert.Contains(t, err.Error(), "cannot apply [task_key='main'] selector to non-array value") } @@ -174,7 +174,7 @@ func TestResolveSelectors_NestedSelectors(t *testing.T) { mutator.DefaultMutators(ctx, b) - result, _, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].libraries[0].pypi.package", b, OperationReplace) + result, _, _, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].libraries[0].pypi.package", b, OperationReplace) require.NoError(t, err) assert.Equal(t, "resources.jobs.test_job.tasks[1].libraries[0].pypi.package", result.String()) } @@ -200,7 +200,7 @@ func TestResolveSelectors_WildcardNotSupported(t *testing.T) { mutator.DefaultMutators(ctx, b) - _, _, err = resolveSelectors("resources.jobs.test_job.tasks.*.task_key", b, OperationReplace) + _, _, _, err = resolveSelectors("resources.jobs.test_job.tasks.*.task_key", b, OperationReplace) require.Error(t, err) assert.Contains(t, err.Error(), "wildcards not allowed in path") } @@ -215,11 +215,13 @@ func TestYamlFileIndex(t *testing.T) { dyn.NewValue(nil, []dyn.Location{{File: "a.yml", Line: 30}}), // pipeline_task dyn.NewValue(nil, []dyn.Location{{File: "a.yml", Line: 20}}), // python_wheel_task } + // Single block: the sequence node is anchored at the block's line in the file. + seqLocations := []dyn.Location{{File: "a.yml", Line: 5}} - assert.Equal(t, 3, yamlFileIndex(seq, 0)) // extra: 3 elements before it in YAML - assert.Equal(t, 0, yamlFileIndex(seq, 1)) // notebook_task: first in YAML - assert.Equal(t, 2, yamlFileIndex(seq, 2)) // pipeline_task: 2 elements before it - assert.Equal(t, 1, yamlFileIndex(seq, 3)) // python_wheel_task: 1 element before it + assert.Equal(t, 3, yamlFileIndex(seq, 0, seqLocations)) // extra: 3 elements before it in YAML + assert.Equal(t, 0, yamlFileIndex(seq, 1, seqLocations)) // notebook_task: first in YAML + assert.Equal(t, 2, yamlFileIndex(seq, 2, seqLocations)) // pipeline_task: 2 elements before it + assert.Equal(t, 1, yamlFileIndex(seq, 3, seqLocations)) // python_wheel_task: 1 element before it } func TestYamlFileIndex_MultipleFiles(t *testing.T) { @@ -233,12 +235,14 @@ func TestYamlFileIndex_MultipleFiles(t *testing.T) { dyn.NewValue(nil, []dyn.Location{{File: "b.yml", Line: 5}}), // task_c dyn.NewValue(nil, []dyn.Location{{File: "b.yml", Line: 15}}), // task_d } + // One anchor per file, each above its file's first element. + seqLocations := []dyn.Location{{File: "a.yml", Line: 3}, {File: "b.yml", Line: 2}} // Indices are relative to each file - assert.Equal(t, 0, yamlFileIndex(seq, 0)) // task_a: first in file A - assert.Equal(t, 1, yamlFileIndex(seq, 1)) // task_b: second in file A - assert.Equal(t, 0, yamlFileIndex(seq, 2)) // task_c: first in file B - assert.Equal(t, 1, yamlFileIndex(seq, 3)) // task_d: second in file B + assert.Equal(t, 0, yamlFileIndex(seq, 0, seqLocations)) // task_a: first in file A + assert.Equal(t, 1, yamlFileIndex(seq, 1, seqLocations)) // task_b: second in file A + assert.Equal(t, 0, yamlFileIndex(seq, 2, seqLocations)) // task_c: first in file B + assert.Equal(t, 1, yamlFileIndex(seq, 3, seqLocations)) // task_d: second in file B } func TestYamlFileIndex_NoLocation(t *testing.T) { @@ -246,6 +250,50 @@ func TestYamlFileIndex_NoLocation(t *testing.T) { dyn.NewValue(nil, nil), dyn.NewValue(nil, nil), } - assert.Equal(t, 0, yamlFileIndex(seq, 0)) - assert.Equal(t, 1, yamlFileIndex(seq, 1)) + assert.Equal(t, 0, yamlFileIndex(seq, 0, nil)) + assert.Equal(t, 1, yamlFileIndex(seq, 1, nil)) +} + +func TestYamlFileIndex_SplitBlocksSameFile(t *testing.T) { + // A job's tasks split across two blocks in the SAME file: a targets override + // block (aaa, near the top) and the top-level resources block (bbb, ccc, lower + // down). MergeJobTasks concatenates and sorts by task_key -> [aaa, bbb, ccc]. + // The tasks sequence node carries one anchor per block: top-level tasks at line + // 20, target tasks at line 11 (order matches the real merge, which keeps the + // top-level location first). Each element's index must be relative to its own + // block, not the whole file. + seq := []dyn.Value{ + dyn.NewValue(nil, []dyn.Location{{File: "databricks.yml", Line: 11}}), // aaa (target block) + dyn.NewValue(nil, []dyn.Location{{File: "databricks.yml", Line: 20}}), // bbb (top-level block) + dyn.NewValue(nil, []dyn.Location{{File: "databricks.yml", Line: 28}}), // ccc (top-level block) + } + seqLocations := []dyn.Location{{File: "databricks.yml", Line: 20}, {File: "databricks.yml", Line: 11}} + + assert.Equal(t, 0, yamlFileIndex(seq, 0, seqLocations)) // aaa: first in target block + assert.Equal(t, 0, yamlFileIndex(seq, 1, seqLocations)) // bbb: first in top-level block (NOT 1) + assert.Equal(t, 1, yamlFileIndex(seq, 2, seqLocations)) // ccc: second in top-level block +} + +func TestElementInOverrideBlock(t *testing.T) { + // seqLocations[0] is always the top-level anchor (merge keeps the reference + // block first); seqLocations[1] is the target override anchor. This holds + // regardless of which block appears first in the file. + topFirst := []dyn.Location{{File: "databricks.yml", Line: 6}, {File: "databricks.yml", Line: 20}} + assert.True(t, elementInOverrideBlock(dyn.Location{File: "databricks.yml", Line: 22}, topFirst)) // target block + assert.False(t, elementInOverrideBlock(dyn.Location{File: "databricks.yml", Line: 8}, topFirst)) // top-level block + + // Target block precedes resources in the file: top-level anchor still first. + targetFirst := []dyn.Location{{File: "databricks.yml", Line: 20}, {File: "databricks.yml", Line: 6}} + assert.True(t, elementInOverrideBlock(dyn.Location{File: "databricks.yml", Line: 8}, targetFirst)) // target block + assert.False(t, elementInOverrideBlock(dyn.Location{File: "databricks.yml", Line: 22}, targetFirst)) // top-level block + + // Single block (no override contribution) is never in an override block. + single := []dyn.Location{{File: "databricks.yml", Line: 6}} + assert.False(t, elementInOverrideBlock(dyn.Location{File: "databricks.yml", Line: 8}, single)) + assert.False(t, elementInOverrideBlock(dyn.Location{}, topFirst)) + + // Multi-file split: a top-level block in a.yml and a target block in b.yml. + multiFile := []dyn.Location{{File: "a.yml", Line: 6}, {File: "b.yml", Line: 4}} + assert.True(t, elementInOverrideBlock(dyn.Location{File: "b.yml", Line: 6}, multiFile)) // target block, other file + assert.False(t, elementInOverrideBlock(dyn.Location{File: "a.yml", Line: 8}, multiFile)) // top-level block }