Package a local directory code_source_path for AI Runtime tasks

.nextchanges/bundles/ai-runtime-code-source-dir.md

@@ -0,0 +1 @@
+An `ai_runtime_task.code_source_path` that points at a local directory is now packaged into a tarball during `bundle deploy` (honoring `.gitignore` and `sync.include`/`sync.exclude`, content-addressed so unchanged code skips re-upload), placed in the bundle and uploaded by bundle file sync, and rewritten to the uploaded workspace path. Pip dependencies are taken from the job's serverless `environments` spec. Complements the existing pre-built-tarball path by letting users point at a source directory directly.

acceptance/bin/list_code_snapshot.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""
+List the entries of each AI Runtime code snapshot tarball uploaded during deploy.
+
+Reads out.requests.txt, takes every ai_runtime_task.code_source_path from the
+jobs/create request, exports each workspace archive via the CLI, and prints its
+sorted tar entries (grouped per archive). Used to assert which local files each
+snapshot includes (gitignore / sync rules), across one or more tasks.
+"""
+
+import gzip
+import io
+import json
+import os
+import subprocess
+import sys
+import tarfile
+
+with open("out.requests.txt") as f:
+    text = f.read()
+
+# out.requests.txt is a stream of concatenated (pretty-printed) JSON objects.
+decoder = json.JSONDecoder()
+requests = []
+pos = 0
+while pos < len(text):
+    while pos < len(text) and text[pos].isspace():
+        pos += 1
+    if pos >= len(text):
+        break
+    obj, pos = decoder.raw_decode(text, pos)
+    requests.append(obj)
+
+# Collect every task's code_source_path from the jobs/create request(s).
+code_source_paths = []
+for req in requests:
+    body = req.get("body")
+    if isinstance(body, dict) and req.get("path", "").endswith("/jobs/create"):
+        for task in body.get("tasks", []):
+            art = task.get("ai_runtime_task")
+            if art and art.get("code_source_path"):
+                code_source_paths.append(art["code_source_path"])
+
+if not code_source_paths:
+    sys.exit("no jobs/create request with code_source_path in out.requests.txt")
+
+cli = os.environ["CLI"]
+# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the /Workspace path.
+env = {**os.environ, "MSYS_NO_PATHCONV": "1"}
+
+# code_source_path is an absolute workspace path (/Workspace/Users/.../files/...).
+# Sort so multi-task output is deterministic.
+for remote in sorted(code_source_paths):
+    local = "code_snapshot.tar.gz"
+    subprocess.run(
+        [cli, "workspace", "export", remote, "--format", "AUTO", "--file", local],
+        check=True,
+        env=env,
+    )
+    with open(local, "rb") as f:
+        data = gzip.decompress(f.read())
+    os.remove(local)
+
+    # Print the archive's sync-relative name (hash tokenized by test.toml repls) so
+    # multi-archive output is legible.
+    print(f"# {remote.split('/files/', 1)[-1]}")
+    with tarfile.open(fileobj=io.BytesIO(data)) as tar:
+        for name in sorted(tar.getnames()):
+            print(name)

acceptance/bin/print_requests.py

@@ -226,9 +226,12 @@ def main():
 
     for req in filtered_requests:
         body = req.get("body")
-        if isinstance(body, dict):
-            for field in del_body_fields:
+        for field in del_body_fields:
+            # Drop the field from the parsed body, and (for non-JSON bodies) the
+            # top-level raw_body — used to strip binary/volatile upload payloads.
+            if isinstance(body, dict):
                 body.pop(field, None)
+            req.pop(field, None)
     if args.verbose:
         print(
             f"Read {len(data)} chars, {len(requests)} requests, {len(filtered_requests)} after filtering",

acceptance/bundle/ai_runtime_task/empty_code_source/databricks.yml

@@ -0,0 +1,22 @@
+bundle:
+  name: ai-runtime-empty
+
+resources:
+  jobs:
+    train:
+      name: "[${bundle.target}] AI Runtime training"
+      tasks:
+        - task_key: train
+          environment_key: default
+          ai_runtime_task:
+            experiment: my-training
+            code_source_path: ./src
+            deployments:
+              - command_path: src/command.sh
+                compute:
+                  accelerator_type: GPU_8xH100
+                  accelerator_count: 8
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "5"

acceptance/bundle/ai_runtime_task/empty_code_source/output.txt

@@ -0,0 +1,6 @@
+
+>>> errcode [CLI] bundle deploy
+Error: code_source_path "./src" has no files to package (all excluded by .gitignore or sync.exclude, or the directory is empty)
+
+
+Exit code: 1

acceptance/bundle/ai_runtime_task/empty_code_source/script

@@ -0,0 +1,4 @@
+# When code_source_path resolves to a directory whose contents are all filtered out
+# (here: a src/.gitignore of "*"), there is nothing to package. Deploy must fail
+# with an actionable message rather than shipping an empty code archive.
+trace errcode $CLI bundle deploy

acceptance/bundle/ai_runtime_task/empty_code_source/src/.gitignore

@@ -0,0 +1 @@
+*

acceptance/bundle/ai_runtime_task/empty_code_source/src/command.sh

@@ -0,0 +1,2 @@
+cd $CODE_SOURCE_PATH
+python train.py

acceptance/bundle/ai_runtime_task/empty_code_source/src/train.py

@@ -0,0 +1 @@
+print("x")

acceptance/bundle/ai_runtime_task/local_code_source/databricks.yml

@@ -0,0 +1,43 @@
+bundle:
+  name: ai-runtime-test
+
+sync:
+  # *.log excluded; data/*.bin force-included despite .gitignore.
+  exclude:
+    - "**/*.log"
+  include:
+    - src/data/*.bin
+
+resources:
+  jobs:
+    train:
+      name: "[${bundle.target}] AI Runtime training"
+      tasks:
+        # Two AI Runtime tasks with distinct local code dirs: each is packaged into
+        # its own content-addressed tarball, both under the repo root's .air_snapshots.
+        - task_key: train
+          environment_key: default
+          ai_runtime_task:
+            experiment: my-training
+            code_source_path: ./src
+            deployments:
+              - command_path: src/command.sh
+                compute:
+                  accelerator_type: GPU_8xH100
+                  accelerator_count: 8
+        - task_key: train2
+          environment_key: default
+          ai_runtime_task:
+            experiment: my-training-2
+            code_source_path: ./src2
+            deployments:
+              - command_path: src2/command.sh
+                compute:
+                  accelerator_type: GPU_8xH100
+                  accelerator_count: 8
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "5"
+            dependencies:
+              - torch>=2.0.0

acceptance/bundle/ai_runtime_task/local_code_source/output.txt

@@ -0,0 +1,135 @@
+
+=== deploy packages and uploads the local code sources
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== each task's tarball holds only synced files (both under the repo root .air_snapshots)
+
+>>> list_code_snapshot.py
+# .air_snapshots/[SNAPSHOT].tar.gz
+src2/command.sh
+src2/train.py
+# .air_snapshots/[SNAPSHOT].tar.gz
+src/.gitignore
+src/command.sh
+src/data/model.bin
+src/train.py
+
+=== both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec
+
+>>> print_requests.py --sort --del-body raw_body //.air_snapshots/ //jobs/create
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
+  "q": {
+    "overwrite": "true"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
+  "q": {
+    "overwrite": "true"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/create",
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "environments": [
+      {
+        "environment_key": "default",
+        "spec": {
+          "dependencies": [
+            "torch>=2.0.0"
+          ],
+          "environment_version": "5"
+        }
+      }
+    ],
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "[default] AI Runtime training",
+    "queue": {
+      "enabled": true
+    },
+    "tasks": [
+      {
+        "ai_runtime_task": {
+          "code_source_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
+          "deployments": [
+            {
+              "command_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/src/command.sh",
+              "compute": {
+                "accelerator_count": 8,
+                "accelerator_type": "GPU_8xH100"
+              }
+            }
+          ],
+          "experiment": "my-training"
+        },
+        "environment_key": "default",
+        "task_key": "train"
+      },
+      {
+        "ai_runtime_task": {
+          "code_source_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
+          "deployments": [
+            {
+              "command_path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/src2/command.sh",
+              "compute": {
+                "accelerator_count": 8,
+                "accelerator_type": "GPU_8xH100"
+              }
+            }
+          ],
+          "experiment": "my-training-2"
+        },
+        "environment_key": "default",
+        "task_key": "train2"
+      }
+    ]
+  }
+}
+
+=== re-planning unchanged code is a no-op (no changes)
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== editing a file changes the snapshot hash (content-addressed name changes)
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py --sort --del-body raw_body //.air_snapshots/
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
+  "q": {
+    "overwrite": "true"
+  }
+}
+
+=== destroy removes the deployed bundle (including the synced snapshots)
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.train
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/ai_runtime_task/local_code_source/script

@@ -0,0 +1,29 @@
+# Two AI Runtime tasks each with a local directory code_source_path. Each is
+# packaged into a content-addressed tarball written into the bundle
+# (.air_snapshots/, at the repo root) and uploaded by normal bundle file sync;
+# code_source_path/command_path are rewritten. Pip deps ride on the job's
+# environments[].spec.dependencies (no requirements.yaml is synthesized).
+
+title "deploy packages and uploads the local code sources\n"
+trace $CLI bundle deploy
+
+title "each task's tarball holds only synced files (both under the repo root .air_snapshots)\n"
+trace list_code_snapshot.py
+
+title "both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec\n"
+# --del-body raw_body drops the binary tarball upload body (kept readable). Filters
+# use a leading // so Git Bash on Windows does not path-convert them. --keep is not
+# passed, so print_requests.py consumes out.requests.txt.
+trace print_requests.py --sort --del-body raw_body '//.air_snapshots/' '//jobs/create'
+
+title "re-planning unchanged code is a no-op (no changes)\n"
+trace $CLI bundle plan
+
+title "editing a file changes the snapshot hash (content-addressed name changes)\n"
+update_file.py src/train.py 'print("training")' 'print("training v2")'
+trace $CLI bundle deploy
+trace print_requests.py --sort --del-body raw_body '//.air_snapshots/'
+
+title "destroy removes the deployed bundle (including the synced snapshots)\n"
+trace $CLI bundle destroy --auto-approve
+rm out.requests.txt

acceptance/bundle/ai_runtime_task/local_code_source/src/.gitignore

@@ -0,0 +1,2 @@
+ignored_by_git.txt
+data/

acceptance/bundle/ai_runtime_task/local_code_source/src/command.sh

@@ -0,0 +1,2 @@
+cd $CODE_SOURCE_PATH
+python train.py

acceptance/bundle/ai_runtime_task/local_code_source/src/data/model.bin

@@ -0,0 +1 @@
+weights

acceptance/bundle/ai_runtime_task/local_code_source/src/data/notes.txt

@@ -0,0 +1 @@
+scratch

acceptance/bundle/ai_runtime_task/local_code_source/src/debug.log

@@ -0,0 +1 @@
+log

acceptance/bundle/ai_runtime_task/local_code_source/src/ignored_by_git.txt

@@ -0,0 +1 @@
+kept

acceptance/bundle/ai_runtime_task/local_code_source/src/train.py

@@ -0,0 +1 @@
+print("training")

acceptance/bundle/ai_runtime_task/local_code_source/src2/command.sh

@@ -0,0 +1,2 @@
+cd $CODE_SOURCE_PATH
+python train.py

acceptance/bundle/ai_runtime_task/local_code_source/src2/train.py

@@ -0,0 +1 @@
+print("train2")

acceptance/bundle/ai_runtime_task/local_code_source/test.toml

@@ -0,0 +1,12 @@
+RecordRequests = true
+
+Ignore = [
+    '.databricks',
+]
+
+# The archive is content-addressed: <dir>_<sha256[:16]>.tar.gz. The hash is stable
+# given the committed inputs, but collapse it to a token so the test does not pin a
+# specific digest. Matches both src_ and src2_ archives.
+[[Repls]]
+Old = '(src2?)_[0-9a-f]{16}\.tar\.gz'
+New = '$1_[SNAPSHOT].tar.gz'