AIR CLI: address review feedback on renderer, hyperlink, and test config

.github/workflows/changelog-preview.yml

@@ -1,12 +1,14 @@
 name: Changelog Preview
 
-# Renders the CHANGELOG.md section the next release would generate from this
-# PR's .nextchanges/ fragments into this check's job summary, so reviewers see
-# the rendered changelog without cutting a release. Fails the check if a
-# fragment is misplaced (so it can't be silently skipped by the renderer).
+# Renders the CHANGELOG.md section the next release would generate from the
+# .nextchanges/ fragments into this check's job summary, so reviewers see the
+# rendered changelog without cutting a release. Fails the check if a fragment
+# is misplaced (so it can't be silently skipped by the renderer).
 #
-# Uses `pull_request` with a read-only token: it renders the PR's own
-# .nextchanges/ content and never needs write credentials.
+# Runs on pull requests (rendering the PR's own .nextchanges/ content) and on
+# pushes to main (so anyone can view the preview for any commit on main). Uses
+# a read-only token: it only reads .nextchanges/ and never needs write
+# credentials.
 on:
   pull_request:
     types: [opened, reopened, synchronize]
@@ -15,6 +17,9 @@ on:
       - "internal/genkit/**"
       - "tools/validate_nextchanges.py"
       - "tools/update_github_links.py"
+  push:
+    branches:
+      - main
 
 permissions:
   contents: read

.nextchanges/bundles/5818.md

@@ -0,0 +1 @@
+* `bundle validate` now reports a clear error when a `sql_warehouse` is missing a `name` (including whitespace-only names), and a warning when a grant is missing a `principal` ([#5818](https://github.com/databricks/cli/pull/5818)).

.nextchanges/bundles/bundle-generate-job-download-script.md

@@ -0,0 +1 @@
+* `bundle generate job` can now download workspace files referenced by `spark_python_task`, rewriting them to a relative path like it already does for notebooks. This is opt-in via the `--download-spark-python-files` flag ([#5799](https://github.com/databricks/cli/pull/5799)).

.nextchanges/cli/filer-gcs-recursive-delete.md

@@ -0,0 +1 @@
+Fixed `databricks fs rm -r` failing on UC Volumes backed by GCS when a directory becomes empty during recursive deletion ([#5958](https://github.com/databricks/cli/pull/5958)).

acceptance/auth/bundle_default_profile/home/.databrickscfg.tmpl

@@ -0,0 +1,10 @@
+[default-target]
+host  = $DATABRICKS_HOST
+token = $DATABRICKS_TOKEN
+
+[other]
+host  = https://other.test
+token = other-token
+
+[__settings__]
+default_profile = default-target

acceptance/auth/bundle_default_profile/script

@@ -3,18 +3,7 @@ sethome "./home"
 # Save the test server host so we can pin a bundle-with-host variant below.
 host_value="$DATABRICKS_HOST"
 
-cat > "./home/.databrickscfg" <<EOF
-[default-target]
-host  = $DATABRICKS_HOST
-token = $DATABRICKS_TOKEN
-
-[other]
-host  = https://other.test
-token = other-token
-
-[__settings__]
-default_profile = default-target
-EOF
+envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg"
 
 unset DATABRICKS_HOST
 unset DATABRICKS_TOKEN

acceptance/auth/host-metadata-cache/home/.databrickscfg.tmpl

@@ -0,0 +1,3 @@
+[cached]
+host = ${DATABRICKS_HOST}
+token = test-token

acceptance/auth/host-metadata-cache/script

@@ -4,11 +4,7 @@ export DATABRICKS_CACHE_DIR="$TEST_TMP_DIR/cache"
 # Point a profile at the mock server so auth profiles triggers a host metadata
 # fetch. Validation must stay on: --skip-validate resolves offline and would
 # never fetch or populate the cache.
-cat > "./home/.databrickscfg" <<EOF
-[cached]
-host = ${DATABRICKS_HOST}
-token = test-token
-EOF
+envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg"
 
 title "First invocation populates the cache\n"
 $CLI auth profiles --output json

acceptance/bundle/artifacts/artifact_path_with_volume/test.toml

@@ -1,3 +1,6 @@
-Local = false
+Local = true
 Cloud = true
+RequiresUnityCatalog = true
+
+# Overrides RecordRequests = true set by the artifacts parent test.toml.
 RecordRequests = false

acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/test.toml

@@ -1,4 +1 @@
 Badness = "Requires two bundles; one for volumes, another for artifacts. Ideally it would just work but today we first upload then deploy, so it does not fit in that architecture."
-Local = true
-Cloud = true
-RequiresUnityCatalog = true

acceptance/bundle/deploy/immutable-no-artifacts/output.txt

@@ -15,13 +15,13 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] jobs get [NUMID]
-"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/src/files/src/main.py"
+"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py"
 
 >>> [CLI] jobs get [NUMID]
-"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/src/files/src/notebook"
+"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook"
 
 >>> [CLI] jobs get [NUMID]
-"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/src/files/some_path"
+"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/some_path"
 
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:

acceptance/bundle/deploy/immutable/output.txt

@@ -13,7 +13,7 @@ Building python_artifact...
 [
   {
     "notebook_task": {
-      "notebook_path": "${workspace.snapshot_path}/src/files/src/notebook"
+      "notebook_path": "${workspace.snapshot_path}/files/src/notebook"
     },
     "task_key": "notebook_task"
   },
@@ -28,7 +28,7 @@ Building python_artifact...
   {
     "environment_key": "env",
     "spark_python_task": {
-      "python_file": "${workspace.snapshot_path}/src/files/src/main.py"
+      "python_file": "${workspace.snapshot_path}/files/src/main.py"
     },
     "task_key": "spark_python_task"
   }
@@ -42,14 +42,14 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] jobs get [NUMID]
-"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/src/files/src/main.py"
+"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/main.py"
 
 >>> [CLI] jobs get [NUMID]
-"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/src/files/src/notebook"
+"/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/files/src/notebook"
 
 >>> [CLI] jobs get [NUMID]
 [
-  "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/src/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
+  "/Workspace/Users/[UUID]/.snapshots/[UUID]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
 ]
 
 >>> [CLI] bundle destroy --auto-approve

acceptance/bundle/deploy/readplan/grants-remove-principal/out.plan.grants.json

@@ -25,15 +25,15 @@
     "full_name": "main.myschema",
     "__embed__": [
       {
-        "principal": "[USERNAME]",
+        "principal": "extra@example.test",
         "privileges": [
-          "USE_SCHEMA"
+          "CREATE_TABLE"
         ]
       },
       {
-        "principal": "extra@example.test",
+        "principal": "[USERNAME]",
         "privileges": [
-          "CREATE_TABLE"
+          "USE_SCHEMA"
         ]
       }
     ]

acceptance/bundle/deploy/readplan/grants-remove-principal/script

@@ -12,8 +12,10 @@ trace print_requests.py //unity-catalog/permissions --sort
 grep -v 'TO_REMOVE' databricks.yml > updated.yml && mv updated.yml databricks.yml
 
 # Generate update plan: entry.RemoteState captures both principals as GrantsState.
+# remote_state.__embed__ principal order is non-deterministic (the backend, and the
+# testserver mirroring it, iterate a map), so sort it by principal for a stable golden.
 trace $CLI bundle plan -o json > tmp.plan.json
-jq '.plan["resources.schemas.myschema.grants"]' tmp.plan.json > out.plan.grants.json
+jq '.plan["resources.schemas.myschema.grants"] | .remote_state.__embed__ |= sort_by(.principal)' tmp.plan.json > out.plan.grants.json
 trace print_requests.py --get //unity-catalog/permissions --sort
 
 # Deploy from serialized plan (READPLAN=1) or without (READPLAN="").

acceptance/bundle/generate/alert/alert.json.tmpl

@@ -1,6 +1,6 @@
 {
   "display_name": "test alert",
-  "parent_path": "/Workspace/test-$UNIQUE_NAME",
+  "parent_path": "/Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAME",
   "query_text": "SELECT 1\n as value",
   "warehouse_id": "$TEST_DEFAULT_WAREHOUSE_ID",
   "evaluation": {

acceptance/bundle/generate/alert/output.txt

@@ -1,5 +1,5 @@
 
->>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME]
+>>> [CLI] workspace mkdirs /Workspace/Users/[USERNAME]/test-[UNIQUE_NAME]
 
 >>> [CLI] bundle generate alert --existing-id [ALERT_ID] --source-dir out/alert --config-dir out/resource
 Alert configuration successfully saved to [TEST_TMP_DIR]/out/resource/test_alert.alert.yml
@@ -12,3 +12,5 @@ so it will not be deployed. Add a matching entry to the 'include' section, for e
 include:
   - out/resource/*.yml
 
+
+>>> [CLI] workspace delete /Workspace/Users/[USERNAME]/test-[UNIQUE_NAME] --recursive

acceptance/bundle/generate/alert/script

@@ -1,4 +1,4 @@
-trace $CLI workspace mkdirs /Workspace/test-$UNIQUE_NAME
+trace $CLI workspace mkdirs /Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAME
 
 # create an alert to import
 envsubst < alert.json.tmpl > alert.json

acceptance/bundle/generate/alert/script.cleanup

@@ -0,0 +1 @@
+trace $CLI workspace delete /Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAME --recursive

acceptance/bundle/generate/auto-bind/databricks.yml.tmpl

@@ -2,7 +2,7 @@ bundle:
   name: auto-bind-test
 
 workspace:
-  root_path: /tmp/${UNIQUE_NAME}
+  root_path: ~/.bundle/auto-bind-test-${UNIQUE_NAME}
 
 include:
   - resources/*.yml

acceptance/bundle/generate/auto-bind/output.txt

@@ -21,7 +21,7 @@ test.py
 
 === Deploy the bound job:
 >>> [CLI] bundle deploy
-Uploading bundle files to /Workspace/tmp/[UNIQUE_NAME]/files...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/auto-bind-test-[UNIQUE_NAME]/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
@@ -31,7 +31,7 @@ Deployment complete!
 The following resources will be deleted:
   delete resources.jobs.test_job
 
-All files and directories at the following location will be deleted: /Workspace/tmp/[UNIQUE_NAME]
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/auto-bind-test-[UNIQUE_NAME]
 
 Deleting files...
 Destroy complete!

acceptance/bundle/generate/dashboard/dashboard.json.tmpl

@@ -1,5 +1,5 @@
 {
     "display_name": "test dashboard",
-    "parent_path": "/Workspace/test-$UNIQUE_NAME",
+    "parent_path": "/Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAME",
     "serialized_dashboard": "{\"pages\":[{\"displayName\":\"New Page\",\"layout\":[],\"name\":\"12345678\"}]}"
 }

acceptance/bundle/generate/dashboard/output.txt

@@ -1,5 +1,5 @@
 
->>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME]
+>>> [CLI] workspace mkdirs /Workspace/Users/[USERNAME]/test-[UNIQUE_NAME]
 
 >>> [CLI] bundle generate dashboard --existing-id [DASHBOARD_ID] --dashboard-dir out/dashboard --resource-dir out/resource
 Writing dashboard to out/dashboard/test_dashboard.lvdash.json
@@ -12,3 +12,5 @@ so it will not be deployed. Add a matching entry to the 'include' section, for e
 include:
   - out/resource/*.yml
 
+
+>>> [CLI] workspace delete /Workspace/Users/[USERNAME]/test-[UNIQUE_NAME] --recursive

acceptance/bundle/generate/dashboard/script

@@ -1,4 +1,4 @@
-trace $CLI workspace mkdirs /Workspace/test-$UNIQUE_NAME
+trace $CLI workspace mkdirs /Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAME
 
 # create a dashboard to import
 envsubst < dashboard.json.tmpl > dashboard.json

acceptance/bundle/generate/dashboard/script.cleanup

@@ -0,0 +1 @@
+trace $CLI workspace delete /Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAME --recursive

acceptance/bundle/generate/genie_space/genie_space.json.tmpl

@@ -1,7 +1,7 @@
 {
     "title": "test genie space",
     "description": "test description",
-    "parent_path": "/Workspace/test-$UNIQUE_NAME",
+    "parent_path": "/Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAME",
     "warehouse_id": "test-warehouse-id",
     "serialized_space": "{\"tables\":[],\"questions\":[]}"
 }

acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml

@@ -5,4 +5,4 @@ resources:
       warehouse_id: test-warehouse-id
       file_path: ../genie_space/test_genie_space.geniespace.json
       description: test description
-      parent_path: /Workspace/test-[UNIQUE_NAME]
+      parent_path: /Workspace/Users/[USERNAME]/test-[UNIQUE_NAME]

acceptance/bundle/generate/genie_space/output.txt

@@ -1,5 +1,5 @@
 
->>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME]
+>>> [CLI] workspace mkdirs /Workspace/Users/[USERNAME]/test-[UNIQUE_NAME]
 
 >>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource
 Writing genie space to out/genie_space/test_genie_space.geniespace.json
@@ -12,3 +12,5 @@ so it will not be deployed. Add a matching entry to the 'include' section, for e
 include:
   - out/resource/*.yml
 
+
+>>> [CLI] workspace delete /Workspace/Users/[USERNAME]/test-[UNIQUE_NAME] --recursive

acceptance/bundle/generate/genie_space/script

@@ -1,4 +1,4 @@
-trace $CLI workspace mkdirs /Workspace/test-$UNIQUE_NAME
+trace $CLI workspace mkdirs /Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAME
 
 # create a genie space to import
 envsubst < genie_space.json.tmpl > genie_space.json

acceptance/bundle/generate/genie_space/script.cleanup

@@ -0,0 +1 @@
+trace $CLI workspace delete /Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAME --recursive

acceptance/bundle/generate/spark_python_task_job/databricks.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: spark_python_task_job

acceptance/bundle/generate/spark_python_task_job/out.job.yml

@@ -0,0 +1,11 @@
+resources:
+  jobs:
+    out:
+      name: sparkpythonjob
+      tasks:
+        - task_key: workspace_file_task
+          spark_python_task:
+            python_file: outjob.py
+        - task_key: cloud_uri_task
+          spark_python_task:
+            python_file: dbfs:/FileStore/job.py

acceptance/bundle/generate/spark_python_task_job/outjob.py

@@ -0,0 +1 @@
+print("Hello, World!")

acceptance/bundle/generate/spark_python_task_job/output.txt

@@ -0,0 +1,10 @@
+File successfully saved to outjob.py
+Job configuration successfully saved to out.job.yml
+Warning: Generated configuration is not included in the bundle
+
+The file out.job.yml is not matched by any pattern in the 'include' section of databricks.yml,
+so it will not be deployed. Add a matching entry to the 'include' section, for example:
+
+include:
+  - *.yml
+

acceptance/bundle/generate/spark_python_task_job/script

@@ -0,0 +1 @@
+$CLI bundle generate job --existing-job-id 1234 --config-dir . --key out --force --source-dir . --download-spark-python-files

acceptance/bundle/generate/spark_python_task_job/test.toml

@@ -0,0 +1,41 @@
+[[Server]]
+Pattern = "GET /api/2.2/jobs/get"
+Response.Body = '''
+{
+    "job_id": 11223344,
+    "settings": {
+        "name": "sparkpythonjob",
+        "tasks": [
+            {
+                "task_key": "workspace_file_task",
+                "spark_python_task": {
+                    "python_file": "/Workspace/Users/tester@databricks.com/outjob.py"
+                }
+            },
+            {
+                "task_key": "cloud_uri_task",
+                "spark_python_task": {
+                    "python_file": "dbfs:/FileStore/job.py"
+                }
+            }
+        ]
+    }
+}
+'''
+
+[[Server]]
+Pattern = "GET /api/2.0/workspace/get-status"
+Response.Body = '''
+{
+    "path": "/Workspace/Users/tester@databricks.com/outjob.py",
+    "object_type": "FILE",
+    "language": "PYTHON",
+    "repos_export_format": "SOURCE"
+}
+'''
+
+[[Server]]
+Pattern = "GET /api/2.0/workspace/export"
+Response.Body = '''
+print("Hello, World!")
+'''