job_runs: expose output, dedupe deploy retries

.nextchanges/bundles/job-runs-idempotency.md

@@ -0,0 +1 @@
+direct: the experimental `job_runs` resource now derives an `idempotency_token` for the run it triggers, so a deploy that fires run-now and then crashes before recording the run rejoins that same run on retry instead of starting a duplicate. The token is computed by the CLI and rejected if set in bundle configuration; set the new `rerun_token` field to a new value to re-run a configuration that already ran. Destroying a `job_run` now leaves the run in place, since the Jobs API keeps its token reserved once the run is deleted.

acceptance/bundle/refschema/out.fields.txt

@@ -861,6 +861,7 @@ resources.job_runs.*.python_params	[]string	ALL
 resources.job_runs.*.python_params[*]	string	ALL
 resources.job_runs.*.queue	*jobs.QueueSettings	ALL
 resources.job_runs.*.queue.enabled	bool	ALL
+resources.job_runs.*.rerun_token	string	ALL
 resources.job_runs.*.resolved_job_id	int64	INPUT
 resources.job_runs.*.run_id	int64	REMOTE
 resources.job_runs.*.run_name	string	REMOTE

acceptance/bundle/resources/job_runs/basic/output.txt

@@ -40,15 +40,16 @@ job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS
 Updating deployment state...
 Deployment complete!
 
+=== redeploy does not trigger a second run
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> [CLI] bundle plan
-Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
-
 >>> [CLI] bundle summary
 Name: job-runs-basic
 Target: default
@@ -71,6 +72,7 @@ Resources:
   "method": "POST",
   "path": "/api/2.2/jobs/run-now",
   "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
     "job_id": [MY_JOB_ID]
   }
 }
@@ -91,12 +93,3 @@ All files and directories at the following location will be deleted: /Workspace/
 
 Deleting files...
 Destroy complete!
-
->>> print_requests.py //jobs/runs/delete
-{
-  "method": "POST",
-  "path": "/api/2.2/jobs/runs/delete",
-  "body": {
-    "run_id": [MY_RUN_ID]
-  }
-}

acceptance/bundle/resources/job_runs/basic/script

@@ -1,7 +1,7 @@
 cleanup() {
+    # destroy leaves the run in place: a run is immutable history.
     trace $CLI bundle destroy --auto-approve
-    # destroy deletes the triggered run via jobs/runs/delete (also unlinks out.requests.txt)
-    trace print_requests.py //jobs/runs/delete
+    rm -f out.requests.txt
 }
 trap cleanup EXIT
 
@@ -11,10 +11,9 @@ trace $CLI bundle plan
 trace $CLI bundle summary
 trace $CLI bundle deploy
 
-# confirm that redeploy does not trigger a second run
-trace $CLI bundle deploy
-
+title "redeploy does not trigger a second run"
 trace $CLI bundle plan
+trace $CLI bundle deploy
 trace $CLI bundle summary
 
 title "exactly one run-now request was made"

acceptance/bundle/resources/job_runs/distinct_runs/databricks.yml

@@ -0,0 +1,21 @@
+bundle:
+  name: job-runs-distinct-runs
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          condition_task:
+            op: EQUAL_TO
+            left: "1"
+            right: "1"
+
+  # Two runs with byte-identical config. Their resource keys go into the
+  # idempotency token, so each gets its own run.
+  job_runs:
+    my_run_a:
+      job_id: ${resources.jobs.my_job.id}
+    my_run_b:
+      job_id: ${resources.jobs.my_job.id}

acceptance/bundle/resources/job_runs/distinct_runs/output.txt

@@ -0,0 +1,40 @@
+
+=== deploy triggers a separate run for each resource
+>>> grep ^job run 
+job run [NUMID]: [TIMESTAMP] "my-job" RUNNING
+job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS
+job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID]
+job run [NUMID]: [TIMESTAMP] "my-job" RUNNING
+job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS
+job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID]
+
+=== two run-now requests were made; each resource stored its own run id
+>>> print_requests.py //jobs/run-now
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
+    "job_id": [NUMID]
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
+    "job_id": [NUMID]
+  }
+}
+run ids differ: identical config did not collapse onto one run
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run_a
+  delete resources.job_runs.my_run_b
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-distinct-runs/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/job_runs/distinct_runs/script

@@ -0,0 +1,31 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+STATE=.databricks/bundle/default/resources.json
+run_id() {
+    jq -r ".state[\"resources.job_runs.$1\"].__id__" "$STATE"
+}
+
+title "deploy triggers a separate run for each resource"
+# The two runs deploy concurrently onto one stream and the SDK jitters its poll
+# interval, so their progress lines arrive in either order: keep just those and
+# sort them.
+$CLI bundle deploy > out.deploy.txt 2>&1
+trace grep '^job run ' < out.deploy.txt | sort
+rm out.deploy.txt
+a=$(run_id my_run_a)
+b=$(run_id my_run_b)
+
+# Both tokens are masked, so the recorded bodies print identically in any order.
+# The resource key is part of each token, so two resources with identical config
+# get distinct tokens and therefore distinct runs.
+title "two run-now requests were made; each resource stored its own run id"
+trace print_requests.py //jobs/run-now
+if [ "$a" != "$b" ]; then
+    echo "run ids differ: identical config did not collapse onto one run"
+else
+    echo "run ids equal: the two resources deduped onto a single run"
+fi

acceptance/bundle/resources/job_runs/failed_run/output.txt

@@ -41,6 +41,7 @@ FAILED
   "method": "POST",
   "path": "/api/2.2/jobs/run-now",
   "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
     "job_id": [NUMID]
   }
 }

acceptance/bundle/resources/job_runs/idempotent_recreate/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: job-runs-idempotent-recreate
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          condition_task:
+            op: EQUAL_TO
+            left: "1"
+            right: "1"
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}

acceptance/bundle/resources/job_runs/idempotent_recreate/output.txt

@@ -0,0 +1,75 @@
+
+=== initial deploy triggers the run
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-idempotent-recreate/default/files...
+Deploying resources...
+job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID]
+job run [NUMID]: [TIMESTAMP] "my-job" RUNNING
+job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS
+Updating deployment state...
+Deployment complete!
+
+=== retry after a lost run id (deploy crashed before the id was recorded)
+>>> [CLI] bundle plan -o json
+{
+  "depends_on": [
+    {
+      "node": "resources.jobs.my_job",
+      "label": "${resources.jobs.my_job.id}"
+    }
+  ],
+  "action": "create",
+  "new_state": {
+    "value": {
+      "job_id": [NUMID]
+    }
+  }
+}
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-idempotent-recreate/default/files...
+Deploying resources...
+job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID]
+job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS
+Updating deployment state...
+Deployment complete!
+
+=== retry after the run id was recorded
+>>> [CLI] bundle plan -o json
+"skip"
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-idempotent-recreate/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== run-now was issued exactly twice, both with the same token, and no duplicate run was created
+>>> print_requests.py //jobs/run-now
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
+    "job_id": [NUMID]
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
+    "job_id": [NUMID]
+  }
+}
+run id stable across both retries ([NUMID]): the idempotency token reused the existing run
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-idempotent-recreate/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/job_runs/idempotent_recreate/script

@@ -0,0 +1,40 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+STATE=.databricks/bundle/default/resources.json
+run_id() {
+    jq -r '.state["resources.job_runs.my_run"].__id__' "$STATE"
+}
+
+title "initial deploy triggers the run"
+trace $CLI bundle deploy
+before=$(run_id)
+
+title "retry after a lost run id (deploy crashed before the id was recorded)"
+# Simulates a deploy that fired run-now but crashed before saving the run id.
+# job_id, and with it the idempotency token, is unchanged, so the retried run-now
+# dedupes onto the existing run.
+jq 'del(.state["resources.job_runs.my_run"])' "$STATE" > "$STATE.new"
+mv "$STATE.new" "$STATE"
+trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]'
+trace $CLI bundle deploy
+after_lost=$(run_id)
+
+title "retry after the run id was recorded"
+# The run id is still in state, so the retry is a no-op: no second run-now.
+trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].action // "none"'
+trace $CLI bundle deploy
+after_kept=$(run_id)
+
+title "run-now was issued exactly twice, both with the same token, and no duplicate run was created"
+# Two run-now calls total, from the initial deploy and the lost-id retry, both
+# with the same token, so a single run exists throughout.
+trace print_requests.py //jobs/run-now
+if [ "$before" = "$after_lost" ] && [ "$after_lost" = "$after_kept" ]; then
+    echo "run id stable across both retries ($before): the idempotency token reused the existing run"
+else
+    echo "run id changed ($before -> $after_lost -> $after_kept): a duplicate run was created"
+fi

acceptance/bundle/resources/job_runs/job_parameters/output.txt

@@ -14,6 +14,7 @@ Deployment complete!
   "method": "POST",
   "path": "/api/2.2/jobs/run-now",
   "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
     "job_id": [NUMID],
     "job_parameters": {
       "env": "prod"

acceptance/bundle/resources/job_runs/recreate_gone/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: job-runs-recreate-gone
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          condition_task:
+            op: EQUAL_TO
+            left: "1"
+            right: "1"
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}