From 78b84027d44c3c61ede6f3fbf138c46f4f6340c5 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 28 Jul 2026 13:05:06 +0000 Subject: [PATCH 01/14] bundle/run: move runPageURL into libs/workspaceurls The legacy-to-path run URL conversion is needed outside `bundle run`, so it moves to libs/workspaceurls next to JobRunPath, which it already used. Pure move: the conversion, its doc comment and its tests are unchanged. --- bundle/run/job.go | 47 ++------------------------------ bundle/run/job_test.go | 48 --------------------------------- libs/workspaceurls/urls.go | 44 ++++++++++++++++++++++++++++++ libs/workspaceurls/urls_test.go | 47 ++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 93 deletions(-) diff --git a/bundle/run/job.go b/bundle/run/job.go index de5457c26e9..2001fffda95 100644 --- a/bundle/run/job.go +++ b/bundle/run/job.go @@ -5,9 +5,7 @@ import ( "encoding/json" "errors" "fmt" - "net/url" "strconv" - "strings" "time" "github.com/databricks/cli/bundle" @@ -99,7 +97,7 @@ func (m *jobRunMonitor) onProgress(info *jobs.Run) { // First time we see this run. if m.prevState == nil { - runURL := runPageURL(m.ctx, info.RunPageUrl) + runURL := workspaceurls.JobRunPageURL(m.ctx, info.RunPageUrl) log.Infof(m.ctx, "Run available at %s", runURL) cmdio.Log(m.ctx, progress.NewJobRunUrlEvent(runURL)) } @@ -126,47 +124,6 @@ func (m *jobRunMonitor) onProgress(info *jobs.Run) { log.Info(m.ctx, event.String()) } -// runPageURL converts the legacy run URL returned by the Jobs API -// -// https:///?o=#job//run/ -// -// into the modern path form -// -// https:///jobs//runs/?o= -// -// so that non-admin users permitted to view the run are not redirected to the -// workspace homepage. See https://github.com/databricks/cli/issues/5142. The -// workspace selector query param (o) is preserved as-is. The conversion is -// cosmetic, so the original URL is returned on the rare chance the format is -// unexpected. -func runPageURL(ctx context.Context, raw string) string { - u, err := url.Parse(raw) - if err != nil { - log.Debugf(ctx, "could not parse run URL %q: %v", raw, err) - return raw - } - - jobID, runID, ok := parseLegacyRunFragment(u.Fragment) - if !ok { - log.Debugf(ctx, "unexpected run URL fragment %q", u.Fragment) - return raw - } - - u.Fragment = "" - u.Path = "/" + workspaceurls.JobRunPath(jobID, runID) - return u.String() -} - -// parseLegacyRunFragment extracts the job and run IDs from a legacy run URL -// fragment of the form "job//run/". -func parseLegacyRunFragment(fragment string) (jobID, runID string, ok bool) { - parts := strings.Split(fragment, "/") - if len(parts) != 4 || parts[0] != "job" || parts[2] != "run" || parts[1] == "" || parts[3] == "" { - return "", "", false - } - return parts[1], parts[3], true -} - func (r *jobRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, error) { jobID, err := strconv.ParseInt(r.job.ID, 10, 64) if err != nil { @@ -205,7 +162,7 @@ func (r *jobRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e if err != nil { return nil, err } - cmdio.Log(ctx, progress.NewJobRunUrlEvent(runPageURL(ctx, details.RunPageUrl))) + cmdio.Log(ctx, progress.NewJobRunUrlEvent(workspaceurls.JobRunPageURL(ctx, details.RunPageUrl))) return nil, nil } diff --git a/bundle/run/job_test.go b/bundle/run/job_test.go index 73fb68757c2..4307f2fae74 100644 --- a/bundle/run/job_test.go +++ b/bundle/run/job_test.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -292,50 +291,3 @@ func TestJobRunnerRestartForContinuousUnpausedJobs(t *testing.T) { _, err := runner.Restart(ctx, &Options{}) require.NoError(t, err) } - -func TestRunPageURL(t *testing.T) { - ctx := t.Context() - tests := []struct { - name string - raw string - expected string - }{ - { - "legacy fragment form preserves workspace selector", - "https://myworkspace.databricks.test/?o=900800700600#job/123/run/456", - "https://myworkspace.databricks.test/jobs/123/runs/456?o=900800700600", - }, - { - "no workspace selector", - "https://myworkspace.databricks.test/#job/123/run/456", - "https://myworkspace.databricks.test/jobs/123/runs/456", - }, - { - "http host with port", - "http://127.0.0.1:8080/?o=900800700600#job/1/run/2", - "http://127.0.0.1:8080/jobs/1/runs/2?o=900800700600", - }, - // Unexpected formats are returned unchanged because the conversion is cosmetic. - { - "already modern path is left as-is", - "https://myworkspace.databricks.test/jobs/123/runs/456?o=900800700600", - "https://myworkspace.databricks.test/jobs/123/runs/456?o=900800700600", - }, - { - "incomplete fragment is left as-is", - "https://myworkspace.databricks.test/?o=900800700600#job/123", - "https://myworkspace.databricks.test/?o=900800700600#job/123", - }, - { - "empty job id is left as-is", - "https://myworkspace.databricks.test/?o=900800700600#job//run/456", - "https://myworkspace.databricks.test/?o=900800700600#job//run/456", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, runPageURL(ctx, tt.raw)) - }) - } -} diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index a1bf973801f..b0248a87a59 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -1,10 +1,13 @@ package workspaceurls import ( + "context" "fmt" "net/url" "slices" "strings" + + "github.com/databricks/cli/libs/log" ) var resourceURLPatterns = map[string]string{ @@ -90,6 +93,47 @@ func JobRunURL(baseURL url.URL, jobID, runID string) string { return baseURL.String() } +// JobRunPageURL converts the legacy run URL returned by the Jobs API +// +// https:///?o=#job//run/ +// +// into the modern path form +// +// https:///jobs//runs/?o= +// +// so that non-admin users permitted to view the run are not redirected to the +// workspace homepage. See https://github.com/databricks/cli/issues/5142. The +// workspace selector query param (o) is preserved as-is. The conversion is +// cosmetic, so the original URL is returned on the rare chance the format is +// unexpected. +func JobRunPageURL(ctx context.Context, raw string) string { + u, err := url.Parse(raw) + if err != nil { + log.Debugf(ctx, "could not parse run URL %q: %v", raw, err) + return raw + } + + jobID, runID, ok := parseLegacyRunFragment(u.Fragment) + if !ok { + log.Debugf(ctx, "unexpected run URL fragment %q", u.Fragment) + return raw + } + + u.Fragment = "" + u.Path = "/" + JobRunPath(jobID, runID) + return u.String() +} + +// parseLegacyRunFragment extracts the job and run IDs from a legacy run URL +// fragment of the form "job//run/". +func parseLegacyRunFragment(fragment string) (jobID, runID string, ok bool) { + parts := strings.Split(fragment, "/") + if len(parts) != 4 || parts[0] != "job" || parts[2] != "run" || parts[1] == "" || parts[3] == "" { + return "", "", false + } + return parts[1], parts[3], true +} + // ResourceURL constructs a workspace URL for a named resource type and ID. func ResourceURL(baseURL url.URL, resourceType, id string) string { resourceType = resolveAlias(resourceType) diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index e39d28d9aaf..7fe4f152b78 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -166,3 +166,50 @@ func TestHasWorkspaceIDInHostname(t *testing.T) { }) } } + +func TestJobRunPageURL(t *testing.T) { + ctx := t.Context() + tests := []struct { + name string + raw string + expected string + }{ + { + "legacy fragment form preserves workspace selector", + "https://myworkspace.databricks.test/?o=900800700600#job/123/run/456", + "https://myworkspace.databricks.test/jobs/123/runs/456?o=900800700600", + }, + { + "no workspace selector", + "https://myworkspace.databricks.test/#job/123/run/456", + "https://myworkspace.databricks.test/jobs/123/runs/456", + }, + { + "http host with port", + "http://127.0.0.1:8080/?o=900800700600#job/1/run/2", + "http://127.0.0.1:8080/jobs/1/runs/2?o=900800700600", + }, + // The conversion is cosmetic, so any other format passes through unchanged. + { + "already modern path is left as-is", + "https://myworkspace.databricks.test/jobs/123/runs/456?o=900800700600", + "https://myworkspace.databricks.test/jobs/123/runs/456?o=900800700600", + }, + { + "incomplete fragment is left as-is", + "https://myworkspace.databricks.test/?o=900800700600#job/123", + "https://myworkspace.databricks.test/?o=900800700600#job/123", + }, + { + "empty job id is left as-is", + "https://myworkspace.databricks.test/?o=900800700600#job//run/456", + "https://myworkspace.databricks.test/?o=900800700600#job//run/456", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, JobRunPageURL(ctx, tt.raw)) + }) + } +} From 9284e90a68f8f8b1d94d09290dea21267bdcf836 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 28 Jul 2026 13:05:17 +0000 Subject: [PATCH 02/14] testserver: roll task outcomes up into the run state The fake workspace reported every run as TERMINATED SUCCESS, overwriting the FAILED state it had just recorded for a task it executed locally. A run now reports the terminal state its tasks add up to, so a failing run can be exercised end to end locally. Tasks whose code the fake workspace does not have are left successful. An immutable deployment, for example, uploads the bundle as a snapshot zip that the fake workspace never unpacks, so there is nothing to execute; that gap is in the fake workspace, not in the job under test. Originally reviewed as #6082. --- libs/testserver/jobs.go | 57 ++++++++++++++++++++--------- libs/testserver/jobs_test.go | 71 ++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 17 deletions(-) diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index 9097d43c086..e8d29877e98 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -20,6 +20,11 @@ import ( const missingJobGitProviderMessage = "git_source.git_provider must be one of: github,gitlab,bitbucketcloud,gitlabenterpriseedition,bitbucketserver,azuredevopsservices,githubenterprise,awscodecommit" +// errNoCodeInWorkspace marks a task there is nothing to execute for, e.g. +// because an immutable deployment uploaded the code as a snapshot zip this +// server never unpacks. The gap is here, not in the job, so the task succeeds. +var errNoCodeInWorkspace = errors.New("task code is not in the workspace") + // venvPython returns the path to the Python executable in a venv. // On Unix: venv/bin/python // On Windows: venv\Scripts\python.exe @@ -387,12 +392,15 @@ func (s *FakeWorkspace) JobsRunNow(req Request) Response { logs, err = s.executeSparkPythonTask(t) } - if err != nil { + switch { + case errors.Is(err, errNoCodeInWorkspace): + // Nothing ran, so the task keeps its SUCCESS state. + case err != nil: taskRun.State.ResultState = jobs.RunResultStateFailed s.JobRunOutputs[taskRunId] = jobs.RunOutput{ Error: err.Error(), } - } else if logs != "" { + case logs != "": s.JobRunOutputs[taskRunId] = jobs.RunOutput{ Logs: logs, } @@ -599,7 +607,7 @@ func (s *FakeWorkspace) executePythonWheelTask(jobSettings *jobs.JobSettings, ta } data := s.files[whlPath].Data if len(data) == 0 { - return "", fmt.Errorf("wheel file not found in workspace: %s", whlPath) + return "", fmt.Errorf("%w: wheel file not found in workspace: %s", errNoCodeInWorkspace, whlPath) } localPath := filepath.Join(env.dir, filepath.Base(whlPath)) if err := os.WriteFile(localPath, data, 0o644); err != nil { @@ -618,7 +626,7 @@ func (s *FakeWorkspace) executePythonWheelTask(jobSettings *jobs.JobSettings, ta } if len(env.installedLibs) == 0 { - return "", errors.New("no wheel libraries found in task") + return "", fmt.Errorf("%w: no wheel libraries found in task", errNoCodeInWorkspace) } // Run the entry point using runpy with sys.argv[0] set to the package name, @@ -664,7 +672,7 @@ func (s *FakeWorkspace) executeNotebookTask(task jobs.Task, notebookParams map[s notebookData = s.files[notebookPath+".py"].Data } if len(notebookData) == 0 { - return "", fmt.Errorf("notebook not found in workspace: %s (also tried .py)", notebookPath) + return "", fmt.Errorf("%w: notebook not found in workspace: %s (also tried .py)", errNoCodeInWorkspace, notebookPath) } // Create a temporary Python environment for notebook execution @@ -750,7 +758,7 @@ func (s *FakeWorkspace) executeSparkPythonTask(task jobs.Task) (string, error) { pythonData := s.files[pythonPath].Data if len(pythonData) == 0 { - return "", fmt.Errorf("python file not found in workspace: %s", pythonPath) + return "", fmt.Errorf("%w: python file not found in workspace: %s", errNoCodeInWorkspace, pythonPath) } env, cleanup, err := s.getOrCreateClusterEnv(task) @@ -848,6 +856,30 @@ func sparkVersionToPython(task jobs.Task) string { return "3.10" } +// terminateRun completes the run, rolling task outcomes up into the run-level +// state the way the Jobs API does: one failed task fails the whole run. +func terminateRun(run *jobs.Run) { + for i := range run.Tasks { + // Tasks that were never executed (jobs/runs/submit) are still running. + if run.Tasks[i].State.LifeCycleState != jobs.RunLifeCycleStateTerminated { + run.Tasks[i].State.LifeCycleState = jobs.RunLifeCycleStateTerminated + run.Tasks[i].State.ResultState = jobs.RunResultStateSuccess + } + } + + run.State = &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + } + for _, task := range run.Tasks { + if task.State.ResultState != jobs.RunResultStateSuccess { + run.State.ResultState = task.State.ResultState + run.State.StateMessage = fmt.Sprintf("task %s failed", task.TaskKey) + return + } + } +} + func (s *FakeWorkspace) JobsGetRun(req Request) Response { runId := req.URL.Query().Get("run_id") runIdInt, err := strconv.ParseInt(runId, 10, 64) @@ -865,19 +897,10 @@ func (s *FakeWorkspace) JobsGetRun(req Request) Response { return Response{StatusCode: 404} } - // Simulate cloud behavior: first poll returns RUNNING, next returns TERMINATED SUCCESS. + // Simulate cloud behavior: first poll returns RUNNING, next the terminal state. if run.State.LifeCycleState == jobs.RunLifeCycleStateRunning { // Transition stored state to TERMINATED for the next poll. - run.State = &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - } - for i := range run.Tasks { - run.Tasks[i].State = &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - } - } + terminateRun(&run) s.JobRuns[runIdInt] = run // Return RUNNING for this poll (before the transition). diff --git a/libs/testserver/jobs_test.go b/libs/testserver/jobs_test.go index 38e47b68b08..27898c3664f 100644 --- a/libs/testserver/jobs_test.go +++ b/libs/testserver/jobs_test.go @@ -87,6 +87,77 @@ func TestJobsSubmit_RunReachesTerminalStateOnPoll(t *testing.T) { assert.Equal(t, jobs.RunResultStateSuccess, second.State.ResultState) } +func createJob(t *testing.T, workspace *FakeWorkspace, tasks ...jobs.Task) int64 { + t.Helper() + body, err := json.Marshal(jobs.CreateJob{Name: "my-job", Tasks: tasks}) + require.NoError(t, err) + + response := workspace.JobsCreate(Request{Body: body}) + require.Equal(t, 0, response.StatusCode) + return response.Body.(jobs.CreateResponse).JobId +} + +func runNow(t *testing.T, workspace *FakeWorkspace, request jobs.RunNow) Response { + t.Helper() + body, err := json.Marshal(request) + require.NoError(t, err) + return workspace.JobsRunNow(Request{Body: body}) +} + +func terminatedTask(taskKey string, result jobs.RunResultState) jobs.RunTask { + return jobs.RunTask{ + TaskKey: taskKey, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: result, + }, + } +} + +func TestTerminateRun_FailedTaskFailsTheRun(t *testing.T) { + run := jobs.Run{Tasks: []jobs.RunTask{ + terminatedTask("first", jobs.RunResultStateSuccess), + terminatedTask("second", jobs.RunResultStateFailed), + }} + + terminateRun(&run) + + assert.Equal(t, jobs.RunLifeCycleStateTerminated, run.State.LifeCycleState) + assert.Equal(t, jobs.RunResultStateFailed, run.State.ResultState) + assert.Equal(t, "task second failed", run.State.StateMessage) +} + +func TestTerminateRun_CompletesTasksThatAreStillRunning(t *testing.T) { + // jobs/runs/submit records its tasks as running: they are never executed. + run := jobs.Run{Tasks: []jobs.RunTask{ + {TaskKey: "main", State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}}, + }} + + terminateRun(&run) + + assert.Equal(t, jobs.RunResultStateSuccess, run.State.ResultState) + assert.Empty(t, run.State.StateMessage) + assert.Equal(t, jobs.RunResultStateSuccess, run.Tasks[0].State.ResultState) +} + +// See errNoCodeInWorkspace: a missing notebook is this server's gap, not a +// failure of the job. +func TestJobsGetRun_TaskWithoutCodeDoesNotFailTheRun(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + jobID := createJob(t, workspace, jobs.Task{ + TaskKey: "main", + NotebookTask: &jobs.NotebookTask{NotebookPath: "/missing-notebook"}, + }) + + response := runNow(t, workspace, jobs.RunNow{JobId: jobID}) + require.Equal(t, 0, response.StatusCode) + runID := response.Body.(jobs.RunNowResponse).RunId + + // The first poll reports RUNNING, the second the terminal state. + require.Equal(t, jobs.RunLifeCycleStateRunning, getRun(t, workspace, runID).State.LifeCycleState) + assert.Equal(t, jobs.RunResultStateSuccess, getRun(t, workspace, runID).State.ResultState) +} + func TestJobsSubmit_RejectsInvalidGitProvider(t *testing.T) { workspace := NewFakeWorkspace("http://test", "dbapi123") From 9e63447f64eae277c9ca1c3066ce8add62d221e4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 28 Jul 2026 13:05:34 +0000 Subject: [PATCH 03/14] job_runs: wait for run completion in WaitAfterCreate Deploying a job_run triggered the run and moved on, so a resource referencing the run's outcome saw whatever state the run happened to be in. The resource now implements the framework's WaitAfterCreate hook, which blocks until the run is terminal and republishes the settled remote state; only SUCCESS lets the deploy continue. While it waits, the deploy reports the run page URL and each state change, the way `bundle run` does, since a run can take hours. A run that does not succeed fails the deploy with the failed task, the message that task reported, and a link to the run page. Bounded by 24h, matching `bundle run`. The framework saves the run id before the wait, so a run that fails stays recorded and an unchanged config plans no second run; failed_run covers that. run_page_url is now normalized to the path form that also resolves for non-admins. invariant/configs/job_run.yml.tmpl loses its notebook task, which deploying it would now actually run, and is excluded from cloud runs: a real workspace reports a condition-task-only run as SKIPPED, and a task that does succeed would add a multi-minute cluster run to every variant of a suite that asserts plan and state invariants. Still covered locally. --- .../bundles/job-runs-wait-for-completion.md | 1 + .../bundle/invariant/configs/job_run.yml.tmpl | 13 +- acceptance/bundle/invariant/test.toml | 5 + .../resources/job_runs/basic/output.txt | 3 + .../job_runs/failed_run/databricks.yml | 34 ++++ .../resources/job_runs/failed_run/fail.py | 4 + .../job_runs/failed_run/out.test.toml | 3 + .../resources/job_runs/failed_run/output.txt | 57 ++++++ .../resources/job_runs/failed_run/script | 24 +++ .../resources/job_runs/failed_run/test.toml | 7 + .../job_runs/job_parameters/output.txt | 3 + .../resources/job_runs/redeploy/output.txt | 11 +- .../job_runs/wait_output/databricks.yml | 30 +++ .../job_runs/wait_output/out.test.toml | 3 + .../resources/job_runs/wait_output/output.txt | 91 +++++++++ .../resources/job_runs/wait_output/script | 18 ++ .../resources/job_runs/wait_output/test.toml | 4 + bundle/direct/dresources/all_test.go | 6 +- bundle/direct/dresources/job_run.go | 163 ++++++++++++++++- bundle/direct/dresources/job_run_test.go | 173 ++++++++++++++++++ bundle/internal/schema/annotations.yml | 2 + bundle/schema/jsonschema.json | 2 +- 22 files changed, 636 insertions(+), 21 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-wait-for-completion.md create mode 100644 acceptance/bundle/resources/job_runs/failed_run/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/failed_run/fail.py create mode 100644 acceptance/bundle/resources/job_runs/failed_run/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/failed_run/output.txt create mode 100644 acceptance/bundle/resources/job_runs/failed_run/script create mode 100644 acceptance/bundle/resources/job_runs/failed_run/test.toml create mode 100644 acceptance/bundle/resources/job_runs/wait_output/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/wait_output/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/wait_output/output.txt create mode 100644 acceptance/bundle/resources/job_runs/wait_output/script create mode 100644 acceptance/bundle/resources/job_runs/wait_output/test.toml create mode 100644 bundle/direct/dresources/job_run_test.go diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md new file mode 100644 index 00000000000..e78497e24cb --- /dev/null +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -0,0 +1 @@ +direct: the experimental `job_runs` resource now waits for the triggered run to finish, and fails the deploy if it does not succeed, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL and each state change while it waits, and names the failed task and the message it reported when the run does not succeed. diff --git a/acceptance/bundle/invariant/configs/job_run.yml.tmpl b/acceptance/bundle/invariant/configs/job_run.yml.tmpl index a09987da02a..2fab76ff020 100644 --- a/acceptance/bundle/invariant/configs/job_run.yml.tmpl +++ b/acceptance/bundle/invariant/configs/job_run.yml.tmpl @@ -6,14 +6,13 @@ resources: foo: name: test-job-$UNIQUE_NAME tasks: + # Deploying a job_run actually runs the job, so use a condition task, + # which needs no workspace files or compute. - task_key: only_task - notebook_task: - notebook_path: /Shared/notebook - new_cluster: - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - instance_pool_id: $TEST_INSTANCE_POOL_ID - num_workers: 1 + condition_task: + op: EQUAL_TO + left: "1" + right: "1" job_runs: foo_run: diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 2723cc9294c..ba471f291e8 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -83,6 +83,11 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] # so this config is local-only (the mock server stores it verbatim). no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] +# Deploying a job_run waits for the run to succeed, and a real workspace reports +# a run of condition tasks alone as SKIPPED. A task that does succeed would add a +# cluster run to every variant of a suite that asserts plan and state invariants. +no_job_run_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run.yml.tmpl"] + # Postgres resources only work on AWS no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt index 14018b93ec6..9ce260e8c85 100644 --- a/acceptance/bundle/resources/job_runs/basic/output.txt +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -34,6 +34,9 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files... Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml new file mode 100644 index 00000000000..8a538daf853 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml @@ -0,0 +1,34 @@ +bundle: + name: job-runs-failed-run + +resources: + jobs: + my_job: + name: my-job + tasks: + # The test server runs this locally; the script exits non-zero, which + # fails the task and with it the run. + - task_key: main + spark_python_task: + python_file: ./fail.py + environment_key: default + + environments: + - environment_key: default + spec: + client: "2" + + # Depends on my_run's result_state, so the failing run aborts the deploy + # before this job is created. + downstream_job: + name: downstream-job + tags: + run_result: ${resources.job_runs.my_run.state.result_state} + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/failed_run/fail.py b/acceptance/bundle/resources/job_runs/failed_run/fail.py new file mode 100644 index 00000000000..3262aa05529 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/fail.py @@ -0,0 +1,4 @@ +import sys + +print("intentional failure", file=sys.stderr) +sys.exit(1) diff --git a/acceptance/bundle/resources/job_runs/failed_run/out.test.toml b/acceptance/bundle/resources/job_runs/failed_run/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt new file mode 100644 index 00000000000..e8a5098a0df --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -0,0 +1,57 @@ + +=== a run that finishes FAILED fails the deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED FAILED task main failed +Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: job run [MY_RUN_ID] did not succeed: FAILED: task main failed +task "main": spark python task execution failed: exit status 1 +intentional failure + +run page: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] + +Error: cannot create resources.jobs.downstream_job: dependency failed: resources.job_runs.my_run + +Updating deployment state... + +=== the failed run stays recorded, so a redeploy starts no new run +>>> read_id.py my_run +[MY_RUN_ID] + +>>> [CLI] bundle plan +create jobs.downstream_job + +Plan: 1 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== downstream_job resolved its tag from the failed run +>>> jq -r select(.path == "/api/2.2/jobs/create" and .body.name == "downstream-job") | .body.tags.run_result out.requests.txt +FAILED + +=== run-now was issued once +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "job_id": [NUMID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.downstream_job + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script new file mode 100644 index 00000000000..40dba04f8b3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -0,0 +1,24 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# The run finishes FAILED, so the deploy aborts: the error names the failed task +# and the message it reported, and downstream_job is reported as a failed +# dependency because it reads the run's result_state. +title "a run that finishes FAILED fails the deploy" +musterr trace $CLI bundle deploy + +# The framework saves the run id before calling WaitAfterCreate, so a run that +# failed stays recorded and an unchanged config plans no second run. +title "the failed run stays recorded, so a redeploy starts no new run" +trace read_id.py my_run +trace $CLI bundle plan +trace $CLI bundle deploy + +title "downstream_job resolved its tag from the failed run" +trace jq -r 'select(.path == "/api/2.2/jobs/create" and .body.name == "downstream-job") | .body.tags.run_result' out.requests.txt + +title "run-now was issued once" +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/failed_run/test.toml b/acceptance/bundle/resources/job_runs/failed_run/test.toml new file mode 100644 index 00000000000..03530b96c6f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_run/test.toml @@ -0,0 +1,7 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true + +# The deploy fails mid-way, leaving local deployment state behind. +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index bcf3f21e017..986caf2b754 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -3,6 +3,9 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files... Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" RUNNING +job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 6ec19be8bce..8662b4880f0 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -3,6 +3,9 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" RUNNING +job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! @@ -64,10 +67,11 @@ Resources: }, "run_id": [NUMID], "run_name": "my-job", - "run_page_url": "[DATABRICKS_URL]/?o=[NUMID]#job/[MY_JOB_ID]/run/[NUMID]", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID]", "run_type": "JOB_RUN", "state": { - "life_cycle_state": "RUNNING" + "life_cycle_state": "TERMINATED", + "result_state": "SUCCESS" } }, "changes": { @@ -84,6 +88,9 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" RUNNING +job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/wait_output/databricks.yml b/acceptance/bundle/resources/job_runs/wait_output/databricks.yml new file mode 100644 index 00000000000..8b9d6691220 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/databricks.yml @@ -0,0 +1,30 @@ +bundle: + name: job-runs-wait-output + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + # Reads the run's output. A job separate from my_job (the run's target), + # since my_run already depends on my_job.id and a reference back would cycle. + downstream_job: + name: downstream-job + tags: + run_result: ${resources.job_runs.my_run.state.result_state} + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/wait_output/out.test.toml b/acceptance/bundle/resources/job_runs/wait_output/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/wait_output/output.txt b/acceptance/bundle/resources/job_runs/wait_output/output.txt new file mode 100644 index 00000000000..bd9def5e990 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/output.txt @@ -0,0 +1,91 @@ + +=== deploy waits for the run to finish, then the downstream job reads its result_state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/files... +Deploying resources... +job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +job run [NUMID]: [TIMESTAMP] "my-job" RUNNING +job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +=== the downstream job was created with the run's result_state resolved into its tag +>>> print_requests.py //jobs/create +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "condition_task": { + "left": "1", + "op": "EQUAL_TO", + "right": "1" + }, + "task_key": "main" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "downstream-job", + "queue": { + "enabled": true + }, + "tags": { + "run_result": "SUCCESS" + }, + "tasks": [ + { + "condition_task": { + "left": "1", + "op": "EQUAL_TO", + "right": "1" + }, + "task_key": "main" + } + ] + } +} + +=== redeploy is a no-op: the resolved result_state tag is stable, not perpetual drift +>>> [CLI] bundle plan -o json +"skip" + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.downstream_job + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/wait_output/script b/acceptance/bundle/resources/job_runs/wait_output/script new file mode 100644 index 00000000000..85812d8c18b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/script @@ -0,0 +1,18 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "deploy waits for the run to finish, then the downstream job reads its result_state" +trace $CLI bundle deploy + +title "the downstream job was created with the run's result_state resolved into its tag" +# A concrete SUCCESS tag proves the run finished and the wait published its +# output before the downstream job was created. +trace print_requests.py //jobs/create + +title "redeploy is a no-op: the resolved result_state tag is stable, not perpetual drift" +# The resolved tag reads back identically, so the job stays as deployed. +trace $CLI bundle plan -o json | jq '.plan["resources.jobs.downstream_job"].action // "none"' +trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/job_runs/wait_output/test.toml b/acceptance/bundle/resources/job_runs/wait_output/test.toml new file mode 100644 index 00000000000..4b94d8b58e9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_output/test.toml @@ -0,0 +1,4 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 54541d94ba9..89d690dfead 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1003,7 +1003,11 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W remoteStateFromWaitCreate, err := adapter.WaitAfterCreate(ctx, createdID, newState) require.NoError(t, err) if remoteStateFromWaitCreate != nil { - require.Equal(t, remote, remoteStateFromWaitCreate) + // WaitAfterCreate returns the settled state; the read right after DoCreate + // may still be non-terminal, so compare against a fresh read, not that one. + remotePostWaitCreate, err := adapter.DoRead(ctx, createdID) + require.NoError(t, err) + require.Equal(t, remotePostWaitCreate, remoteStateFromWaitCreate) } if adapter.HasDoUpdate() { diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 0a2ae0ea6af..fd783dfd2ea 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -1,16 +1,28 @@ package dresources import ( + "cmp" "context" + "errors" "fmt" "strconv" + "strings" + "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/run/progress" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/jobs" ) +// jobRunTimeout bounds the wait for a run to finish, matching `bundle run` +// (jobRunTimeout in bundle/run/job.go). +const jobRunTimeout = 24 * time.Hour + // JobRunState is what we persist for a triggered run: the RunNow request. type JobRunState struct { jobs.RunNow @@ -29,11 +41,13 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { type JobRunRemote struct { jobs.RunNow - RunId int64 `json:"run_id,omitempty"` - RunName string `json:"run_name,omitempty"` - State *jobs.RunState `json:"state,omitempty"` - RunPageUrl string `json:"run_page_url,omitempty"` - RunType jobs.RunType `json:"run_type,omitempty"` + RunId int64 `json:"run_id,omitempty"` + RunName string `json:"run_name,omitempty"` + State *jobs.RunState `json:"state,omitempty"` + // Normalized to the path form that also resolves for non-admins; see + // workspaceurls.JobRunPageURL. + RunPageUrl string `json:"run_page_url,omitempty"` + RunType jobs.RunType `json:"run_type,omitempty"` } // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise @@ -65,7 +79,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { // makeJobRunRemote maps the GetRun response into the RunNow-shaped remote: GET // nests the params under overriding_parameters and returns job_parameters as a // list, so both are flattened back into RunNow. -func makeJobRunRemote(run *jobs.Run) *JobRunRemote { +func makeJobRunRemote(ctx context.Context, run *jobs.Run) *JobRunRemote { var overriding jobs.RunParameters if run.OverridingParameters != nil { overriding = *run.OverridingParameters @@ -100,7 +114,7 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { RunId: run.RunId, RunName: run.RunName, State: run.State, - RunPageUrl: run.RunPageUrl, + RunPageUrl: workspaceurls.JobRunPageURL(ctx, run.RunPageUrl), RunType: run.RunType, } } @@ -120,7 +134,7 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, if err != nil { return nil, err } - return makeJobRunRemote(run), nil + return makeJobRunRemote(ctx, run), nil } // RemapState extracts the embedded RunNow as the state used for diffing. @@ -138,12 +152,141 @@ func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (str return strconv.FormatInt(wait.RunId, 10), nil, nil } +// WaitAfterCreate blocks until the triggered run finishes, so a resource that +// references this run's output (e.g. state.result_state) is created only once the +// run has produced it. Only a SUCCESS lets the deploy continue. +func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobRunState) (*JobRunRemote, error) { + runID, err := parseRunID(id) + if err != nil { + return nil, err + } + return r.waitForRun(ctx, runID) +} + +// waitForRun blocks until the run reaches a terminal state and returns its +// remote view; only SUCCESS returns a nil error. +func (r *ResourceJobRun) waitForRun(ctx context.Context, runID int64) (*JobRunRemote, error) { + // A run can take hours, so report progress like `bundle run` does. pageURL + // outlives the callback so an abandoned wait can still link the run. + var prevState *jobs.RunState + var pageURL string + run, err := r.client.Jobs.WaitGetRunJobTerminatedOrSkipped(ctx, runID, jobRunTimeout, func(run *jobs.Run) { + pageURL = run.RunPageUrl + prevState = logRunProgress(ctx, run, prevState) + }) + if err != nil { + // The run hit INTERNAL_ERROR, or we gave up on timeout or interrupt while it + // kept going; either way the run id is what makes the error actionable. + return nil, fmt.Errorf("waiting for job run %d: %w%s", runID, err, runPageLine(ctx, pageURL)) + } + // FAILED, TIMEDOUT, CANCELED, SUCCESS_WITH_FAILURES and SKIPPED all fail the + // deploy; the waiter already errored on INTERNAL_ERROR and on timeout. + if run.State.ResultState != jobs.RunResultStateSuccess { + return nil, r.runFailedError(ctx, run) + } + return makeJobRunRemote(ctx, run), nil +} + +// runFailedError reports why the run did not succeed, naming each failed task +// and the error it reported. +func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) error { + outcome := string(run.State.ResultState) + if outcome == "" { + // A skipped run has no result_state; report the lifecycle state. + outcome = string(run.State.LifeCycleState) + } + var msg strings.Builder + fmt.Fprintf(&msg, "job run %d did not succeed: %s", run.RunId, outcome) + if run.State.StateMessage != "" { + fmt.Fprintf(&msg, ": %s", run.State.StateMessage) + } + for _, task := range run.Tasks { + if taskFailed(task) { + fmt.Fprintf(&msg, "\ntask %q: %s", task.TaskKey, r.taskError(ctx, task)) + } + } + msg.WriteString(runPageLine(ctx, run.RunPageUrl)) + return errors.New(msg.String()) +} + +// taskFailed reports whether a task caused the run to fail rather than being a +// casualty of it. Tasks left SKIPPED or UPSTREAM_FAILED by an earlier failure +// add noise without naming the problem. +func taskFailed(task jobs.RunTask) bool { + // State is deprecated in favour of Status, so it may be absent. + if task.State == nil { + return false + } + return task.State.LifeCycleState == jobs.RunLifeCycleStateInternalError || + task.State.ResultState == jobs.RunResultStateFailed || + task.State.ResultState == jobs.RunResultStateTimedout +} + +// taskError returns the message the task reported, from the same place +// `bundle run` reads it. Only called for tasks that taskFailed accepted, so +// State is set. +func (r *ResourceJobRun) taskError(ctx context.Context, task jobs.RunTask) string { + var reported string + output, err := r.client.Jobs.GetRunOutput(ctx, jobs.GetRunOutputRequest{RunId: task.RunId}) + if err != nil { + log.Debugf(ctx, "could not read output of task %s: %v", task.TaskKey, err) + } else { + reported = output.Error + } + // Not every task type reports an error through GetRunOutput, so fall back to + // what the run itself says about the task. + return cmp.Or(reported, task.State.StateMessage, string(task.State.ResultState), string(task.State.LifeCycleState)) +} + +// runPageLine returns a line linking the run page, or an empty string when the +// URL is unknown. +func runPageLine(ctx context.Context, rawURL string) string { + if rawURL == "" { + return "" + } + return "\nrun page: " + workspaceurls.JobRunPageURL(ctx, rawURL) +} + +// logRunProgress mirrors `bundle run`'s monitor: the run page URL once, then +// each state change. It returns the state to remember for the next poll. +func logRunProgress(ctx context.Context, run *jobs.Run, prev *jobs.RunState) *jobs.RunState { + if run.State == nil { + return prev + } + if prev != nil && + prev.LifeCycleState == run.State.LifeCycleState && + prev.ResultState == run.State.ResultState { + return prev + } + if prev == nil && run.RunPageUrl != "" { + logRunLine(ctx, run.RunId, "Run URL: "+workspaceurls.JobRunPageURL(ctx, run.RunPageUrl)) + } + logRunLine(ctx, run.RunId, (&progress.JobProgressEvent{ + Timestamp: time.Now(), + JobId: run.JobId, + RunId: run.RunId, + RunName: run.RunName, + State: *run.State, + }).String()) + return run.State +} + +// logRunLine reports one line about a run to the user and the log. Resources +// deploy concurrently onto one stream, so the user-facing copy names the run it +// describes; the log already carries the resource key via log.WithPrefix. +func logRunLine(ctx context.Context, runID int64, msg string) { + log.Info(ctx, msg) + if cmdio.HasIO(ctx) { + cmdio.LogString(ctx, fmt.Sprintf("job run %d: %s", runID, msg)) + } +} + // DoUpdate is intentionally not implemented: a run can't be modified in place, // so any change recreates it (delete + a fresh RunNow). // DoDelete deletes the run via jobs/runs/delete, on both destroy and the -// recreate path. The API rejects a still-active run; this milestone doesn't -// await completion, so that error surfaces to the user. +// recreate path. The API rejects a still-active run, which WaitAfterCreate +// leaves terminal; that error surfaces for a run whose wait was interrupted. func (r *ResourceJobRun) DoDelete(ctx context.Context, id string, _ *JobRunState) error { runID, err := parseRunID(id) if err != nil { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go new file mode 100644 index 00000000000..0177b077f2a --- /dev/null +++ b/bundle/direct/dresources/job_run_test.go @@ -0,0 +1,173 @@ +package dresources + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// jobRunClientFor returns a client talking to server. Call it after the test +// registers its own handlers: first registration wins, so the defaults added here +// only fill the gaps. +func jobRunClientFor(t *testing.T, server *testserver.Server) *databricks.WorkspaceClient { + t.Helper() + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + return client +} + +// jobRunServer returns a test server whose runs/get handler is the given one, +// so a wait can be exercised without a real run. +func jobRunServer(t *testing.T, getRun testserver.HandlerFunc) *databricks.WorkspaceClient { + t.Helper() + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", getRun) + return jobRunClientFor(t, server) +} + +// jobRunClient returns a client whose GetRun always reports the given run state. +func jobRunClient(t *testing.T, state *jobs.RunState) *databricks.WorkspaceClient { + t.Helper() + return jobRunServer(t, func(req testserver.Request) any { + return jobs.Run{RunId: 123, JobId: 456, State: state} + }) +} + +// waitForTestRun drives the framework hook, so it covers parsing the id the +// framework hands back from DoCreate along with the wait itself. +func waitForTestRun(t *testing.T, ctx context.Context, client *databricks.WorkspaceClient) (*JobRunRemote, error) { + t.Helper() + r := (&ResourceJobRun{}).New(client) + return r.WaitAfterCreate(ctx, "123", &JobRunState{}) +} + +func TestJobRunWaitSucceeds(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }) + + remote, err := waitForTestRun(t, t.Context(), client) + + require.NoError(t, err) + require.NotNil(t, remote.State) + assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) +} + +func TestJobRunWaitFailsOnFailedResult(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateFailed, + StateMessage: "task failed", + }) + + _, err := waitForTestRun(t, t.Context(), client) + + // Only SUCCESS completes the deploy; a FAILED result fails it. + require.ErrorContains(t, err, "did not succeed: FAILED: task failed") +} + +func TestJobRunWaitReportsFailedTask(t *testing.T) { + failed := &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateFailed, + } + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { + return jobs.Run{ + RunId: 123, + JobId: 456, + State: failed, + Tasks: []jobs.RunTask{ + {TaskKey: "ok", RunId: 998, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }}, + {TaskKey: "main", RunId: 999, State: failed}, + }, + } + }) + server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { + return jobs.RunOutput{Error: "notebook not found"} + }) + + _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) + + // The error names the failing task and the message it reported, and leaves out + // the tasks that did not fail. + require.ErrorContains(t, err, `task "main": notebook not found`) + assert.NotContains(t, err.Error(), `task "ok"`) +} + +func TestJobRunWaitFailsOnSkipped(t *testing.T) { + // A skipped run has no result_state, so the lifecycle state is reported. + client := jobRunClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateSkipped, + }) + + _, err := waitForTestRun(t, t.Context(), client) + + require.ErrorContains(t, err, "did not succeed: SKIPPED") +} + +func TestJobRunWaitFailsOnInternalError(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateInternalError, + }) + + _, err := waitForTestRun(t, t.Context(), client) + + // The SDK waiter errors on INTERNAL_ERROR, ahead of the result check, so the + // wrapping is all that names the run. + require.ErrorContains(t, err, "waiting for job run 123") + require.ErrorContains(t, err, "INTERNAL_ERROR") +} + +func TestJobRunWaitAbandonedNamesTheRun(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) + + ctx, cancel := context.WithTimeout(t.Context(), time.Millisecond) + defer cancel() + + _, err := waitForTestRun(t, ctx, client) + + // Giving up on the wait does not stop the run, so the error has to name it. + require.ErrorContains(t, err, "waiting for job run 123") +} + +// Reporting RUNNING for the first two polls exercises the poll loop; the other +// tests stub an already-terminal state. +func TestJobRunWaitPollsUntilTerminal(t *testing.T) { + var gets atomic.Int32 + client := jobRunServer(t, func(req testserver.Request) any { + if gets.Add(1) <= 2 { + return jobs.Run{RunId: 123, JobId: 456, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateRunning, + }} + } + return jobs.Run{RunId: 123, JobId: 456, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }} + }) + + remote, err := waitForTestRun(t, t.Context(), client) + require.NoError(t, err) + + // SUCCESS is only reachable by polling past the RUNNING reads. + require.NotNil(t, remote.State) + assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) + assert.GreaterOrEqual(t, gets.Load(), int32(2), "expected the wait to poll more than once") +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 10832fe04a6..5d47c44e289 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -968,6 +968,8 @@ resources: "job_runs": "description": |- The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment. + + The deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs..state.result_state}`. A run that did not succeed is still recorded, so an unchanged configuration is not run again. "$fields": "lifecycle": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4c78bd7c384..bf6039da58d 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3228,7 +3228,7 @@ "markdownDescription": "The instance pool definitions for the bundle, where each key is the name of the instance pool. See [instance_pools](https://docs.databricks.com/dev-tools/bundles/resources.html#instance_pools)." }, "job_runs": { - "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.", + "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.\n\nThe deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs.\u003cname\u003e.state.result_state}`. A run that did not succeed is still recorded, so an unchanged configuration is not run again.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.JobRun" }, "jobs": { From b3ba749c3ceaa1fbc2d23f2ee52fc86b4da3f6ed Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 12:02:50 +0000 Subject: [PATCH 04/14] job_runs: report run progress through a tracker shared with bundle run The wait reimplemented what `bundle run`'s monitor already does: report the run page URL once, then each state change. Both now go through progress.JobStateTracker, which decides what a poll is worth reporting; the two callers keep their own sinks, since a concurrent deploy reports plain prefixed lines where `bundle run` reports progress events. The failure the deploy reports no longer repeats the run id that the framework's wrapper already carries; what the wait adds is the link to a run that outlives it. --- .../bundles/job-runs-wait-for-completion.md | 2 +- .../resources/job_runs/failed_run/output.txt | 2 +- bundle/direct/dresources/job_run.go | 59 +++++++------------ bundle/direct/dresources/job_run_test.go | 23 +++++--- bundle/run/job.go | 27 ++------- bundle/run/progress/job.go | 27 +++++++++ bundle/run/progress/job_test.go | 33 +++++++++++ 7 files changed, 103 insertions(+), 70 deletions(-) diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md index e78497e24cb..4520d1d9180 100644 --- a/.nextchanges/bundles/job-runs-wait-for-completion.md +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -1 +1 @@ -direct: the experimental `job_runs` resource now waits for the triggered run to finish, and fails the deploy if it does not succeed, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL and each state change while it waits, and names the failed task and the message it reported when the run does not succeed. +direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). A run that does not succeed fails the deploy, naming the failed task and the message it reported; while waiting, the deploy reports the run page URL and each state change. diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index e8a5098a0df..78b95c98da1 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -6,7 +6,7 @@ Deploying resources... job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED FAILED task main failed -Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: job run [MY_RUN_ID] did not succeed: FAILED: task main failed +Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: task main failed task "main": spark python task execution failed: exit status 1 intentional failure diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index fd783dfd2ea..16441ef3e4a 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -152,32 +152,27 @@ func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (str return strconv.FormatInt(wait.RunId, 10), nil, nil } -// WaitAfterCreate blocks until the triggered run finishes, so a resource that -// references this run's output (e.g. state.result_state) is created only once the -// run has produced it. Only a SUCCESS lets the deploy continue. +// WaitAfterCreate blocks until the run finishes, so a resource referencing its +// output (e.g. state.result_state) sees a settled run. Only SUCCESS lets the +// deploy continue. func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobRunState) (*JobRunRemote, error) { runID, err := parseRunID(id) if err != nil { return nil, err } - return r.waitForRun(ctx, runID) -} -// waitForRun blocks until the run reaches a terminal state and returns its -// remote view; only SUCCESS returns a nil error. -func (r *ResourceJobRun) waitForRun(ctx context.Context, runID int64) (*JobRunRemote, error) { // A run can take hours, so report progress like `bundle run` does. pageURL // outlives the callback so an abandoned wait can still link the run. - var prevState *jobs.RunState + var tracker progress.JobStateTracker var pageURL string run, err := r.client.Jobs.WaitGetRunJobTerminatedOrSkipped(ctx, runID, jobRunTimeout, func(run *jobs.Run) { pageURL = run.RunPageUrl - prevState = logRunProgress(ctx, run, prevState) + logRunProgress(ctx, run, &tracker) }) if err != nil { - // The run hit INTERNAL_ERROR, or we gave up on timeout or interrupt while it - // kept going; either way the run id is what makes the error actionable. - return nil, fmt.Errorf("waiting for job run %d: %w%s", runID, err, runPageLine(ctx, pageURL)) + // The wait can end with the run still going (timeout, interrupt), so link + // the run page; the framework's wrapper carries the id. + return nil, fmt.Errorf("%w%s", err, runPageLine(ctx, pageURL)) } // FAILED, TIMEDOUT, CANCELED, SUCCESS_WITH_FAILURES and SKIPPED all fail the // deploy; the waiter already errored on INTERNAL_ERROR and on timeout. @@ -196,7 +191,8 @@ func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) erro outcome = string(run.State.LifeCycleState) } var msg strings.Builder - fmt.Fprintf(&msg, "job run %d did not succeed: %s", run.RunId, outcome) + // The framework already prefixes the resource key and the run id. + fmt.Fprintf(&msg, "run did not succeed: %s", outcome) if run.State.StateMessage != "" { fmt.Fprintf(&msg, ": %s", run.State.StateMessage) } @@ -210,8 +206,7 @@ func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) erro } // taskFailed reports whether a task caused the run to fail rather than being a -// casualty of it. Tasks left SKIPPED or UPSTREAM_FAILED by an earlier failure -// add noise without naming the problem. +// casualty of it: tasks left SKIPPED or UPSTREAM_FAILED add noise. func taskFailed(task jobs.RunTask) bool { // State is deprecated in favour of Status, so it may be absent. if task.State == nil { @@ -222,9 +217,8 @@ func taskFailed(task jobs.RunTask) bool { task.State.ResultState == jobs.RunResultStateTimedout } -// taskError returns the message the task reported, from the same place -// `bundle run` reads it. Only called for tasks that taskFailed accepted, so -// State is set. +// taskError returns the message the task reported, from the same place `bundle +// run` reads it. Only called for tasks taskFailed accepted, so State is set. func (r *ResourceJobRun) taskError(ctx context.Context, task jobs.RunTask) string { var reported string output, err := r.client.Jobs.GetRunOutput(ctx, jobs.GetRunOutputRequest{RunId: task.RunId}) @@ -247,28 +241,17 @@ func runPageLine(ctx context.Context, rawURL string) string { return "\nrun page: " + workspaceurls.JobRunPageURL(ctx, rawURL) } -// logRunProgress mirrors `bundle run`'s monitor: the run page URL once, then -// each state change. It returns the state to remember for the next poll. -func logRunProgress(ctx context.Context, run *jobs.Run, prev *jobs.RunState) *jobs.RunState { - if run.State == nil { - return prev - } - if prev != nil && - prev.LifeCycleState == run.State.LifeCycleState && - prev.ResultState == run.State.ResultState { - return prev +// logRunProgress reports what `bundle run` reports: the run page URL once, then +// each state change. +func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobStateTracker) { + event, first := tracker.Poll(run) + if event == nil { + return } - if prev == nil && run.RunPageUrl != "" { + if first && run.RunPageUrl != "" { logRunLine(ctx, run.RunId, "Run URL: "+workspaceurls.JobRunPageURL(ctx, run.RunPageUrl)) } - logRunLine(ctx, run.RunId, (&progress.JobProgressEvent{ - Timestamp: time.Now(), - JobId: run.JobId, - RunId: run.RunId, - RunName: run.RunName, - State: *run.State, - }).String()) - return run.State + logRunLine(ctx, run.RunId, event.String()) } // logRunLine reports one line about a run to the user and the log. Resources diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 0177b077f2a..e9c29366136 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -37,11 +37,18 @@ func jobRunServer(t *testing.T, getRun testserver.HandlerFunc) *databricks.Works return jobRunClientFor(t, server) } +// The Jobs API reports the run page in the legacy fragment form; errors and +// progress lines carry the path form it converts to. +const ( + testRunPageURL = "https://myworkspace.databricks.test/?o=900800700600#job/456/run/123" + testRunPageLink = "run page: https://myworkspace.databricks.test/jobs/456/runs/123?o=900800700600" +) + // jobRunClient returns a client whose GetRun always reports the given run state. func jobRunClient(t *testing.T, state *jobs.RunState) *databricks.WorkspaceClient { t.Helper() return jobRunServer(t, func(req testserver.Request) any { - return jobs.Run{RunId: 123, JobId: 456, State: state} + return jobs.Run{RunId: 123, JobId: 456, State: state, RunPageUrl: testRunPageURL} }) } @@ -129,13 +136,12 @@ func TestJobRunWaitFailsOnInternalError(t *testing.T) { _, err := waitForTestRun(t, t.Context(), client) - // The SDK waiter errors on INTERNAL_ERROR, ahead of the result check, so the - // wrapping is all that names the run. - require.ErrorContains(t, err, "waiting for job run 123") + // The SDK waiter errors on INTERNAL_ERROR, ahead of the result check. require.ErrorContains(t, err, "INTERNAL_ERROR") + require.ErrorContains(t, err, testRunPageLink) } -func TestJobRunWaitAbandonedNamesTheRun(t *testing.T) { +func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) ctx, cancel := context.WithTimeout(t.Context(), time.Millisecond) @@ -143,8 +149,9 @@ func TestJobRunWaitAbandonedNamesTheRun(t *testing.T) { _, err := waitForTestRun(t, ctx, client) - // Giving up on the wait does not stop the run, so the error has to name it. - require.ErrorContains(t, err, "waiting for job run 123") + // Giving up on the wait does not stop the run, so the error links to it. + require.Error(t, err) + require.ErrorContains(t, err, testRunPageLink) } // Reporting RUNNING for the first two polls exercises the poll loop; the other @@ -169,5 +176,5 @@ func TestJobRunWaitPollsUntilTerminal(t *testing.T) { // SUCCESS is only reachable by polling past the RUNNING reads. require.NotNil(t, remote.State) assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) - assert.GreaterOrEqual(t, gets.Load(), int32(2), "expected the wait to poll more than once") + assert.Equal(t, int32(3), gets.Load(), "expected the wait to poll past both RUNNING reads") } diff --git a/bundle/run/job.go b/bundle/run/job.go index 2001fffda95..b61c98e4479 100644 --- a/bundle/run/job.go +++ b/bundle/run/job.go @@ -84,42 +84,25 @@ func (r *jobRunner) logFailedTasks(ctx context.Context, runId int64) { // jobRunMonitor tracks state for a single job run and provides callbacks // for monitoring progress. type jobRunMonitor struct { - ctx context.Context - prevState *jobs.RunState + ctx context.Context + tracker progress.JobStateTracker } // onProgress is the single callback that handles all state tracking and logging. func (m *jobRunMonitor) onProgress(info *jobs.Run) { - state := info.State - if state == nil { + event, first := m.tracker.Poll(info) + if event == nil { return } // First time we see this run. - if m.prevState == nil { + if first { runURL := workspaceurls.JobRunPageURL(m.ctx, info.RunPageUrl) log.Infof(m.ctx, "Run available at %s", runURL) cmdio.Log(m.ctx, progress.NewJobRunUrlEvent(runURL)) } - // No state change: do not log. - if m.prevState != nil && - m.prevState.LifeCycleState == state.LifeCycleState && - m.prevState.ResultState == state.ResultState { - return - } - - // Capture current state as previous state for next call. - m.prevState = state - // Log progress event both to the terminal (in place or append), and to the logger. - event := &progress.JobProgressEvent{ - Timestamp: time.Now(), - JobId: info.JobId, - RunId: info.RunId, - RunName: info.RunName, - State: *info.State, - } cmdio.Log(m.ctx, event) log.Info(m.ctx, event.String()) } diff --git a/bundle/run/progress/job.go b/bundle/run/progress/job.go index 6deee451ad6..11183681743 100644 --- a/bundle/run/progress/job.go +++ b/bundle/run/progress/job.go @@ -16,6 +16,33 @@ type JobProgressEvent struct { State jobs.RunState `json:"state"` } +// JobStateTracker turns the polls of a job run into one event per state change, +// for callers that report a run's progress as it goes. +type JobStateTracker struct { + prev *jobs.RunState +} + +// Poll returns the event to report for this poll of run, or nil when the state +// has not changed since the last one. first is true for the state a run is seen +// in initially, where callers also report the run page URL. +func (t *JobStateTracker) Poll(run *jobs.Run) (event *JobProgressEvent, first bool) { + if run.State == nil { + return nil, false + } + first = t.prev == nil + if !first && t.prev.LifeCycleState == run.State.LifeCycleState && t.prev.ResultState == run.State.ResultState { + return nil, false + } + t.prev = run.State + return &JobProgressEvent{ + Timestamp: time.Now(), + JobId: run.JobId, + RunId: run.RunId, + RunName: run.RunName, + State: *run.State, + }, first +} + func (event *JobProgressEvent) String() string { result := strings.Builder{} result.WriteString(event.Timestamp.Format("2006-01-02 15:04:05") + " ") diff --git a/bundle/run/progress/job_test.go b/bundle/run/progress/job_test.go index 31196520305..521c54f357a 100644 --- a/bundle/run/progress/job_test.go +++ b/bundle/run/progress/job_test.go @@ -22,3 +22,36 @@ func TestJobProgressEventString(t *testing.T) { } assert.Equal(t, "-0001-11-30 00:00:00 \"run_name\" TERMINATED SUCCESS state_message", event.String()) } + +func TestJobStateTrackerPoll(t *testing.T) { + running := &jobs.Run{RunId: 456, State: &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}} + terminated := &jobs.Run{RunId: 456, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }} + + var tracker JobStateTracker + + event, first := tracker.Poll(running) + assert.True(t, first) + assert.Equal(t, jobs.RunLifeCycleStateRunning, event.State.LifeCycleState) + + // The same state again is not worth reporting, and is no longer the first one. + event, first = tracker.Poll(running) + assert.Nil(t, event) + assert.False(t, first) + + event, first = tracker.Poll(terminated) + assert.False(t, first) + assert.Equal(t, jobs.RunResultStateSuccess, event.State.ResultState) +} + +func TestJobStateTrackerPollWithoutState(t *testing.T) { + var tracker JobStateTracker + + event, first := tracker.Poll(&jobs.Run{RunId: 456}) + + // A run reported without a state has no progress to report. + assert.Nil(t, event) + assert.False(t, first) +} From c3216b8647f2c1034aba673a09ca8efc42ebede4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 12:03:07 +0000 Subject: [PATCH 05/14] acc: run a job_run against a real workspace Excluding job_run.yml.tmpl from cloud left job_runs with no cloud coverage at all: every test under resources/job_runs inherits Cloud=false. The behaviour this milestone adds is the one that depends most on the real Jobs API, so wait_cloud triggers a run for real and reads the outcome back out of the downstream job, the way the vector search exclusion points at a dedicated test. Serverless keeps the run to about a minute, and the deploy's progress stream stays out of the golden: a real run reports an unpredictable number of intermediate states. --- acceptance/bundle/invariant/test.toml | 6 ++- .../job_runs/wait_cloud/databricks.yml.tmpl | 39 +++++++++++++++++++ .../resources/job_runs/wait_cloud/hello.py | 1 + .../job_runs/wait_cloud/out.test.toml | 4 ++ .../resources/job_runs/wait_cloud/output.txt | 22 +++++++++++ .../resources/job_runs/wait_cloud/script | 20 ++++++++++ .../resources/job_runs/wait_cloud/test.toml | 20 ++++++++++ 7 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/hello.py create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/output.txt create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/script create mode 100644 acceptance/bundle/resources/job_runs/wait_cloud/test.toml diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index ba471f291e8..1983b647e8c 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -84,8 +84,10 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] # Deploying a job_run waits for the run to succeed, and a real workspace reports -# a run of condition tasks alone as SKIPPED. A task that does succeed would add a -# cluster run to every variant of a suite that asserts plan and state invariants. +# a run of condition tasks alone as SKIPPED. A task that does succeed would run in +# every variant of a suite that asserts plan and state invariants, so +# resources/job_runs/wait_cloud covers the real run on cloud instead. Still +# exercised locally here. no_job_run_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run.yml.tmpl"] # Postgres resources only work on AWS diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl new file mode 100644 index 00000000000..c72555f11ae --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl @@ -0,0 +1,39 @@ +bundle: + name: job-runs-wait-cloud + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + jobs: + my_job: + name: test-job-$UNIQUE_NAME + tasks: + # Serverless keeps the run to about a minute; a job cluster would take + # several. + - task_key: main + spark_python_task: + python_file: ./hello.py + environment_key: default + + environments: + - environment_key: default + spec: + environment_version: "2" + + # Reads the run's outcome, so the tag it is created with shows whether the + # deploy waited for the run before creating resources that depend on it. + downstream_job: + name: test-downstream-job-$UNIQUE_NAME + tags: + run_result: ${resources.job_runs.my_run.state.result_state} + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/hello.py b/acceptance/bundle/resources/job_runs/wait_cloud/hello.py new file mode 100644 index 00000000000..93e0cef4a92 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/hello.py @@ -0,0 +1 @@ +print("hello from a job_run") diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml b/acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml new file mode 100644 index 00000000000..fe4076cdf9b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt new file mode 100644 index 00000000000..673337551c3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt @@ -0,0 +1,22 @@ + +=== the deploy waits for the run to finish +>>> grep -c Run URL: deploy.log +1 + +>>> grep -o TERMINATED SUCCESS deploy.log +TERMINATED SUCCESS + +=== the downstream job was created with the run's result_state +>>> [CLI] jobs get [DOWNSTREAM_JOB_ID] -o json +SUCCESS + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.downstream_job + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/script b/acceptance/bundle/resources/job_runs/wait_cloud/script new file mode 100644 index 00000000000..937e9e18bd6 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# A real run reports a variable number of intermediate states (PENDING while +# serverless compute starts), so keep the deploy's progress stream out of the +# golden and assert on the terminal state it waited for. +title "the deploy waits for the run to finish" +if ! $CLI bundle deploy > deploy.log 2>&1; then + cat deploy.log +fi +trace grep -c "Run URL: " deploy.log +trace grep -o "TERMINATED SUCCESS" deploy.log + +title "the downstream job was created with the run's result_state" +downstream_id=$(read_id.py downstream_job) +trace $CLI jobs get $downstream_id -o json | jq -r ".settings.tags.run_result" diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml new file mode 100644 index 00000000000..63977f7d2e9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml @@ -0,0 +1,20 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The cloud counterpart of wait_output: it runs the job for real, which is what +# invariant/configs/job_run.yml.tmpl no longer does on cloud. Serverless needs +# Unity Catalog. +Cloud = true +RequiresUnityCatalog = true + +# A real workspace is not proxied, so there are no recorded requests to assert +# on; this test reads the deployed job back instead. +RecordRequests = false + +Ignore = [ + "databricks.yml", + "databricks.yml.tmpl", + "hello.py", + "deploy.log", +] From a57b619383ef390ae9a4b5d5392a3933a3bee8c2 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 12:24:09 +0000 Subject: [PATCH 06/14] acc: check the resolved job parameters are not drift on cloud The ignore_remote_changes for job_parameters assumes GetRun reports every parameter the job defines, not just the ones the run overrode. Assert that on the run wait_cloud already triggers by overriding one of two parameters and planning after the deploy. --- .../resources/job_runs/wait_cloud/databricks.yml.tmpl | 10 ++++++++++ .../bundle/resources/job_runs/wait_cloud/output.txt | 4 ++++ acceptance/bundle/resources/job_runs/wait_cloud/script | 3 +++ 3 files changed, 17 insertions(+) diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl index c72555f11ae..3457f638246 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl +++ b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl @@ -8,6 +8,11 @@ resources: jobs: my_job: name: test-job-$UNIQUE_NAME + parameters: + - name: env + default: dev + - name: region + default: us tasks: # Serverless keeps the run to about a minute; a job cluster would take # several. @@ -37,3 +42,8 @@ resources: job_runs: my_run: job_id: ${resources.jobs.my_job.id} + # Override one of the job's two parameters. GetRun reports the full resolved + # set, including the region default this run does not override, so a real + # workspace is where the ignore_remote_changes for job_parameters can regress. + job_parameters: + env: prod diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt index 673337551c3..92b6ad11f0a 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt +++ b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt @@ -10,6 +10,10 @@ TERMINATED SUCCESS >>> [CLI] jobs get [DOWNSTREAM_JOB_ID] -o json SUCCESS +=== the parameters the run resolved are not drift +>>> [CLI] bundle plan -o json +skip + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/script b/acceptance/bundle/resources/job_runs/wait_cloud/script index 937e9e18bd6..d047e5294dc 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/script +++ b/acceptance/bundle/resources/job_runs/wait_cloud/script @@ -18,3 +18,6 @@ trace grep -o "TERMINATED SUCCESS" deploy.log title "the downstream job was created with the run's result_state" downstream_id=$(read_id.py downstream_job) trace $CLI jobs get $downstream_id -o json | jq -r ".settings.tags.run_result" + +title "the parameters the run resolved are not drift" +trace $CLI bundle plan -o json | jq -r '.plan["resources.job_runs.my_run"].action' From f5453350d927db20d392413c4fc4f971dd1f67d6 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 12:32:02 +0000 Subject: [PATCH 07/14] job_runs: tighten the comments added by this branch Drop the comments that only restate the code, the duplicated note about the framework prefixing the run id, and the filler in the ones that carry a reason. --- acceptance/bundle/invariant/test.toml | 9 ++++--- .../job_runs/failed_run/databricks.yml | 4 ++-- .../job_runs/wait_cloud/databricks.yml.tmpl | 6 ++--- bundle/direct/dresources/all_test.go | 2 +- bundle/direct/dresources/job_run.go | 24 +++++++------------ bundle/run/progress/job.go | 9 ++++--- 6 files changed, 23 insertions(+), 31 deletions(-) diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 1983b647e8c..3135fd958c3 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -83,11 +83,10 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] # so this config is local-only (the mock server stores it verbatim). no_run_job_ref_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run_job_ref.yml.tmpl"] -# Deploying a job_run waits for the run to succeed, and a real workspace reports -# a run of condition tasks alone as SKIPPED. A task that does succeed would run in -# every variant of a suite that asserts plan and state invariants, so -# resources/job_runs/wait_cloud covers the real run on cloud instead. Still -# exercised locally here. +# Deploying a job_run waits for the run to succeed, and a real workspace reports a +# run of condition tasks alone as SKIPPED. A task that does succeed would run in +# every variant of this suite, so resources/job_runs/wait_cloud covers the real run +# on cloud instead. Still exercised locally here. no_job_run_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=job_run.yml.tmpl"] # Postgres resources only work on AWS diff --git a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml index 8a538daf853..756474a90e2 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/databricks.yml +++ b/acceptance/bundle/resources/job_runs/failed_run/databricks.yml @@ -6,8 +6,8 @@ resources: my_job: name: my-job tasks: - # The test server runs this locally; the script exits non-zero, which - # fails the task and with it the run. + # The test server runs this locally: it exits non-zero, failing the task + # and with it the run. - task_key: main spark_python_task: python_file: ./fail.py diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl index 3457f638246..c2e8d3fd919 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl +++ b/acceptance/bundle/resources/job_runs/wait_cloud/databricks.yml.tmpl @@ -42,8 +42,8 @@ resources: job_runs: my_run: job_id: ${resources.jobs.my_job.id} - # Override one of the job's two parameters. GetRun reports the full resolved - # set, including the region default this run does not override, so a real - # workspace is where the ignore_remote_changes for job_parameters can regress. + # Override one of the job's two parameters: a real GetRun reports the full + # resolved set, including the region default, which ignore_remote_changes + # has to absorb to avoid perpetual drift. job_parameters: env: prod diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 89d690dfead..f00ec21ed93 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1004,7 +1004,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) if remoteStateFromWaitCreate != nil { // WaitAfterCreate returns the settled state; the read right after DoCreate - // may still be non-terminal, so compare against a fresh read, not that one. + // may still be non-terminal, so compare against a fresh read. remotePostWaitCreate, err := adapter.DoRead(ctx, createdID) require.NoError(t, err) require.Equal(t, remotePostWaitCreate, remoteStateFromWaitCreate) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 16441ef3e4a..362a8c4cf64 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -19,8 +19,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/jobs" ) -// jobRunTimeout bounds the wait for a run to finish, matching `bundle run` -// (jobRunTimeout in bundle/run/job.go). +// jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour // JobRunState is what we persist for a triggered run: the RunNow request. @@ -44,8 +43,7 @@ type JobRunRemote struct { RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` - // Normalized to the path form that also resolves for non-admins; see - // workspaceurls.JobRunPageURL. + // Normalized by workspaceurls.JobRunPageURL so it resolves for non-admins too. RunPageUrl string `json:"run_page_url,omitempty"` RunType jobs.RunType `json:"run_type,omitempty"` } @@ -171,7 +169,7 @@ func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobR }) if err != nil { // The wait can end with the run still going (timeout, interrupt), so link - // the run page; the framework's wrapper carries the id. + // the run page. return nil, fmt.Errorf("%w%s", err, runPageLine(ctx, pageURL)) } // FAILED, TIMEDOUT, CANCELED, SUCCESS_WITH_FAILURES and SKIPPED all fail the @@ -218,7 +216,7 @@ func taskFailed(task jobs.RunTask) bool { } // taskError returns the message the task reported, from the same place `bundle -// run` reads it. Only called for tasks taskFailed accepted, so State is set. +// run` reads it. Only reached for a task taskFailed accepted, so State is set. func (r *ResourceJobRun) taskError(ctx context.Context, task jobs.RunTask) string { var reported string output, err := r.client.Jobs.GetRunOutput(ctx, jobs.GetRunOutputRequest{RunId: task.RunId}) @@ -232,8 +230,6 @@ func (r *ResourceJobRun) taskError(ctx context.Context, task jobs.RunTask) strin return cmp.Or(reported, task.State.StateMessage, string(task.State.ResultState), string(task.State.LifeCycleState)) } -// runPageLine returns a line linking the run page, or an empty string when the -// URL is unknown. func runPageLine(ctx context.Context, rawURL string) string { if rawURL == "" { return "" @@ -241,8 +237,7 @@ func runPageLine(ctx context.Context, rawURL string) string { return "\nrun page: " + workspaceurls.JobRunPageURL(ctx, rawURL) } -// logRunProgress reports what `bundle run` reports: the run page URL once, then -// each state change. +// logRunProgress mirrors `bundle run`: the run page URL once, then each state change. func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobStateTracker) { event, first := tracker.Poll(run) if event == nil { @@ -254,9 +249,8 @@ func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobSta logRunLine(ctx, run.RunId, event.String()) } -// logRunLine reports one line about a run to the user and the log. Resources -// deploy concurrently onto one stream, so the user-facing copy names the run it -// describes; the log already carries the resource key via log.WithPrefix. +// logRunLine names the run in the user-facing copy, since resources deploy +// concurrently onto one stream; the log carries the resource key already. func logRunLine(ctx context.Context, runID int64, msg string) { log.Info(ctx, msg) if cmdio.HasIO(ctx) { @@ -268,8 +262,8 @@ func logRunLine(ctx context.Context, runID int64, msg string) { // so any change recreates it (delete + a fresh RunNow). // DoDelete deletes the run via jobs/runs/delete, on both destroy and the -// recreate path. The API rejects a still-active run, which WaitAfterCreate -// leaves terminal; that error surfaces for a run whose wait was interrupted. +// recreate path. The API rejects a still-active run; WaitAfterCreate leaves it +// terminal, so that error only surfaces when a wait was interrupted. func (r *ResourceJobRun) DoDelete(ctx context.Context, id string, _ *JobRunState) error { runID, err := parseRunID(id) if err != nil { diff --git a/bundle/run/progress/job.go b/bundle/run/progress/job.go index 11183681743..fb29a7fadd6 100644 --- a/bundle/run/progress/job.go +++ b/bundle/run/progress/job.go @@ -16,15 +16,14 @@ type JobProgressEvent struct { State jobs.RunState `json:"state"` } -// JobStateTracker turns the polls of a job run into one event per state change, -// for callers that report a run's progress as it goes. +// JobStateTracker turns the polls of a job run into one event per state change. type JobStateTracker struct { prev *jobs.RunState } -// Poll returns the event to report for this poll of run, or nil when the state -// has not changed since the last one. first is true for the state a run is seen -// in initially, where callers also report the run page URL. +// Poll returns the event to report for this poll, or nil when the state has not +// changed. first is true for the state the run is seen in initially, where +// callers also report the run page URL. func (t *JobStateTracker) Poll(run *jobs.Run) (event *JobProgressEvent, first bool) { if run.State == nil { return nil, false From e8d0207fd18dabded64d9ede7e8d42dbbe1b3f38 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 12:58:48 +0000 Subject: [PATCH 08/14] workspaceurls: drop the context from JobRunPageURL The context only reached two debug lines about a URL that did not parse, and it had to be threaded through makeJobRunRemote, which is otherwise a pure mapping of the GetRun response. --- bundle/direct/dresources/job_run.go | 33 ++++++++++++++--------------- bundle/run/job.go | 4 ++-- libs/workspaceurls/urls.go | 7 +----- libs/workspaceurls/urls_test.go | 3 +-- 4 files changed, 20 insertions(+), 27 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 362a8c4cf64..e11321870f4 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -40,12 +40,11 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { type JobRunRemote struct { jobs.RunNow - RunId int64 `json:"run_id,omitempty"` - RunName string `json:"run_name,omitempty"` - State *jobs.RunState `json:"state,omitempty"` - // Normalized by workspaceurls.JobRunPageURL so it resolves for non-admins too. - RunPageUrl string `json:"run_page_url,omitempty"` - RunType jobs.RunType `json:"run_type,omitempty"` + RunId int64 `json:"run_id,omitempty"` + RunName string `json:"run_name,omitempty"` + State *jobs.RunState `json:"state,omitempty"` + RunPageUrl string `json:"run_page_url,omitempty"` + RunType jobs.RunType `json:"run_type,omitempty"` } // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise @@ -77,7 +76,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { // makeJobRunRemote maps the GetRun response into the RunNow-shaped remote: GET // nests the params under overriding_parameters and returns job_parameters as a // list, so both are flattened back into RunNow. -func makeJobRunRemote(ctx context.Context, run *jobs.Run) *JobRunRemote { +func makeJobRunRemote(run *jobs.Run) *JobRunRemote { var overriding jobs.RunParameters if run.OverridingParameters != nil { overriding = *run.OverridingParameters @@ -112,7 +111,7 @@ func makeJobRunRemote(ctx context.Context, run *jobs.Run) *JobRunRemote { RunId: run.RunId, RunName: run.RunName, State: run.State, - RunPageUrl: workspaceurls.JobRunPageURL(ctx, run.RunPageUrl), + RunPageUrl: workspaceurls.JobRunPageURL(run.RunPageUrl), RunType: run.RunType, } } @@ -132,7 +131,7 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, if err != nil { return nil, err } - return makeJobRunRemote(ctx, run), nil + return makeJobRunRemote(run), nil } // RemapState extracts the embedded RunNow as the state used for diffing. @@ -170,14 +169,14 @@ func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobR if err != nil { // The wait can end with the run still going (timeout, interrupt), so link // the run page. - return nil, fmt.Errorf("%w%s", err, runPageLine(ctx, pageURL)) + return nil, fmt.Errorf("%w%s", err, runPageLine(pageURL)) } - // FAILED, TIMEDOUT, CANCELED, SUCCESS_WITH_FAILURES and SKIPPED all fail the - // deploy; the waiter already errored on INTERNAL_ERROR and on timeout. + // The waiter already errored on INTERNAL_ERROR and on its timeout, so only the + // terminal results are left to reject. if run.State.ResultState != jobs.RunResultStateSuccess { return nil, r.runFailedError(ctx, run) } - return makeJobRunRemote(ctx, run), nil + return makeJobRunRemote(run), nil } // runFailedError reports why the run did not succeed, naming each failed task @@ -199,7 +198,7 @@ func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) erro fmt.Fprintf(&msg, "\ntask %q: %s", task.TaskKey, r.taskError(ctx, task)) } } - msg.WriteString(runPageLine(ctx, run.RunPageUrl)) + msg.WriteString(runPageLine(run.RunPageUrl)) return errors.New(msg.String()) } @@ -230,11 +229,11 @@ func (r *ResourceJobRun) taskError(ctx context.Context, task jobs.RunTask) strin return cmp.Or(reported, task.State.StateMessage, string(task.State.ResultState), string(task.State.LifeCycleState)) } -func runPageLine(ctx context.Context, rawURL string) string { +func runPageLine(rawURL string) string { if rawURL == "" { return "" } - return "\nrun page: " + workspaceurls.JobRunPageURL(ctx, rawURL) + return "\nrun page: " + workspaceurls.JobRunPageURL(rawURL) } // logRunProgress mirrors `bundle run`: the run page URL once, then each state change. @@ -244,7 +243,7 @@ func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobSta return } if first && run.RunPageUrl != "" { - logRunLine(ctx, run.RunId, "Run URL: "+workspaceurls.JobRunPageURL(ctx, run.RunPageUrl)) + logRunLine(ctx, run.RunId, "Run URL: "+workspaceurls.JobRunPageURL(run.RunPageUrl)) } logRunLine(ctx, run.RunId, event.String()) } diff --git a/bundle/run/job.go b/bundle/run/job.go index b61c98e4479..121bcef38db 100644 --- a/bundle/run/job.go +++ b/bundle/run/job.go @@ -97,7 +97,7 @@ func (m *jobRunMonitor) onProgress(info *jobs.Run) { // First time we see this run. if first { - runURL := workspaceurls.JobRunPageURL(m.ctx, info.RunPageUrl) + runURL := workspaceurls.JobRunPageURL(info.RunPageUrl) log.Infof(m.ctx, "Run available at %s", runURL) cmdio.Log(m.ctx, progress.NewJobRunUrlEvent(runURL)) } @@ -145,7 +145,7 @@ func (r *jobRunner) Run(ctx context.Context, opts *Options) (output.RunOutput, e if err != nil { return nil, err } - cmdio.Log(ctx, progress.NewJobRunUrlEvent(workspaceurls.JobRunPageURL(ctx, details.RunPageUrl))) + cmdio.Log(ctx, progress.NewJobRunUrlEvent(workspaceurls.JobRunPageURL(details.RunPageUrl))) return nil, nil } diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index b0248a87a59..ff24080c09a 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -1,13 +1,10 @@ package workspaceurls import ( - "context" "fmt" "net/url" "slices" "strings" - - "github.com/databricks/cli/libs/log" ) var resourceURLPatterns = map[string]string{ @@ -106,16 +103,14 @@ func JobRunURL(baseURL url.URL, jobID, runID string) string { // workspace selector query param (o) is preserved as-is. The conversion is // cosmetic, so the original URL is returned on the rare chance the format is // unexpected. -func JobRunPageURL(ctx context.Context, raw string) string { +func JobRunPageURL(raw string) string { u, err := url.Parse(raw) if err != nil { - log.Debugf(ctx, "could not parse run URL %q: %v", raw, err) return raw } jobID, runID, ok := parseLegacyRunFragment(u.Fragment) if !ok { - log.Debugf(ctx, "unexpected run URL fragment %q", u.Fragment) return raw } diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index 7fe4f152b78..59dceb844ef 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -168,7 +168,6 @@ func TestHasWorkspaceIDInHostname(t *testing.T) { } func TestJobRunPageURL(t *testing.T) { - ctx := t.Context() tests := []struct { name string raw string @@ -209,7 +208,7 @@ func TestJobRunPageURL(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, JobRunPageURL(ctx, tt.raw)) + assert.Equal(t, tt.expected, JobRunPageURL(tt.raw)) }) } } From c575577b5b69a4359e2a706ac41c38f66e1b55aa Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 13:29:36 +0000 Subject: [PATCH 09/14] job_runs: report only the run URL and the state the run ends in Reporting every state change made a deploy's output depend on how many states the run passed through, which varies with how long its compute takes to start. That cost the cloud test its coverage: it had to grep the deploy log instead of comparing it. The full state history is still in the log. Also record that a failed run is not run again in the changelog entry. --- .../bundles/job-runs-wait-for-completion.md | 2 +- .../resources/job_runs/basic/output.txt | 1 - .../resources/job_runs/failed_run/output.txt | 1 - .../job_runs/job_parameters/output.txt | 1 - .../resources/job_runs/redeploy/output.txt | 2 -- .../resources/job_runs/wait_cloud/output.txt | 13 +++++--- .../resources/job_runs/wait_cloud/script | 13 +++----- .../resources/job_runs/wait_cloud/test.toml | 7 +++- .../resources/job_runs/wait_output/output.txt | 1 - bundle/direct/dresources/job_run.go | 33 ++++++++++++++----- 10 files changed, 45 insertions(+), 29 deletions(-) diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md index 4520d1d9180..438070dde4a 100644 --- a/.nextchanges/bundles/job-runs-wait-for-completion.md +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -1 +1 @@ -direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). A run that does not succeed fails the deploy, naming the failed task and the message it reported; while waiting, the deploy reports the run page URL and each state change. +direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL while it waits, and a run that does not succeed fails the deploy, naming the failed task and the message it reported. A failed run stays recorded, so the next deploy does not run the job again. diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt index 9ce260e8c85..67d2a32021b 100644 --- a/acceptance/bundle/resources/job_runs/basic/output.txt +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -35,7 +35,6 @@ Resources: Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files... Deploying resources... job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 78b95c98da1..7b0f33c5574 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files... Deploying resources... job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: [TIMESTAMP] "my-job" RUNNING job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED FAILED task main failed Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: task main failed task "main": spark python task execution failed: exit status 1 diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index 986caf2b754..1f793b6f01b 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files... Deploying resources... job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" RUNNING job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 8662b4880f0..e75196469df 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" RUNNING job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! @@ -89,7 +88,6 @@ Resources: Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Deploying resources... job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" RUNNING job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt index 92b6ad11f0a..a38ed72002e 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/output.txt +++ b/acceptance/bundle/resources/job_runs/wait_cloud/output.txt @@ -1,10 +1,15 @@ === the deploy waits for the run to finish ->>> grep -c Run URL: deploy.log -1 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [RUN_URL] +job run [MY_RUN_ID]: [TIMESTAMP] "test-job-[UNIQUE_NAME]" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! ->>> grep -o TERMINATED SUCCESS deploy.log -TERMINATED SUCCESS +>>> read_id.py my_run +[MY_RUN_ID] === the downstream job was created with the run's result_state >>> [CLI] jobs get [DOWNSTREAM_JOB_ID] -o json diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/script b/acceptance/bundle/resources/job_runs/wait_cloud/script index d047e5294dc..dc58273b5e8 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/script +++ b/acceptance/bundle/resources/job_runs/wait_cloud/script @@ -5,15 +5,12 @@ cleanup() { } trap cleanup EXIT -# A real run reports a variable number of intermediate states (PENDING while -# serverless compute starts), so keep the deploy's progress stream out of the -# golden and assert on the terminal state it waited for. title "the deploy waits for the run to finish" -if ! $CLI bundle deploy > deploy.log 2>&1; then - cat deploy.log -fi -trace grep -c "Run URL: " deploy.log -trace grep -o "TERMINATED SUCCESS" deploy.log +trace $CLI bundle deploy + +# Registers the run id as a replacement, so the deploy output above compares the +# same way against a real workspace as against the test server. +trace read_id.py my_run title "the downstream job was created with the run's result_state" downstream_id=$(read_id.py downstream_job) diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml index 63977f7d2e9..9653ec13c31 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml +++ b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml @@ -16,5 +16,10 @@ Ignore = [ "databricks.yml", "databricks.yml.tmpl", "hello.py", - "deploy.log", ] + +# The host and the workspace selector in the run URL differ per workspace, and the +# URL form itself is covered by libs/workspaceurls; assert only that it is reported. +[[Repls]] +Old = 'Run URL: .*' +New = 'Run URL: [RUN_URL]' diff --git a/acceptance/bundle/resources/job_runs/wait_output/output.txt b/acceptance/bundle/resources/job_runs/wait_output/output.txt index bd9def5e990..5a2bcfa4892 100644 --- a/acceptance/bundle/resources/job_runs/wait_output/output.txt +++ b/acceptance/bundle/resources/job_runs/wait_output/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-wait-output/default/files... Deploying resources... job run [NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -job run [NUMID]: [TIMESTAMP] "my-job" RUNNING job run [NUMID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index e11321870f4..9addde32b8b 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -202,8 +202,8 @@ func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) erro return errors.New(msg.String()) } -// taskFailed reports whether a task caused the run to fail rather than being a -// casualty of it: tasks left SKIPPED or UPSTREAM_FAILED add noise. +// taskFailed reports whether a task is a cause of the run's failure. A task left +// SKIPPED or UPSTREAM_FAILED never ran, so it has no error to report. func taskFailed(task jobs.RunTask) bool { // State is deprecated in favour of Status, so it may be absent. if task.State == nil { @@ -236,22 +236,37 @@ func runPageLine(rawURL string) string { return "\nrun page: " + workspaceurls.JobRunPageURL(rawURL) } -// logRunProgress mirrors `bundle run`: the run page URL once, then each state change. +// logRunProgress logs every state change like `bundle run` does, but reports only +// the run page URL and the state the run ends in to the user: how many states a +// run passes through depends on how long its compute takes to start, which would +// make a deploy's output differ between runs of the same bundle. func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobStateTracker) { event, first := tracker.Poll(run) if event == nil { return } + log.Info(ctx, event.String()) if first && run.RunPageUrl != "" { - logRunLine(ctx, run.RunId, "Run URL: "+workspaceurls.JobRunPageURL(run.RunPageUrl)) + line := "Run URL: " + workspaceurls.JobRunPageURL(run.RunPageUrl) + log.Info(ctx, line) + reportRunLine(ctx, run.RunId, line) } - logRunLine(ctx, run.RunId, event.String()) + if runIsTerminal(run.State.LifeCycleState) { + reportRunLine(ctx, run.RunId, event.String()) + } +} + +// runIsTerminal reports whether a run is done, i.e. in one of the states the SDK +// waiter stops on. +func runIsTerminal(state jobs.RunLifeCycleState) bool { + return state == jobs.RunLifeCycleStateTerminated || + state == jobs.RunLifeCycleStateSkipped || + state == jobs.RunLifeCycleStateInternalError } -// logRunLine names the run in the user-facing copy, since resources deploy -// concurrently onto one stream; the log carries the resource key already. -func logRunLine(ctx context.Context, runID int64, msg string) { - log.Info(ctx, msg) +// reportRunLine names the run it describes, since resources deploy concurrently +// onto one output stream. +func reportRunLine(ctx context.Context, runID int64, msg string) { if cmdio.HasIO(ctx) { cmdio.LogString(ctx, fmt.Sprintf("job run %d: %s", runID, msg)) } From b7d86401db0367c3f1934f9063abcd22d170292e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 13:51:55 +0000 Subject: [PATCH 10/14] acc: run a failing job_run against a real workspace The message the deploy names a failed task with came from an error the test server writes itself, so nothing checked that a real workspace reports one at all. Assert it does, and that we are not falling back to the states the run reports. --- .../job_runs/failed_cloud/databricks.yml.tmpl | 26 +++++++++++++++++++ .../resources/job_runs/failed_cloud/fail.py | 1 + .../job_runs/failed_cloud/out.test.toml | 4 +++ .../job_runs/failed_cloud/output.txt | 20 ++++++++++++++ .../resources/job_runs/failed_cloud/script | 19 ++++++++++++++ .../resources/job_runs/failed_cloud/test.toml | 21 +++++++++++++++ 6 files changed, 91 insertions(+) create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/fail.py create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/output.txt create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/script create mode 100644 acceptance/bundle/resources/job_runs/failed_cloud/test.toml diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl b/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl new file mode 100644 index 00000000000..1393bdaf33e --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/databricks.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: job-runs-failed-cloud + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + jobs: + my_job: + name: test-job-$UNIQUE_NAME + tasks: + # Serverless keeps the run to about a minute; a job cluster would take + # several. + - task_key: main + spark_python_task: + python_file: ./fail.py + environment_key: default + + environments: + - environment_key: default + spec: + environment_version: "2" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/fail.py b/acceptance/bundle/resources/job_runs/failed_cloud/fail.py new file mode 100644 index 00000000000..fa56481ece5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/fail.py @@ -0,0 +1 @@ +raise RuntimeError("intentional failure") diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml new file mode 100644 index 00000000000..fe4076cdf9b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/output.txt b/acceptance/bundle/resources/job_runs/failed_cloud/output.txt new file mode 100644 index 00000000000..bc4e275e341 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/output.txt @@ -0,0 +1,20 @@ + +=== a run that fails on a real workspace fails the deploy +>>> contains.py run did not succeed: FAILED run page: http !task "main": FAILED + +>>> grep -cE task "main": .+ deploy.log +1 + +=== the failed run is recorded, so it is destroyed rather than left behind +>>> read_id.py my_run +[MY_RUN_ID] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/script b/acceptance/bundle/resources/job_runs/failed_cloud/script new file mode 100644 index 00000000000..662d43fc738 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/script @@ -0,0 +1,19 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# A traceback from a real task is words we do not control, so keep the deploy's +# output out of the golden and assert the parts the CLI is responsible for. +title "a run that fails on a real workspace fails the deploy" +musterr $CLI bundle deploy > deploy.log 2>&1 +trace contains.py 'run did not succeed: FAILED' 'run page: http' '!task "main": FAILED' < deploy.log > /dev/null + +# The message comes from the task, not from our fallback to the states the run +# itself reports, which is what the last assertion above rules out. +trace grep -cE 'task "main": .+' deploy.log + +title "the failed run is recorded, so it is destroyed rather than left behind" +trace read_id.py my_run diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml new file mode 100644 index 00000000000..dd06a0daee3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml @@ -0,0 +1,21 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The failure counterpart of wait_cloud: only a real workspace shows whether a +# failed task reports a message the deploy can name, since the error the test +# server reports is one it writes itself. Serverless needs Unity Catalog. +Cloud = true +RequiresUnityCatalog = true + +# A real workspace is not proxied, so there are no recorded requests to assert on. +RecordRequests = false + +# The deploy fails mid-way, leaving local deployment state behind. +Ignore = [ + ".databricks", + "databricks.yml", + "databricks.yml.tmpl", + "fail.py", + "deploy.log", +] From 01c5202c59d9054f1c1f913921e4e6f3cdcdd614 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 29 Jul 2026 13:57:11 +0000 Subject: [PATCH 11/14] job_runs: shorten the comments added by this branch Also fix jobRunServer's doc, which said it returns a server when it returns a client. --- .../bundle/resources/job_runs/failed_cloud/script | 8 ++++---- .../resources/job_runs/failed_cloud/test.toml | 6 +++--- .../bundle/resources/job_runs/wait_cloud/script | 2 +- bundle/direct/dresources/job_run.go | 15 ++++++--------- bundle/direct/dresources/job_run_test.go | 4 ++-- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/script b/acceptance/bundle/resources/job_runs/failed_cloud/script index 662d43fc738..ce49c3a9aa2 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/script +++ b/acceptance/bundle/resources/job_runs/failed_cloud/script @@ -5,14 +5,14 @@ cleanup() { } trap cleanup EXIT -# A traceback from a real task is words we do not control, so keep the deploy's -# output out of the golden and assert the parts the CLI is responsible for. +# A real task's traceback is text we do not control, so keep the deploy output out +# of the golden and assert the parts the CLI produces. title "a run that fails on a real workspace fails the deploy" musterr $CLI bundle deploy > deploy.log 2>&1 trace contains.py 'run did not succeed: FAILED' 'run page: http' '!task "main": FAILED' < deploy.log > /dev/null -# The message comes from the task, not from our fallback to the states the run -# itself reports, which is what the last assertion above rules out. +# A non-empty message means the task reported one: the negative assertion above +# rules out the fallback to the states the run itself reports. trace grep -cE 'task "main": .+' deploy.log title "the failed run is recorded, so it is destroyed rather than left behind" diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml index dd06a0daee3..9d396b2cab8 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml +++ b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml @@ -2,9 +2,9 @@ # equivalent, so restrict the matrix to direct. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# The failure counterpart of wait_cloud: only a real workspace shows whether a -# failed task reports a message the deploy can name, since the error the test -# server reports is one it writes itself. Serverless needs Unity Catalog. +# The failure counterpart of wait_cloud: the error the test server reports is one it +# writes itself, so only a real workspace shows whether a failed task reports a +# message the deploy can name. Serverless needs Unity Catalog. Cloud = true RequiresUnityCatalog = true diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/script b/acceptance/bundle/resources/job_runs/wait_cloud/script index dc58273b5e8..22879da2149 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/script +++ b/acceptance/bundle/resources/job_runs/wait_cloud/script @@ -9,7 +9,7 @@ title "the deploy waits for the run to finish" trace $CLI bundle deploy # Registers the run id as a replacement, so the deploy output above compares the -# same way against a real workspace as against the test server. +# same against a real workspace as against the test server. trace read_id.py my_run title "the downstream job was created with the run's result_state" diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9addde32b8b..aeea0f8fa56 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -150,8 +150,7 @@ func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (str } // WaitAfterCreate blocks until the run finishes, so a resource referencing its -// output (e.g. state.result_state) sees a settled run. Only SUCCESS lets the -// deploy continue. +// output (e.g. state.result_state) sees a settled run. Only SUCCESS continues the deploy. func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobRunState) (*JobRunRemote, error) { runID, err := parseRunID(id) if err != nil { @@ -237,9 +236,9 @@ func runPageLine(rawURL string) string { } // logRunProgress logs every state change like `bundle run` does, but reports only -// the run page URL and the state the run ends in to the user: how many states a -// run passes through depends on how long its compute takes to start, which would -// make a deploy's output differ between runs of the same bundle. +// the run page URL and the run's final state to the user: how many states a run +// passes through varies with how long its compute takes to start, so a deploy's +// output would not be reproducible. func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobStateTracker) { event, first := tracker.Poll(run) if event == nil { @@ -256,16 +255,14 @@ func logRunProgress(ctx context.Context, run *jobs.Run, tracker *progress.JobSta } } -// runIsTerminal reports whether a run is done, i.e. in one of the states the SDK -// waiter stops on. +// runIsTerminal reports whether a run is done, i.e. in a state the SDK waiter stops on. func runIsTerminal(state jobs.RunLifeCycleState) bool { return state == jobs.RunLifeCycleStateTerminated || state == jobs.RunLifeCycleStateSkipped || state == jobs.RunLifeCycleStateInternalError } -// reportRunLine names the run it describes, since resources deploy concurrently -// onto one output stream. +// reportRunLine names the run, since resources deploy concurrently onto one stream. func reportRunLine(ctx context.Context, runID int64, msg string) { if cmdio.HasIO(ctx) { cmdio.LogString(ctx, fmt.Sprintf("job run %d: %s", runID, msg)) diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index e9c29366136..2625ef406c2 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -28,8 +28,8 @@ func jobRunClientFor(t *testing.T, server *testserver.Server) *databricks.Worksp return client } -// jobRunServer returns a test server whose runs/get handler is the given one, -// so a wait can be exercised without a real run. +// jobRunServer returns a client whose runs/get is the given handler, so a wait can +// be driven without a real run. func jobRunServer(t *testing.T, getRun testserver.HandlerFunc) *databricks.WorkspaceClient { t.Helper() server := testserver.New(t) From 202e9f99df25a07735e765c1fad4ffef6d2ae630 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 30 Jul 2026 08:14:24 +0000 Subject: [PATCH 12/14] job_runs: handle a wait the user interrupted Interrupting a deploy mid-wait leaves the run going, and jobs/runs/delete rejects an active run, so destroy failed and the bundle could not be torn down without cancelling the run by hand. Delete now cancels it first, and waits for the cancellation to settle since the API cancels asynchronously. The interrupt itself was reported as a timeout, blaming the 24h bound for something the user did. It now says it was interrupted, and still links the run, whose page URL is pinned to the first poll that reported one. The run left going is what the next deploy reads, and it triggers no second run, so a reference to the outcome resolves to an empty string. Recorded in a test rather than fixed here: stopping a run the user did not ask to stop is a departure from `bundle run`, which leaves interrupted runs alive. --- .../bundles/job-runs-wait-for-completion.md | 2 +- bundle/direct/dresources/job_run.go | 36 +++++++- bundle/direct/dresources/job_run_test.go | 84 ++++++++++++++++++- 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md index 438070dde4a..9c7d3941c56 100644 --- a/.nextchanges/bundles/job-runs-wait-for-completion.md +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -1 +1 @@ -direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL while it waits, and a run that does not succeed fails the deploy, naming the failed task and the message it reported. A failed run stays recorded, so the next deploy does not run the job again. +direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL while it waits, and a run that does not succeed fails the deploy, naming the failed task and the message it reported. A failed run stays recorded, so the next deploy does not run the job again. Destroying a run that has not finished, which is what interrupting a deploy mid-wait leaves behind, cancels it first. diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index aeea0f8fa56..546f8bea785 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -162,12 +162,16 @@ func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobR var tracker progress.JobStateTracker var pageURL string run, err := r.client.Jobs.WaitGetRunJobTerminatedOrSkipped(ctx, runID, jobRunTimeout, func(run *jobs.Run) { - pageURL = run.RunPageUrl + pageURL = cmp.Or(pageURL, run.RunPageUrl) logRunProgress(ctx, run, &tracker) }) if err != nil { // The wait can end with the run still going (timeout, interrupt), so link - // the run page. + // the run page: the next deploy triggers no second run, finished or not. + if ctx.Err() != nil { + // The waiter reports a cancelled context as a timeout. + return nil, fmt.Errorf("interrupted while waiting for the run to finish: %w%s", ctx.Err(), runPageLine(pageURL)) + } return nil, fmt.Errorf("%w%s", err, runPageLine(pageURL)) } // The waiter already errored on INTERNAL_ERROR and on its timeout, so only the @@ -273,16 +277,40 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // so any change recreates it (delete + a fresh RunNow). // DoDelete deletes the run via jobs/runs/delete, on both destroy and the -// recreate path. The API rejects a still-active run; WaitAfterCreate leaves it -// terminal, so that error only surfaces when a wait was interrupted. +// recreate path. The API rejects a still-active run, which an interrupted wait +// leaves behind, so cancel it first. func (r *ResourceJobRun) DoDelete(ctx context.Context, id string, _ *JobRunState) error { runID, err := parseRunID(id) if err != nil { return err } + remote, err := r.DoRead(ctx, id) + if err != nil { + return err + } + if !runIsTerminal(remote.State.LifeCycleState) { + err = r.cancelRun(ctx, runID) + if err != nil { + return err + } + } return r.client.Jobs.DeleteRunByRunId(ctx, runID) } +// cancelRun cancels a run and waits for it to settle. Cancellation is +// asynchronous, so a delete issued right after would still be rejected. +func (r *ResourceJobRun) cancelRun(ctx context.Context, runID int64) error { + waiter, err := r.client.Jobs.CancelRun(ctx, jobs.CancelRun{RunId: runID}) + if err != nil { + return fmt.Errorf("cancelling run %d before deleting it: %w", runID, err) + } + _, err = waiter.Get() + if err != nil { + return fmt.Errorf("waiting for run %d to be cancelled: %w", runID, err) + } + return nil +} + func parseRunID(id string) (int64, error) { result, err := strconv.ParseInt(id, 10, 64) if err != nil { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 2625ef406c2..9dce5dbf2ca 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -149,11 +149,25 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { _, err := waitForTestRun(t, ctx, client) - // Giving up on the wait does not stop the run, so the error links to it. - require.Error(t, err) + // The run keeps going, so the error links to it and names the interrupt rather + // than the 24h bound. + require.ErrorContains(t, err, "interrupted while waiting for the run to finish") require.ErrorContains(t, err, testRunPageLink) } +// After an abandoned wait the next deploy triggers no second run: it reads this +// one, so a reference to the outcome resolves to an empty string. +func TestJobRunReadOfUnfinishedRunReportsNoResult(t *testing.T) { + client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) + + remote, err := (&ResourceJobRun{}).New(client).DoRead(t.Context(), "123") + + require.NoError(t, err) + require.NotNil(t, remote.State) + assert.Equal(t, jobs.RunLifeCycleStateRunning, remote.State.LifeCycleState) + assert.Empty(t, remote.State.ResultState) +} + // Reporting RUNNING for the first two polls exercises the poll loop; the other // tests stub an already-terminal state. func TestJobRunWaitPollsUntilTerminal(t *testing.T) { @@ -178,3 +192,69 @@ func TestJobRunWaitPollsUntilTerminal(t *testing.T) { assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) assert.Equal(t, int32(3), gets.Load(), "expected the wait to poll past both RUNNING reads") } + +// jobRunDeletion records what the fake workspace saw while a run was deleted. +type jobRunDeletion struct { + cancelled atomic.Bool + settled atomic.Bool + settledAtDelete atomic.Bool +} + +// jobRunDeleteClient returns a client for a run in the given state, whose cancel +// settles one poll late the way the API's asynchronous cancellation does. +func jobRunDeleteClient(t *testing.T, state *jobs.RunState) (*databricks.WorkspaceClient, *jobRunDeletion) { + t.Helper() + var deletion jobRunDeletion + cancelled := &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateCanceled, + } + + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { + current := state + switch { + case deletion.settled.Load(): + current = cancelled + case deletion.cancelled.Load(): + // Report the run's old state once more, then settle on the next poll. + deletion.settled.Store(true) + } + return jobs.Run{RunId: 123, JobId: 456, State: current} + }) + server.Handle("POST", "/api/2.2/jobs/runs/cancel", func(req testserver.Request) any { + deletion.cancelled.Store(true) + return testserver.Response{} + }) + server.Handle("POST", "/api/2.2/jobs/runs/delete", func(req testserver.Request) any { + deletion.settledAtDelete.Store(deletion.settled.Load()) + return testserver.Response{} + }) + return jobRunClientFor(t, server), &deletion +} + +func deleteTestRun(t *testing.T, client *databricks.WorkspaceClient) error { + t.Helper() + return (&ResourceJobRun{}).New(client).DoDelete(t.Context(), "123", &JobRunState{}) +} + +func TestJobRunDeleteCancelsUnfinishedRun(t *testing.T) { + // An interrupted wait leaves the run going, and jobs/runs/delete rejects it. + client, deletion := jobRunDeleteClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) + + require.NoError(t, deleteTestRun(t, client)) + + assert.True(t, deletion.cancelled.Load(), "expected the run to be cancelled") + assert.True(t, deletion.settledAtDelete.Load(), "expected the delete to wait for the cancellation to settle") +} + +func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { + client, deletion := jobRunDeleteClient(t, &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }) + + require.NoError(t, deleteTestRun(t, client)) + + assert.False(t, deletion.cancelled.Load(), "a run that already finished has nothing to cancel") +} From ea0a8b3d38e95b3a04ad183911e465176da0f932 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 30 Jul 2026 11:11:45 +0000 Subject: [PATCH 13/14] job_runs: wait for any terminal state, not the two the SDK stops on A real workspace reports a run whose task failed as INTERNAL_ERROR in the deprecated life_cycle_state, though status.state is TERMINATED with termination code RUN_EXECUTION_ERROR. The SDK waiter halts on INTERNAL_ERROR with an error of its own, so the deploy blamed the run for an internal failure instead of naming the task that failed and the message it reported. The wait now polls for any state runIsTerminal accepts, the definition the delete path already used, and leaves the verdict to the run's result. The Jobs API retries a task that failed and reports it once per attempt, so the same task was named twice over. Only its last attempt is reported now. The fake workspace rolls a failed task up to TERMINATED FAILED, so failed_cloud was the only test that saw either of these; both are now covered by unit tests. --- bundle/direct/dresources/job_run.go | 55 ++++++++++++++++---- bundle/direct/dresources/job_run_test.go | 66 +++++++++++++++++++++++- 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 546f8bea785..2ae6219f4bc 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -158,24 +159,37 @@ func (r *ResourceJobRun) WaitAfterCreate(ctx context.Context, id string, _ *JobR } // A run can take hours, so report progress like `bundle run` does. pageURL - // outlives the callback so an abandoned wait can still link the run. + // outlives the poll so an abandoned wait can still link the run. var tracker progress.JobStateTracker var pageURL string - run, err := r.client.Jobs.WaitGetRunJobTerminatedOrSkipped(ctx, runID, jobRunTimeout, func(run *jobs.Run) { + // Polled here rather than through Jobs.WaitGetRunJobTerminatedOrSkipped: a run + // whose task failed reports the deprecated life_cycle_state as INTERNAL_ERROR + // (status.state is TERMINATED, termination code RUN_EXECUTION_ERROR), and the + // SDK waiter halts on it with an error of its own, which hides the task that + // failed. + run, err := retries.Poll(ctx, jobRunTimeout, func() (*jobs.Run, *retries.Err) { + var req jobs.GetRunRequest + req.RunId = runID + run, err := r.client.Jobs.GetRun(ctx, req) + if err != nil { + return nil, retries.Halt(err) + } pageURL = cmp.Or(pageURL, run.RunPageUrl) logRunProgress(ctx, run, &tracker) + if !runIsTerminal(run.State.LifeCycleState) { + return nil, retries.Continues(run.State.StateMessage) + } + return run, nil }) if err != nil { // The wait can end with the run still going (timeout, interrupt), so link // the run page: the next deploy triggers no second run, finished or not. if ctx.Err() != nil { - // The waiter reports a cancelled context as a timeout. + // A cancelled context is reported as a timeout. return nil, fmt.Errorf("interrupted while waiting for the run to finish: %w%s", ctx.Err(), runPageLine(pageURL)) } return nil, fmt.Errorf("%w%s", err, runPageLine(pageURL)) } - // The waiter already errored on INTERNAL_ERROR and on its timeout, so only the - // terminal results are left to reject. if run.State.ResultState != jobs.RunResultStateSuccess { return nil, r.runFailedError(ctx, run) } @@ -196,15 +210,38 @@ func (r *ResourceJobRun) runFailedError(ctx context.Context, run *jobs.Run) erro if run.State.StateMessage != "" { fmt.Fprintf(&msg, ": %s", run.State.StateMessage) } - for _, task := range run.Tasks { - if taskFailed(task) { - fmt.Fprintf(&msg, "\ntask %q: %s", task.TaskKey, r.taskError(ctx, task)) - } + for _, task := range lastFailedAttempts(run.Tasks) { + fmt.Fprintf(&msg, "\ntask %q: %s", task.TaskKey, r.taskError(ctx, task)) } msg.WriteString(runPageLine(run.RunPageUrl)) return errors.New(msg.String()) } +// lastFailedAttempts returns the failed tasks in the order the run reports them, +// one per task key: a task the Jobs API retried is reported once per attempt, and +// only its last one says how the run ended up. +func lastFailedAttempts(tasks []jobs.RunTask) []jobs.RunTask { + latest := make(map[string]jobs.RunTask) + var keys []string + for _, task := range tasks { + if !taskFailed(task) { + continue + } + previous, seen := latest[task.TaskKey] + if !seen { + keys = append(keys, task.TaskKey) + } + if !seen || task.AttemptNumber > previous.AttemptNumber { + latest[task.TaskKey] = task + } + } + result := make([]jobs.RunTask, 0, len(keys)) + for _, key := range keys { + result = append(result, latest[key]) + } + return result +} + // taskFailed reports whether a task is a cause of the run's failure. A task left // SKIPPED or UPSTREAM_FAILED never ran, so it has no error to report. func taskFailed(task jobs.RunTask) bool { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 9dce5dbf2ca..50580104437 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -136,11 +136,73 @@ func TestJobRunWaitFailsOnInternalError(t *testing.T) { _, err := waitForTestRun(t, t.Context(), client) - // The SDK waiter errors on INTERNAL_ERROR, ahead of the result check. - require.ErrorContains(t, err, "INTERNAL_ERROR") + require.ErrorContains(t, err, "run did not succeed: INTERNAL_ERROR") require.ErrorContains(t, err, testRunPageLink) } +// A real workspace reports a run whose task failed as INTERNAL_ERROR in the +// deprecated life_cycle_state, which the SDK waiter halts on with an error of its +// own. The failing task still has to be named. +func TestJobRunWaitReportsFailedTaskOfInternalErrorRun(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { + return jobs.Run{ + RunId: 123, + JobId: 456, + RunPageUrl: testRunPageURL, + State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateInternalError, + ResultState: jobs.RunResultStateFailed, + StateMessage: "Task main failed with message: Workload failed, see run output for details.", + }, + Tasks: []jobs.RunTask{ + {TaskKey: "main", RunId: 999, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateFailed, + }}, + }, + } + }) + server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { + return jobs.RunOutput{Error: "RuntimeError: intentional failure"} + }) + + _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) + + require.ErrorContains(t, err, "run did not succeed: FAILED") + require.ErrorContains(t, err, `task "main": RuntimeError: intentional failure`) + require.ErrorContains(t, err, testRunPageLink) +} + +func TestJobRunWaitReportsOnlyTheLastAttemptOfATask(t *testing.T) { + failed := &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateFailed, + } + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { + return jobs.Run{ + RunId: 123, + JobId: 456, + State: failed, + Tasks: []jobs.RunTask{ + {TaskKey: "main", RunId: 998, AttemptNumber: 0, State: failed}, + {TaskKey: "main", RunId: 999, AttemptNumber: 1, State: failed}, + }, + } + }) + server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { + return jobs.RunOutput{Error: "output of run " + req.URL.Query().Get("run_id")} + }) + + _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) + + // The Jobs API reports a retried task once per attempt; only the last one says + // how the run ended up. + require.ErrorContains(t, err, `task "main": output of run 999`) + assert.NotContains(t, err.Error(), "output of run 998") +} + func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) From 8e63c56641ab24bf66aa2d6acd274ca4cb7a92f6 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 30 Jul 2026 13:00:16 +0000 Subject: [PATCH 14/14] job_runs: make a triggered run idempotent across deploy retries DoCreate derives idempotency_token as a SHA-256 of the run config plus a create identity (workspace root path, resource key, and the prior run id when re-creating a vanished run), so a deploy that fires run-now and then crashes before recording the id rejoins that same run on retry instead of starting a duplicate. The token is computed by the CLI and rejected if set in bundle configuration; the new bundle-only rerun_token field folds into it, so bumping it re-runs a configuration that already ran. DoDelete becomes a noop: the Jobs API keeps an idempotency_token reserved once the run it triggered is deleted, so deleting the run would tombstone the token and break re-running the same configuration. That drops the cancel-before-delete path this branch added earlier: with no delete there is nothing to make room for, and cancelling would settle the run on CANCELED, which no later deploy that dedupes onto it could complete. A run whose wait was interrupted keeps going, and the error that ended the wait links its run page. --- .nextchanges/bundles/job-runs-idempotency.md | 1 + .../bundles/job-runs-wait-for-completion.md | 2 +- acceptance/bundle/refschema/out.fields.txt | 1 + .../resources/job_runs/basic/output.txt | 17 +- .../bundle/resources/job_runs/basic/script | 9 +- .../bundle/resources/job_runs/basic/test.toml | 4 - .../job_runs/distinct_runs/databricks.yml | 21 ++ .../job_runs/distinct_runs/out.test.toml | 3 + .../job_runs/distinct_runs/output.txt | 43 ++++ .../resources/job_runs/distinct_runs/script | 19 ++ .../resources/job_runs/failed_cloud/test.toml | 4 - .../resources/job_runs/failed_run/output.txt | 1 + .../idempotent_recreate/databricks.yml | 17 ++ .../idempotent_recreate/out.test.toml | 3 + .../job_runs/idempotent_recreate/output.txt | 79 ++++++ .../job_runs/idempotent_recreate/script | 32 +++ .../job_runs/job_parameters/output.txt | 1 + .../job_runs/job_parameters/test.toml | 4 - .../job_runs/recreate_gone/databricks.yml | 17 ++ .../job_runs/recreate_gone/out.test.toml | 3 + .../job_runs/recreate_gone/output.txt | 53 ++++ .../resources/job_runs/recreate_gone/script | 23 ++ .../resources/job_runs/redeploy/output.txt | 11 +- .../bundle/resources/job_runs/redeploy/script | 4 +- .../resources/job_runs/redeploy/test.toml | 4 - .../resources/job_runs/rerun/databricks.yml | 20 ++ .../resources/job_runs/rerun/out.test.toml | 3 + .../resources/job_runs/rerun/output.txt | 65 +++++ .../bundle/resources/job_runs/rerun/script | 23 ++ .../bundle/resources/job_runs/test.toml | 14 ++ .../token_survives_delete/databricks.yml | 19 ++ .../token_survives_delete/out.test.toml | 3 + .../job_runs/token_survives_delete/output.txt | 73 ++++++ .../job_runs/token_survives_delete/script | 36 +++ .../job_runs/validate/databricks.yml | 20 ++ .../resources/job_runs/validate/out.test.toml | 3 + .../resources/job_runs/validate/output.txt | 14 ++ .../bundle/resources/job_runs/validate/script | 2 + .../resources/job_runs/validate/test.toml | 2 + .../resources/job_runs/wait_cloud/test.toml | 4 - bundle/config/resources/job_run.go | 9 +- bundle/config/validate/validate_job_runs.go | 45 ++++ .../config/validate/validate_job_runs_test.go | 48 ++++ bundle/direct/apply.go | 9 + bundle/direct/bundle_apply.go | 11 +- bundle/direct/dresources/all_test.go | 10 +- bundle/direct/dresources/identity.go | 38 +++ bundle/direct/dresources/job_run.go | 127 ++++++---- bundle/direct/dresources/job_run_test.go | 230 +++++++++++++----- bundle/direct/pkg.go | 3 + bundle/internal/schema/annotations.yml | 7 +- bundle/phases/deploy.go | 2 +- bundle/phases/destroy.go | 2 +- bundle/phases/initialize.go | 4 + bundle/schema/jsonschema.json | 6 +- libs/testserver/fake_workspace.go | 2 + libs/testserver/jobs.go | 21 ++ libs/testserver/jobs_test.go | 30 +++ 58 files changed, 1113 insertions(+), 168 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-idempotency.md delete mode 100644 acceptance/bundle/resources/job_runs/basic/test.toml create mode 100644 acceptance/bundle/resources/job_runs/distinct_runs/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/distinct_runs/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/distinct_runs/output.txt create mode 100644 acceptance/bundle/resources/job_runs/distinct_runs/script create mode 100644 acceptance/bundle/resources/job_runs/idempotent_recreate/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/idempotent_recreate/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/idempotent_recreate/output.txt create mode 100644 acceptance/bundle/resources/job_runs/idempotent_recreate/script delete mode 100644 acceptance/bundle/resources/job_runs/job_parameters/test.toml create mode 100644 acceptance/bundle/resources/job_runs/recreate_gone/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/recreate_gone/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/recreate_gone/output.txt create mode 100644 acceptance/bundle/resources/job_runs/recreate_gone/script delete mode 100644 acceptance/bundle/resources/job_runs/redeploy/test.toml create mode 100644 acceptance/bundle/resources/job_runs/rerun/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/rerun/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/rerun/output.txt create mode 100644 acceptance/bundle/resources/job_runs/rerun/script create mode 100644 acceptance/bundle/resources/job_runs/test.toml create mode 100644 acceptance/bundle/resources/job_runs/token_survives_delete/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/token_survives_delete/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/token_survives_delete/output.txt create mode 100644 acceptance/bundle/resources/job_runs/token_survives_delete/script create mode 100644 acceptance/bundle/resources/job_runs/validate/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/validate/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/validate/output.txt create mode 100644 acceptance/bundle/resources/job_runs/validate/script create mode 100644 acceptance/bundle/resources/job_runs/validate/test.toml create mode 100644 bundle/config/validate/validate_job_runs.go create mode 100644 bundle/config/validate/validate_job_runs_test.go create mode 100644 bundle/direct/dresources/identity.go diff --git a/.nextchanges/bundles/job-runs-idempotency.md b/.nextchanges/bundles/job-runs-idempotency.md new file mode 100644 index 00000000000..b3d258b9f54 --- /dev/null +++ b/.nextchanges/bundles/job-runs-idempotency.md @@ -0,0 +1 @@ +direct: the experimental `job_runs` resource now derives an `idempotency_token` for the run it triggers, so a deploy that fires run-now and then crashes before recording the run rejoins that same run on retry instead of starting a duplicate. The token is computed by the CLI and rejected if set in bundle configuration; set the new `rerun_token` field to a new value to re-run a configuration that already ran. Destroying a `job_run` now leaves the run in place, since the Jobs API keeps its token reserved once the run is deleted: a run that has not finished, which is what interrupting a deploy mid-wait leaves behind, keeps going, and the error that ended the wait links its run page. diff --git a/.nextchanges/bundles/job-runs-wait-for-completion.md b/.nextchanges/bundles/job-runs-wait-for-completion.md index 9c7d3941c56..438070dde4a 100644 --- a/.nextchanges/bundles/job-runs-wait-for-completion.md +++ b/.nextchanges/bundles/job-runs-wait-for-completion.md @@ -1 +1 @@ -direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL while it waits, and a run that does not succeed fails the deploy, naming the failed task and the message it reported. A failed run stays recorded, so the next deploy does not run the job again. Destroying a run that has not finished, which is what interrupting a deploy mid-wait leaves behind, cancels it first. +direct: the experimental `job_runs` resource now waits for the triggered run to finish, so other resources can reference the run's outcome (e.g. `${resources.job_runs.nightly.state.result_state}`). The deploy reports the run page URL while it waits, and a run that does not succeed fails the deploy, naming the failed task and the message it reported. A failed run stays recorded, so the next deploy does not run the job again. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d35bfa350f0..c1db9d02405 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -861,6 +861,7 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL +resources.job_runs.*.rerun_token string ALL resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.run_id int64 REMOTE resources.job_runs.*.run_name string REMOTE diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt index 67d2a32021b..fc0fc23a8ac 100644 --- a/acceptance/bundle/resources/job_runs/basic/output.txt +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -39,15 +39,16 @@ job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS Updating deployment state... Deployment complete! +=== redeploy does not trigger a second run +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files... Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - >>> [CLI] bundle summary Name: job-runs-basic Target: default @@ -70,6 +71,7 @@ Resources: "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN]", "job_id": [MY_JOB_ID] } } @@ -90,12 +92,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! - ->>> print_requests.py //jobs/runs/delete -{ - "method": "POST", - "path": "/api/2.2/jobs/runs/delete", - "body": { - "run_id": [MY_RUN_ID] - } -} diff --git a/acceptance/bundle/resources/job_runs/basic/script b/acceptance/bundle/resources/job_runs/basic/script index bb52888ea2b..3df85f03213 100644 --- a/acceptance/bundle/resources/job_runs/basic/script +++ b/acceptance/bundle/resources/job_runs/basic/script @@ -1,7 +1,7 @@ cleanup() { + # destroy leaves the run in place: a run is immutable history. trace $CLI bundle destroy --auto-approve - # destroy deletes the triggered run via jobs/runs/delete (also unlinks out.requests.txt) - trace print_requests.py //jobs/runs/delete + rm -f out.requests.txt } trap cleanup EXIT @@ -11,10 +11,9 @@ trace $CLI bundle plan trace $CLI bundle summary trace $CLI bundle deploy -# confirm that redeploy does not trigger a second run -trace $CLI bundle deploy - +title "redeploy does not trigger a second run" trace $CLI bundle plan +trace $CLI bundle deploy trace $CLI bundle summary title "exactly one run-now request was made" diff --git a/acceptance/bundle/resources/job_runs/basic/test.toml b/acceptance/bundle/resources/job_runs/basic/test.toml deleted file mode 100644 index 4b94d8b58e9..00000000000 --- a/acceptance/bundle/resources/job_runs/basic/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true diff --git a/acceptance/bundle/resources/job_runs/distinct_runs/databricks.yml b/acceptance/bundle/resources/job_runs/distinct_runs/databricks.yml new file mode 100644 index 00000000000..b70d9a29d2f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/distinct_runs/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: job-runs-distinct-runs + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + # Two runs with byte-identical config. Their resource keys go into the + # idempotency token, so each gets its own run. + job_runs: + my_run_a: + job_id: ${resources.jobs.my_job.id} + my_run_b: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/distinct_runs/out.test.toml b/acceptance/bundle/resources/job_runs/distinct_runs/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/distinct_runs/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/distinct_runs/output.txt b/acceptance/bundle/resources/job_runs/distinct_runs/output.txt new file mode 100644 index 00000000000..f13488497ea --- /dev/null +++ b/acceptance/bundle/resources/job_runs/distinct_runs/output.txt @@ -0,0 +1,43 @@ + +=== deploy triggers a separate run for each resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-distinct-runs/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> read_id.py my_run_a +[MY_RUN_A_ID] + +>>> read_id.py my_run_b +[MY_RUN_B_ID] + +=== two run-now requests were made, one per resource +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][0]", + "job_id": [NUMID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][1]", + "job_id": [NUMID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run_a + delete resources.job_runs.my_run_b + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-distinct-runs/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/distinct_runs/script b/acceptance/bundle/resources/job_runs/distinct_runs/script new file mode 100644 index 00000000000..bbf0e492ff0 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/distinct_runs/script @@ -0,0 +1,19 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "deploy triggers a separate run for each resource" +# The two runs deploy concurrently onto one stream, so their progress lines +# interleave in either order; drop them and read each resource's run id from +# state instead. +trace $CLI bundle deploy 2>&1 | grep -v '^job run ' +trace read_id.py my_run_a +trace read_id.py my_run_b + +title "two run-now requests were made, one per resource" +# The resource key is part of each token, so identical config still gets distinct +# tokens, numbered [0] and [1] below. The two bodies are otherwise identical, so +# the recorded order does not matter. +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml index 9d396b2cab8..0cf64378571 100644 --- a/acceptance/bundle/resources/job_runs/failed_cloud/test.toml +++ b/acceptance/bundle/resources/job_runs/failed_cloud/test.toml @@ -1,7 +1,3 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - # The failure counterpart of wait_cloud: the error the test server reports is one it # writes itself, so only a real workspace shows whether a failed task reports a # message the deploy can name. Serverless needs Unity Catalog. diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 7b0f33c5574..20d60bf9084 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -40,6 +40,7 @@ FAILED "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN]", "job_id": [NUMID] } } diff --git a/acceptance/bundle/resources/job_runs/idempotent_recreate/databricks.yml b/acceptance/bundle/resources/job_runs/idempotent_recreate/databricks.yml new file mode 100644 index 00000000000..76df3319ef2 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/idempotent_recreate/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: job-runs-idempotent-recreate + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/idempotent_recreate/out.test.toml b/acceptance/bundle/resources/job_runs/idempotent_recreate/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/idempotent_recreate/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/idempotent_recreate/output.txt b/acceptance/bundle/resources/job_runs/idempotent_recreate/output.txt new file mode 100644 index 00000000000..c7764920135 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/idempotent_recreate/output.txt @@ -0,0 +1,79 @@ + +=== initial deploy triggers the run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-idempotent-recreate/default/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +>>> read_id.py my_run +[MY_RUN_ID] + +=== retry after a lost run id (deploy crashed before the id was recorded) +>>> [CLI] bundle plan -o json +{ + "depends_on": [ + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "job_id": [NUMID] + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-idempotent-recreate/default/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +=== retry after the run id was recorded +>>> [CLI] bundle plan -o json +"skip" + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-idempotent-recreate/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== run-now was issued exactly twice, both with the same token +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][0]", + "job_id": [NUMID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][0]", + "job_id": [NUMID] + } +} + +>>> read_id.py my_run +[MY_RUN_ID] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-idempotent-recreate/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/idempotent_recreate/script b/acceptance/bundle/resources/job_runs/idempotent_recreate/script new file mode 100644 index 00000000000..258f5f83e84 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/idempotent_recreate/script @@ -0,0 +1,32 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +STATE=.databricks/bundle/default/resources.json + +title "initial deploy triggers the run" +trace $CLI bundle deploy +trace read_id.py my_run + +title "retry after a lost run id (deploy crashed before the id was recorded)" +# Simulates a deploy that fired run-now but crashed before saving the run id. +# job_id, and with it the idempotency token, is unchanged, so the retried run-now +# dedupes onto the existing run: the deploy below reports [MY_RUN_ID] again +# instead of a second run. +jq 'del(.state["resources.job_runs.my_run"])' "$STATE" > "$STATE.new" +mv "$STATE.new" "$STATE" +trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]' +trace $CLI bundle deploy + +title "retry after the run id was recorded" +# The run id is still in state, so the retry is a no-op: no second run-now. +trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].action // "none"' +trace $CLI bundle deploy + +title "run-now was issued exactly twice, both with the same token" +# The initial deploy and the lost-id retry. The second deduped, so state still +# holds the run the first deploy triggered. +trace print_requests.py //jobs/run-now +trace read_id.py my_run diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index 1f793b6f01b..fdbc76f02b4 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -13,6 +13,7 @@ Deployment complete! "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN]", "job_id": [NUMID], "job_parameters": { "env": "prod" diff --git a/acceptance/bundle/resources/job_runs/job_parameters/test.toml b/acceptance/bundle/resources/job_runs/job_parameters/test.toml deleted file mode 100644 index 4b94d8b58e9..00000000000 --- a/acceptance/bundle/resources/job_runs/job_parameters/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true diff --git a/acceptance/bundle/resources/job_runs/recreate_gone/databricks.yml b/acceptance/bundle/resources/job_runs/recreate_gone/databricks.yml new file mode 100644 index 00000000000..4788270e4d6 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/recreate_gone/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: job-runs-recreate-gone + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/recreate_gone/out.test.toml b/acceptance/bundle/resources/job_runs/recreate_gone/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/recreate_gone/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/recreate_gone/output.txt b/acceptance/bundle/resources/job_runs/recreate_gone/output.txt new file mode 100644 index 00000000000..0789d5d0a0e --- /dev/null +++ b/acceptance/bundle/resources/job_runs/recreate_gone/output.txt @@ -0,0 +1,53 @@ + +=== initial deploy triggers the run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-recreate-gone/default/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +=== delete the run out-of-band, tombstoning its idempotency token +>>> [CLI] jobs delete-run [MY_RUN_ID] + +=== redeploy re-triggers a fresh run instead of replaying the tombstoned token +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-recreate-gone/default/files... +Deploying resources... +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +>>> read_id.py my_run +[MY_RUN_ID_2] + +=== run-now was issued twice, with a different token each time +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][0]", + "job_id": [NUMID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][1]", + "job_id": [NUMID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-recreate-gone/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/recreate_gone/script b/acceptance/bundle/resources/job_runs/recreate_gone/script new file mode 100644 index 00000000000..77cefd270a1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/recreate_gone/script @@ -0,0 +1,23 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "initial deploy triggers the run" +trace $CLI bundle deploy +original=$(read_id.py my_run) + +title "delete the run out-of-band, tombstoning its idempotency token" +# Simulates the workspace garbage-collecting run history: the run now 404s and +# its token stays reserved, so the Jobs API errors on reuse. +trace $CLI jobs delete-run "$original" + +title "redeploy re-triggers a fresh run instead of replaying the tombstoned token" +# The planner sees the run gone and re-creates it, folding the prior run id into +# the token so run-now starts a new run, reported below as [MY_RUN_ID_2]. +trace $CLI bundle deploy +trace read_id.py my_run + +title "run-now was issued twice, with a different token each time" +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index e75196469df..150c275e230 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -32,6 +32,7 @@ Resources: "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][0]", "job_id": [MY_JOB_ID], "job_parameters": { "env": "dev" @@ -108,21 +109,15 @@ Resources: Name: my-job URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[NUMID] -=== the config change deleted the previous run and triggered a second, different run +=== the config change triggered a second, different run; the previous run is left in place >>> print_requests.py --keep //jobs/runs/delete -{ - "method": "POST", - "path": "/api/2.2/jobs/runs/delete", - "body": { - "run_id": [NUMID] - } -} >>> print_requests.py //jobs/run-now { "method": "POST", "path": "/api/2.2/jobs/run-now", "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][1]", "job_id": [MY_JOB_ID], "job_parameters": { "env": "prod" diff --git a/acceptance/bundle/resources/job_runs/redeploy/script b/acceptance/bundle/resources/job_runs/redeploy/script index 87d035b9eee..1eed52ca460 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/script +++ b/acceptance/bundle/resources/job_runs/redeploy/script @@ -16,6 +16,8 @@ trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]' trace $CLI bundle deploy trace $CLI bundle summary -title "the config change deleted the previous run and triggered a second, different run" +title "the config change triggered a second, different run; the previous run is left in place" +# recreate leaves the old run untouched (no jobs/runs/delete) and fires a fresh +# run-now with the new parameters. trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/redeploy/test.toml b/acceptance/bundle/resources/job_runs/redeploy/test.toml deleted file mode 100644 index 4b94d8b58e9..00000000000 --- a/acceptance/bundle/resources/job_runs/redeploy/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true diff --git a/acceptance/bundle/resources/job_runs/rerun/databricks.yml b/acceptance/bundle/resources/job_runs/rerun/databricks.yml new file mode 100644 index 00000000000..2b4479a8e02 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/rerun/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: job-runs-rerun + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + # rerun_token feeds the computed idempotency_token and stays local. Bumping + # it forces a fresh run of otherwise-unchanged config. + rerun_token: v1 diff --git a/acceptance/bundle/resources/job_runs/rerun/out.test.toml b/acceptance/bundle/resources/job_runs/rerun/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/rerun/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/rerun/output.txt b/acceptance/bundle/resources/job_runs/rerun/output.txt new file mode 100644 index 00000000000..04e5114acc7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/rerun/output.txt @@ -0,0 +1,65 @@ + +=== deploy triggers the first run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-rerun/default/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +>>> read_id.py my_run +[MY_RUN_ID] + +=== redeploy with identical config is a no-op: the run is reused, no new run-now +>>> [CLI] bundle plan -o json +"skip" + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-rerun/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== bump rerun_token v1 -> v2: the config changed, so a fresh, different run is triggered +>>> update_file.py databricks.yml rerun_token: v1 rerun_token: v2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-rerun/default/files... +Deploying resources... +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +>>> read_id.py my_run +[MY_RUN_ID_2] + +=== run-now was issued exactly twice (initial + bump); rerun_token itself is never sent to the API +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][0]", + "job_id": [NUMID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[IDEMPOTENCY_TOKEN][1]", + "job_id": [NUMID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-rerun/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/rerun/script b/acceptance/bundle/resources/job_runs/rerun/script new file mode 100644 index 00000000000..ccaa24f1fbc --- /dev/null +++ b/acceptance/bundle/resources/job_runs/rerun/script @@ -0,0 +1,23 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "deploy triggers the first run" +trace $CLI bundle deploy +trace read_id.py my_run + +title "redeploy with identical config is a no-op: the run is reused, no new run-now" +# rerun_token is unchanged, so the token is unchanged and the planner skips the run. +trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].action // "none"' +trace $CLI bundle deploy + +title "bump rerun_token v1 -> v2: the config changed, so a fresh, different run is triggered" +trace update_file.py databricks.yml "rerun_token: v1" "rerun_token: v2" +trace $CLI bundle deploy +trace read_id.py my_run + +title "run-now was issued exactly twice (initial + bump); rerun_token itself is never sent to the API" +# rerun_token stays local: it only feeds the masked idempotency_token. +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/test.toml b/acceptance/bundle/resources/job_runs/test.toml new file mode 100644 index 00000000000..8bd40e528e0 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/test.toml @@ -0,0 +1,14 @@ +# job_runs is a direct-engine-only resource; the Terraform provider has no +# equivalent, so restrict the matrix to direct. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true + +# idempotency_token is a SHA-256 that differs every run (job_id comes from a +# time-based counter). Mask just that field so run-now goldens stay stable. +# Distinct numbers each token value, so a golden with more than one run-now shows +# whether the calls shared a token or rotated to a new one. The closing quote is +# outside the match, so the number lands inside the JSON string. +[[Repls]] +Old = '"idempotency_token": "[0-9a-f]{64}' +New = '"idempotency_token": "[IDEMPOTENCY_TOKEN]' +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/token_survives_delete/databricks.yml b/acceptance/bundle/resources/job_runs/token_survives_delete/databricks.yml new file mode 100644 index 00000000000..712ec6a971e --- /dev/null +++ b/acceptance/bundle/resources/job_runs/token_survives_delete/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: job-runs-token-survives-delete + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + job_parameters: + env: dev diff --git a/acceptance/bundle/resources/job_runs/token_survives_delete/out.test.toml b/acceptance/bundle/resources/job_runs/token_survives_delete/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/token_survives_delete/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/token_survives_delete/output.txt b/acceptance/bundle/resources/job_runs/token_survives_delete/output.txt new file mode 100644 index 00000000000..aeed5a089a3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/token_survives_delete/output.txt @@ -0,0 +1,73 @@ + +=== deploy with env=dev triggers the first run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-token-survives-delete/default/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +>>> read_id.py my_run +[MY_RUN_ID] + +=== flip env dev -> prod: the config changed, so a fresh, different run is triggered +>>> update_file.py databricks.yml env: dev env: prod + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-token-survives-delete/default/files... +Deploying resources... +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +>>> read_id.py my_run +[MY_RUN_ID_2] + +=== flip env prod -> dev: dedupes back to the original dev run +>>> update_file.py databricks.yml env: prod env: dev + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-token-survives-delete/default/files... +Deploying resources... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +=== destroy removes the resource from state but leaves the run in place +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-token-survives-delete/default + +Deleting files... +Destroy complete! + +=== redeploy after destroy triggers a fresh run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-token-survives-delete/default/files... +Deploying resources... +job run [MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[MY_RUN_ID_3]?o=[NUMID] +job run [MY_RUN_ID_3]: [TIMESTAMP] "my-job" TERMINATED SUCCESS +Updating deployment state... +Deployment complete! + +>>> read_id.py my_run +[MY_RUN_ID_3] + +=== no run was ever deleted +>>> print_requests.py --keep //jobs/runs/delete + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-token-survives-delete/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/job_runs/token_survives_delete/script b/acceptance/bundle/resources/job_runs/token_survives_delete/script new file mode 100644 index 00000000000..79c294c3796 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/token_survives_delete/script @@ -0,0 +1,36 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# A run is immutable history that recreate and destroy leave in place, so its +# idempotency token stays live and keeps deduping. + +title "deploy with env=dev triggers the first run" +trace $CLI bundle deploy +trace read_id.py my_run + +title "flip env dev -> prod: the config changed, so a fresh, different run is triggered" +trace update_file.py databricks.yml "env: dev" "env: prod" +trace $CLI bundle deploy +trace read_id.py my_run + +title "flip env prod -> dev: dedupes back to the original dev run" +# The dev->prod recreate left the original dev run in place, so its token is +# still live: flipping back re-issues run-now with that token and dedupes onto +# that run, reported below as [MY_RUN_ID] rather than a third run. +trace update_file.py databricks.yml "env: prod" "env: dev" +trace $CLI bundle deploy + +title "destroy removes the resource from state but leaves the run in place" +trace $CLI bundle destroy --auto-approve + +title "redeploy after destroy triggers a fresh run" +# The job is recreated with a new id, so the run config, and with it the token, +# differs from the destroyed deployment's. +trace $CLI bundle deploy +trace read_id.py my_run + +title "no run was ever deleted" +trace print_requests.py --keep //jobs/runs/delete diff --git a/acceptance/bundle/resources/job_runs/validate/databricks.yml b/acceptance/bundle/resources/job_runs/validate/databricks.yml new file mode 100644 index 00000000000..1703477f59d --- /dev/null +++ b/acceptance/bundle/resources/job_runs/validate/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: job-runs-validate + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + condition_task: + op: EQUAL_TO + left: "1" + right: "1" + + job_runs: + bad_token: + job_id: ${resources.jobs.my_job.id} + # Rejected: the token is computed from the run's configuration, and + # rerun_token is the supported way to force a new run. + idempotency_token: mine diff --git a/acceptance/bundle/resources/job_runs/validate/out.test.toml b/acceptance/bundle/resources/job_runs/validate/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/job_runs/validate/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/validate/output.txt b/acceptance/bundle/resources/job_runs/validate/output.txt new file mode 100644 index 00000000000..74d5da9451b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/validate/output.txt @@ -0,0 +1,14 @@ + +=== unsupported job_runs configuration is rejected before deploy +>>> [CLI] bundle validate +Error: idempotency_token is computed automatically and must not be set in bundle configuration; set `rerun_token` to force a new run + at resources.job_runs.bad_token.idempotency_token + in databricks.yml:20:26 + +Name: job-runs-validate +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-validate/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/validate/script b/acceptance/bundle/resources/job_runs/validate/script new file mode 100644 index 00000000000..024b2ebb1e5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/validate/script @@ -0,0 +1,2 @@ +title "unsupported job_runs configuration is rejected before deploy" +musterr trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/validate/test.toml b/acceptance/bundle/resources/job_runs/validate/test.toml new file mode 100644 index 00000000000..3af07cca9aa --- /dev/null +++ b/acceptance/bundle/resources/job_runs/validate/test.toml @@ -0,0 +1,2 @@ +# This test never reaches the API, so the recorded requests are noise. +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml index 9653ec13c31..bfd4edaa836 100644 --- a/acceptance/bundle/resources/job_runs/wait_cloud/test.toml +++ b/acceptance/bundle/resources/job_runs/wait_cloud/test.toml @@ -1,7 +1,3 @@ -# job_runs is a direct-engine-only resource; the Terraform provider has no -# equivalent, so restrict the matrix to direct. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - # The cloud counterpart of wait_output: it runs the job for real, which is what # invariant/configs/job_run.yml.tmpl no longer does on cloud. Serverless needs # Unity Catalog. diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 8db6ced76e1..38c31dad35b 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -14,12 +14,17 @@ import ( ) // JobRun is the bundle config for a triggered job run, described by the same -// fields as the Jobs RunNow request (embedded). It re-triggers only when its own -// config changes, not when the targeted job (stable job_id) changes. +// fields as the Jobs RunNow request (embedded). It re-triggers when its own +// config changes or when the workspace no longer has the tracked run, not when +// the targeted job (stable job_id) changes. type JobRun struct { BaseResource jobs.RunNow + // RerunToken feeds the computed idempotency_token: set a new value to re-run a + // configuration that already ran. + RerunToken string `json:"rerun_token,omitempty"` + // ResolvedJobID holds the run's job_id loaded from state, used only to build // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. diff --git a/bundle/config/validate/validate_job_runs.go b/bundle/config/validate/validate_job_runs.go new file mode 100644 index 00000000000..e7f9162057a --- /dev/null +++ b/bundle/config/validate/validate_job_runs.go @@ -0,0 +1,45 @@ +package validate + +import ( + "context" + "maps" + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +func ValidateJobRuns() bundle.ReadOnlyMutator { + return &validateJobRuns{} +} + +type validateJobRuns struct{ bundle.RO } + +func (v *validateJobRuns) Name() string { + return "validate:job_runs" +} + +func (v *validateJobRuns) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // Sorted so the reported order does not depend on map iteration order. + for _, name := range slices.Sorted(maps.Keys(b.Config.Resources.JobRuns)) { + jr := b.Config.Resources.JobRuns[name] + // An empty `job_runs.:` entry loads as a present key with a nil value. + if jr == nil || jr.IdempotencyToken == "" { + continue + } + // DoCreate computes the token so a retried deploy rejoins the run it already + // triggered; a user-set one would break that. + path := "resources.job_runs." + name + ".idempotency_token" + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "idempotency_token is computed automatically and must not be set in bundle configuration; set `rerun_token` to force a new run", + Paths: []dyn.Path{dyn.MustPathFromString(path)}, + Locations: b.Config.GetLocations(path), + }) + } + + return diags +} diff --git a/bundle/config/validate/validate_job_runs_test.go b/bundle/config/validate/validate_job_runs_test.go new file mode 100644 index 00000000000..ab8d4aa0f3b --- /dev/null +++ b/bundle/config/validate/validate_job_runs_test.go @@ -0,0 +1,48 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func jobRunsBundle(runs map[string]*resources.JobRun) *bundle.Bundle { + return &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{JobRuns: runs}, + }, + } +} + +func TestValidateJobRunsAllowsValidConfig(t *testing.T) { + b := jobRunsBundle(map[string]*resources.JobRun{ + // An empty `job_runs.:` entry unmarshals to a nil pointer, which the + // validator skips. + "empty": nil, + "minimal": {RunNow: jobs.RunNow{JobId: 1}}, + "rerun": {RunNow: jobs.RunNow{JobId: 2}, RerunToken: "v2"}, + }) + + require.Empty(t, ValidateJobRuns().Apply(t.Context(), b)) +} + +func TestValidateJobRunsRejectsIdempotencyToken(t *testing.T) { + b := jobRunsBundle(map[string]*resources.JobRun{ + "b_run": {RunNow: jobs.RunNow{JobId: 1, IdempotencyToken: "x"}}, + "a_run": {RunNow: jobs.RunNow{JobId: 2, IdempotencyToken: "y"}}, + "ok": {RunNow: jobs.RunNow{JobId: 3}}, + }) + + diags := ValidateJobRuns().Apply(t.Context(), b) + + require.Len(t, diags, 2) + // Sorted by name, so a_run comes before b_run regardless of map order. + assert.Equal(t, "resources.job_runs.a_run.idempotency_token", diags[0].Paths[0].String()) + assert.Equal(t, "resources.job_runs.b_run.idempotency_token", diags[1].Paths[0].String()) + assert.Contains(t, diags[0].Summary, "idempotency_token is computed automatically") +} diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index b3c46036c53..2b513f808e0 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -51,6 +51,15 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, } func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error { + // A leftover state id means the tracked remote is gone and the plan turned + // recreate into create; Recreate drops the state entry before calling this, so + // on that path the prior id is empty. See dresources.CreateIdentity. + ctx = dresources.WithCreateIdentity(ctx, dresources.CreateIdentity{ + Deployment: d.DeploymentRoot, + ResourceKey: d.ResourceKey, + PriorID: db.GetResourceID(d.ResourceKey), + }) + var newID string var remoteState any _, err := retryWith(ctx, func(err error) bool { diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c4178c4e601..32f2489e2c6 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -15,7 +15,9 @@ import ( "github.com/databricks/databricks-sdk-go" ) -func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) { +// Apply executes plan. deploymentRoot is the bundle's workspace root path, which +// scopes the create-time keys resources derive (see dresources.CreateIdentity). +func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan, deploymentRoot string) { if plan == nil { panic("Planning is not done") } @@ -71,9 +73,10 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } d := &DeploymentUnit{ - ResourceKey: resourceKey, - Adapter: adapter, - DependsOn: entry.DependsOn, + ResourceKey: resourceKey, + DeploymentRoot: deploymentRoot, + Adapter: adapter, + DependsOn: entry.DependsOn, } if action == deployplan.Delete { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f00ec21ed93..815e717f5f6 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -966,7 +966,11 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W newState, err := adapter.PrepareState(inputConfig) require.NoError(t, err, "PrepareState failed") - ctx := t.Context() + // Stand in for the framework, which always attaches a create identity. + ctx := WithCreateIdentity(t.Context(), CreateIdentity{ + Deployment: "/Workspace/Users/user@example.com/.bundle/test/default", + ResourceKey: "resources." + group + ".test", + }) // initial DoRead() cannot find the resource remote, err := adapter.DoRead(ctx, "1234") @@ -1079,7 +1083,9 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) } - deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") + // job_runs, like permissions/grants, has a noop DoDelete: a run is immutable + // history left in place, so DoRead still finds it after delete. + deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") || group == "job_runs" // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. // A GET on the DELETING app returns the app, not 404 -- the testserver diff --git a/bundle/direct/dresources/identity.go b/bundle/direct/dresources/identity.go new file mode 100644 index 00000000000..95a0680b2bd --- /dev/null +++ b/bundle/direct/dresources/identity.go @@ -0,0 +1,38 @@ +package dresources + +import "context" + +// CreateIdentity identifies the resource being created, for resources that +// derive a stable create-time key from it (e.g. a run-now idempotency token). +// +// Every part is reconstructible from config and state, so a deploy that crashes +// mid-create derives the same identity on the next attempt. +type CreateIdentity struct { + // Deployment is the bundle's workspace root path. It keeps deployments that + // share a workspace (different targets, different users) apart. + Deployment string + + // ResourceKey is the deployment-local key, e.g. "resources.job_runs.nightly". + ResourceKey string + + // PriorID is the id of a resource whose remote copy is gone, set only when + // re-creating it. Folding it in keeps the new key clear of the old one, which + // the backend may still reserve (e.g. a deleted run's idempotency token). + PriorID string +} + +type createIdentityKey struct{} + +// WithCreateIdentity records the identity of the resource being created. Set by +// the framework in DeploymentUnit.Create; see CreateIdentity. +func WithCreateIdentity(ctx context.Context, id CreateIdentity) context.Context { + return context.WithValue(ctx, createIdentityKey{}, id) +} + +// GetCreateIdentity returns the identity set by WithCreateIdentity. ok is false +// only when the caller bypassed DeploymentUnit.Create, which a resource that +// needs the identity should report as an error rather than paper over. +func GetCreateIdentity(ctx context.Context) (CreateIdentity, bool) { + id, ok := ctx.Value(createIdentityKey{}).(CreateIdentity) + return id, ok +} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 2ae6219f4bc..f3fb57f9c2c 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,6 +3,9 @@ package dresources import ( "cmp" "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" "fmt" "strconv" @@ -26,6 +29,10 @@ const jobRunTimeout = 24 * time.Hour // JobRunState is what we persist for a triggered run: the RunNow request. type JobRunState struct { jobs.RunNow + + // RerunToken folds into the idempotency token, so changing it recreates the + // run. Bundle-only, never sent to the API. + RerunToken string `json:"rerun_token,omitempty"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { @@ -37,10 +44,14 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { } // JobRunRemote embeds RunNow so every StateType path is a valid RemoteType path -// (see TestRemoteSuperset), plus the run's output-only fields for a faithful view. +// (see TestRemoteSuperset), plus the run's output-only fields. type JobRunRemote struct { jobs.RunNow + // RerunToken is bundle-only. GetRun never reports it, so it stays empty here + // and root ignore_remote_changes hides the drift against state. + RerunToken string `json:"rerun_token,omitempty"` + RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -70,7 +81,8 @@ func (*ResourceJobRun) New(client *databricks.WorkspaceClient) *ResourceJobRun { func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { return &JobRunState{ - RunNow: input.RunNow, + RunNow: input.RunNow, + RerunToken: input.RerunToken, } } @@ -101,14 +113,15 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { PythonParams: overriding.PythonParams, SparkSubmitParams: overriding.SparkSubmitParams, SqlParams: overriding.SqlParams, - // Request-only fields GetRun never reports; listed so exhaustruct - // flags any new SDK field. + // Request-only fields, listed so exhaustruct flags any new SDK field. IdempotencyToken: "", Only: nil, PerformanceTarget: "", Queue: nil, ForceSendFields: nil, }, + // Bundle-only; see its doc comment. + RerunToken: "", RunId: run.RunId, RunName: run.RunName, State: run.State, @@ -118,14 +131,14 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { } // DoRead returns the run as GetRun reports it; a 404 lets the planner -// re-trigger. Root ignore_remote_changes suppresses all remote drift, so a run -// is recreated only on a local config change. +// re-trigger. Root ignore_remote_changes suppresses remote drift, so a local +// config change is what recreates a run. func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, error) { runID, err := parseRunID(id) if err != nil { return nil, err } - // var + field set (not a literal) avoids listing GetRunRequest's other fields. + // Assigned through a var to satisfy exhaustruct without listing every field. var req jobs.GetRunRequest req.RunId = runID run, err := r.client.Jobs.GetRun(ctx, req) @@ -135,19 +148,39 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, return makeJobRunRemote(run), nil } -// RemapState extracts the embedded RunNow as the state used for diffing. +// RemapState maps remote into the state shape for diffing. RerunToken is empty +// in remote; root ignore_remote_changes hides that drift. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { - return &JobRunState{RunNow: remote.RunNow} + return &JobRunState{ + RunNow: remote.RunNow, + RerunToken: remote.RerunToken, + } } func (r *ResourceJobRun) DoCreate(ctx context.Context, config *JobRunState) (string, *JobRunRemote, error) { - // RunNow returns only the new run id, so we return a nil remote and let the - // framework read it back via DoRead. - wait, err := r.client.Jobs.RunNow(ctx, config.RunNow) + // The framework always attaches an identity on create, so its absence is a + // wiring bug rather than a user error. + identity, ok := GetCreateIdentity(ctx) + if !ok { + return "", nil, errors.New("internal error: job_run created without a create identity") + } + token, err := idempotencyToken(identity, config) if err != nil { return "", nil, err } - return strconv.FormatInt(wait.RunId, 10), nil, nil + + // Copy so the token reaches the API and stays out of state. The token makes a + // deploy that fired run-now and then crashed before recording the id rejoin + // that same run on retry, instead of triggering a second one. + req := config.RunNow + req.IdempotencyToken = token + triggered, err := r.client.Jobs.RunNow(ctx, req) + if err != nil { + return "", nil, fmt.Errorf("triggering a run of job %d: %w", req.JobId, err) + } + // RunNow returns only the run id, so return a nil remote and let the framework + // read it back via DoRead. + return strconv.FormatInt(triggered.RunId, 10), nil, nil } // WaitAfterCreate blocks until the run finishes, so a resource referencing its @@ -310,41 +343,15 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { } } -// DoUpdate is intentionally not implemented: a run can't be modified in place, -// so any change recreates it (delete + a fresh RunNow). +// DoUpdate is intentionally not implemented: a run is immutable, so any config +// change recreates the resource as a fresh RunNow. -// DoDelete deletes the run via jobs/runs/delete, on both destroy and the -// recreate path. The API rejects a still-active run, which an interrupted wait -// leaves behind, so cancel it first. -func (r *ResourceJobRun) DoDelete(ctx context.Context, id string, _ *JobRunState) error { - runID, err := parseRunID(id) - if err != nil { - return err - } - remote, err := r.DoRead(ctx, id) - if err != nil { - return err - } - if !runIsTerminal(remote.State.LifeCycleState) { - err = r.cancelRun(ctx, runID) - if err != nil { - return err - } - } - return r.client.Jobs.DeleteRunByRunId(ctx, runID) -} - -// cancelRun cancels a run and waits for it to settle. Cancellation is -// asynchronous, so a delete issued right after would still be rejected. -func (r *ResourceJobRun) cancelRun(ctx context.Context, runID int64) error { - waiter, err := r.client.Jobs.CancelRun(ctx, jobs.CancelRun{RunId: runID}) - if err != nil { - return fmt.Errorf("cancelling run %d before deleting it: %w", runID, err) - } - _, err = waiter.Get() - if err != nil { - return fmt.Errorf("waiting for run %d to be cancelled: %w", runID, err) - } +// DoDelete is a noop: a run is immutable history, so destroy and recreate leave +// it in place. That also keeps its idempotency_token usable, which the Jobs API +// rejects once the run it triggered is deleted. There is nothing to cancel +// either: a run whose wait was interrupted keeps going, and the error that ended +// the wait links its run page. +func (*ResourceJobRun) DoDelete(_ context.Context, _ string, _ *JobRunState) error { return nil } @@ -355,3 +362,29 @@ func parseRunID(id string) (int64, error) { } return result, nil } + +// idempotencyToken hashes the create identity and the run config into a hex +// SHA-256 (64 chars, the Jobs API maximum). It is stable, so a retried create +// dedupes onto the same run; it changes with the config, so an edited run (or a +// bumped rerun_token) starts a new one; and the identity keeps identical configs +// apart across resource keys and deployments. +func idempotencyToken(identity CreateIdentity, state *JobRunState) (string, error) { + toHash := *state + // The token itself is not part of the identity being hashed. + toHash.IdempotencyToken = "" + canonical, err := json.Marshal(toHash) + if err != nil { + return "", err + } + + h := sha256.New() + // NUL separators keep adjacent parts from colliding on a shifted boundary. + h.Write([]byte(identity.Deployment)) + h.Write([]byte{0}) + h.Write([]byte(identity.ResourceKey)) + h.Write([]byte{0}) + h.Write([]byte(identity.PriorID)) + h.Write([]byte{0}) + h.Write(canonical) + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 50580104437..d3a5b42b259 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -2,6 +2,7 @@ package dresources import ( "context" + "encoding/json" "sync/atomic" "testing" "time" @@ -13,6 +14,89 @@ import ( "github.com/stretchr/testify/require" ) +// testIdentity is what the framework attaches on a first create. +var testIdentity = CreateIdentity{ + Deployment: "/Workspace/Users/me/.bundle/mybundle/default", + ResourceKey: "resources.job_runs.nightly", +} + +func tokenFor(t *testing.T, identity CreateIdentity, state *JobRunState) string { + t.Helper() + token, err := idempotencyToken(identity, state) + require.NoError(t, err) + return token +} + +func TestIdempotencyTokenIsStableHex(t *testing.T) { + run := jobs.RunNow{JobId: 123, JobParameters: map[string]string{"env": "prod"}} + + got := tokenFor(t, testIdentity, &JobRunState{RunNow: run}) + + // hex SHA-256 is always 64 lowercase hex chars (the Jobs API maximum). + assert.Regexp(t, "^[0-9a-f]{64}$", got) + + // Deterministic: the same config yields the same token, so a retry dedupes. + assert.Equal(t, got, tokenFor(t, testIdentity, &JobRunState{RunNow: run})) +} + +func TestIdempotencyTokenIgnoresPresetToken(t *testing.T) { + base := tokenFor(t, testIdentity, &JobRunState{RunNow: jobs.RunNow{JobId: 123}}) + preset := tokenFor(t, testIdentity, &JobRunState{RunNow: jobs.RunNow{JobId: 123, IdempotencyToken: "user-supplied"}}) + + // The token is cleared before hashing, so a preset value cannot change it. + assert.Equal(t, base, preset) +} + +func TestIdempotencyTokenChangesWithConfig(t *testing.T) { + dev := tokenFor(t, testIdentity, &JobRunState{RunNow: jobs.RunNow{JobId: 123, JobParameters: map[string]string{"env": "dev"}}}) + prod := tokenFor(t, testIdentity, &JobRunState{RunNow: jobs.RunNow{JobId: 123, JobParameters: map[string]string{"env": "prod"}}}) + otherJob := tokenFor(t, testIdentity, &JobRunState{RunNow: jobs.RunNow{JobId: 456}}) + + assert.NotEqual(t, dev, prod) // different params --> different token + assert.NotEqual(t, dev, otherJob) // different job_id --> different token +} + +func TestIdempotencyTokenChangesWithRerunToken(t *testing.T) { + run := jobs.RunNow{JobId: 123} + + base := tokenFor(t, testIdentity, &JobRunState{RunNow: run}) + bumped := tokenFor(t, testIdentity, &JobRunState{RunNow: run, RerunToken: "v2"}) + bumpedAgain := tokenFor(t, testIdentity, &JobRunState{RunNow: run, RerunToken: "v2"}) + + assert.NotEqual(t, base, bumped) // changing rerun_token forces a new run + assert.Equal(t, bumped, bumpedAgain) // the same rerun_token value stays stable +} + +func TestIdempotencyTokenChangesWithIdentity(t *testing.T) { + state := &JobRunState{RunNow: jobs.RunNow{JobId: 123}} + + base := tokenFor(t, testIdentity, state) + + otherKey := testIdentity + otherKey.ResourceKey = "resources.job_runs.other" + otherDeployment := testIdentity + otherDeployment.Deployment = "/Workspace/Users/me/.bundle/mybundle/prod" + + // The identity keeps identical config apart across resource keys and across + // deployments sharing a workspace. + assert.NotEqual(t, base, tokenFor(t, otherKey, state)) + assert.NotEqual(t, base, tokenFor(t, otherDeployment, state)) +} + +func TestIdempotencyTokenRotatesWithPriorID(t *testing.T) { + state := &JobRunState{RunNow: jobs.RunNow{JobId: 123}} + recreate := testIdentity + recreate.PriorID = "555" + + base := tokenFor(t, testIdentity, state) + rotated := tokenFor(t, recreate, state) + + // Re-creating a vanished run rotates the token off the tombstoned one... + assert.NotEqual(t, base, rotated) + // ...but stays deterministic so a retry dedupes onto the fresh run. + assert.Equal(t, rotated, tokenFor(t, recreate, state)) +} + // jobRunClientFor returns a client talking to server. Call it after the test // registers its own handlers: first registration wins, so the defaults added here // only fill the gaps. @@ -28,6 +112,77 @@ func jobRunClientFor(t *testing.T, server *testserver.Server) *databricks.Worksp return client } +// fakeRunNow models the run-now deduplication the Jobs API does: a token already +// spent on a run returns that same run instead of starting a new one. +type fakeRunNow struct { + // spent maps an already-used idempotency token to the run it triggered. It is + // fixed at construction, so the handler reads it without locking. + spent map[string]int64 + + // nextRunID is the id given to the next run this fake starts. + nextRunID atomic.Int64 + + // calls counts run-now requests, so a test can tell a dedupe from a second + // trigger. + calls atomic.Int64 +} + +func (f *fakeRunNow) client(t *testing.T) *databricks.WorkspaceClient { + t.Helper() + server := testserver.New(t) + server.Handle("POST", "/api/2.2/jobs/run-now", func(req testserver.Request) any { + var body jobs.RunNow + require.NoError(t, json.Unmarshal(req.Body, &body)) + f.calls.Add(1) + + if runID, ok := f.spent[body.IdempotencyToken]; ok { + return jobs.RunNowResponse{RunId: runID} + } + return jobs.RunNowResponse{RunId: f.nextRunID.Add(1)} + }) + return jobRunClientFor(t, server) +} + +func createTestRun(t *testing.T, fake *fakeRunNow, state *JobRunState) (string, error) { + t.Helper() + ctx := WithCreateIdentity(t.Context(), testIdentity) + id, _, err := (&ResourceJobRun{}).New(fake.client(t)).DoCreate(ctx, state) + return id, err +} + +func TestJobRunCreateRequiresIdentity(t *testing.T) { + r := (&ResourceJobRun{}).New((&fakeRunNow{}).client(t)) + + // The framework always sets one, so its absence is a wiring bug and errors. + _, _, err := r.DoCreate(t.Context(), &JobRunState{RunNow: jobs.RunNow{JobId: 123}}) + require.ErrorContains(t, err, "without a create identity") +} + +func TestJobRunCreateStartsAFreshRun(t *testing.T) { + fake := &fakeRunNow{} + + id, err := createTestRun(t, fake, &JobRunState{RunNow: jobs.RunNow{JobId: 456}}) + + // No token was spent, so run-now starts a run. + require.NoError(t, err) + assert.Equal(t, "1", id) + assert.Equal(t, int64(1), fake.calls.Load()) +} + +func TestJobRunCreateRejoinsTheRunItAlreadyTriggered(t *testing.T) { + state := &JobRunState{RunNow: jobs.RunNow{JobId: 456}} + const triggered = int64(111) + fake := &fakeRunNow{spent: map[string]int64{tokenFor(t, testIdentity, state): triggered}} + + id, err := createTestRun(t, fake, state) + + // A deploy that crashed before recording the id rejoins its own run on retry + // instead of triggering a second one. + require.NoError(t, err) + assert.Equal(t, "111", id) + assert.Equal(t, int64(1), fake.calls.Load()) +} + // jobRunServer returns a client whose runs/get is the given handler, so a wait can // be driven without a real run. func jobRunServer(t *testing.T, getRun testserver.HandlerFunc) *databricks.WorkspaceClient { @@ -255,68 +410,19 @@ func TestJobRunWaitPollsUntilTerminal(t *testing.T) { assert.Equal(t, int32(3), gets.Load(), "expected the wait to poll past both RUNNING reads") } -// jobRunDeletion records what the fake workspace saw while a run was deleted. -type jobRunDeletion struct { - cancelled atomic.Bool - settled atomic.Bool - settledAtDelete atomic.Bool -} - -// jobRunDeleteClient returns a client for a run in the given state, whose cancel -// settles one poll late the way the API's asynchronous cancellation does. -func jobRunDeleteClient(t *testing.T, state *jobs.RunState) (*databricks.WorkspaceClient, *jobRunDeletion) { - t.Helper() - var deletion jobRunDeletion - cancelled := &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateCanceled, - } - +// The token is only worth deriving if it stays usable, so a run must survive +// destroy and recreate whatever state it is in: deleting it reserves its token +// against any later run, and cancelling it settles it on CANCELED, which no +// deploy that dedupes onto it could ever complete. +func TestJobRunDeleteTouchesNothing(t *testing.T) { server := testserver.New(t) - server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - current := state - switch { - case deletion.settled.Load(): - current = cancelled - case deletion.cancelled.Load(): - // Report the run's old state once more, then settle on the next poll. - deletion.settled.Store(true) - } - return jobs.Run{RunId: 123, JobId: 456, State: current} - }) - server.Handle("POST", "/api/2.2/jobs/runs/cancel", func(req testserver.Request) any { - deletion.cancelled.Store(true) - return testserver.Response{} - }) - server.Handle("POST", "/api/2.2/jobs/runs/delete", func(req testserver.Request) any { - deletion.settledAtDelete.Store(deletion.settled.Load()) - return testserver.Response{} - }) - return jobRunClientFor(t, server), &deletion -} - -func deleteTestRun(t *testing.T, client *databricks.WorkspaceClient) error { - t.Helper() - return (&ResourceJobRun{}).New(client).DoDelete(t.Context(), "123", &JobRunState{}) -} - -func TestJobRunDeleteCancelsUnfinishedRun(t *testing.T) { - // An interrupted wait leaves the run going, and jobs/runs/delete rejects it. - client, deletion := jobRunDeleteClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) - - require.NoError(t, deleteTestRun(t, client)) - - assert.True(t, deletion.cancelled.Load(), "expected the run to be cancelled") - assert.True(t, deletion.settledAtDelete.Load(), "expected the delete to wait for the cancellation to settle") -} - -func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { - client, deletion := jobRunDeleteClient(t, &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }) - - require.NoError(t, deleteTestRun(t, client)) + for _, path := range []string{"/api/2.2/jobs/runs/delete", "/api/2.2/jobs/runs/cancel"} { + server.Handle("POST", path, func(req testserver.Request) any { + assert.Fail(t, "DoDelete called "+path) + return testserver.Response{} + }) + } - assert.False(t, deletion.cancelled.Load(), "a run that already finished has nothing to cancel") + r := (&ResourceJobRun{}).New(jobRunClientFor(t, server)) + require.NoError(t, r.DoDelete(t.Context(), "123", &JobRunState{})) } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 48a9c5a2ff7..92f48e1ca4d 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -21,6 +21,9 @@ type DeploymentUnit struct { // Resource identifier: "resources.jobs.foo" or "resources.jobs.foo.permissions" ResourceKey string + // Workspace root path of this deployment; see dresources.CreateIdentity. + DeploymentRoot string + // Implementation for this resource; all deployments from the same group share the adapter Adapter *dresources.Adapter diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 5d47c44e289..36135b17210 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -969,7 +969,9 @@ resources: "description": |- The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment. - The deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs..state.result_state}`. A run that did not succeed is still recorded, so an unchanged configuration is not run again. + The deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs..state.result_state}`. A run that did not succeed is still recorded, so change `rerun_token` to try the same configuration again. + + A run is triggered once per configuration and is then tracked by its id. If the workspace no longer has that run, which happens once it ages out of run history after about 60 days, the next deployment triggers a fresh run of the same configuration. Avoid `job_runs` for a job that must run exactly once ever. "$fields": "lifecycle": "description": |- @@ -977,6 +979,9 @@ resources: "python_named_params": "description": |- PLACEHOLDER + "rerun_token": + "description": |- + An arbitrary token that forces a new run when its value changes. Leave it unset for normal deploys; change it to re-run a configuration that already ran. Each distinct value maps to one run for about 60 days, so restoring a previous value within that window rejoins the run it originally triggered instead of starting a new one. "jobs": "description": |- The job definitions for the bundle, where each key is the name of the job. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f65e50a940e..6d8886e54cc 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -89,7 +89,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st err error ) if stateEngine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, b.Config.Workspace.RootPath) state, err = b.DeploymentBundle.StateDB.Finalize(ctx) // Capture the finalized state for deploy telemetry. It carries each // resource's state-size in bytes (from the WAL replay Finalize just diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2496c7033ad..dd5db399573 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -82,7 +82,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { if engine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, b.Config.Workspace.RootPath) } else { // Core destructive mutators for destroy. These require informed user consent. bundle.ApplyContext(ctx, b, terraform.Apply()) diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..6f9e879b5f9 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,6 +177,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Reads (typed): b.Config.Resources.JobRuns + // Rejects a user-set idempotency_token, which the CLI computes itself. + validate.ValidateJobRuns(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index bf6039da58d..50b90d1db9a 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1232,6 +1232,10 @@ "description": "The queue settings of the run.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.QueueSettings" }, + "rerun_token": { + "description": "An arbitrary token that forces a new run when its value changes. Leave it unset for normal deploys; change it to re-run a configuration that already ran. Each distinct value maps to one run for about 60 days, so restoring a previous value within that window rejoins the run it originally triggered instead of starting a new one.", + "$ref": "#/$defs/string" + }, "spark_submit_params": { "description": "[Private Preview] A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\n⚠ **Deprecation note** Use [job parameters](https://docs.databricks.com/jobs/job-parameters.html#job-parameter-pushdown) to pass information down to tasks.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", "$ref": "#/$defs/slice/string", @@ -3228,7 +3232,7 @@ "markdownDescription": "The instance pool definitions for the bundle, where each key is the name of the instance pool. See [instance_pools](https://docs.databricks.com/dev-tools/bundles/resources.html#instance_pools)." }, "job_runs": { - "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.\n\nThe deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs.\u003cname\u003e.state.result_state}`. A run that did not succeed is still recorded, so an unchanged configuration is not run again.", + "description": "The job run definitions for the bundle, where each key is the name of the job run. Each job run triggers a run of an existing job as part of bundle deployment.\n\nThe deployment waits for the run to finish and fails if it does not succeed, so other resources can reference the run's outcome, for example `${resources.job_runs.\u003cname\u003e.state.result_state}`. A run that did not succeed is still recorded, so change `rerun_token` to try the same configuration again.\n\nA run is triggered once per configuration and is then tracked by its id. If the workspace no longer has that run, which happens once it ages out of run history after about 60 days, the next deployment triggers a fresh run of the same configuration. Avoid `job_runs` for a job that must run exactly once ever.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.JobRun" }, "jobs": { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 8d6e8ee0dd3..cf7d928a9ae 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -169,6 +169,7 @@ type FakeWorkspace struct { Jobs map[int64]jobs.Job JobRuns map[int64]jobs.Run JobRunOutputs map[int64]jobs.RunOutput + JobRunsByToken map[string]int64 Pipelines map[string]pipelines.GetPipelineResponse PipelineUpdates map[string]bool Monitors map[string]catalog.MonitorInfo @@ -335,6 +336,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Jobs: map[int64]jobs.Job{}, JobRuns: map[int64]jobs.Run{}, JobRunOutputs: map[int64]jobs.RunOutput{}, + JobRunsByToken: map[string]int64{}, Grants: map[string][]catalog.PrivilegeAssignment{}, Pipelines: map[string]pipelines.GetPipelineResponse{}, PipelineUpdates: map[string]bool{}, diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index e8d29877e98..59e7ee591c3 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -354,6 +354,20 @@ func (s *FakeWorkspace) JobsRunNow(req Request) Response { return Response{StatusCode: 404} } + // run-now is idempotent: the same token returns the existing run, and the + // token stays reserved once that run is deleted, so reuse errors. Only + // non-empty tokens are recorded, so a request without one falls through. + // https://docs.databricks.com/api/workspace/jobs/runnow + if existing, ok := s.JobRunsByToken[request.IdempotencyToken]; ok { + if _, alive := s.JobRuns[existing]; alive { + return Response{Body: jobs.RunNowResponse{RunId: existing}} + } + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("idempotency_token %q was used for run %d, which has been deleted", request.IdempotencyToken, existing), + } + } + runId := nextID() runName := "run-name" if job.Settings != nil && job.Settings.Name != "" { @@ -420,6 +434,10 @@ func (s *FakeWorkspace) JobsRunNow(req Request) Response { OverridingParameters: runOverridingParameters(request), } + if request.IdempotencyToken != "" { + s.JobRunsByToken[request.IdempotencyToken] = runId + } + return Response{Body: jobs.RunNowResponse{RunId: runId}} } @@ -922,6 +940,9 @@ func (s *FakeWorkspace) JobsDeleteRun(req Request) Response { Body: fmt.Sprintf("request parsing error: %s", err), } } + + // The Jobs API keeps an idempotency_token reserved after its run is deleted, + // so JobRunsByToken keeps its entry as a tombstone (see JobsRunNow). return MapDelete(s, s.JobRuns, request.RunId) } diff --git a/libs/testserver/jobs_test.go b/libs/testserver/jobs_test.go index 27898c3664f..83d1c5595f6 100644 --- a/libs/testserver/jobs_test.go +++ b/libs/testserver/jobs_test.go @@ -104,6 +104,36 @@ func runNow(t *testing.T, workspace *FakeWorkspace, request jobs.RunNow) Respons return workspace.JobsRunNow(Request{Body: body}) } +// An idempotency_token stays reserved after its run is deleted, so a later reuse +// errors. This is why job_runs' DoDelete is a noop: keeping the run keeps its +// deterministic token usable for re-running the same config. +func TestJobsRunNow_IdempotencyTokenTombstonedAfterDelete(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + jobID := createJob(t, workspace) + + const token = "stable-token" + + first := runNow(t, workspace, jobs.RunNow{JobId: jobID, IdempotencyToken: token}) + require.Equal(t, 0, first.StatusCode) + runID := first.Body.(jobs.RunNowResponse).RunId + require.NotZero(t, runID) + + // A retry with the same token dedupes onto the existing run. + second := runNow(t, workspace, jobs.RunNow{JobId: jobID, IdempotencyToken: token}) + require.Equal(t, 0, second.StatusCode) + assert.Equal(t, runID, second.Body.(jobs.RunNowResponse).RunId) + + body, err := json.Marshal(jobs.DeleteRun{RunId: runID}) + require.NoError(t, err) + del := workspace.JobsDeleteRun(Request{Body: body}) + require.Equal(t, 0, del.StatusCode) + + // The token stays reserved after the delete, so reuse errors. + third := runNow(t, workspace, jobs.RunNow{JobId: jobID, IdempotencyToken: token}) + assert.Equal(t, 400, third.StatusCode) + assert.Contains(t, third.Body.(string), "has been deleted") +} + func terminatedTask(taskKey string, result jobs.RunResultState) jobs.RunTask { return jobs.RunTask{ TaskKey: taskKey,