Skip to content

Commit 668ab10

Browse files
authored
feat(storage): read active work unit resources in Spanner and Postgres (#1597)
* feat(storage): read active work unit resources in Spanner and Postgres Implement WorkUnitStore.FetchActiveWorkUnitResources for both backends: one query over active work units (IsPendingOrRunningJob) in the requested queues joined to Experiments.Creator, returning each work unit's creator, executable and state. Spanner uses a single-use read; Postgres a single statement on the repository's DBTX. GetRunningAndPendingJobs.user_allocations is now populated. Tests share storagetest.ActiveResourceSpecs and derive the expected rows from them (every non-terminal spec in the queue), using postgrestest.SeedExperimentWithCreator and a shared Spanner insert helper that the existing pagination tests reuse. Bug: 582253355 TAG=agy CONV=4ed50232-a6f2-4c76-aa6f-ef1c3833a550 * docs(rod): describe generic accelerator attribution precisely Bug: 582253355 TAG=agy CONV=4ed50232-a6f2-4c76-aa6f-ef1c3833a550 * docs(rod): document user_allocations next to the ROD job data notes Describe user_allocations as its own short list in section 5 instead of an 'exception' paragraph, and leave the RPC table unchanged. Bug: 582253355 TAG=agy CONV=4ed50232-a6f2-4c76-aa6f-ef1c3833a550
1 parent da96db1 commit 668ab10

11 files changed

Lines changed: 333 additions & 40 deletions

‎docs/flows/metrics_and_observability.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,18 @@ translate into API server load proportional to the number of viewers. Watch once
286286
Unlike the Reconciler, ROD persists nothing — its cache is derived state, rebuilt from a full
287287
re-list on restart.
288288

289+
`GetRunningAndPendingJobs` also returns `user_allocations` on every page: the requested units (not
290+
measured utilization) of each resource type, summed per experiment creator across *all* active jobs
291+
in the queue, not just the current page.
292+
293+
- Storage returns each active work unit's creator, executable and state; ROD canonicalizes the
294+
resource types (CPU in cores, memory in GiB, GPU devices, TPU chips) and sums them.
295+
- `RUNNING` and `STARTING` count as running (admitted by Kueue); `SCHEDULING` and `INITIALIZING`
296+
count as pending. `INITIALIZING` precedes admission, even though the jobs table orders it with
297+
running jobs.
298+
- A generic accelerator request (e.g. `nvidia.com/gpu`) is attributed to a model only when the
299+
queue's flavors resolve that accelerator to exactly one model.
300+
289301
---
290302

291303
## 6. Related Reading

‎internal/storage/postgres/testing/testdb.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -336,12 +336,18 @@ func applySchema(ctx context.Context, t *testing.T, pool *pgxpool.Pool, s schema
336336

337337
// SeedExperiment inserts a minimal Experiment row for foreign-key-dependent repository tests.
338338
func SeedExperiment(ctx context.Context, t testing.TB, pool *pgxpool.Pool, xid int64) {
339+
t.Helper()
340+
SeedExperimentWithCreator(ctx, t, pool, xid, "user")
341+
}
342+
343+
// SeedExperimentWithCreator inserts a minimal Experiments row created by creator.
344+
func SeedExperimentWithCreator(ctx context.Context, t testing.TB, pool *pgxpool.Pool, xid int64, creator string) {
339345
t.Helper()
340346
const seedExpSQL = `INSERT INTO Experiments (
341347
Xid, Title, Creator, Priority, Urls, Tags, Notes, Resources, Status, ActiveState, LaunchState, CreateTime, UpdateTime
342-
) VALUES ($1, 'test', 'user', 0, '{}', '{}', '', '{}', '\x', 0, 0, now(), now())
348+
) VALUES ($1, 'test', $2, 0, '{}', '{}', '', '{}', '\x', 0, 0, now(), now())
343349
ON CONFLICT (Xid) DO NOTHING`
344-
if _, err := pool.Exec(ctx, seedExpSQL, xid); err != nil {
350+
if _, err := pool.Exec(ctx, seedExpSQL, xid, creator); err != nil {
345351
t.Fatalf("seed Experiments(%d) failed: %v", xid, err)
346352
}
347353
}

‎internal/storage/postgres/workunit/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ go_library(
4949
go_test(
5050
name = "workunit_test",
5151
srcs = [
52+
"active_work_unit_resources_test.go",
5253
"queries_test.go",
5354
"repository_test.go",
5455
],
@@ -60,6 +61,7 @@ go_test(
6061
"//internal/storage",
6162
"//internal/storage/postgres/pgdb",
6263
"//internal/storage/postgres/testing",
64+
"//internal/storage/testing",
6365
"//internal/validator",
6466
"@com_github_google_go_cmp//cmp",
6567
"@com_github_jackc_pgx_v5//:pgx",

‎internal/storage/postgres/workunit/active_work_unit_resources.go‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,51 @@ package workunit
1616

1717
import (
1818
"context"
19+
"fmt"
1920

21+
essmsgpb "xmanager_cloud/api/protos/experiment"
22+
"xmanager_cloud/internal/safesql"
2023
"xmanager_cloud/internal/storage"
24+
postgresdata "xmanager_cloud/internal/storage/postgres/data"
25+
"xmanager_cloud/internal/storage/postgres/pgdb"
2126
)
2227

23-
// FetchActiveWorkUnitResources returns no rows until active work units are queried.
24-
// TODO(#1597): query the creator, executable and state of active work units in queueNames.
28+
type activeWorkUnitResourcesRow struct {
29+
Executable []byte `db:"executable"`
30+
ExecutionState int64 `db:"executionstate"`
31+
Creator string `db:"creator"`
32+
}
33+
34+
var fetchActiveWorkUnitResourcesSQL = safesql.New(`SELECT wu.Executable, wu.ExecutionState, e.Creator
35+
FROM WorkUnits wu
36+
JOIN Experiments e ON wu.Xid = e.Xid
37+
WHERE wu.QueueName = ANY($1) AND wu.IsPendingOrRunningJob IS NOT NULL`)
38+
39+
// FetchActiveWorkUnitResources reads the creator, executable and state of every active work unit
40+
// in queueNames in a single statement.
2541
func (r *Repository) FetchActiveWorkUnitResources(ctx context.Context, queueNames []string) ([]storage.ActiveWorkUnitResources, error) {
26-
return []storage.ActiveWorkUnitResources{}, nil
42+
return fetchActiveWorkUnitResourcesFromDB(ctx, r.db, queueNames)
43+
}
44+
45+
func fetchActiveWorkUnitResourcesFromDB(ctx context.Context, db pgdb.DBTX, queueNames []string) ([]storage.ActiveWorkUnitResources, error) {
46+
if len(queueNames) == 0 {
47+
return []storage.ActiveWorkUnitResources{}, nil
48+
}
49+
rows, err := pgdb.QueryRows[activeWorkUnitResourcesRow](ctx, db, fetchActiveWorkUnitResourcesSQL.String(), entityName, queueNames)
50+
if err != nil {
51+
return nil, err
52+
}
53+
result := make([]storage.ActiveWorkUnitResources, 0, len(rows))
54+
for _, row := range rows {
55+
exec, err := postgresdata.UnmarshalProto[essmsgpb.Executable](row.Executable)
56+
if err != nil {
57+
return nil, fmt.Errorf("failed to unmarshal Executable: %w", err)
58+
}
59+
result = append(result, storage.ActiveWorkUnitResources{
60+
Creator: row.Creator,
61+
Executable: exec,
62+
ExecutionState: essmsgpb.WorkUnitStatus_WorkUnitState(row.ExecutionState),
63+
})
64+
}
65+
return result, nil
2766
}
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package workunit
16+
17+
import (
18+
"testing"
19+
"time"
20+
21+
"github.com/google/go-cmp/cmp"
22+
"github.com/jackc/pgx/v5/pgxpool"
23+
"google.golang.org/protobuf/testing/protocmp"
24+
postgrestest "xmanager_cloud/internal/storage/postgres/testing"
25+
storagetest "xmanager_cloud/internal/storage/testing"
26+
)
27+
28+
func insertActiveResourceWorkUnits(t *testing.T, pool *pgxpool.Pool, repo *Repository, specs []storagetest.ActiveResourceSpec) {
29+
t.Helper()
30+
for i, spec := range specs {
31+
wu := blankWorkUnitProto(spec.XID, spec.WID, baseTestTime.Add(time.Duration(i)*time.Second))
32+
wu.CurrentWorkUnitExecution.Status.State = spec.State
33+
wu.Executable = spec.Executable()
34+
mustInsertWorkUnit(t, pool, repo, wu)
35+
}
36+
}
37+
38+
func TestRepository_FetchActiveWorkUnitResources(t *testing.T) {
39+
ctx := t.Context()
40+
_, essPool := postgrestest.Setup(ctx, t)
41+
repo := NewRepository(essPool)
42+
postgrestest.SeedExperimentWithCreator(ctx, t, essPool, 701, "alice@example.com")
43+
postgrestest.SeedExperimentWithCreator(ctx, t, essPool, 702, "bob@example.com")
44+
specs := storagetest.ActiveResourceSpecs(701, 702)
45+
insertActiveResourceWorkUnits(t, essPool, repo, specs)
46+
47+
if got, err := repo.FetchActiveWorkUnitResources(ctx, nil); err != nil || len(got) != 0 {
48+
t.Errorf("FetchActiveWorkUnitResources(nil queues) = (%v, %v), want (empty, nil)", got, err)
49+
}
50+
got, err := repo.FetchActiveWorkUnitResources(ctx, []string{storagetest.ActiveResourceQueue})
51+
if err != nil {
52+
t.Fatalf("FetchActiveWorkUnitResources failed: %v", err)
53+
}
54+
want := storagetest.WantActiveWorkUnitResources(specs, map[int64]string{701: "alice@example.com", 702: "bob@example.com"})
55+
if diff := cmp.Diff(want, got, storagetest.ActiveWorkUnitResourcesOrder, protocmp.Transform()); diff != "" {
56+
t.Errorf("FetchActiveWorkUnitResources() mismatch (-want +got):\n%s", diff)
57+
}
58+
}

‎internal/storage/spanner/workunit/BUILD.bazel‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,16 +61,20 @@ go_library(
6161
go_test(
6262
name = "workunit_unit_test",
6363
srcs = [
64+
"active_work_unit_resources_test.go",
6465
"createworkunit_unit_test.go",
6566
"fetch_active_work_units_test.go",
6667
],
6768
embed = [":workunit"],
6869
deps = [
6970
"//api/protos/experiment:messages_go_proto",
71+
"//internal/grpcerr",
7072
"//internal/storage",
7173
"//internal/storage/spanner",
7274
"//internal/storage/spanner/emulator",
75+
"//internal/storage/testing",
7376
"@com_github_google_go_cmp//cmp",
77+
"@org_golang_google_protobuf//testing/protocmp",
7478
"@org_golang_google_protobuf//types/known/timestamppb",
7579
],
7680
)

‎internal/storage/spanner/workunit/active_work_unit_resources.go‎

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,52 @@ package workunit
1616

1717
import (
1818
"context"
19+
"fmt"
1920

21+
"cloud.google.com/go/spanner"
22+
"xmanager_cloud/internal/grpcerr"
23+
"xmanager_cloud/internal/safesql"
2024
"xmanager_cloud/internal/storage"
2125
)
2226

23-
// FetchActiveWorkUnitResources returns no rows until active work units are queried.
24-
// TODO(#1597): query the creator, executable and state of active work units in queueNames.
27+
var fetchActiveWorkUnitResourcesSQL = safesql.New(
28+
`SELECT e.Creator, wu.Executable, wu.ExecutionState
29+
FROM WorkUnits wu
30+
JOIN Experiments e ON wu.BitReversedXid = e.BitReversedXid
31+
WHERE wu.QueueName IN UNNEST(@queueNames)
32+
AND wu.IsPendingOrRunningJob IS NOT NULL`,
33+
)
34+
35+
// FetchActiveWorkUnitResources reads the creator, executable and state of every active work unit
36+
// in queueNames in a single read.
2537
func (r *Repository) FetchActiveWorkUnitResources(ctx context.Context, queueNames []string) ([]storage.ActiveWorkUnitResources, error) {
26-
return []storage.ActiveWorkUnitResources{}, nil
38+
if r == nil || r.client == nil {
39+
return nil, fmt.Errorf("spanner storage client is uninitialized: %w", grpcerr.ErrFailedPrecondition)
40+
}
41+
return fetchActiveWorkUnitResourcesInTxn(ctx, r.client.Single(), queueNames)
42+
}
43+
44+
func fetchActiveWorkUnitResourcesInTxn(ctx context.Context, txn *spanner.ReadOnlyTransaction, queueNames []string) ([]storage.ActiveWorkUnitResources, error) {
45+
result := []storage.ActiveWorkUnitResources{}
46+
if len(queueNames) == 0 {
47+
return result, nil
48+
}
49+
stmt := spanner.Statement{
50+
SQL: fetchActiveWorkUnitResourcesSQL.String(),
51+
Params: map[string]any{"queueNames": queueNames},
52+
}
53+
iter := txn.Query(ctx, stmt)
54+
defer iter.Stop()
55+
err := iter.Do(func(row *spanner.Row) error {
56+
var wu storage.ActiveWorkUnitResources
57+
if err := row.Columns(&wu.Creator, &wu.Executable, &wu.ExecutionState); err != nil {
58+
return fmt.Errorf("failed to read active work unit resources row: %w", err)
59+
}
60+
result = append(result, wu)
61+
return nil
62+
})
63+
if err != nil {
64+
return nil, fmt.Errorf("failed to query active work unit resources: %w", err)
65+
}
66+
return result, nil
2767
}
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package workunit_test
16+
17+
import (
18+
"context"
19+
"errors"
20+
"testing"
21+
"time"
22+
23+
"github.com/google/go-cmp/cmp"
24+
"google.golang.org/protobuf/testing/protocmp"
25+
essmsgpb "xmanager_cloud/api/protos/experiment"
26+
"xmanager_cloud/internal/grpcerr"
27+
"xmanager_cloud/internal/storage/spanner"
28+
"xmanager_cloud/internal/storage/spanner/emulator"
29+
storagetest "xmanager_cloud/internal/storage/testing"
30+
)
31+
32+
func newEmulatorStorage(ctx context.Context, t *testing.T) *spanner.Storage {
33+
t.Helper()
34+
_, essClient := emulator.Setup(ctx, t)
35+
store, err := spanner.NewStorage(essClient)
36+
if err != nil {
37+
t.Fatalf("spanner.NewStorage failed: %v", err)
38+
}
39+
return store
40+
}
41+
42+
func seedExperimentWithCreator(ctx context.Context, t *testing.T, store *spanner.Storage, xid int64, creator string) {
43+
t.Helper()
44+
exp := spanner.BlankExperimentProto(xid)
45+
exp.Creator = creator
46+
_ = spanner.CreateAndStoreExperiment(ctx, t, store, exp)
47+
}
48+
49+
func seedActiveResourceWorkUnits(ctx context.Context, t *testing.T, store *spanner.Storage, specs []storagetest.ActiveResourceSpec) {
50+
t.Helper()
51+
wus := make([]*essmsgpb.WorkUnit, 0, len(specs))
52+
for _, spec := range specs {
53+
wu := spanner.BlankWorkUnitProto(spec.XID, spec.WID)
54+
wu.CurrentWorkUnitExecution.Status.State = spec.State
55+
wu.Executable = spec.Executable()
56+
wus = append(wus, wu)
57+
}
58+
insertWorkUnitsAt(ctx, t, store, time.Date(2026, 2, 1, 12, 0, 0, 0, time.UTC), wus)
59+
}
60+
61+
func TestFetchActiveWorkUnitResources_GuardsAndEmpty(t *testing.T) {
62+
if _, err := (&spanner.Storage{}).FetchActiveWorkUnitResources(t.Context(), []string{"q"}); !errors.Is(err, grpcerr.ErrFailedPrecondition) {
63+
t.Errorf("FetchActiveWorkUnitResources(nil client) = %v, want ErrFailedPrecondition", err)
64+
}
65+
66+
ctx := t.Context()
67+
store := newEmulatorStorage(ctx, t)
68+
got, err := store.FetchActiveWorkUnitResources(ctx, nil)
69+
if err != nil || len(got) != 0 {
70+
t.Errorf("FetchActiveWorkUnitResources(nil queues) = (%v, %v), want (empty, nil)", got, err)
71+
}
72+
}
73+
74+
func TestFetchActiveWorkUnitResources_AllActiveWorkUnits(t *testing.T) {
75+
ctx := t.Context()
76+
store := newEmulatorStorage(ctx, t)
77+
seedExperimentWithCreator(ctx, t, store, 201, "alice@example.com")
78+
seedExperimentWithCreator(ctx, t, store, 202, "bob@example.com")
79+
specs := storagetest.ActiveResourceSpecs(201, 202)
80+
seedActiveResourceWorkUnits(ctx, t, store, specs)
81+
82+
got, err := store.FetchActiveWorkUnitResources(ctx, []string{storagetest.ActiveResourceQueue})
83+
if err != nil {
84+
t.Fatalf("FetchActiveWorkUnitResources failed: %v", err)
85+
}
86+
want := storagetest.WantActiveWorkUnitResources(specs, map[int64]string{201: "alice@example.com", 202: "bob@example.com"})
87+
if diff := cmp.Diff(want, got, storagetest.ActiveWorkUnitResourcesOrder, protocmp.Transform()); diff != "" {
88+
t.Errorf("FetchActiveWorkUnitResources() mismatch (-want +got):\n%s", diff)
89+
}
90+
}

0 commit comments

Comments
 (0)