From 6b976fc15b4dee8ff78a50c0a82c9621b6d679c1 Mon Sep 17 00:00:00 2001 From: Vlad Lukiantsev Date: Tue, 17 Nov 2026 13:42:41 +0000 Subject: [PATCH 1/2] feat(study-controller): guard study reconcile ticks with per-study Redis lock Introduce distributed per-study lease locking in the Study Controller so multi-replica deployments never reconcile the same study concurrently when consuming at-least-once ticks from xmc-study-reconcile-stream. - Add internal/redislock to acquire non-blocking leases with SET ... NX PX and random 16-byte hex tokens, releasing keys via a token-checked Lua DEL script so an expired lease never deletes a newer holder's lock. - Replace the raw consumer task callback in cmd/study_controller/worker with Handler and NewRedisHandler, sharing the consumer's single *redis.Client connection pool across stream consumption, per-study locking, and /readyz. - Acknowledge contended ticks (ErrNotAcquired) immediately without retrying, dead-letter malformed payloads before lock acquisition, and retry transient lock or reconcile failures under a 60s reconcile timeout (< 2m stream process timeout). - Detach lease release from parent shutdown cancellation with a 2s bound and extend consumer drain timeout to 10s so in-flight ticks release their lock and settle cleanly on SIGTERM. - Expose xmc_study_controller_reconcile_ticks_total{outcome} on /metrics. BUG=b/583731472 TAG=agy CONV=dafb63f4-0f3c-404e-b8fc-511e6baf7d1d --- cmd/study_controller/BUILD.bazel | 2 + cmd/study_controller/health_test.go | 3 +- cmd/study_controller/main.go | 16 +- cmd/study_controller/main_test.go | 37 ++- cmd/study_controller/worker/BUILD.bazel | 5 + cmd/study_controller/worker/consumer.go | 22 +- cmd/study_controller/worker/consumer_test.go | 22 +- cmd/study_controller/worker/handler.go | 166 ++++++++++- cmd/study_controller/worker/handler_test.go | 269 ++++++++++++++++-- .../stable-2026b/components/hpo/README.md | 16 +- docs/architecture/components.md | 1 + docs/flows/experiment_lifecycle.md | 8 +- docs/flows/metrics_and_observability.md | 3 +- internal/redislock/BUILD.bazel | 33 +++ internal/redislock/redislock.go | 119 ++++++++ internal/redislock/redislock_test.go | 177 ++++++++++++ 16 files changed, 845 insertions(+), 54 deletions(-) create mode 100644 internal/redislock/BUILD.bazel create mode 100644 internal/redislock/redislock.go create mode 100644 internal/redislock/redislock_test.go diff --git a/cmd/study_controller/BUILD.bazel b/cmd/study_controller/BUILD.bazel index efd849640..ab4819e56 100644 --- a/cmd/study_controller/BUILD.bazel +++ b/cmd/study_controller/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "//cmd/study_controller/config", "//cmd/study_controller/worker", "//internal/logging", + "//internal/task_queue_redis/redisclient", "@com_github_prometheus_client_golang//prometheus/promhttp", ], ) @@ -55,5 +56,6 @@ go_test( "//cmd/study_controller/config", "//cmd/study_controller/worker", "//internal/logging", + "//internal/task_queue_redis/redisclient", ], ) diff --git a/cmd/study_controller/health_test.go b/cmd/study_controller/health_test.go index 8970ed178..cef207355 100644 --- a/cmd/study_controller/health_test.go +++ b/cmd/study_controller/health_test.go @@ -53,7 +53,8 @@ func TestNewHTTPHandler(t *testing.T) { {name: "readiness with Redis reachable", method: http.MethodGet, path: "/readyz", wantStatus: http.StatusOK, wantContains: "OK"}, {name: "readiness with Redis unreachable", method: http.MethodGet, path: "/readyz", pingErr: unreachable, wantStatus: http.StatusServiceUnavailable, wantContains: "Service Unavailable"}, {name: "readiness with Redis ping timeout", method: http.MethodGet, path: "/readyz", pingErr: context.DeadlineExceeded, wantStatus: http.StatusServiceUnavailable, wantContains: "Service Unavailable"}, - {name: "metrics", method: http.MethodGet, path: "/metrics", wantStatus: http.StatusOK, wantContains: "go_goroutines"}, + {name: "metrics exposes runtime stats", method: http.MethodGet, path: "/metrics", wantStatus: http.StatusOK, wantContains: "go_goroutines"}, + {name: "metrics exposes reconcile ticks counter", method: http.MethodGet, path: "/metrics", wantStatus: http.StatusOK, wantContains: "xmc_study_controller_reconcile_ticks_total"}, {name: "unknown path", method: http.MethodGet, path: "/unknown", wantStatus: http.StatusNotFound}, {name: "non-GET liveness", method: http.MethodPost, path: "/healthz", wantStatus: http.StatusMethodNotAllowed}, {name: "non-GET readiness", method: http.MethodPost, path: "/readyz", wantStatus: http.StatusMethodNotAllowed}, diff --git a/cmd/study_controller/main.go b/cmd/study_controller/main.go index 9a0f368f0..f39c4bd06 100644 --- a/cmd/study_controller/main.go +++ b/cmd/study_controller/main.go @@ -32,6 +32,7 @@ import ( "xmanager_cloud/cmd/study_controller/config" "xmanager_cloud/cmd/study_controller/worker" "xmanager_cloud/internal/logging" + "xmanager_cloud/internal/task_queue_redis/redisclient" ) // HTTP server timeouts bound per-connection resource usage (e.g. slow-client attacks). @@ -82,8 +83,19 @@ func listenAndServe(ctx context.Context, cfg config.Config) error { if err != nil { return fmt.Errorf("failed to listen on port %s: %w", cfg.Port, err) } - client := worker.NewRedisConsumer(cfg.RedisAddr) - consumer := worker.Start(ctx, client) + return serveWithClient(ctx, lis, worker.NewRedisConsumer(cfg.RedisAddr)) +} + +func serveWithClient(ctx context.Context, lis net.Listener, client *redisclient.ConsumerClient) error { + handler, err := worker.NewRedisHandler(client) + if err != nil { + _ = lis.Close() + if client != nil && client.Client != nil { + _ = client.Close(context.WithoutCancel(ctx)) + } + return fmt.Errorf("failed to create study reconcile handler: %w", err) + } + consumer := worker.Start(ctx, client, handler) defer stopConsumer(ctx, consumer) return serve(ctx, lis, newHTTPHandler(client)) } diff --git a/cmd/study_controller/main_test.go b/cmd/study_controller/main_test.go index 07f6b63ed..c3297a721 100644 --- a/cmd/study_controller/main_test.go +++ b/cmd/study_controller/main_test.go @@ -30,6 +30,7 @@ import ( "xmanager_cloud/cmd/study_controller/config" "xmanager_cloud/cmd/study_controller/worker" "xmanager_cloud/internal/logging" + "xmanager_cloud/internal/task_queue_redis/redisclient" ) // waitHealthy polls GET /healthz at addr until it returns 200 OK or the deadline passes. @@ -101,6 +102,36 @@ func TestRun_CancelledContextExitsZero(t *testing.T) { } } +type noopRedisClient struct{ redisclient.Client } + +func (noopRedisClient) Close() error { return nil } + +func TestServeWithClient_RejectsInvalidClients(t *testing.T) { + nonLockClient := worker.NewRedisConsumer("127.0.0.1:" + freePort(t)) + _ = nonLockClient.Close(context.Background()) + nonLockClient.Client = noopRedisClient{} + + tests := []struct { + name string + client *redisclient.ConsumerClient + }{ + {name: "nil consumer client", client: nil}, + {name: "non-locking underlying client", client: nonLockClient}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen() error = %v", err) + } + defer lis.Close() + if err := serveWithClient(context.Background(), lis, tc.client); err == nil { + t.Errorf("serveWithClient(%+v) error = nil, want non-nil", tc.client) + } + }) + } +} + func TestReadiness_UnreachableRedis(t *testing.T) { client := worker.NewRedisConsumer("127.0.0.1:" + freePort(t)) defer client.Close(context.Background()) @@ -170,7 +201,11 @@ func TestStopConsumer_LogsCloseFailure(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() client := worker.NewRedisConsumer("127.0.0.1:" + freePort(t)) - consumer := worker.Start(ctx, client) + handler, err := worker.NewRedisHandler(client) + if err != nil { + t.Fatalf("NewRedisHandler() error = %v", err) + } + consumer := worker.Start(ctx, client, handler) // Closing the client first makes the consumer's own close fail. if err := client.Close(context.Background()); err != nil { t.Fatalf("client.Close() error = %v", err) diff --git a/cmd/study_controller/worker/BUILD.bazel b/cmd/study_controller/worker/BUILD.bazel index c0a18729f..c22ae00d8 100644 --- a/cmd/study_controller/worker/BUILD.bazel +++ b/cmd/study_controller/worker/BUILD.bazel @@ -28,8 +28,11 @@ go_library( "//api/protos/task_queue:task_go_proto", "//internal/kubernetes/common/constants", "//internal/logging", + "//internal/redislock", "//internal/task_queue_redis/redisclient", "//internal/validator", + "@com_github_prometheus_client_golang//prometheus", + "@com_github_prometheus_client_golang//prometheus/promauto", "@org_golang_google_protobuf//encoding/protojson", ], ) @@ -45,8 +48,10 @@ go_test( deps = [ "//internal/kubernetes/common/constants", "//internal/logging", + "//internal/redislock", "//internal/task_queue_redis/redisclient", "@com_github_google_go_cmp//cmp", + "@com_github_prometheus_client_golang//prometheus/testutil", "@com_github_redis_go_redis_v9//:go-redis", ], ) diff --git a/cmd/study_controller/worker/consumer.go b/cmd/study_controller/worker/consumer.go index 3487300a5..a91d8da2b 100644 --- a/cmd/study_controller/worker/consumer.go +++ b/cmd/study_controller/worker/consumer.go @@ -25,9 +25,9 @@ import ( ) // defaultDrainTimeout bounds how long Close waits for the worker and cleanup loops. It covers the -// 2s XREADGROUP block of the worker loop plus redisclient.SettleTimeout, so a tick being settled -// when Close is called finishes before the client is closed. -const defaultDrainTimeout = redisclient.SettleTimeout + 3*time.Second +// 2s XREADGROUP block of the worker loop, defaultReleaseTimeout, and redisclient.SettleTimeout, so +// a tick being settled when Close is called finishes before the client is closed. +const defaultDrainTimeout = redisclient.SettleTimeout + defaultReleaseTimeout + 3*time.Second // NewRedisConsumer returns a Redis consumer of the study reconcile stream for the Redis server at // addr. It reads the password from REDIS_PASSWORD and connects lazily. Context deadlines bound its @@ -46,7 +46,7 @@ func NewRedisConsumer(addr string) *redisclient.ConsumerClient { ) } -// Consumer runs the worker and cleanup loops of a Redis consumer with handleTask as its handler. +// Consumer runs the worker and cleanup loops of a Redis consumer with Handler.HandleTask. type Consumer struct { client redisclient.Consumer cancel context.CancelFunc @@ -56,12 +56,12 @@ type Consumer struct { closeErr error // result of the first Close, returned by every call } -// Start sets handleTask on client and runs its worker and cleanup loops until ctx is done or Close -// is called. The loops create the stream and consumer group themselves, at start-up and again -// whenever Redis reports them missing (NOGROUP), so Start never blocks on Redis and consumption -// resumes after Redis comes back without them. -func Start(ctx context.Context, client redisclient.Consumer) *Consumer { - client.SetTaskHandler(handleTask) +// Start sets handler.HandleTask on client and runs its worker and cleanup loops until ctx is done +// or Close is called. The loops create the stream and consumer group themselves, at start-up and +// again whenever Redis reports them missing (NOGROUP), so Start never blocks on Redis and +// consumption resumes after Redis comes back without them. +func Start(ctx context.Context, client redisclient.Consumer, handler *Handler) *Consumer { + client.SetTaskHandler(handler.HandleTask) loopCtx, cancel := context.WithCancel(ctx) c := &Consumer{client: client, cancel: cancel, drainTimeout: defaultDrainTimeout} workerID := redisclient.HostConsumerName(constants.StudyControllerWorkerID) @@ -84,7 +84,7 @@ func (c *Consumer) runLoop(loop func()) { } // Close stops the loops and closes the Redis client. It waits for the loops to return, but at -// most the drain timeout (8s) and only until ctx is done; after that it logs a warning and closes +// most the drain timeout (10s) and only until ctx is done; after that it logs a warning and closes // the client anyway, leaving a stuck loop to fail on the closed client. A tick being handled when // Close is called is still acknowledged or retried. Every call returns the first call's error. func (c *Consumer) Close(ctx context.Context) error { diff --git a/cmd/study_controller/worker/consumer_test.go b/cmd/study_controller/worker/consumer_test.go index f5273e38f..4806d5ee4 100644 --- a/cmd/study_controller/worker/consumer_test.go +++ b/cmd/study_controller/worker/consumer_test.go @@ -93,9 +93,18 @@ func (f *fakeConsumer) waitForLoops(t *testing.T) []string { return ids } +func newStubHandler(t *testing.T) *Handler { + t.Helper() + h, err := NewHandler(HandlerParams{Locker: &fakeLocker{lease: &fakeLease{}}}) + if err != nil { + t.Fatalf("NewHandler() error = %v", err) + } + return h +} + func TestStart_RunsLoopsWithHandleTask(t *testing.T) { fake := newFakeConsumer() - c := Start(context.Background(), fake) + c := Start(context.Background(), fake, newStubHandler(t)) defer c.Close(context.Background()) ids := fake.waitForLoops(t) @@ -111,7 +120,7 @@ func TestStart_RunsLoopsWithHandleTask(t *testing.T) { t.Fatal("Start() did not set a task handler") } if got := handler(context.Background(), "w", `{"experiment":"experiments/7"}`); !got.WasRunSuccessful { - t.Errorf("task handler on a valid tick = %+v, want success (handleTask)", got) + t.Errorf("task handler on a valid tick = %+v, want success (HandleTask)", got) } } @@ -127,7 +136,7 @@ func containsPrefix(ids []string, prefix string) bool { func TestConsumer_CloseStopsLoopsAndClosesClient(t *testing.T) { fake := newFakeConsumer() fake.closeErr = errors.New("close failed") - c := Start(context.Background(), fake) + c := Start(context.Background(), fake, newStubHandler(t)) fake.waitForLoops(t) if err := c.Close(context.Background()); !errors.Is(err, fake.closeErr) { @@ -146,7 +155,7 @@ func TestConsumer_CloseStopsLoopsAndClosesClient(t *testing.T) { func TestStart_ContextCancelStopsLoops(t *testing.T) { fake := newFakeConsumer() ctx, cancel := context.WithCancel(context.Background()) - c := Start(ctx, fake) + c := Start(ctx, fake, newStubHandler(t)) fake.waitForLoops(t) cancel() @@ -179,7 +188,7 @@ func TestConsumer_CloseWithStuckLoops(t *testing.T) { fake := newFakeConsumer() fake.ignoreCancel = true defer close(fake.release) - c := Start(context.Background(), fake) + c := Start(context.Background(), fake, newStubHandler(t)) fake.waitForLoops(t) c.drainTimeout = tc.drainTimeout @@ -204,6 +213,7 @@ type consumerSettings struct { Addr, Stream, Group, DLQ string MaxRetries int ContextTimeoutEnabled bool + DrainTimeout time.Duration } func TestNewRedisConsumer(t *testing.T) { @@ -217,10 +227,12 @@ func TestNewRedisConsumer(t *testing.T) { got := consumerSettings{ Addr: client.Options().Addr, Stream: c.StreamName, Group: c.GroupName, DLQ: c.DLQStreamName, MaxRetries: c.MaxRetries, ContextTimeoutEnabled: client.Options().ContextTimeoutEnabled, + DrainTimeout: defaultDrainTimeout, } want := consumerSettings{ Addr: "redis-service:6379", Stream: "xmc-study-reconcile-stream", Group: "study-controller-group", DLQ: "xmc-study-reconcile-dlq-stream", MaxRetries: constants.RedisMaxRetries, ContextTimeoutEnabled: true, + DrainTimeout: 10 * time.Second, } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("NewRedisConsumer() settings mismatch (-want +got):\n%s", diff) diff --git a/cmd/study_controller/worker/handler.go b/cmd/study_controller/worker/handler.go index a96417c1a..fc82ebfd4 100644 --- a/cmd/study_controller/worker/handler.go +++ b/cmd/study_controller/worker/handler.go @@ -17,23 +17,177 @@ package worker import ( "context" + "errors" + "fmt" + "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "xmanager_cloud/internal/kubernetes/common/constants" "xmanager_cloud/internal/logging" + "xmanager_cloud/internal/redislock" "xmanager_cloud/internal/task_queue_redis/redisclient" "xmanager_cloud/internal/validator" ) -// handleTask handles one study reconcile tick. A payload that does not decode to a valid -// experiment name can never succeed, so it is moved to the dead-letter stream without a retry. -// A valid tick is logged at debug level (it repeats per active study) with the study's XID and -// acknowledged; reconciling the study is not implemented yet. -func handleTask(ctx context.Context, workerID, data string) redisclient.TaskResult { +const ( + defaultLeaseTTL = 15 * time.Second + defaultReconcileTimeout = 60 * time.Second + defaultReleaseTimeout = 2 * time.Second + studyLockKeyPrefix = "xmc:study-reconcile-lock:" + outcomeReconciled = "reconciled" + outcomeContended = "contended" + outcomeRetryError = "retry_error" + outcomeInvalidPayload = "invalid_payload" +) + +var reconcileTicksTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "xmc_study_controller_reconcile_ticks_total", + Help: "Total number of study reconcile ticks processed by the Study Controller, partitioned by outcome.", +}, []string{"outcome"}) + +func init() { + for _, o := range []string{outcomeReconciled, outcomeContended, outcomeRetryError, outcomeInvalidPayload} { + reconcileTicksTotal.WithLabelValues(o) + } +} + +// Lease represents an acquired per-study reconcile lock. *redislock.Lease satisfies it. +type Lease interface { + Release(ctx context.Context) error +} + +var _ Lease = (*redislock.Lease)(nil) + +// Locker acquires per-study reconcile leases without blocking. +type Locker interface { + Acquire(ctx context.Context, key string) (Lease, error) +} + +type redisLocker struct{ locker *redislock.Locker } + +func (r redisLocker) Acquire(ctx context.Context, key string) (Lease, error) { + lease, err := r.locker.Acquire(ctx, key) + if err != nil { + return nil, err + } + return lease, nil +} + +// ReconcileFunc reconciles the study identified by xid under a held per-study lease. +type ReconcileFunc func(ctx context.Context, xid int64) error + +func noopReconcile(context.Context, int64) error { return nil } +func studyLockKey(xid int64) string { return fmt.Sprintf("%s%d", studyLockKeyPrefix, xid) } + +// HandlerParams configures a study reconcile Handler. +type HandlerParams struct { + Locker Locker + Reconcile ReconcileFunc + ReconcileTimeout, ReleaseTimeout time.Duration +} + +// Handler processes study reconcile ticks under a distributed per-study lease lock. +type Handler struct { + locker Locker + reconcile ReconcileFunc + reconcileTimeout, releaseTimeout time.Duration +} + +// NewRedisHandler constructs a Handler sharing consumer's underlying *redis.Client pool. +func NewRedisHandler(consumer *redisclient.ConsumerClient) (*Handler, error) { + if consumer == nil { + return nil, errors.New("redis consumer client must not be nil") + } + lockClient, ok := consumer.Client.(redislock.Client) + if !ok { + return nil, errors.New("redis consumer client does not implement redislock.Client") + } + locker, err := redislock.New(lockClient, defaultLeaseTTL) + if err != nil { + return nil, err + } + return NewHandler(HandlerParams{Locker: redisLocker{locker: locker}}) +} + +// NewHandler validates p and returns a Handler with defaults applied to zero-valued parameters. +func NewHandler(p HandlerParams) (*Handler, error) { + if p.Locker == nil { + return nil, errors.New("study reconcile locker must not be nil") + } + if p.Reconcile == nil { + p.Reconcile = noopReconcile + } + if p.ReconcileTimeout == 0 { + p.ReconcileTimeout = defaultReconcileTimeout + } + if p.ReleaseTimeout == 0 { + p.ReleaseTimeout = defaultReleaseTimeout + } + if p.ReconcileTimeout < 0 || p.ReleaseTimeout < 0 { + return nil, errors.New("study reconcile handler durations must be positive") + } + if p.ReconcileTimeout >= constants.RedisProcessTimeout { + return nil, fmt.Errorf("reconcile timeout (%v) must be less than redis process timeout (%v)", p.ReconcileTimeout, constants.RedisProcessTimeout) + } + return &Handler{ + locker: p.Locker, + reconcile: p.Reconcile, + reconcileTimeout: p.ReconcileTimeout, + releaseTimeout: p.ReleaseTimeout, + }, nil +} + +// HandleTask handles one study reconcile tick under a per-study Redis lease. +func (h *Handler) HandleTask(ctx context.Context, workerID, data string) redisclient.TaskResult { xid, err := parseReconcilePayload(data) if err != nil { + reconcileTicksTotal.WithLabelValues(outcomeInvalidPayload).Inc() logging.Warn(ctx, "Dropping invalid study reconcile tick", "worker_id", workerID, "payload_bytes", len(data), "error", err) return redisclient.TaskResult{WasRunSuccessful: false, ShouldBeRetried: false, Error: err} } ctx = logging.WithExperimentName(ctx, validator.ConstructExperimentName(xid)) - logging.Debug(ctx, "Received study reconcile tick", "worker_id", workerID) + lease, err := h.locker.Acquire(ctx, studyLockKey(xid)) + if errors.Is(err, redislock.ErrNotAcquired) { + reconcileTicksTotal.WithLabelValues(outcomeContended).Inc() + logging.Info(ctx, "Skipped contended study reconcile tick", "worker_id", workerID) + return redisclient.TaskResult{WasRunSuccessful: true} + } + if err != nil { + reconcileTicksTotal.WithLabelValues(outcomeRetryError).Inc() + logging.Warn(ctx, "Failed to acquire study reconcile lock", "worker_id", workerID, "error", err) + return redisclient.TaskResult{WasRunSuccessful: false, ShouldBeRetried: true, Error: err} + } + if err := h.reconcileWithLease(ctx, workerID, xid, lease); err != nil { + reconcileTicksTotal.WithLabelValues(outcomeRetryError).Inc() + logging.Warn(ctx, "Failed to process study reconcile tick", "worker_id", workerID, "error", err) + return redisclient.TaskResult{WasRunSuccessful: false, ShouldBeRetried: true, Error: err} + } + reconcileTicksTotal.WithLabelValues(outcomeReconciled).Inc() + logging.Info(ctx, "Processed study reconcile tick", "worker_id", workerID) return redisclient.TaskResult{WasRunSuccessful: true} } + +func (h *Handler) reconcileWithLease(ctx context.Context, workerID string, xid int64, lease Lease) error { + reconcileCtx, cancel := context.WithTimeout(ctx, h.reconcileTimeout) + defer cancel() + defer h.releaseLease(ctx, workerID, lease) + if err := h.reconcile(reconcileCtx, xid); err != nil { + return err + } + return reconcileCtx.Err() +} + +func (h *Handler) releaseLease(ctx context.Context, workerID string, lease Lease) { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), h.releaseTimeout) + defer cancel() + err := lease.Release(releaseCtx) + if err == nil { + return + } + msg := "Failed to release study reconcile lock" + if errors.Is(err, redislock.ErrLeaseLost) { + msg = "Study reconcile lock lease was already lost before release" + } + logging.Warn(ctx, msg, "worker_id", workerID, "error", err) +} diff --git a/cmd/study_controller/worker/handler_test.go b/cmd/study_controller/worker/handler_test.go index 6933741ef..8114ea85e 100644 --- a/cmd/study_controller/worker/handler_test.go +++ b/cmd/study_controller/worker/handler_test.go @@ -17,46 +17,273 @@ package worker import ( "bytes" "context" + "errors" "log/slog" "strings" + "sync" "testing" + "time" + "github.com/google/go-cmp/cmp" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/redis/go-redis/v9" + "xmanager_cloud/internal/kubernetes/common/constants" "xmanager_cloud/internal/logging" + "xmanager_cloud/internal/redislock" + "xmanager_cloud/internal/task_queue_redis/redisclient" ) -func TestHandleTask(t *testing.T) { - t.Parallel() +type fakeLease struct { + mu sync.Mutex + releaseFn func(ctx context.Context) error + releaseCalls int + releaseCtxErr error + releaseHasDeadline bool +} + +func (l *fakeLease) Release(ctx context.Context) error { + l.mu.Lock() + l.releaseCalls++ + l.releaseCtxErr = ctx.Err() + _, l.releaseHasDeadline = ctx.Deadline() + fn := l.releaseFn + l.mu.Unlock() + if fn != nil { + return fn(ctx) + } + return nil +} + +type fakeLocker struct { + mu sync.Mutex + lease Lease + err error + lastKey string + calls int +} + +func (f *fakeLocker) Acquire(_ context.Context, key string) (Lease, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + f.lastKey = key + if f.err != nil { + return nil, f.err + } + return f.lease, nil +} + +type stubLockClient struct { + redisclient.Client + redis.Scripter + setErr error +} + +func (s *stubLockClient) SetArgs(_ context.Context, _ string, _ any, _ redis.SetArgs) *redis.StatusCmd { + if s.setErr != nil { + return redis.NewStatusResult("", s.setErr) + } + return redis.NewStatusResult("OK", nil) +} + +func (s *stubLockClient) EvalSha(_ context.Context, _ string, _ []string, _ ...any) *redis.Cmd { + cmd := redis.NewCmd(context.Background()) + cmd.SetVal(int64(1)) + return cmd +} +func snapshotOutcomes() map[string]float64 { + outcomes := []string{outcomeReconciled, outcomeContended, outcomeRetryError, outcomeInvalidPayload} + snap := make(map[string]float64, len(outcomes)) + for _, o := range outcomes { + snap[o] = testutil.ToFloat64(reconcileTicksTotal.WithLabelValues(o)) + } + return snap +} + +func diffOutcomes(before map[string]float64) map[string]float64 { + deltas := make(map[string]float64) + for k, v := range snapshotOutcomes() { + if delta := v - before[k]; delta != 0 { + deltas[k] = delta + } + } + return deltas +} + +func newLogContext(buf *bytes.Buffer) context.Context { + return logging.WithLogger(context.Background(), slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) +} + +func TestNewHandler_ValidationAndRedisWiring(t *testing.T) { + valid := &fakeLocker{lease: &fakeLease{}} tests := []struct { - name string - data string - wantSuccess bool - wantLogs []string + name string + params HandlerParams + wantErr bool }{ - {name: "valid tick is acknowledged", data: `{"experiment":"experiments/123"}`, wantSuccess: true, - wantLogs: []string{"Received study reconcile tick", "worker_id=worker-1", "xid=123"}}, - {name: "malformed payload is dead-lettered", data: `not json`, wantLogs: []string{"Dropping invalid study reconcile tick", "payload_bytes=8"}}, - {name: "invalid experiment is dead-lettered", data: `{"experiment":"xids/123"}`, wantLogs: []string{"Dropping invalid study reconcile tick"}}, + {name: "nil locker", params: HandlerParams{}, wantErr: true}, + {name: "negative reconcile timeout", params: HandlerParams{Locker: valid, ReconcileTimeout: -time.Second}, wantErr: true}, + {name: "negative release timeout", params: HandlerParams{Locker: valid, ReleaseTimeout: -time.Second}, wantErr: true}, + {name: "reconcile timeout at process timeout", params: HandlerParams{Locker: valid, ReconcileTimeout: constants.RedisProcessTimeout}, wantErr: true}, + {name: "valid defaults", params: HandlerParams{Locker: valid}}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - t.Parallel() - var logs bytes.Buffer - ctx := logging.WithLogger(context.Background(), slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) + got, err := NewHandler(tc.params) + if (err != nil) != tc.wantErr || (!tc.wantErr && got == nil) { + t.Fatalf("NewHandler(%+v) = (%v, %v), wantErr %v", tc.params, got, err, tc.wantErr) + } + }) + } + + var nonLockConsumer redisclient.ConsumerClient + typedNilConsumer := NewRedisConsumer("127.0.0.1:6379") + _ = typedNilConsumer.Close(context.Background()) + typedNilConsumer.Client = (*redis.Client)(nil) + for _, bad := range []*redisclient.ConsumerClient{nil, &nonLockConsumer, typedNilConsumer} { + if _, err := NewRedisHandler(bad); err == nil { + t.Fatalf("NewRedisHandler(%+v) error = nil, want error", bad) + } + } + + okConsumer := NewRedisConsumer("127.0.0.1:6379") + _ = okConsumer.Close(context.Background()) + okConsumer.Client = &stubLockClient{} + h, err := NewRedisHandler(okConsumer) + if err != nil || !h.HandleTask(context.Background(), "w", `{"experiment":"experiments/55"}`).WasRunSuccessful { + t.Fatalf("NewRedisHandler(ok) err = %v or HandleTask failed", err) + } + okConsumer.Client = &stubLockClient{setErr: redis.Nil} + hContended, err := NewRedisHandler(okConsumer) + if err != nil || !hContended.HandleTask(context.Background(), "w", `{"experiment":"experiments/55"}`).WasRunSuccessful { + t.Fatalf("NewRedisHandler(contended) err = %v or HandleTask failed", err) + } +} + +func TestHandleTask_OutcomesAndLockAcquisition(t *testing.T) { + acquireErr, reconcileErr := errors.New("redis connection refused"), errors.New("vizier unavailable") + const validTick = `{"experiment":"experiments/123"}` + const wantKey = "xmc:study-reconcile-lock:123" + + tests := []struct { + name string + data string + acquireErr, recErr, releaseErr error + wantSuccess, wantRetry, wantRec bool + wantErrIs error + wantAcquireCalls int + wantKey string + wantMetrics map[string]float64 + wantLogs []string + }{ + {name: "fast reconcile", data: validTick, wantSuccess: true, wantAcquireCalls: 1, wantKey: wantKey, wantRec: true, + wantMetrics: map[string]float64{outcomeReconciled: 1}, wantLogs: []string{"Processed study reconcile tick", "worker_id=worker-1", "xid=123"}}, + {name: "contended tick acked", data: validTick, acquireErr: redislock.ErrNotAcquired, wantSuccess: true, wantAcquireCalls: 1, wantKey: wantKey, + wantMetrics: map[string]float64{outcomeContended: 1}, wantLogs: []string{"Skipped contended study reconcile tick", "xid=123"}}, + {name: "acquire error retried", data: validTick, acquireErr: acquireErr, wantRetry: true, wantErrIs: acquireErr, wantAcquireCalls: 1, wantKey: wantKey, + wantMetrics: map[string]float64{outcomeRetryError: 1}, wantLogs: []string{"Failed to acquire study reconcile lock", "xid=123"}}, + {name: "reconcile error retried", data: validTick, recErr: reconcileErr, wantRetry: true, wantErrIs: reconcileErr, wantAcquireCalls: 1, wantKey: wantKey, wantRec: true, + wantMetrics: map[string]float64{outcomeRetryError: 1}, wantLogs: []string{"Failed to process study reconcile tick", "xid=123"}}, + {name: "release lost lease logs warn", data: validTick, releaseErr: redislock.ErrLeaseLost, wantSuccess: true, wantAcquireCalls: 1, wantKey: wantKey, wantRec: true, + wantMetrics: map[string]float64{outcomeReconciled: 1}, wantLogs: []string{"Study reconcile lock lease was already lost before release"}}, + {name: "release transport error logs warn", data: validTick, releaseErr: acquireErr, wantSuccess: true, wantAcquireCalls: 1, wantKey: wantKey, wantRec: true, + wantMetrics: map[string]float64{outcomeReconciled: 1}, wantLogs: []string{"Failed to release study reconcile lock"}}, + {name: "empty payload", data: "", wantMetrics: map[string]float64{outcomeInvalidPayload: 1}, wantLogs: []string{"Dropping invalid study reconcile tick", "payload_bytes=0"}}, + {name: "malformed json", data: "not json", wantMetrics: map[string]float64{outcomeInvalidPayload: 1}, wantLogs: []string{"Dropping invalid study reconcile tick", "payload_bytes=8"}}, + {name: "multibyte invalid xid", data: `{"experiment":"experiments/123"}`, wantMetrics: map[string]float64{outcomeInvalidPayload: 1}, wantLogs: []string{"Dropping invalid study reconcile tick"}}, + {name: "wrong resource", data: `{"experiment":"xids/123"}`, wantMetrics: map[string]float64{outcomeInvalidPayload: 1}, wantLogs: []string{"Dropping invalid study reconcile tick"}}, + } - got := handleTask(ctx, "worker-1", tc.data) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + lease := &fakeLease{releaseFn: func(context.Context) error { return tc.releaseErr }} + locker := &fakeLocker{lease: lease, err: tc.acquireErr} + var reconciled bool + h, _ := NewHandler(HandlerParams{ + Locker: locker, + Reconcile: func(_ context.Context, xid int64) error { + reconciled = xid == 123 + return tc.recErr + }, + }) + + var logs bytes.Buffer + before := snapshotOutcomes() + got := h.HandleTask(newLogContext(&logs), "worker-1", tc.data) - if got.WasRunSuccessful != tc.wantSuccess || got.ShouldBeRetried { - t.Errorf("handleTask(%q) = %+v, want WasRunSuccessful=%v ShouldBeRetried=false", tc.data, got, tc.wantSuccess) + if got.WasRunSuccessful != tc.wantSuccess || got.ShouldBeRetried != tc.wantRetry { + t.Errorf("HandleTask() = %+v, want success=%v retry=%v", got, tc.wantSuccess, tc.wantRetry) } - if (got.Error == nil) != tc.wantSuccess { - t.Errorf("handleTask(%q) error = %v, want error only on failure", tc.data, got.Error) + if tc.wantErrIs != nil && !errors.Is(got.Error, tc.wantErrIs) { + t.Errorf("error = %v, want %v", got.Error, tc.wantErrIs) } - for _, want := range tc.wantLogs { - if !strings.Contains(logs.String(), want) { - t.Errorf("handleTask(%q) logs = %q, want them to contain %q", tc.data, logs.String(), want) + if locker.calls != tc.wantAcquireCalls || locker.lastKey != tc.wantKey || reconciled != tc.wantRec { + t.Errorf("calls=%d key=%q rec=%v, want %d %q %v", locker.calls, locker.lastKey, reconciled, tc.wantAcquireCalls, tc.wantKey, tc.wantRec) + } + if tc.wantRec && (lease.releaseCalls != 1 || !lease.releaseHasDeadline) { + t.Errorf("releaseCalls=%d hasDeadline=%v, want 1 bounded Release", lease.releaseCalls, lease.releaseHasDeadline) + } + if diff := cmp.Diff(tc.wantMetrics, diffOutcomes(before)); diff != "" { + t.Errorf("metric deltas mismatch (-want +got):\n%s", diff) + } + for _, wantLog := range tc.wantLogs { + if !strings.Contains(logs.String(), wantLog) { + t.Errorf("logs = %q, want %q", logs.String(), wantLog) } } }) } } + +func TestHandleTask_DeadlineAndShutdownTransitions(t *testing.T) { + tests := []struct { + name string + reconcileTimeout time.Duration + ignoreCancel bool + cancelParent bool + wantErrIs error + }{ + {name: "reconcile timeout deadline", reconcileTimeout: 15 * time.Millisecond, wantErrIs: context.DeadlineExceeded}, + {name: "reconcile timeout exact boundary when reconcile returns nil", reconcileTimeout: 15 * time.Millisecond, + ignoreCancel: true, wantErrIs: context.DeadlineExceeded}, + {name: "parent shutdown cancels reconcile and releases on detached ctx", cancelParent: true, wantErrIs: context.Canceled}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + parentCtx, cancelParent := context.WithCancel(context.Background()) + defer cancelParent() + started := make(chan struct{}) + if tc.cancelParent { + go func() { <-started; cancelParent() }() + } + lease := &fakeLease{} + h, _ := NewHandler(HandlerParams{ + Locker: &fakeLocker{lease: lease}, + ReconcileTimeout: tc.reconcileTimeout, + Reconcile: func(ctx context.Context, _ int64) error { + close(started) + <-ctx.Done() + if tc.ignoreCancel { + return nil + } + return ctx.Err() + }, + }) + + before := snapshotOutcomes() + got := h.HandleTask(parentCtx, "worker-1", `{"experiment":"experiments/14"}`) + + if got.WasRunSuccessful || !got.ShouldBeRetried || !errors.Is(got.Error, tc.wantErrIs) { + t.Fatalf("got = %+v, want retryable %v", got, tc.wantErrIs) + } + if lease.releaseCalls != 1 || lease.releaseCtxErr != nil || !lease.releaseHasDeadline { + t.Errorf("releaseCalls=%d ctxErr=%v hasDeadline=%v", lease.releaseCalls, lease.releaseCtxErr, lease.releaseHasDeadline) + } + if diff := cmp.Diff(map[string]float64{outcomeRetryError: 1}, diffOutcomes(before)); diff != "" { + t.Errorf("metric deltas mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/deploy/profiles/stable-2026b/components/hpo/README.md b/deploy/profiles/stable-2026b/components/hpo/README.md index b26d88c78..4e97bfb64 100644 --- a/deploy/profiles/stable-2026b/components/hpo/README.md +++ b/deploy/profiles/stable-2026b/components/hpo/README.md @@ -3,7 +3,7 @@ This Helm sub-chart (`deploy/profiles/stable-2026b/components/hpo/`) defines the XMC 1P Native Component blueprint ([`templates/component.yaml`](templates/component.yaml)) for **Hyperparameter Optimization (HPO)** (`hyperparameter-optimization`) and the `xmc-study-controller` workload that `xmc-bridge` actuates for it ([`templates/deployment.yaml`](templates/deployment.yaml), [`templates/serviceaccount.yaml`](templates/serviceaccount.yaml)). > [!NOTE] -> The `xmc-study-controller` workload is a `Deployment` and a plain `ServiceAccount` only. It consumes study reconcile ticks from Redis and logs them, and serves health and metrics endpoints; it does not yet reconcile studies. The chart renders no `Service`, `Role`/`RoleBinding`, `NetworkPolicy` or `ConfigMap`, and the `ServiceAccount` carries no Workload Identity annotation. +> The `xmc-study-controller` workload is a `Deployment` and a plain `ServiceAccount` only. It consumes study reconcile ticks from Redis under a per-study Redis lease lock and logs them, and serves health and metrics endpoints; it does not yet reconcile studies. The chart renders no `Service`, `Role`/`RoleBinding`, `NetworkPolicy` or `ConfigMap`, and the `ServiceAccount` carries no Workload Identity annotation. --- @@ -72,12 +72,22 @@ The Study Controller connects to the XMC Redis server: ### Study Reconcile Ticks -The Study Controller consumes the `xmc-study-reconcile-stream` Redis stream as the `study-controller-group` consumer group. Each message is a protojson `StudyReconcilePayload` (`{"experiment": "experiments/{xid}"}`) that names a study; ticks are unsigned hints, delivered at least once. The controller logs each valid tick at debug level with the study's XID (`Received study reconcile tick`) and acknowledges it; it does not reconcile the study yet. A payload that does not name a valid experiment, or a message without data, goes to the `xmc-study-reconcile-dlq-stream` dead-letter stream without a retry. The controller re-creates the consumer group when Redis loses it, for example after a restart without persistence. +The Study Controller consumes the `xmc-study-reconcile-stream` Redis stream as the `study-controller-group` consumer group, sharing a single Redis connection pool across stream consumption, per-study locking, and `/readyz`. Each message is a protojson `StudyReconcilePayload` (`{"experiment": "experiments/{xid}"}`) that names a study; ticks are unsigned hints, delivered at least once. + +Each delivered tick is classified by outcome (`xmc_study_controller_reconcile_ticks_total{outcome}`); for valid payloads, the controller acquires a distributed per-study lease at `xmc:study-reconcile-lock:{xid}` (`SET ... NX PX` with a random 16-byte token, 15s lease TTL up to a 60s reconcile timeout, and token-checked Lua `DEL`): + +- **Acquired (`reconciled`)**: reconciles the study (currently a stub), releases the lease, logs `Processed study reconcile tick` at info level with the study's XID, and acknowledges the tick. +- **Already held (`contended`)**: logs `Skipped contended study reconcile tick` at info level and acknowledges the tick without reconciling so multi-replica deployments never reconcile the same study concurrently. +- **Lock or reconcile failure (`retry_error`)**: releases any acquired lease if still owned and retries the tick via stream redelivery. +- **Malformed payload (`invalid_payload`)**: a payload whose `data` does not decode to a valid `experiments/{xid}` resource name goes to the `xmc-study-reconcile-dlq-stream` dead-letter stream without acquiring the lock or retrying (raw stream entries missing a `data` field are dead-lettered directly by the stream consumer before the handler runs). + +The controller re-creates the consumer group when Redis loses it, for example after a restart without persistence. The Experiment State Server publishes one tick when it creates a `STUDY` experiment; see [Create Experiment](../../../../../docs/flows/experiment_lifecycle.md#1-create-experiment). +- **Metrics**: `GET /metrics` exposes `xmc_study_controller_reconcile_ticks_total{outcome="reconciled|contended|retry_error|invalid_payload"}`. - **Troubleshooting**: to inspect dead-lettered ticks and the reason each was rejected, run `XRANGE xmc-study-reconcile-dlq-stream - +` in `redis-cli`. -- **Shutdown**: on SIGTERM the controller stops its HTTP server (up to 5s), then drains the consumer (up to 8s) so a tick being handled is still acknowledged. Ticks it has not started stay pending and are reclaimed later. If you set the Component's `lifecycleControl.drainTimeoutSeconds`, set it to at least 8: the pod's termination grace period is that value plus 5s. +- **Shutdown**: on SIGTERM the controller stops its HTTP server (up to 5s), then drains the consumer (up to 10s) so an in-flight tick releases its per-study lock (2s detached timeout) and settles on the stream (5s detached timeout). Ticks it has not started stay pending and are reclaimed later. If you set the Component's `lifecycleControl.drainTimeoutSeconds`, set it to at least 10: the pod's termination grace period is that value plus 5s. ### Image Resolution diff --git a/docs/architecture/components.md b/docs/architecture/components.md index 7cbad6092..f9b38e646 100644 --- a/docs/architecture/components.md +++ b/docs/architecture/components.md @@ -320,6 +320,7 @@ declarations fail the build rather than a deployment. | [`internal/storage`](../../internal/storage) | Spanner data access, schema migrations, and the storage interfaces used by ESS, Actuator, Reconciler, Health Service, and Log Monitor. | | [`internal/task_queue`](../../internal/task_queue) | Builds, signs, and enqueues actuator payloads. Owns the idempotency key and JWT/OIDC signing scheme. | | [`internal/task_queue_redis`](../../internal/task_queue_redis) | Redis Streams publisher and consumer with consumer groups, retries, abandoned-message reclamation, and dead-lettering. | +| [`internal/redislock`](../../internal/redislock) | Distributed Redis lease lock (`SET ... NX PX` with token-checked Lua release). | | [`internal/broadcast`](../../internal/broadcast) | Redis Streams broadcast publisher and consumer used to fan out scheduling pause and resume events (`xmc-pause-stream`) to Actuator replicas. | | [`internal/featureflags`](../../internal/featureflags) | Unified feature flag definitions (`XMC_PAUSE_SCHEDULING_ENABLED`) and environment-variable loading across services. | | [`internal/auth`](../../internal/auth) | Authentication context extraction and authorization decisions shared across services. | diff --git a/docs/flows/experiment_lifecycle.md b/docs/flows/experiment_lifecycle.md index 522cbf77a..87393c62f 100644 --- a/docs/flows/experiment_lifecycle.md +++ b/docs/flows/experiment_lifecycle.md @@ -38,7 +38,7 @@ sequenceDiagram ESS-->>Client: 11. Experiment opt STUDY Note over Redis,SC: Asynchronously, after the RPC returns - Redis-->>SC: Reconcile tick (logged) + Redis-->>SC: Reconcile tick (per-study lease lock, logged) end ``` @@ -99,8 +99,10 @@ when the client cancels the RPC: when it fails, ESS logs the error, increments failing the RPC would make the client retry and create a duplicate study. Such a study is not reconciled until a future sweeper recovers it. `STANDARD` experiments enqueue nothing. The Study Controller ([`cmd/study_controller`](../../cmd/study_controller)) consumes the stream -asynchronously as the `study-controller-group` consumer group and logs each tick; a payload that -does not name a valid experiment goes to the `xmc-study-reconcile-dlq-stream` dead-letter stream. +asynchronously as the `study-controller-group` consumer group under a per-study Redis lease lock +(`xmc:study-reconcile-lock:{xid}`), acknowledging contended duplicate ticks without reconciling; a +payload that does not name a valid experiment goes to the `xmc-study-reconcile-dlq-stream` +dead-letter stream. Each submission enqueues one tick (delivered at least once); periodic ticks are not implemented yet. Domain rules for assembling the experiment — defaults, derived fields, validation — live in diff --git a/docs/flows/metrics_and_observability.md b/docs/flows/metrics_and_observability.md index a81280e3f..0a0e03b9b 100644 --- a/docs/flows/metrics_and_observability.md +++ b/docs/flows/metrics_and_observability.md @@ -253,11 +253,12 @@ When `XMC_PAUSE_SCHEDULING_ENABLED` is enabled, the Health Service, Actuator, an ### 4.2 Study Reconcile Observability -ESS exposes the following Prometheus metric under `/metrics`: +ESS and the Study Controller expose the following Prometheus metrics under `/metrics`: | Metric | Type | Component | Labels | Description | | :--- | :--- | :--- | :--- | :--- | | `xmc_ess_study_reconcile_enqueue_failures_total` | Counter | ESS | — | Created `STUDY` experiments whose initial reconcile tick could not be enqueued to `xmc-study-reconcile-stream`. Such a study is not reconciled until a future sweeper recovers it, so alert on any increase. | +| `xmc_study_controller_reconcile_ticks_total` | Counter | Study Controller | `outcome` | Study reconcile ticks processed, partitioned by `outcome` (`reconciled`, `contended`, `retry_error`, `invalid_payload`). | --- diff --git a/internal/redislock/BUILD.bazel b/internal/redislock/BUILD.bazel new file mode 100644 index 000000000..c4921539c --- /dev/null +++ b/internal/redislock/BUILD.bazel @@ -0,0 +1,33 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "redislock", + srcs = ["redislock.go"], + importpath = "xmanager_cloud/internal/redislock", + visibility = ["//:__subpackages__"], + deps = ["@com_github_redis_go_redis_v9//:go-redis"], +) + +go_test( + name = "redislock_test", + srcs = ["redislock_test.go"], + embed = [":redislock"], + deps = [ + "@com_github_google_go_cmp//cmp", + "@com_github_redis_go_redis_v9//:go-redis", + ], +) diff --git a/internal/redislock/redislock.go b/internal/redislock/redislock.go new file mode 100644 index 000000000..a0f14f82d --- /dev/null +++ b/internal/redislock/redislock.go @@ -0,0 +1,119 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package redislock provides single-instance distributed Redis lease locks backed by SET NX PX +// acquisition and a token-checked Lua release script. +package redislock + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const tokenBytes = 16 + +var ( + // ErrNotAcquired is returned by Locker.Acquire when the lock key is already held. + ErrNotAcquired = errors.New("redis lock not acquired") + // ErrLeaseLost is returned by Lease.Release when the lock key expired or changed owner. + ErrLeaseLost = errors.New("redis lock lease lost") + + releaseScript = redis.NewScript(`if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) else return 0 end`) +) + +// Client is the minimal go-redis interface required by Locker. *redis.Client satisfies it. +type Client interface { + SetArgs(ctx context.Context, key string, value any, a redis.SetArgs) *redis.StatusCmd + redis.Scripter +} + +var _ Client = (*redis.Client)(nil) + +// Locker acquires time-bounded distributed leases in Redis. +type Locker struct { + client Client + ttl time.Duration + randRead func([]byte) (int, error) +} + +// New validates client and ttl and returns a Locker whose leases expire after ttl. +func New(client Client, ttl time.Duration) (*Locker, error) { + if isNilClient(client) { + return nil, errors.New("redis lock client must not be nil") + } + if ttl < time.Millisecond { + return nil, fmt.Errorf("redis lock ttl must be at least 1ms, got %v", ttl) + } + return &Locker{client: client, ttl: ttl, randRead: rand.Read}, nil +} + +func isNilClient(client Client) bool { + if client == nil { + return true + } + c, ok := client.(*redis.Client) + return ok && c == nil +} + +// Acquire attempts once to acquire the lock at key without blocking, returning ErrNotAcquired if held. +func (l *Locker) Acquire(ctx context.Context, key string) (*Lease, error) { + if key == "" { + return nil, errors.New("redis lock key must not be empty") + } + token, err := l.generateToken() + if err != nil { + return nil, err + } + err = l.client.SetArgs(ctx, key, token, redis.SetArgs{Mode: "NX", TTL: l.ttl}).Err() + if errors.Is(err, redis.Nil) { + return nil, ErrNotAcquired + } + if err != nil { + return nil, fmt.Errorf("failed to acquire redis lock %q: %w", key, err) + } + return &Lease{client: l.client, key: key, token: token}, nil +} + +func (l *Locker) generateToken() (string, error) { + buf := make([]byte, tokenBytes) + if _, err := l.randRead(buf); err != nil { + return "", fmt.Errorf("failed to generate redis lock token: %w", err) + } + return hex.EncodeToString(buf), nil +} + +// Lease represents an acquired lock on a single Redis key. +type Lease struct { + client Client + key string + token string +} + +// Release deletes the lock key if and only if the lease token still owns it. +func (l *Lease) Release(ctx context.Context) error { + deleted, err := releaseScript.Run(ctx, l.client, []string{l.key}, l.token).Int64() + if err != nil { + return fmt.Errorf("failed to release redis lock %q: %w", l.key, err) + } + if deleted == 0 { + return ErrLeaseLost + } + return nil +} diff --git a/internal/redislock/redislock_test.go b/internal/redislock/redislock_test.go new file mode 100644 index 000000000..6f5623dfa --- /dev/null +++ b/internal/redislock/redislock_test.go @@ -0,0 +1,177 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package redislock + +import ( + "context" + "encoding/hex" + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/redis/go-redis/v9" +) + +type setCall struct { + Key, Mode string + TTL time.Duration +} + +type evalCall struct { + Keys []string + Args []any +} + +type fakeClient struct { + redis.Scripter + setErr, evalErr error + evalVal int64 + lastSet setCall + lastValue any + lastEval evalCall +} + +func (f *fakeClient) SetArgs(_ context.Context, key string, value any, a redis.SetArgs) *redis.StatusCmd { + f.lastSet, f.lastValue = setCall{Key: key, Mode: a.Mode, TTL: a.TTL}, value + if f.setErr != nil { + return redis.NewStatusResult("", f.setErr) + } + return redis.NewStatusResult("OK", nil) +} + +func (f *fakeClient) EvalSha(_ context.Context, _ string, keys []string, args ...any) *redis.Cmd { + f.lastEval = evalCall{Keys: keys, Args: args} + cmd := redis.NewCmd(context.Background()) + if f.evalErr != nil { + cmd.SetErr(f.evalErr) + return cmd + } + cmd.SetVal(f.evalVal) + return cmd +} + +func (f *fakeClient) Eval(ctx context.Context, _ string, keys []string, args ...any) *redis.Cmd { + return f.EvalSha(ctx, "", keys, args...) +} + +func TestNew_Validation(t *testing.T) { + t.Parallel() + valid := &fakeClient{} + tests := []struct { + name string + client Client + ttl time.Duration + wantErr bool + }{ + {name: "nil client", client: nil, ttl: time.Second, wantErr: true}, + {name: "typed nil redis client", client: (*redis.Client)(nil), ttl: time.Second, wantErr: true}, + {name: "zero ttl", client: valid, ttl: 0, wantErr: true}, + {name: "negative ttl", client: valid, ttl: -time.Second, wantErr: true}, + {name: "sub-millisecond ttl", client: valid, ttl: 999 * time.Microsecond, wantErr: true}, + {name: "exact 1ms boundary", client: valid, ttl: time.Millisecond}, + {name: "standard 15s ttl", client: valid, ttl: 15 * time.Second}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := New(tc.client, tc.ttl) + if (err != nil) != tc.wantErr || (!tc.wantErr && got == nil) { + t.Fatalf("New(_, %v) = (%v, %v), wantErr %v", tc.ttl, got, err, tc.wantErr) + } + }) + } +} + +func TestLocker_Acquire(t *testing.T) { + t.Parallel() + transportErr, randErr := errors.New("connection refused"), errors.New("entropy unavailable") + tests := []struct { + name string + key string + setErr error + randRead func([]byte) (int, error) + wantErr error + anyErr bool + }{ + {name: "acquired", key: "xmc:study-reconcile-lock:42"}, + {name: "multibyte key", key: "xmc:study-reconcile-lock:実験-42"}, + {name: "empty key", key: "", anyErr: true}, + {name: "already held", key: "xmc:study-reconcile-lock:42", setErr: redis.Nil, wantErr: ErrNotAcquired}, + {name: "transport error", key: "xmc:study-reconcile-lock:42", setErr: transportErr, wantErr: transportErr}, + {name: "token rng error", key: "xmc:study-reconcile-lock:42", randRead: func([]byte) (int, error) { return 0, randErr }, wantErr: randErr}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + fc := &fakeClient{setErr: tc.setErr} + locker, _ := New(fc, 15*time.Second) + if tc.randRead != nil { + locker.randRead = tc.randRead + } + lease, err := locker.Acquire(context.Background(), tc.key) + if tc.wantErr != nil { + if !errors.Is(err, tc.wantErr) { + t.Fatalf("Acquire(%q) error = %v, want %v", tc.key, err, tc.wantErr) + } + return + } + if tc.anyErr || err != nil { + if !tc.anyErr || err == nil { + t.Fatalf("Acquire(%q) error = %v, anyErr %v", tc.key, err, tc.anyErr) + } + return + } + if diff := cmp.Diff(setCall{Key: tc.key, Mode: "NX", TTL: 15 * time.Second}, fc.lastSet); diff != "" { + t.Errorf("SetArgs mismatch (-want +got):\n%s", diff) + } + token, _ := fc.lastValue.(string) + decoded, decErr := hex.DecodeString(token) + if decErr != nil || len(decoded) != tokenBytes || lease.token != token || lease.key != tc.key { + t.Errorf("lease={%q,%q} decoded=%d decErr=%v, want %d hex bytes", lease.key, lease.token, len(decoded), decErr, tokenBytes) + } + }) + } +} + +func TestLease_Release(t *testing.T) { + t.Parallel() + transportErr := errors.New("broken pipe") + tests := []struct { + name string + evalVal int64 + evalErr error + wantErr error + }{ + {name: "release succeeds", evalVal: 1}, + {name: "release detects lost lease", evalVal: 0, wantErr: ErrLeaseLost}, + {name: "release wraps transport error", evalErr: transportErr, wantErr: transportErr}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + fc := &fakeClient{evalVal: tc.evalVal, evalErr: tc.evalErr} + locker, _ := New(fc, 15*time.Second) + lease, _ := locker.Acquire(context.Background(), "xmc:study-reconcile-lock:99") + opErr := lease.Release(context.Background()) + if (tc.wantErr != nil && !errors.Is(opErr, tc.wantErr)) || (tc.wantErr == nil && opErr != nil) { + t.Fatalf("Release() error = %v, want %v", opErr, tc.wantErr) + } + if diff := cmp.Diff(evalCall{Keys: []string{"xmc:study-reconcile-lock:99"}, Args: []any{lease.token}}, fc.lastEval); diff != "" { + t.Errorf("script call mismatch (-want +got):\n%s", diff) + } + }) + } +} From ffd5c2118ba089c0c46085a6446b5338ce39b286 Mon Sep 17 00:00:00 2001 From: Vlad Lukiantsev Date: Tue, 17 Nov 2026 14:47:47 +0000 Subject: [PATCH 2/2] fix(study-controller): cancel reconcile context before releasing lease Swap LIFO defer ordering in reconcileWithLease so the reconcile context timer is cancelled immediately upon reconcile return before the detached Redis lease release RPC executes. Comment link: https://github.com/google/xmc/pull/1707#discussion_r4601048125 TAG=agy CONV=dafb63f4-0f3c-404e-b8fc-511e6baf7d1d --- cmd/study_controller/worker/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/study_controller/worker/handler.go b/cmd/study_controller/worker/handler.go index fc82ebfd4..1a238b887 100644 --- a/cmd/study_controller/worker/handler.go +++ b/cmd/study_controller/worker/handler.go @@ -170,8 +170,8 @@ func (h *Handler) HandleTask(ctx context.Context, workerID, data string) rediscl func (h *Handler) reconcileWithLease(ctx context.Context, workerID string, xid int64, lease Lease) error { reconcileCtx, cancel := context.WithTimeout(ctx, h.reconcileTimeout) - defer cancel() defer h.releaseLease(ctx, workerID, lease) + defer cancel() if err := h.reconcile(reconcileCtx, xid); err != nil { return err }