diff --git a/cmd/study_controller/BUILD.bazel b/cmd/study_controller/BUILD.bazel index efd849640..ab4819e56 100644 --- a/cmd/study_controller/BUILD.bazel +++ b/cmd/study_controller/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "//cmd/study_controller/config", "//cmd/study_controller/worker", "//internal/logging", + "//internal/task_queue_redis/redisclient", "@com_github_prometheus_client_golang//prometheus/promhttp", ], ) @@ -55,5 +56,6 @@ go_test( "//cmd/study_controller/config", "//cmd/study_controller/worker", "//internal/logging", + "//internal/task_queue_redis/redisclient", ], ) diff --git a/cmd/study_controller/health_test.go b/cmd/study_controller/health_test.go index 8970ed178..cef207355 100644 --- a/cmd/study_controller/health_test.go +++ b/cmd/study_controller/health_test.go @@ -53,7 +53,8 @@ func TestNewHTTPHandler(t *testing.T) { {name: "readiness with Redis reachable", method: http.MethodGet, path: "/readyz", wantStatus: http.StatusOK, wantContains: "OK"}, {name: "readiness with Redis unreachable", method: http.MethodGet, path: "/readyz", pingErr: unreachable, wantStatus: http.StatusServiceUnavailable, wantContains: "Service Unavailable"}, {name: "readiness with Redis ping timeout", method: http.MethodGet, path: "/readyz", pingErr: context.DeadlineExceeded, wantStatus: http.StatusServiceUnavailable, wantContains: "Service Unavailable"}, - {name: "metrics", method: http.MethodGet, path: "/metrics", wantStatus: http.StatusOK, wantContains: "go_goroutines"}, + {name: "metrics exposes runtime stats", method: http.MethodGet, path: "/metrics", wantStatus: http.StatusOK, wantContains: "go_goroutines"}, + {name: "metrics exposes reconcile ticks counter", method: http.MethodGet, path: "/metrics", wantStatus: http.StatusOK, wantContains: "xmc_study_controller_reconcile_ticks_total"}, {name: "unknown path", method: http.MethodGet, path: "/unknown", wantStatus: http.StatusNotFound}, {name: "non-GET liveness", method: http.MethodPost, path: "/healthz", wantStatus: http.StatusMethodNotAllowed}, {name: "non-GET readiness", method: http.MethodPost, path: "/readyz", wantStatus: http.StatusMethodNotAllowed}, diff --git a/cmd/study_controller/main.go b/cmd/study_controller/main.go index 9a0f368f0..f39c4bd06 100644 --- a/cmd/study_controller/main.go +++ b/cmd/study_controller/main.go @@ -32,6 +32,7 @@ import ( "xmanager_cloud/cmd/study_controller/config" "xmanager_cloud/cmd/study_controller/worker" "xmanager_cloud/internal/logging" + "xmanager_cloud/internal/task_queue_redis/redisclient" ) // HTTP server timeouts bound per-connection resource usage (e.g. slow-client attacks). @@ -82,8 +83,19 @@ func listenAndServe(ctx context.Context, cfg config.Config) error { if err != nil { return fmt.Errorf("failed to listen on port %s: %w", cfg.Port, err) } - client := worker.NewRedisConsumer(cfg.RedisAddr) - consumer := worker.Start(ctx, client) + return serveWithClient(ctx, lis, worker.NewRedisConsumer(cfg.RedisAddr)) +} + +func serveWithClient(ctx context.Context, lis net.Listener, client *redisclient.ConsumerClient) error { + handler, err := worker.NewRedisHandler(client) + if err != nil { + _ = lis.Close() + if client != nil && client.Client != nil { + _ = client.Close(context.WithoutCancel(ctx)) + } + return fmt.Errorf("failed to create study reconcile handler: %w", err) + } + consumer := worker.Start(ctx, client, handler) defer stopConsumer(ctx, consumer) return serve(ctx, lis, newHTTPHandler(client)) } diff --git a/cmd/study_controller/main_test.go b/cmd/study_controller/main_test.go index 07f6b63ed..c3297a721 100644 --- a/cmd/study_controller/main_test.go +++ b/cmd/study_controller/main_test.go @@ -30,6 +30,7 @@ import ( "xmanager_cloud/cmd/study_controller/config" "xmanager_cloud/cmd/study_controller/worker" "xmanager_cloud/internal/logging" + "xmanager_cloud/internal/task_queue_redis/redisclient" ) // waitHealthy polls GET /healthz at addr until it returns 200 OK or the deadline passes. @@ -101,6 +102,36 @@ func TestRun_CancelledContextExitsZero(t *testing.T) { } } +type noopRedisClient struct{ redisclient.Client } + +func (noopRedisClient) Close() error { return nil } + +func TestServeWithClient_RejectsInvalidClients(t *testing.T) { + nonLockClient := worker.NewRedisConsumer("127.0.0.1:" + freePort(t)) + _ = nonLockClient.Close(context.Background()) + nonLockClient.Client = noopRedisClient{} + + tests := []struct { + name string + client *redisclient.ConsumerClient + }{ + {name: "nil consumer client", client: nil}, + {name: "non-locking underlying client", client: nonLockClient}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen() error = %v", err) + } + defer lis.Close() + if err := serveWithClient(context.Background(), lis, tc.client); err == nil { + t.Errorf("serveWithClient(%+v) error = nil, want non-nil", tc.client) + } + }) + } +} + func TestReadiness_UnreachableRedis(t *testing.T) { client := worker.NewRedisConsumer("127.0.0.1:" + freePort(t)) defer client.Close(context.Background()) @@ -170,7 +201,11 @@ func TestStopConsumer_LogsCloseFailure(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() client := worker.NewRedisConsumer("127.0.0.1:" + freePort(t)) - consumer := worker.Start(ctx, client) + handler, err := worker.NewRedisHandler(client) + if err != nil { + t.Fatalf("NewRedisHandler() error = %v", err) + } + consumer := worker.Start(ctx, client, handler) // Closing the client first makes the consumer's own close fail. if err := client.Close(context.Background()); err != nil { t.Fatalf("client.Close() error = %v", err) diff --git a/cmd/study_controller/worker/BUILD.bazel b/cmd/study_controller/worker/BUILD.bazel index c0a18729f..c22ae00d8 100644 --- a/cmd/study_controller/worker/BUILD.bazel +++ b/cmd/study_controller/worker/BUILD.bazel @@ -28,8 +28,11 @@ go_library( "//api/protos/task_queue:task_go_proto", "//internal/kubernetes/common/constants", "//internal/logging", + "//internal/redislock", "//internal/task_queue_redis/redisclient", "//internal/validator", + "@com_github_prometheus_client_golang//prometheus", + "@com_github_prometheus_client_golang//prometheus/promauto", "@org_golang_google_protobuf//encoding/protojson", ], ) @@ -45,8 +48,10 @@ go_test( deps = [ "//internal/kubernetes/common/constants", "//internal/logging", + "//internal/redislock", "//internal/task_queue_redis/redisclient", "@com_github_google_go_cmp//cmp", + "@com_github_prometheus_client_golang//prometheus/testutil", "@com_github_redis_go_redis_v9//:go-redis", ], ) diff --git a/cmd/study_controller/worker/consumer.go b/cmd/study_controller/worker/consumer.go index 3487300a5..a91d8da2b 100644 --- a/cmd/study_controller/worker/consumer.go +++ b/cmd/study_controller/worker/consumer.go @@ -25,9 +25,9 @@ import ( ) // defaultDrainTimeout bounds how long Close waits for the worker and cleanup loops. It covers the -// 2s XREADGROUP block of the worker loop plus redisclient.SettleTimeout, so a tick being settled -// when Close is called finishes before the client is closed. -const defaultDrainTimeout = redisclient.SettleTimeout + 3*time.Second +// 2s XREADGROUP block of the worker loop, defaultReleaseTimeout, and redisclient.SettleTimeout, so +// a tick being settled when Close is called finishes before the client is closed. +const defaultDrainTimeout = redisclient.SettleTimeout + defaultReleaseTimeout + 3*time.Second // NewRedisConsumer returns a Redis consumer of the study reconcile stream for the Redis server at // addr. It reads the password from REDIS_PASSWORD and connects lazily. Context deadlines bound its @@ -46,7 +46,7 @@ func NewRedisConsumer(addr string) *redisclient.ConsumerClient { ) } -// Consumer runs the worker and cleanup loops of a Redis consumer with handleTask as its handler. +// Consumer runs the worker and cleanup loops of a Redis consumer with Handler.HandleTask. type Consumer struct { client redisclient.Consumer cancel context.CancelFunc @@ -56,12 +56,12 @@ type Consumer struct { closeErr error // result of the first Close, returned by every call } -// Start sets handleTask on client and runs its worker and cleanup loops until ctx is done or Close -// is called. The loops create the stream and consumer group themselves, at start-up and again -// whenever Redis reports them missing (NOGROUP), so Start never blocks on Redis and consumption -// resumes after Redis comes back without them. -func Start(ctx context.Context, client redisclient.Consumer) *Consumer { - client.SetTaskHandler(handleTask) +// Start sets handler.HandleTask on client and runs its worker and cleanup loops until ctx is done +// or Close is called. The loops create the stream and consumer group themselves, at start-up and +// again whenever Redis reports them missing (NOGROUP), so Start never blocks on Redis and +// consumption resumes after Redis comes back without them. +func Start(ctx context.Context, client redisclient.Consumer, handler *Handler) *Consumer { + client.SetTaskHandler(handler.HandleTask) loopCtx, cancel := context.WithCancel(ctx) c := &Consumer{client: client, cancel: cancel, drainTimeout: defaultDrainTimeout} workerID := redisclient.HostConsumerName(constants.StudyControllerWorkerID) @@ -84,7 +84,7 @@ func (c *Consumer) runLoop(loop func()) { } // Close stops the loops and closes the Redis client. It waits for the loops to return, but at -// most the drain timeout (8s) and only until ctx is done; after that it logs a warning and closes +// most the drain timeout (10s) and only until ctx is done; after that it logs a warning and closes // the client anyway, leaving a stuck loop to fail on the closed client. A tick being handled when // Close is called is still acknowledged or retried. Every call returns the first call's error. func (c *Consumer) Close(ctx context.Context) error { diff --git a/cmd/study_controller/worker/consumer_test.go b/cmd/study_controller/worker/consumer_test.go index f5273e38f..4806d5ee4 100644 --- a/cmd/study_controller/worker/consumer_test.go +++ b/cmd/study_controller/worker/consumer_test.go @@ -93,9 +93,18 @@ func (f *fakeConsumer) waitForLoops(t *testing.T) []string { return ids } +func newStubHandler(t *testing.T) *Handler { + t.Helper() + h, err := NewHandler(HandlerParams{Locker: &fakeLocker{lease: &fakeLease{}}}) + if err != nil { + t.Fatalf("NewHandler() error = %v", err) + } + return h +} + func TestStart_RunsLoopsWithHandleTask(t *testing.T) { fake := newFakeConsumer() - c := Start(context.Background(), fake) + c := Start(context.Background(), fake, newStubHandler(t)) defer c.Close(context.Background()) ids := fake.waitForLoops(t) @@ -111,7 +120,7 @@ func TestStart_RunsLoopsWithHandleTask(t *testing.T) { t.Fatal("Start() did not set a task handler") } if got := handler(context.Background(), "w", `{"experiment":"experiments/7"}`); !got.WasRunSuccessful { - t.Errorf("task handler on a valid tick = %+v, want success (handleTask)", got) + t.Errorf("task handler on a valid tick = %+v, want success (HandleTask)", got) } } @@ -127,7 +136,7 @@ func containsPrefix(ids []string, prefix string) bool { func TestConsumer_CloseStopsLoopsAndClosesClient(t *testing.T) { fake := newFakeConsumer() fake.closeErr = errors.New("close failed") - c := Start(context.Background(), fake) + c := Start(context.Background(), fake, newStubHandler(t)) fake.waitForLoops(t) if err := c.Close(context.Background()); !errors.Is(err, fake.closeErr) { @@ -146,7 +155,7 @@ func TestConsumer_CloseStopsLoopsAndClosesClient(t *testing.T) { func TestStart_ContextCancelStopsLoops(t *testing.T) { fake := newFakeConsumer() ctx, cancel := context.WithCancel(context.Background()) - c := Start(ctx, fake) + c := Start(ctx, fake, newStubHandler(t)) fake.waitForLoops(t) cancel() @@ -179,7 +188,7 @@ func TestConsumer_CloseWithStuckLoops(t *testing.T) { fake := newFakeConsumer() fake.ignoreCancel = true defer close(fake.release) - c := Start(context.Background(), fake) + c := Start(context.Background(), fake, newStubHandler(t)) fake.waitForLoops(t) c.drainTimeout = tc.drainTimeout @@ -204,6 +213,7 @@ type consumerSettings struct { Addr, Stream, Group, DLQ string MaxRetries int ContextTimeoutEnabled bool + DrainTimeout time.Duration } func TestNewRedisConsumer(t *testing.T) { @@ -217,10 +227,12 @@ func TestNewRedisConsumer(t *testing.T) { got := consumerSettings{ Addr: client.Options().Addr, Stream: c.StreamName, Group: c.GroupName, DLQ: c.DLQStreamName, MaxRetries: c.MaxRetries, ContextTimeoutEnabled: client.Options().ContextTimeoutEnabled, + DrainTimeout: defaultDrainTimeout, } want := consumerSettings{ Addr: "redis-service:6379", Stream: "xmc-study-reconcile-stream", Group: "study-controller-group", DLQ: "xmc-study-reconcile-dlq-stream", MaxRetries: constants.RedisMaxRetries, ContextTimeoutEnabled: true, + DrainTimeout: 10 * time.Second, } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("NewRedisConsumer() settings mismatch (-want +got):\n%s", diff) diff --git a/cmd/study_controller/worker/handler.go b/cmd/study_controller/worker/handler.go index a96417c1a..1a238b887 100644 --- a/cmd/study_controller/worker/handler.go +++ b/cmd/study_controller/worker/handler.go @@ -17,23 +17,177 @@ package worker import ( "context" + "errors" + "fmt" + "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "xmanager_cloud/internal/kubernetes/common/constants" "xmanager_cloud/internal/logging" + "xmanager_cloud/internal/redislock" "xmanager_cloud/internal/task_queue_redis/redisclient" "xmanager_cloud/internal/validator" ) -// handleTask handles one study reconcile tick. A payload that does not decode to a valid -// experiment name can never succeed, so it is moved to the dead-letter stream without a retry. -// A valid tick is logged at debug level (it repeats per active study) with the study's XID and -// acknowledged; reconciling the study is not implemented yet. -func handleTask(ctx context.Context, workerID, data string) redisclient.TaskResult { +const ( + defaultLeaseTTL = 15 * time.Second + defaultReconcileTimeout = 60 * time.Second + defaultReleaseTimeout = 2 * time.Second + studyLockKeyPrefix = "xmc:study-reconcile-lock:" + outcomeReconciled = "reconciled" + outcomeContended = "contended" + outcomeRetryError = "retry_error" + outcomeInvalidPayload = "invalid_payload" +) + +var reconcileTicksTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "xmc_study_controller_reconcile_ticks_total", + Help: "Total number of study reconcile ticks processed by the Study Controller, partitioned by outcome.", +}, []string{"outcome"}) + +func init() { + for _, o := range []string{outcomeReconciled, outcomeContended, outcomeRetryError, outcomeInvalidPayload} { + reconcileTicksTotal.WithLabelValues(o) + } +} + +// Lease represents an acquired per-study reconcile lock. *redislock.Lease satisfies it. +type Lease interface { + Release(ctx context.Context) error +} + +var _ Lease = (*redislock.Lease)(nil) + +// Locker acquires per-study reconcile leases without blocking. +type Locker interface { + Acquire(ctx context.Context, key string) (Lease, error) +} + +type redisLocker struct{ locker *redislock.Locker } + +func (r redisLocker) Acquire(ctx context.Context, key string) (Lease, error) { + lease, err := r.locker.Acquire(ctx, key) + if err != nil { + return nil, err + } + return lease, nil +} + +// ReconcileFunc reconciles the study identified by xid under a held per-study lease. +type ReconcileFunc func(ctx context.Context, xid int64) error + +func noopReconcile(context.Context, int64) error { return nil } +func studyLockKey(xid int64) string { return fmt.Sprintf("%s%d", studyLockKeyPrefix, xid) } + +// HandlerParams configures a study reconcile Handler. +type HandlerParams struct { + Locker Locker + Reconcile ReconcileFunc + ReconcileTimeout, ReleaseTimeout time.Duration +} + +// Handler processes study reconcile ticks under a distributed per-study lease lock. +type Handler struct { + locker Locker + reconcile ReconcileFunc + reconcileTimeout, releaseTimeout time.Duration +} + +// NewRedisHandler constructs a Handler sharing consumer's underlying *redis.Client pool. +func NewRedisHandler(consumer *redisclient.ConsumerClient) (*Handler, error) { + if consumer == nil { + return nil, errors.New("redis consumer client must not be nil") + } + lockClient, ok := consumer.Client.(redislock.Client) + if !ok { + return nil, errors.New("redis consumer client does not implement redislock.Client") + } + locker, err := redislock.New(lockClient, defaultLeaseTTL) + if err != nil { + return nil, err + } + return NewHandler(HandlerParams{Locker: redisLocker{locker: locker}}) +} + +// NewHandler validates p and returns a Handler with defaults applied to zero-valued parameters. +func NewHandler(p HandlerParams) (*Handler, error) { + if p.Locker == nil { + return nil, errors.New("study reconcile locker must not be nil") + } + if p.Reconcile == nil { + p.Reconcile = noopReconcile + } + if p.ReconcileTimeout == 0 { + p.ReconcileTimeout = defaultReconcileTimeout + } + if p.ReleaseTimeout == 0 { + p.ReleaseTimeout = defaultReleaseTimeout + } + if p.ReconcileTimeout < 0 || p.ReleaseTimeout < 0 { + return nil, errors.New("study reconcile handler durations must be positive") + } + if p.ReconcileTimeout >= constants.RedisProcessTimeout { + return nil, fmt.Errorf("reconcile timeout (%v) must be less than redis process timeout (%v)", p.ReconcileTimeout, constants.RedisProcessTimeout) + } + return &Handler{ + locker: p.Locker, + reconcile: p.Reconcile, + reconcileTimeout: p.ReconcileTimeout, + releaseTimeout: p.ReleaseTimeout, + }, nil +} + +// HandleTask handles one study reconcile tick under a per-study Redis lease. +func (h *Handler) HandleTask(ctx context.Context, workerID, data string) redisclient.TaskResult { xid, err := parseReconcilePayload(data) if err != nil { + reconcileTicksTotal.WithLabelValues(outcomeInvalidPayload).Inc() logging.Warn(ctx, "Dropping invalid study reconcile tick", "worker_id", workerID, "payload_bytes", len(data), "error", err) return redisclient.TaskResult{WasRunSuccessful: false, ShouldBeRetried: false, Error: err} } ctx = logging.WithExperimentName(ctx, validator.ConstructExperimentName(xid)) - logging.Debug(ctx, "Received study reconcile tick", "worker_id", workerID) + lease, err := h.locker.Acquire(ctx, studyLockKey(xid)) + if errors.Is(err, redislock.ErrNotAcquired) { + reconcileTicksTotal.WithLabelValues(outcomeContended).Inc() + logging.Info(ctx, "Skipped contended study reconcile tick", "worker_id", workerID) + return redisclient.TaskResult{WasRunSuccessful: true} + } + if err != nil { + reconcileTicksTotal.WithLabelValues(outcomeRetryError).Inc() + logging.Warn(ctx, "Failed to acquire study reconcile lock", "worker_id", workerID, "error", err) + return redisclient.TaskResult{WasRunSuccessful: false, ShouldBeRetried: true, Error: err} + } + if err := h.reconcileWithLease(ctx, workerID, xid, lease); err != nil { + reconcileTicksTotal.WithLabelValues(outcomeRetryError).Inc() + logging.Warn(ctx, "Failed to process study reconcile tick", "worker_id", workerID, "error", err) + return redisclient.TaskResult{WasRunSuccessful: false, ShouldBeRetried: true, Error: err} + } + reconcileTicksTotal.WithLabelValues(outcomeReconciled).Inc() + logging.Info(ctx, "Processed study reconcile tick", "worker_id", workerID) return redisclient.TaskResult{WasRunSuccessful: true} } + +func (h *Handler) reconcileWithLease(ctx context.Context, workerID string, xid int64, lease Lease) error { + reconcileCtx, cancel := context.WithTimeout(ctx, h.reconcileTimeout) + defer h.releaseLease(ctx, workerID, lease) + defer cancel() + if err := h.reconcile(reconcileCtx, xid); err != nil { + return err + } + return reconcileCtx.Err() +} + +func (h *Handler) releaseLease(ctx context.Context, workerID string, lease Lease) { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), h.releaseTimeout) + defer cancel() + err := lease.Release(releaseCtx) + if err == nil { + return + } + msg := "Failed to release study reconcile lock" + if errors.Is(err, redislock.ErrLeaseLost) { + msg = "Study reconcile lock lease was already lost before release" + } + logging.Warn(ctx, msg, "worker_id", workerID, "error", err) +} diff --git a/cmd/study_controller/worker/handler_test.go b/cmd/study_controller/worker/handler_test.go index 6933741ef..8114ea85e 100644 --- a/cmd/study_controller/worker/handler_test.go +++ b/cmd/study_controller/worker/handler_test.go @@ -17,46 +17,273 @@ package worker import ( "bytes" "context" + "errors" "log/slog" "strings" + "sync" "testing" + "time" + "github.com/google/go-cmp/cmp" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/redis/go-redis/v9" + "xmanager_cloud/internal/kubernetes/common/constants" "xmanager_cloud/internal/logging" + "xmanager_cloud/internal/redislock" + "xmanager_cloud/internal/task_queue_redis/redisclient" ) -func TestHandleTask(t *testing.T) { - t.Parallel() +type fakeLease struct { + mu sync.Mutex + releaseFn func(ctx context.Context) error + releaseCalls int + releaseCtxErr error + releaseHasDeadline bool +} + +func (l *fakeLease) Release(ctx context.Context) error { + l.mu.Lock() + l.releaseCalls++ + l.releaseCtxErr = ctx.Err() + _, l.releaseHasDeadline = ctx.Deadline() + fn := l.releaseFn + l.mu.Unlock() + if fn != nil { + return fn(ctx) + } + return nil +} + +type fakeLocker struct { + mu sync.Mutex + lease Lease + err error + lastKey string + calls int +} + +func (f *fakeLocker) Acquire(_ context.Context, key string) (Lease, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + f.lastKey = key + if f.err != nil { + return nil, f.err + } + return f.lease, nil +} + +type stubLockClient struct { + redisclient.Client + redis.Scripter + setErr error +} + +func (s *stubLockClient) SetArgs(_ context.Context, _ string, _ any, _ redis.SetArgs) *redis.StatusCmd { + if s.setErr != nil { + return redis.NewStatusResult("", s.setErr) + } + return redis.NewStatusResult("OK", nil) +} + +func (s *stubLockClient) EvalSha(_ context.Context, _ string, _ []string, _ ...any) *redis.Cmd { + cmd := redis.NewCmd(context.Background()) + cmd.SetVal(int64(1)) + return cmd +} +func snapshotOutcomes() map[string]float64 { + outcomes := []string{outcomeReconciled, outcomeContended, outcomeRetryError, outcomeInvalidPayload} + snap := make(map[string]float64, len(outcomes)) + for _, o := range outcomes { + snap[o] = testutil.ToFloat64(reconcileTicksTotal.WithLabelValues(o)) + } + return snap +} + +func diffOutcomes(before map[string]float64) map[string]float64 { + deltas := make(map[string]float64) + for k, v := range snapshotOutcomes() { + if delta := v - before[k]; delta != 0 { + deltas[k] = delta + } + } + return deltas +} + +func newLogContext(buf *bytes.Buffer) context.Context { + return logging.WithLogger(context.Background(), slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) +} + +func TestNewHandler_ValidationAndRedisWiring(t *testing.T) { + valid := &fakeLocker{lease: &fakeLease{}} tests := []struct { - name string - data string - wantSuccess bool - wantLogs []string + name string + params HandlerParams + wantErr bool }{ - {name: "valid tick is acknowledged", data: `{"experiment":"experiments/123"}`, wantSuccess: true, - wantLogs: []string{"Received study reconcile tick", "worker_id=worker-1", "xid=123"}}, - {name: "malformed payload is dead-lettered", data: `not json`, wantLogs: []string{"Dropping invalid study reconcile tick", "payload_bytes=8"}}, - {name: "invalid experiment is dead-lettered", data: `{"experiment":"xids/123"}`, wantLogs: []string{"Dropping invalid study reconcile tick"}}, + {name: "nil locker", params: HandlerParams{}, wantErr: true}, + {name: "negative reconcile timeout", params: HandlerParams{Locker: valid, ReconcileTimeout: -time.Second}, wantErr: true}, + {name: "negative release timeout", params: HandlerParams{Locker: valid, ReleaseTimeout: -time.Second}, wantErr: true}, + {name: "reconcile timeout at process timeout", params: HandlerParams{Locker: valid, ReconcileTimeout: constants.RedisProcessTimeout}, wantErr: true}, + {name: "valid defaults", params: HandlerParams{Locker: valid}}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - t.Parallel() - var logs bytes.Buffer - ctx := logging.WithLogger(context.Background(), slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) + got, err := NewHandler(tc.params) + if (err != nil) != tc.wantErr || (!tc.wantErr && got == nil) { + t.Fatalf("NewHandler(%+v) = (%v, %v), wantErr %v", tc.params, got, err, tc.wantErr) + } + }) + } + + var nonLockConsumer redisclient.ConsumerClient + typedNilConsumer := NewRedisConsumer("127.0.0.1:6379") + _ = typedNilConsumer.Close(context.Background()) + typedNilConsumer.Client = (*redis.Client)(nil) + for _, bad := range []*redisclient.ConsumerClient{nil, &nonLockConsumer, typedNilConsumer} { + if _, err := NewRedisHandler(bad); err == nil { + t.Fatalf("NewRedisHandler(%+v) error = nil, want error", bad) + } + } + + okConsumer := NewRedisConsumer("127.0.0.1:6379") + _ = okConsumer.Close(context.Background()) + okConsumer.Client = &stubLockClient{} + h, err := NewRedisHandler(okConsumer) + if err != nil || !h.HandleTask(context.Background(), "w", `{"experiment":"experiments/55"}`).WasRunSuccessful { + t.Fatalf("NewRedisHandler(ok) err = %v or HandleTask failed", err) + } + okConsumer.Client = &stubLockClient{setErr: redis.Nil} + hContended, err := NewRedisHandler(okConsumer) + if err != nil || !hContended.HandleTask(context.Background(), "w", `{"experiment":"experiments/55"}`).WasRunSuccessful { + t.Fatalf("NewRedisHandler(contended) err = %v or HandleTask failed", err) + } +} + +func TestHandleTask_OutcomesAndLockAcquisition(t *testing.T) { + acquireErr, reconcileErr := errors.New("redis connection refused"), errors.New("vizier unavailable") + const validTick = `{"experiment":"experiments/123"}` + const wantKey = "xmc:study-reconcile-lock:123" + + tests := []struct { + name string + data string + acquireErr, recErr, releaseErr error + wantSuccess, wantRetry, wantRec bool + wantErrIs error + wantAcquireCalls int + wantKey string + wantMetrics map[string]float64 + wantLogs []string + }{ + {name: "fast reconcile", data: validTick, wantSuccess: true, wantAcquireCalls: 1, wantKey: wantKey, wantRec: true, + wantMetrics: map[string]float64{outcomeReconciled: 1}, wantLogs: []string{"Processed study reconcile tick", "worker_id=worker-1", "xid=123"}}, + {name: "contended tick acked", data: validTick, acquireErr: redislock.ErrNotAcquired, wantSuccess: true, wantAcquireCalls: 1, wantKey: wantKey, + wantMetrics: map[string]float64{outcomeContended: 1}, wantLogs: []string{"Skipped contended study reconcile tick", "xid=123"}}, + {name: "acquire error retried", data: validTick, acquireErr: acquireErr, wantRetry: true, wantErrIs: acquireErr, wantAcquireCalls: 1, wantKey: wantKey, + wantMetrics: map[string]float64{outcomeRetryError: 1}, wantLogs: []string{"Failed to acquire study reconcile lock", "xid=123"}}, + {name: "reconcile error retried", data: validTick, recErr: reconcileErr, wantRetry: true, wantErrIs: reconcileErr, wantAcquireCalls: 1, wantKey: wantKey, wantRec: true, + wantMetrics: map[string]float64{outcomeRetryError: 1}, wantLogs: []string{"Failed to process study reconcile tick", "xid=123"}}, + {name: "release lost lease logs warn", data: validTick, releaseErr: redislock.ErrLeaseLost, wantSuccess: true, wantAcquireCalls: 1, wantKey: wantKey, wantRec: true, + wantMetrics: map[string]float64{outcomeReconciled: 1}, wantLogs: []string{"Study reconcile lock lease was already lost before release"}}, + {name: "release transport error logs warn", data: validTick, releaseErr: acquireErr, wantSuccess: true, wantAcquireCalls: 1, wantKey: wantKey, wantRec: true, + wantMetrics: map[string]float64{outcomeReconciled: 1}, wantLogs: []string{"Failed to release study reconcile lock"}}, + {name: "empty payload", data: "", wantMetrics: map[string]float64{outcomeInvalidPayload: 1}, wantLogs: []string{"Dropping invalid study reconcile tick", "payload_bytes=0"}}, + {name: "malformed json", data: "not json", wantMetrics: map[string]float64{outcomeInvalidPayload: 1}, wantLogs: []string{"Dropping invalid study reconcile tick", "payload_bytes=8"}}, + {name: "multibyte invalid xid", data: `{"experiment":"experiments/123"}`, wantMetrics: map[string]float64{outcomeInvalidPayload: 1}, wantLogs: []string{"Dropping invalid study reconcile tick"}}, + {name: "wrong resource", data: `{"experiment":"xids/123"}`, wantMetrics: map[string]float64{outcomeInvalidPayload: 1}, wantLogs: []string{"Dropping invalid study reconcile tick"}}, + } - got := handleTask(ctx, "worker-1", tc.data) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + lease := &fakeLease{releaseFn: func(context.Context) error { return tc.releaseErr }} + locker := &fakeLocker{lease: lease, err: tc.acquireErr} + var reconciled bool + h, _ := NewHandler(HandlerParams{ + Locker: locker, + Reconcile: func(_ context.Context, xid int64) error { + reconciled = xid == 123 + return tc.recErr + }, + }) + + var logs bytes.Buffer + before := snapshotOutcomes() + got := h.HandleTask(newLogContext(&logs), "worker-1", tc.data) - if got.WasRunSuccessful != tc.wantSuccess || got.ShouldBeRetried { - t.Errorf("handleTask(%q) = %+v, want WasRunSuccessful=%v ShouldBeRetried=false", tc.data, got, tc.wantSuccess) + if got.WasRunSuccessful != tc.wantSuccess || got.ShouldBeRetried != tc.wantRetry { + t.Errorf("HandleTask() = %+v, want success=%v retry=%v", got, tc.wantSuccess, tc.wantRetry) } - if (got.Error == nil) != tc.wantSuccess { - t.Errorf("handleTask(%q) error = %v, want error only on failure", tc.data, got.Error) + if tc.wantErrIs != nil && !errors.Is(got.Error, tc.wantErrIs) { + t.Errorf("error = %v, want %v", got.Error, tc.wantErrIs) } - for _, want := range tc.wantLogs { - if !strings.Contains(logs.String(), want) { - t.Errorf("handleTask(%q) logs = %q, want them to contain %q", tc.data, logs.String(), want) + if locker.calls != tc.wantAcquireCalls || locker.lastKey != tc.wantKey || reconciled != tc.wantRec { + t.Errorf("calls=%d key=%q rec=%v, want %d %q %v", locker.calls, locker.lastKey, reconciled, tc.wantAcquireCalls, tc.wantKey, tc.wantRec) + } + if tc.wantRec && (lease.releaseCalls != 1 || !lease.releaseHasDeadline) { + t.Errorf("releaseCalls=%d hasDeadline=%v, want 1 bounded Release", lease.releaseCalls, lease.releaseHasDeadline) + } + if diff := cmp.Diff(tc.wantMetrics, diffOutcomes(before)); diff != "" { + t.Errorf("metric deltas mismatch (-want +got):\n%s", diff) + } + for _, wantLog := range tc.wantLogs { + if !strings.Contains(logs.String(), wantLog) { + t.Errorf("logs = %q, want %q", logs.String(), wantLog) } } }) } } + +func TestHandleTask_DeadlineAndShutdownTransitions(t *testing.T) { + tests := []struct { + name string + reconcileTimeout time.Duration + ignoreCancel bool + cancelParent bool + wantErrIs error + }{ + {name: "reconcile timeout deadline", reconcileTimeout: 15 * time.Millisecond, wantErrIs: context.DeadlineExceeded}, + {name: "reconcile timeout exact boundary when reconcile returns nil", reconcileTimeout: 15 * time.Millisecond, + ignoreCancel: true, wantErrIs: context.DeadlineExceeded}, + {name: "parent shutdown cancels reconcile and releases on detached ctx", cancelParent: true, wantErrIs: context.Canceled}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + parentCtx, cancelParent := context.WithCancel(context.Background()) + defer cancelParent() + started := make(chan struct{}) + if tc.cancelParent { + go func() { <-started; cancelParent() }() + } + lease := &fakeLease{} + h, _ := NewHandler(HandlerParams{ + Locker: &fakeLocker{lease: lease}, + ReconcileTimeout: tc.reconcileTimeout, + Reconcile: func(ctx context.Context, _ int64) error { + close(started) + <-ctx.Done() + if tc.ignoreCancel { + return nil + } + return ctx.Err() + }, + }) + + before := snapshotOutcomes() + got := h.HandleTask(parentCtx, "worker-1", `{"experiment":"experiments/14"}`) + + if got.WasRunSuccessful || !got.ShouldBeRetried || !errors.Is(got.Error, tc.wantErrIs) { + t.Fatalf("got = %+v, want retryable %v", got, tc.wantErrIs) + } + if lease.releaseCalls != 1 || lease.releaseCtxErr != nil || !lease.releaseHasDeadline { + t.Errorf("releaseCalls=%d ctxErr=%v hasDeadline=%v", lease.releaseCalls, lease.releaseCtxErr, lease.releaseHasDeadline) + } + if diff := cmp.Diff(map[string]float64{outcomeRetryError: 1}, diffOutcomes(before)); diff != "" { + t.Errorf("metric deltas mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/deploy/profiles/stable-2026b/components/hpo/README.md b/deploy/profiles/stable-2026b/components/hpo/README.md index b26d88c78..4e97bfb64 100644 --- a/deploy/profiles/stable-2026b/components/hpo/README.md +++ b/deploy/profiles/stable-2026b/components/hpo/README.md @@ -3,7 +3,7 @@ This Helm sub-chart (`deploy/profiles/stable-2026b/components/hpo/`) defines the XMC 1P Native Component blueprint ([`templates/component.yaml`](templates/component.yaml)) for **Hyperparameter Optimization (HPO)** (`hyperparameter-optimization`) and the `xmc-study-controller` workload that `xmc-bridge` actuates for it ([`templates/deployment.yaml`](templates/deployment.yaml), [`templates/serviceaccount.yaml`](templates/serviceaccount.yaml)). > [!NOTE] -> The `xmc-study-controller` workload is a `Deployment` and a plain `ServiceAccount` only. It consumes study reconcile ticks from Redis and logs them, and serves health and metrics endpoints; it does not yet reconcile studies. The chart renders no `Service`, `Role`/`RoleBinding`, `NetworkPolicy` or `ConfigMap`, and the `ServiceAccount` carries no Workload Identity annotation. +> The `xmc-study-controller` workload is a `Deployment` and a plain `ServiceAccount` only. It consumes study reconcile ticks from Redis under a per-study Redis lease lock and logs them, and serves health and metrics endpoints; it does not yet reconcile studies. The chart renders no `Service`, `Role`/`RoleBinding`, `NetworkPolicy` or `ConfigMap`, and the `ServiceAccount` carries no Workload Identity annotation. --- @@ -72,12 +72,22 @@ The Study Controller connects to the XMC Redis server: ### Study Reconcile Ticks -The Study Controller consumes the `xmc-study-reconcile-stream` Redis stream as the `study-controller-group` consumer group. Each message is a protojson `StudyReconcilePayload` (`{"experiment": "experiments/{xid}"}`) that names a study; ticks are unsigned hints, delivered at least once. The controller logs each valid tick at debug level with the study's XID (`Received study reconcile tick`) and acknowledges it; it does not reconcile the study yet. A payload that does not name a valid experiment, or a message without data, goes to the `xmc-study-reconcile-dlq-stream` dead-letter stream without a retry. The controller re-creates the consumer group when Redis loses it, for example after a restart without persistence. +The Study Controller consumes the `xmc-study-reconcile-stream` Redis stream as the `study-controller-group` consumer group, sharing a single Redis connection pool across stream consumption, per-study locking, and `/readyz`. Each message is a protojson `StudyReconcilePayload` (`{"experiment": "experiments/{xid}"}`) that names a study; ticks are unsigned hints, delivered at least once. + +Each delivered tick is classified by outcome (`xmc_study_controller_reconcile_ticks_total{outcome}`); for valid payloads, the controller acquires a distributed per-study lease at `xmc:study-reconcile-lock:{xid}` (`SET ... NX PX` with a random 16-byte token, 15s lease TTL up to a 60s reconcile timeout, and token-checked Lua `DEL`): + +- **Acquired (`reconciled`)**: reconciles the study (currently a stub), releases the lease, logs `Processed study reconcile tick` at info level with the study's XID, and acknowledges the tick. +- **Already held (`contended`)**: logs `Skipped contended study reconcile tick` at info level and acknowledges the tick without reconciling so multi-replica deployments never reconcile the same study concurrently. +- **Lock or reconcile failure (`retry_error`)**: releases any acquired lease if still owned and retries the tick via stream redelivery. +- **Malformed payload (`invalid_payload`)**: a payload whose `data` does not decode to a valid `experiments/{xid}` resource name goes to the `xmc-study-reconcile-dlq-stream` dead-letter stream without acquiring the lock or retrying (raw stream entries missing a `data` field are dead-lettered directly by the stream consumer before the handler runs). + +The controller re-creates the consumer group when Redis loses it, for example after a restart without persistence. The Experiment State Server publishes one tick when it creates a `STUDY` experiment; see [Create Experiment](../../../../../docs/flows/experiment_lifecycle.md#1-create-experiment). +- **Metrics**: `GET /metrics` exposes `xmc_study_controller_reconcile_ticks_total{outcome="reconciled|contended|retry_error|invalid_payload"}`. - **Troubleshooting**: to inspect dead-lettered ticks and the reason each was rejected, run `XRANGE xmc-study-reconcile-dlq-stream - +` in `redis-cli`. -- **Shutdown**: on SIGTERM the controller stops its HTTP server (up to 5s), then drains the consumer (up to 8s) so a tick being handled is still acknowledged. Ticks it has not started stay pending and are reclaimed later. If you set the Component's `lifecycleControl.drainTimeoutSeconds`, set it to at least 8: the pod's termination grace period is that value plus 5s. +- **Shutdown**: on SIGTERM the controller stops its HTTP server (up to 5s), then drains the consumer (up to 10s) so an in-flight tick releases its per-study lock (2s detached timeout) and settles on the stream (5s detached timeout). Ticks it has not started stay pending and are reclaimed later. If you set the Component's `lifecycleControl.drainTimeoutSeconds`, set it to at least 10: the pod's termination grace period is that value plus 5s. ### Image Resolution diff --git a/docs/architecture/components.md b/docs/architecture/components.md index 7cbad6092..f9b38e646 100644 --- a/docs/architecture/components.md +++ b/docs/architecture/components.md @@ -320,6 +320,7 @@ declarations fail the build rather than a deployment. | [`internal/storage`](../../internal/storage) | Spanner data access, schema migrations, and the storage interfaces used by ESS, Actuator, Reconciler, Health Service, and Log Monitor. | | [`internal/task_queue`](../../internal/task_queue) | Builds, signs, and enqueues actuator payloads. Owns the idempotency key and JWT/OIDC signing scheme. | | [`internal/task_queue_redis`](../../internal/task_queue_redis) | Redis Streams publisher and consumer with consumer groups, retries, abandoned-message reclamation, and dead-lettering. | +| [`internal/redislock`](../../internal/redislock) | Distributed Redis lease lock (`SET ... NX PX` with token-checked Lua release). | | [`internal/broadcast`](../../internal/broadcast) | Redis Streams broadcast publisher and consumer used to fan out scheduling pause and resume events (`xmc-pause-stream`) to Actuator replicas. | | [`internal/featureflags`](../../internal/featureflags) | Unified feature flag definitions (`XMC_PAUSE_SCHEDULING_ENABLED`) and environment-variable loading across services. | | [`internal/auth`](../../internal/auth) | Authentication context extraction and authorization decisions shared across services. | diff --git a/docs/flows/experiment_lifecycle.md b/docs/flows/experiment_lifecycle.md index 522cbf77a..87393c62f 100644 --- a/docs/flows/experiment_lifecycle.md +++ b/docs/flows/experiment_lifecycle.md @@ -38,7 +38,7 @@ sequenceDiagram ESS-->>Client: 11. Experiment opt STUDY Note over Redis,SC: Asynchronously, after the RPC returns - Redis-->>SC: Reconcile tick (logged) + Redis-->>SC: Reconcile tick (per-study lease lock, logged) end ``` @@ -99,8 +99,10 @@ when the client cancels the RPC: when it fails, ESS logs the error, increments failing the RPC would make the client retry and create a duplicate study. Such a study is not reconciled until a future sweeper recovers it. `STANDARD` experiments enqueue nothing. The Study Controller ([`cmd/study_controller`](../../cmd/study_controller)) consumes the stream -asynchronously as the `study-controller-group` consumer group and logs each tick; a payload that -does not name a valid experiment goes to the `xmc-study-reconcile-dlq-stream` dead-letter stream. +asynchronously as the `study-controller-group` consumer group under a per-study Redis lease lock +(`xmc:study-reconcile-lock:{xid}`), acknowledging contended duplicate ticks without reconciling; a +payload that does not name a valid experiment goes to the `xmc-study-reconcile-dlq-stream` +dead-letter stream. Each submission enqueues one tick (delivered at least once); periodic ticks are not implemented yet. Domain rules for assembling the experiment — defaults, derived fields, validation — live in diff --git a/docs/flows/metrics_and_observability.md b/docs/flows/metrics_and_observability.md index a81280e3f..0a0e03b9b 100644 --- a/docs/flows/metrics_and_observability.md +++ b/docs/flows/metrics_and_observability.md @@ -253,11 +253,12 @@ When `XMC_PAUSE_SCHEDULING_ENABLED` is enabled, the Health Service, Actuator, an ### 4.2 Study Reconcile Observability -ESS exposes the following Prometheus metric under `/metrics`: +ESS and the Study Controller expose the following Prometheus metrics under `/metrics`: | Metric | Type | Component | Labels | Description | | :--- | :--- | :--- | :--- | :--- | | `xmc_ess_study_reconcile_enqueue_failures_total` | Counter | ESS | — | Created `STUDY` experiments whose initial reconcile tick could not be enqueued to `xmc-study-reconcile-stream`. Such a study is not reconciled until a future sweeper recovers it, so alert on any increase. | +| `xmc_study_controller_reconcile_ticks_total` | Counter | Study Controller | `outcome` | Study reconcile ticks processed, partitioned by `outcome` (`reconciled`, `contended`, `retry_error`, `invalid_payload`). | --- diff --git a/internal/redislock/BUILD.bazel b/internal/redislock/BUILD.bazel new file mode 100644 index 000000000..c4921539c --- /dev/null +++ b/internal/redislock/BUILD.bazel @@ -0,0 +1,33 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "redislock", + srcs = ["redislock.go"], + importpath = "xmanager_cloud/internal/redislock", + visibility = ["//:__subpackages__"], + deps = ["@com_github_redis_go_redis_v9//:go-redis"], +) + +go_test( + name = "redislock_test", + srcs = ["redislock_test.go"], + embed = [":redislock"], + deps = [ + "@com_github_google_go_cmp//cmp", + "@com_github_redis_go_redis_v9//:go-redis", + ], +) diff --git a/internal/redislock/redislock.go b/internal/redislock/redislock.go new file mode 100644 index 000000000..a0f14f82d --- /dev/null +++ b/internal/redislock/redislock.go @@ -0,0 +1,119 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package redislock provides single-instance distributed Redis lease locks backed by SET NX PX +// acquisition and a token-checked Lua release script. +package redislock + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +const tokenBytes = 16 + +var ( + // ErrNotAcquired is returned by Locker.Acquire when the lock key is already held. + ErrNotAcquired = errors.New("redis lock not acquired") + // ErrLeaseLost is returned by Lease.Release when the lock key expired or changed owner. + ErrLeaseLost = errors.New("redis lock lease lost") + + releaseScript = redis.NewScript(`if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) else return 0 end`) +) + +// Client is the minimal go-redis interface required by Locker. *redis.Client satisfies it. +type Client interface { + SetArgs(ctx context.Context, key string, value any, a redis.SetArgs) *redis.StatusCmd + redis.Scripter +} + +var _ Client = (*redis.Client)(nil) + +// Locker acquires time-bounded distributed leases in Redis. +type Locker struct { + client Client + ttl time.Duration + randRead func([]byte) (int, error) +} + +// New validates client and ttl and returns a Locker whose leases expire after ttl. +func New(client Client, ttl time.Duration) (*Locker, error) { + if isNilClient(client) { + return nil, errors.New("redis lock client must not be nil") + } + if ttl < time.Millisecond { + return nil, fmt.Errorf("redis lock ttl must be at least 1ms, got %v", ttl) + } + return &Locker{client: client, ttl: ttl, randRead: rand.Read}, nil +} + +func isNilClient(client Client) bool { + if client == nil { + return true + } + c, ok := client.(*redis.Client) + return ok && c == nil +} + +// Acquire attempts once to acquire the lock at key without blocking, returning ErrNotAcquired if held. +func (l *Locker) Acquire(ctx context.Context, key string) (*Lease, error) { + if key == "" { + return nil, errors.New("redis lock key must not be empty") + } + token, err := l.generateToken() + if err != nil { + return nil, err + } + err = l.client.SetArgs(ctx, key, token, redis.SetArgs{Mode: "NX", TTL: l.ttl}).Err() + if errors.Is(err, redis.Nil) { + return nil, ErrNotAcquired + } + if err != nil { + return nil, fmt.Errorf("failed to acquire redis lock %q: %w", key, err) + } + return &Lease{client: l.client, key: key, token: token}, nil +} + +func (l *Locker) generateToken() (string, error) { + buf := make([]byte, tokenBytes) + if _, err := l.randRead(buf); err != nil { + return "", fmt.Errorf("failed to generate redis lock token: %w", err) + } + return hex.EncodeToString(buf), nil +} + +// Lease represents an acquired lock on a single Redis key. +type Lease struct { + client Client + key string + token string +} + +// Release deletes the lock key if and only if the lease token still owns it. +func (l *Lease) Release(ctx context.Context) error { + deleted, err := releaseScript.Run(ctx, l.client, []string{l.key}, l.token).Int64() + if err != nil { + return fmt.Errorf("failed to release redis lock %q: %w", l.key, err) + } + if deleted == 0 { + return ErrLeaseLost + } + return nil +} diff --git a/internal/redislock/redislock_test.go b/internal/redislock/redislock_test.go new file mode 100644 index 000000000..6f5623dfa --- /dev/null +++ b/internal/redislock/redislock_test.go @@ -0,0 +1,177 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package redislock + +import ( + "context" + "encoding/hex" + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/redis/go-redis/v9" +) + +type setCall struct { + Key, Mode string + TTL time.Duration +} + +type evalCall struct { + Keys []string + Args []any +} + +type fakeClient struct { + redis.Scripter + setErr, evalErr error + evalVal int64 + lastSet setCall + lastValue any + lastEval evalCall +} + +func (f *fakeClient) SetArgs(_ context.Context, key string, value any, a redis.SetArgs) *redis.StatusCmd { + f.lastSet, f.lastValue = setCall{Key: key, Mode: a.Mode, TTL: a.TTL}, value + if f.setErr != nil { + return redis.NewStatusResult("", f.setErr) + } + return redis.NewStatusResult("OK", nil) +} + +func (f *fakeClient) EvalSha(_ context.Context, _ string, keys []string, args ...any) *redis.Cmd { + f.lastEval = evalCall{Keys: keys, Args: args} + cmd := redis.NewCmd(context.Background()) + if f.evalErr != nil { + cmd.SetErr(f.evalErr) + return cmd + } + cmd.SetVal(f.evalVal) + return cmd +} + +func (f *fakeClient) Eval(ctx context.Context, _ string, keys []string, args ...any) *redis.Cmd { + return f.EvalSha(ctx, "", keys, args...) +} + +func TestNew_Validation(t *testing.T) { + t.Parallel() + valid := &fakeClient{} + tests := []struct { + name string + client Client + ttl time.Duration + wantErr bool + }{ + {name: "nil client", client: nil, ttl: time.Second, wantErr: true}, + {name: "typed nil redis client", client: (*redis.Client)(nil), ttl: time.Second, wantErr: true}, + {name: "zero ttl", client: valid, ttl: 0, wantErr: true}, + {name: "negative ttl", client: valid, ttl: -time.Second, wantErr: true}, + {name: "sub-millisecond ttl", client: valid, ttl: 999 * time.Microsecond, wantErr: true}, + {name: "exact 1ms boundary", client: valid, ttl: time.Millisecond}, + {name: "standard 15s ttl", client: valid, ttl: 15 * time.Second}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := New(tc.client, tc.ttl) + if (err != nil) != tc.wantErr || (!tc.wantErr && got == nil) { + t.Fatalf("New(_, %v) = (%v, %v), wantErr %v", tc.ttl, got, err, tc.wantErr) + } + }) + } +} + +func TestLocker_Acquire(t *testing.T) { + t.Parallel() + transportErr, randErr := errors.New("connection refused"), errors.New("entropy unavailable") + tests := []struct { + name string + key string + setErr error + randRead func([]byte) (int, error) + wantErr error + anyErr bool + }{ + {name: "acquired", key: "xmc:study-reconcile-lock:42"}, + {name: "multibyte key", key: "xmc:study-reconcile-lock:実験-42"}, + {name: "empty key", key: "", anyErr: true}, + {name: "already held", key: "xmc:study-reconcile-lock:42", setErr: redis.Nil, wantErr: ErrNotAcquired}, + {name: "transport error", key: "xmc:study-reconcile-lock:42", setErr: transportErr, wantErr: transportErr}, + {name: "token rng error", key: "xmc:study-reconcile-lock:42", randRead: func([]byte) (int, error) { return 0, randErr }, wantErr: randErr}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + fc := &fakeClient{setErr: tc.setErr} + locker, _ := New(fc, 15*time.Second) + if tc.randRead != nil { + locker.randRead = tc.randRead + } + lease, err := locker.Acquire(context.Background(), tc.key) + if tc.wantErr != nil { + if !errors.Is(err, tc.wantErr) { + t.Fatalf("Acquire(%q) error = %v, want %v", tc.key, err, tc.wantErr) + } + return + } + if tc.anyErr || err != nil { + if !tc.anyErr || err == nil { + t.Fatalf("Acquire(%q) error = %v, anyErr %v", tc.key, err, tc.anyErr) + } + return + } + if diff := cmp.Diff(setCall{Key: tc.key, Mode: "NX", TTL: 15 * time.Second}, fc.lastSet); diff != "" { + t.Errorf("SetArgs mismatch (-want +got):\n%s", diff) + } + token, _ := fc.lastValue.(string) + decoded, decErr := hex.DecodeString(token) + if decErr != nil || len(decoded) != tokenBytes || lease.token != token || lease.key != tc.key { + t.Errorf("lease={%q,%q} decoded=%d decErr=%v, want %d hex bytes", lease.key, lease.token, len(decoded), decErr, tokenBytes) + } + }) + } +} + +func TestLease_Release(t *testing.T) { + t.Parallel() + transportErr := errors.New("broken pipe") + tests := []struct { + name string + evalVal int64 + evalErr error + wantErr error + }{ + {name: "release succeeds", evalVal: 1}, + {name: "release detects lost lease", evalVal: 0, wantErr: ErrLeaseLost}, + {name: "release wraps transport error", evalErr: transportErr, wantErr: transportErr}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + fc := &fakeClient{evalVal: tc.evalVal, evalErr: tc.evalErr} + locker, _ := New(fc, 15*time.Second) + lease, _ := locker.Acquire(context.Background(), "xmc:study-reconcile-lock:99") + opErr := lease.Release(context.Background()) + if (tc.wantErr != nil && !errors.Is(opErr, tc.wantErr)) || (tc.wantErr == nil && opErr != nil) { + t.Fatalf("Release() error = %v, want %v", opErr, tc.wantErr) + } + if diff := cmp.Diff(evalCall{Keys: []string{"xmc:study-reconcile-lock:99"}, Args: []any{lease.token}}, fc.lastEval); diff != "" { + t.Errorf("script call mismatch (-want +got):\n%s", diff) + } + }) + } +}