Skip to content
Draft
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmd/study_controller/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ go_library(
"//cmd/study_controller/config",
"//cmd/study_controller/worker",
"//internal/logging",
"//internal/task_queue_redis/redisclient",
"@com_github_prometheus_client_golang//prometheus/promhttp",
],
)
Expand Down Expand Up @@ -55,5 +56,6 @@ go_test(
"//cmd/study_controller/config",
"//cmd/study_controller/worker",
"//internal/logging",
"//internal/task_queue_redis/redisclient",
],
)
3 changes: 2 additions & 1 deletion cmd/study_controller/health_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ func TestNewHTTPHandler(t *testing.T) {
{name: "readiness with Redis reachable", method: http.MethodGet, path: "/readyz", wantStatus: http.StatusOK, wantContains: "OK"},
{name: "readiness with Redis unreachable", method: http.MethodGet, path: "/readyz", pingErr: unreachable, wantStatus: http.StatusServiceUnavailable, wantContains: "Service Unavailable"},
{name: "readiness with Redis ping timeout", method: http.MethodGet, path: "/readyz", pingErr: context.DeadlineExceeded, wantStatus: http.StatusServiceUnavailable, wantContains: "Service Unavailable"},
{name: "metrics", method: http.MethodGet, path: "/metrics", wantStatus: http.StatusOK, wantContains: "go_goroutines"},
{name: "metrics exposes runtime stats", method: http.MethodGet, path: "/metrics", wantStatus: http.StatusOK, wantContains: "go_goroutines"},
{name: "metrics exposes reconcile ticks counter", method: http.MethodGet, path: "/metrics", wantStatus: http.StatusOK, wantContains: "xmc_study_controller_reconcile_ticks_total"},
{name: "unknown path", method: http.MethodGet, path: "/unknown", wantStatus: http.StatusNotFound},
{name: "non-GET liveness", method: http.MethodPost, path: "/healthz", wantStatus: http.StatusMethodNotAllowed},
{name: "non-GET readiness", method: http.MethodPost, path: "/readyz", wantStatus: http.StatusMethodNotAllowed},
Expand Down
16 changes: 14 additions & 2 deletions cmd/study_controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"xmanager_cloud/cmd/study_controller/config"
"xmanager_cloud/cmd/study_controller/worker"
"xmanager_cloud/internal/logging"
"xmanager_cloud/internal/task_queue_redis/redisclient"
)

// HTTP server timeouts bound per-connection resource usage (e.g. slow-client attacks).
Expand Down Expand Up @@ -82,8 +83,19 @@ func listenAndServe(ctx context.Context, cfg config.Config) error {
if err != nil {
return fmt.Errorf("failed to listen on port %s: %w", cfg.Port, err)
}
client := worker.NewRedisConsumer(cfg.RedisAddr)
consumer := worker.Start(ctx, client)
return serveWithClient(ctx, lis, worker.NewRedisConsumer(cfg.RedisAddr))
}

func serveWithClient(ctx context.Context, lis net.Listener, client *redisclient.ConsumerClient) error {
handler, err := worker.NewRedisHandler(client)
if err != nil {
_ = lis.Close()
if client != nil && client.Client != nil {
_ = client.Close(context.WithoutCancel(ctx))
}
return fmt.Errorf("failed to create study reconcile handler: %w", err)
}
consumer := worker.Start(ctx, client, handler)
defer stopConsumer(ctx, consumer)
return serve(ctx, lis, newHTTPHandler(client))
}
Expand Down
37 changes: 36 additions & 1 deletion cmd/study_controller/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"xmanager_cloud/cmd/study_controller/config"
"xmanager_cloud/cmd/study_controller/worker"
"xmanager_cloud/internal/logging"
"xmanager_cloud/internal/task_queue_redis/redisclient"
)

// waitHealthy polls GET /healthz at addr until it returns 200 OK or the deadline passes.
Expand Down Expand Up @@ -101,6 +102,36 @@ func TestRun_CancelledContextExitsZero(t *testing.T) {
}
}

type noopRedisClient struct{ redisclient.Client }

func (noopRedisClient) Close() error { return nil }

func TestServeWithClient_RejectsInvalidClients(t *testing.T) {
nonLockClient := worker.NewRedisConsumer("127.0.0.1:" + freePort(t))
_ = nonLockClient.Close(context.Background())
nonLockClient.Client = noopRedisClient{}

tests := []struct {
name string
client *redisclient.ConsumerClient
}{
{name: "nil consumer client", client: nil},
{name: "non-locking underlying client", client: nonLockClient},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("net.Listen() error = %v", err)
}
defer lis.Close()
if err := serveWithClient(context.Background(), lis, tc.client); err == nil {
t.Errorf("serveWithClient(%+v) error = nil, want non-nil", tc.client)
}
})
}
}

func TestReadiness_UnreachableRedis(t *testing.T) {
client := worker.NewRedisConsumer("127.0.0.1:" + freePort(t))
defer client.Close(context.Background())
Expand Down Expand Up @@ -170,7 +201,11 @@ func TestStopConsumer_LogsCloseFailure(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
client := worker.NewRedisConsumer("127.0.0.1:" + freePort(t))
consumer := worker.Start(ctx, client)
handler, err := worker.NewRedisHandler(client)
if err != nil {
t.Fatalf("NewRedisHandler() error = %v", err)
}
consumer := worker.Start(ctx, client, handler)
// Closing the client first makes the consumer's own close fail.
if err := client.Close(context.Background()); err != nil {
t.Fatalf("client.Close() error = %v", err)
Expand Down
5 changes: 5 additions & 0 deletions cmd/study_controller/worker/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,11 @@ go_library(
"//api/protos/task_queue:task_go_proto",
"//internal/kubernetes/common/constants",
"//internal/logging",
"//internal/redislock",
"//internal/task_queue_redis/redisclient",
"//internal/validator",
"@com_github_prometheus_client_golang//prometheus",
"@com_github_prometheus_client_golang//prometheus/promauto",
"@org_golang_google_protobuf//encoding/protojson",
],
)
Expand All @@ -45,8 +48,10 @@ go_test(
deps = [
"//internal/kubernetes/common/constants",
"//internal/logging",
"//internal/redislock",
"//internal/task_queue_redis/redisclient",
"@com_github_google_go_cmp//cmp",
"@com_github_prometheus_client_golang//prometheus/testutil",
"@com_github_redis_go_redis_v9//:go-redis",
],
)
22 changes: 11 additions & 11 deletions cmd/study_controller/worker/consumer.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ import (
)

// defaultDrainTimeout bounds how long Close waits for the worker and cleanup loops. It covers the
// 2s XREADGROUP block of the worker loop plus redisclient.SettleTimeout, so a tick being settled
// when Close is called finishes before the client is closed.
const defaultDrainTimeout = redisclient.SettleTimeout + 3*time.Second
// 2s XREADGROUP block of the worker loop, defaultReleaseTimeout, and redisclient.SettleTimeout, so
// a tick being settled when Close is called finishes before the client is closed.
const defaultDrainTimeout = redisclient.SettleTimeout + defaultReleaseTimeout + 3*time.Second

// NewRedisConsumer returns a Redis consumer of the study reconcile stream for the Redis server at
// addr. It reads the password from REDIS_PASSWORD and connects lazily. Context deadlines bound its
Expand All @@ -46,7 +46,7 @@ func NewRedisConsumer(addr string) *redisclient.ConsumerClient {
)
}

// Consumer runs the worker and cleanup loops of a Redis consumer with handleTask as its handler.
// Consumer runs the worker and cleanup loops of a Redis consumer with Handler.HandleTask.
type Consumer struct {
client redisclient.Consumer
cancel context.CancelFunc
Expand All @@ -56,12 +56,12 @@ type Consumer struct {
closeErr error // result of the first Close, returned by every call
}

// Start sets handleTask on client and runs its worker and cleanup loops until ctx is done or Close
// is called. The loops create the stream and consumer group themselves, at start-up and again
// whenever Redis reports them missing (NOGROUP), so Start never blocks on Redis and consumption
// resumes after Redis comes back without them.
func Start(ctx context.Context, client redisclient.Consumer) *Consumer {
client.SetTaskHandler(handleTask)
// Start sets handler.HandleTask on client and runs its worker and cleanup loops until ctx is done
// or Close is called. The loops create the stream and consumer group themselves, at start-up and
// again whenever Redis reports them missing (NOGROUP), so Start never blocks on Redis and
// consumption resumes after Redis comes back without them.
func Start(ctx context.Context, client redisclient.Consumer, handler *Handler) *Consumer {
client.SetTaskHandler(handler.HandleTask)
loopCtx, cancel := context.WithCancel(ctx)
c := &Consumer{client: client, cancel: cancel, drainTimeout: defaultDrainTimeout}
workerID := redisclient.HostConsumerName(constants.StudyControllerWorkerID)
Expand All @@ -84,7 +84,7 @@ func (c *Consumer) runLoop(loop func()) {
}

// Close stops the loops and closes the Redis client. It waits for the loops to return, but at
// most the drain timeout (8s) and only until ctx is done; after that it logs a warning and closes
// most the drain timeout (10s) and only until ctx is done; after that it logs a warning and closes
// the client anyway, leaving a stuck loop to fail on the closed client. A tick being handled when
// Close is called is still acknowledged or retried. Every call returns the first call's error.
func (c *Consumer) Close(ctx context.Context) error {
Expand Down
22 changes: 17 additions & 5 deletions cmd/study_controller/worker/consumer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,18 @@ func (f *fakeConsumer) waitForLoops(t *testing.T) []string {
return ids
}

func newStubHandler(t *testing.T) *Handler {
t.Helper()
h, err := NewHandler(HandlerParams{Locker: &fakeLocker{lease: &fakeLease{}}})
if err != nil {
t.Fatalf("NewHandler() error = %v", err)
}
return h
}

func TestStart_RunsLoopsWithHandleTask(t *testing.T) {
fake := newFakeConsumer()
c := Start(context.Background(), fake)
c := Start(context.Background(), fake, newStubHandler(t))
defer c.Close(context.Background())
ids := fake.waitForLoops(t)

Expand All @@ -111,7 +120,7 @@ func TestStart_RunsLoopsWithHandleTask(t *testing.T) {
t.Fatal("Start() did not set a task handler")
}
if got := handler(context.Background(), "w", `{"experiment":"experiments/7"}`); !got.WasRunSuccessful {
t.Errorf("task handler on a valid tick = %+v, want success (handleTask)", got)
t.Errorf("task handler on a valid tick = %+v, want success (HandleTask)", got)
}
}

Expand All @@ -127,7 +136,7 @@ func containsPrefix(ids []string, prefix string) bool {
func TestConsumer_CloseStopsLoopsAndClosesClient(t *testing.T) {
fake := newFakeConsumer()
fake.closeErr = errors.New("close failed")
c := Start(context.Background(), fake)
c := Start(context.Background(), fake, newStubHandler(t))
fake.waitForLoops(t)

if err := c.Close(context.Background()); !errors.Is(err, fake.closeErr) {
Expand All @@ -146,7 +155,7 @@ func TestConsumer_CloseStopsLoopsAndClosesClient(t *testing.T) {
func TestStart_ContextCancelStopsLoops(t *testing.T) {
fake := newFakeConsumer()
ctx, cancel := context.WithCancel(context.Background())
c := Start(ctx, fake)
c := Start(ctx, fake, newStubHandler(t))
fake.waitForLoops(t)

cancel()
Expand Down Expand Up @@ -179,7 +188,7 @@ func TestConsumer_CloseWithStuckLoops(t *testing.T) {
fake := newFakeConsumer()
fake.ignoreCancel = true
defer close(fake.release)
c := Start(context.Background(), fake)
c := Start(context.Background(), fake, newStubHandler(t))
fake.waitForLoops(t)
c.drainTimeout = tc.drainTimeout

Expand All @@ -204,6 +213,7 @@ type consumerSettings struct {
Addr, Stream, Group, DLQ string
MaxRetries int
ContextTimeoutEnabled bool
DrainTimeout time.Duration
}

func TestNewRedisConsumer(t *testing.T) {
Expand All @@ -217,10 +227,12 @@ func TestNewRedisConsumer(t *testing.T) {
got := consumerSettings{
Addr: client.Options().Addr, Stream: c.StreamName, Group: c.GroupName, DLQ: c.DLQStreamName,
MaxRetries: c.MaxRetries, ContextTimeoutEnabled: client.Options().ContextTimeoutEnabled,
DrainTimeout: defaultDrainTimeout,
}
want := consumerSettings{
Addr: "redis-service:6379", Stream: "xmc-study-reconcile-stream", Group: "study-controller-group",
DLQ: "xmc-study-reconcile-dlq-stream", MaxRetries: constants.RedisMaxRetries, ContextTimeoutEnabled: true,
DrainTimeout: 10 * time.Second,
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("NewRedisConsumer() settings mismatch (-want +got):\n%s", diff)
Expand Down
Loading
Loading