From 039b079f1989a1e032c54cc70f514bc87a7b0fd5 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 22 Jul 2026 01:18:30 +0300 Subject: [PATCH 01/12] Add Wave 0 security foundations for untrusted MCP servers Lays the security groundwork for the untrusted-MCP-server egress credential broker (ADR-0001): four independent hardening mechanisms that close gaps the broker design depends on. Read-side binding validation: upstream token storage now validates caller-asserted UserID/ClientID before releasing tokens, making the documented ErrInvalidBinding contract real. Binding is checked before expiry so a mismatched row never releases refresh material; bulk reads exclude mismatched rows per-provider. Strict mode added for untrusted workloads to fail closed on legacy rows. Operator untrusted-env gate: annotated MCPServer workloads (toolhive.stacklok.dev/untrusted=true, interim until the spec field lands) are rejected at reconcile time when the backend container would receive credentials via env valueFrom (Secret/ConfigMap), envFrom, or Secret/ConfigMap/projected/CSI volumes. Terminal error surfacing with latch-clearing on recovery so warnings re-arm. Token encryption at rest: upstream OAuth tokens in Redis are now AES-256-GCM envelope-encrypted with per-record DEKs, KEK keyring with rotation (lazy re-seal on read), and ciphertext bound to the Redis key via AAD to defeat cut-and-paste row swaps. Legacy plaintext rows read through unchanged; encrypted fleet without a keyring fails loudly. Operator RBAC: staged networkpolicies permissions for the Wave 3 egress lockdown (deploy-time pre-staging rationale in ADR). --- .../api/v1beta1/mcpserver_types.go | 7 + .../controllers/mcpserver_controller.go | 179 +++++ .../mcpserver_untrusted_env_test.go | 328 ++++++++ .../untrusted_env_validation.go | 149 ++++ .../untrusted_env_validation_test.go | 372 +++++++++ .../operator/templates/clusterrole/role.yaml | 12 + .../adr/0001-untrusted-mcp-egress-broker.md | 332 ++++++++ pkg/auth/token.go | 33 +- pkg/auth/token_test.go | 126 +++- .../upstreamtoken/mocks/mock_token_reader.go | 9 +- pkg/auth/upstreamtoken/service.go | 10 +- pkg/auth/upstreamtoken/service_test.go | 74 +- pkg/auth/upstreamtoken/types.go | 22 +- pkg/authserver/integration_test.go | 233 +++++- pkg/authserver/runner/embeddedauthserver.go | 46 +- .../runner/embeddedauthserver_test.go | 222 ++++++ pkg/authserver/server/handlers/callback.go | 18 +- pkg/authserver/server/handlers/handler.go | 18 +- .../server/handlers/handler_chain_test.go | 2 +- .../server/handlers/helpers_test.go | 4 +- pkg/authserver/storage/binding.go | 138 ++++ pkg/authserver/storage/binding_test.go | 162 ++++ pkg/authserver/storage/config.go | 23 + pkg/authserver/storage/memory.go | 51 +- pkg/authserver/storage/memory_test.go | 357 ++++++++- pkg/authserver/storage/mocks/mock_storage.go | 32 +- pkg/authserver/storage/redis.go | 370 ++++++--- .../storage/redis_integration_test.go | 24 +- pkg/authserver/storage/redis_migrate.go | 135 ++++ pkg/authserver/storage/redis_test.go | 373 +++++++-- pkg/authserver/storage/redis_tokenenc_test.go | 713 ++++++++++++++++++ pkg/authserver/storage/types.go | 39 +- pkg/authserver/tokenenc/tokenenc.go | 293 +++++++ pkg/authserver/tokenenc/tokenenc_test.go | 358 +++++++++ .../auth/factory/incoming_upstream_test.go | 2 +- 35 files changed, 4977 insertions(+), 289 deletions(-) create mode 100644 cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go create mode 100644 cmd/thv-operator/pkg/controllerutil/untrusted_env_validation.go create mode 100644 cmd/thv-operator/pkg/controllerutil/untrusted_env_validation_test.go create mode 100644 docs/arch/adr/0001-untrusted-mcp-egress-broker.md create mode 100644 pkg/authserver/storage/binding.go create mode 100644 pkg/authserver/storage/binding_test.go create mode 100644 pkg/authserver/storage/redis_tokenenc_test.go create mode 100644 pkg/authserver/tokenenc/tokenenc.go create mode 100644 pkg/authserver/tokenenc/tokenenc_test.go diff --git a/cmd/thv-operator/api/v1beta1/mcpserver_types.go b/cmd/thv-operator/api/v1beta1/mcpserver_types.go index d2762f0013..48784fddcb 100644 --- a/cmd/thv-operator/api/v1beta1/mcpserver_types.go +++ b/cmd/thv-operator/api/v1beta1/mcpserver_types.go @@ -48,6 +48,13 @@ const ( ConditionReasonPodTemplateInvalid = "InvalidPodTemplateSpec" ) +const ( + // ConditionReasonSecretEnvRejected indicates an untrusted-flagged workload + // attempted to source backend (mcp) container env from a Secret or ConfigMap, + // or mount a Secret volume — rejected by the untrusted env gate. + ConditionReasonSecretEnvRejected = "SecretEnvRejected" +) + // Condition type for CA bundle validation const ( // ConditionCABundleRefValidated indicates whether the CABundleRef is valid diff --git a/cmd/thv-operator/controllers/mcpserver_controller.go b/cmd/thv-operator/controllers/mcpserver_controller.go index 06eb9cc126..3b6e75d1e9 100644 --- a/cmd/thv-operator/controllers/mcpserver_controller.go +++ b/cmd/thv-operator/controllers/mcpserver_controller.go @@ -167,6 +167,7 @@ func (r *MCPServerReconciler) detectPlatform(ctx context.Context) (kubernetes.Pl // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups="",resources=pods/attach,verbs=create;get // +kubebuilder:rbac:groups="",resources=pods/log,verbs=get @@ -227,6 +228,12 @@ func (r *MCPServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{Requeue: true}, nil } + // Copy the status before the advisory validators run so the terminal gate + // below can skip its status write when the in-memory object is unchanged + // (advisory validators persist their own conditions and leave the object + // current when nothing changed). + statusBeforeGate := mcpServer.Status.DeepCopy() + // Check if the GroupRef is valid if specified r.validateGroupRef(ctx, mcpServer) @@ -249,6 +256,12 @@ func (r *MCPServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, nil } + // Enforce the untrusted-workload env gate before any workload resources are + // created. Terminal spec error: condition + one-shot Warning, no requeue. + if !r.validateUntrustedSecretEnv(ctx, mcpServer, statusBeforeGate) { + return ctrl.Result{}, nil + } + // Check if MCPToolConfig is referenced and handle it if err := r.handleToolConfig(ctx, mcpServer); err != nil { ctxLogger.Error(err, "Failed to handle MCPToolConfig") @@ -398,6 +411,22 @@ func (r *MCPServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Define a new deployment dep, err := r.deploymentForMCPServer(ctx, mcpServer, runConfigChecksum) if err != nil { + var specErr *SpecValidationError + if stderrors.As(err, &specErr) { + // Terminal spec error (e.g. the untrusted env gate firing on a spec + // that changed after the reconcile-time gate passed): surface the + // condition and stop without requeue — returning the error would + // requeue forever with backoff. Not expected on the normal path: + // validateUntrustedSecretEnv already terminates earlier. + ctxLogger.Error(err, "Deployment build rejected by spec validation") + mcpServer.Status.Phase = mcpv1beta1.MCPServerPhaseFailed + mcpServer.Status.Message = fmt.Sprintf("Failed to build Deployment: %s", specErr.Error()) + setReadyCondition(mcpServer, metav1.ConditionFalse, mcpv1beta1.ConditionReasonNotReady, mcpServer.Status.Message) + if statusErr := r.Status().Update(ctx, mcpServer); statusErr != nil { + ctxLogger.Error(statusErr, "Failed to update MCPServer status after Deployment spec validation failure") + } + return ctrl.Result{}, nil + } ctxLogger.Error(err, "Failed to build Deployment object") mcpServer.Status.Phase = mcpv1beta1.MCPServerPhaseFailed mcpServer.Status.Message = fmt.Sprintf("Failed to build Deployment: %s", err.Error()) @@ -493,6 +522,18 @@ func (r *MCPServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // kubectl scale remain in control. newDeployment, err := r.deploymentForMCPServer(ctx, mcpServer, runConfigChecksum) if err != nil { + var specErr *SpecValidationError + if stderrors.As(err, &specErr) { + // Terminal spec error: see the create-path branch above. + ctxLogger.Error(err, "Deployment update build rejected by spec validation") + mcpServer.Status.Phase = mcpv1beta1.MCPServerPhaseFailed + mcpServer.Status.Message = fmt.Sprintf("Failed to build Deployment: %s", specErr.Error()) + setReadyCondition(mcpServer, metav1.ConditionFalse, mcpv1beta1.ConditionReasonNotReady, mcpServer.Status.Message) + if statusErr := r.Status().Update(ctx, mcpServer); statusErr != nil { + ctxLogger.Error(statusErr, "Failed to update MCPServer status after Deployment spec validation failure") + } + return ctrl.Result{}, nil + } ctxLogger.Error(err, "Failed to build updated Deployment object") mcpServer.Status.Phase = mcpv1beta1.MCPServerPhaseFailed mcpServer.Status.Message = fmt.Sprintf("Failed to build Deployment: %s", err.Error()) @@ -1039,6 +1080,132 @@ func logOBOSecretEnvVarError(ctx context.Context, err error) { "see the referenced MCPExternalAuthConfig status for details") } +// untrustedAnnotationKey is the interim (Wave 0) opt-in annotation that marks an +// MCPServer as untrusted. Wave 1 replaces this with the spec.untrusted field and +// deletes the annotation. +const untrustedAnnotationKey = "toolhive.stacklok.dev/untrusted" + +// isUntrusted reports whether the workload must be treated as untrusted. +// Wave 1 replaces the body with `return m.Spec.Untrusted`. +// Interim (Wave 0): annotation opt-in so the gate is exercisable and e2e-testable +// before the CRD field exists. The annotation is not a documented user feature. +func isUntrusted(m *mcpv1beta1.MCPServer) bool { + return m.Annotations[untrustedAnnotationKey] == "true" +} + +// untrustedGateSuffix documents the gate's semantics on every rejection +// message: the gate is admission-style, NOT eviction — it blocks creation of +// future pods but does NOT evict a currently-running deployment (a workload +// already running with the now-rejected config keeps running until its next +// rollout). +const untrustedGateSuffix = "The gate blocks creation of new pods until the spec is fixed; " + + "it does not evict a currently-running deployment." + +// validateUntrustedSecretEnv enforces the untrusted-workload env gate before any +// workload resources are created: an untrusted-flagged MCPServer must never +// receive a credential through backend (mcp) container env or Secret volumes. +// It validates the fully-built pod-template patch (covering both spec.secrets and +// raw spec.podTemplateSpec, which converge in the builder output). +// +// This is a terminal spec error — NOT retryable: it records a Valid=False +// condition with ObservedGeneration, emits a one-shot Warning event on the +// false-transition, and returns false so the caller returns ctrl.Result{}, nil +// (no error → no forever-backoff), per .claude/rules/operator.md "Separate +// terminal from transient errors". Wave 1 adds a CEL admission rule mirroring +// this check for defense-in-depth. +// +// On the pass path the gate clears any Valid condition it previously latched +// (mirroring the validateAuthzPrimaryUpstreamProviderIgnored RemoveStatusCondition +// convention), so a fixed spec does not stay poisoned: clearing the condition +// re-arms the one-shot Warning, which must fire again if the workload later +// re-enters the rejected state. +// +// statusBefore is the status snapshotted at the start of Reconcile, before the +// advisory validators ran. They persist their own condition writes, so if the +// object is unchanged after they ran it is already current in the API server — +// combined with SetStatusCondition's no-op-on-unchanged semantics this makes +// re-observing a rejected spec write nothing (idempotent, per operator rules). +func (r *MCPServerReconciler) validateUntrustedSecretEnv( + ctx context.Context, mcpServer *mcpv1beta1.MCPServer, statusBefore *mcpv1beta1.MCPServerStatus, +) bool { + ctxLogger := log.FromContext(ctx) + + // Fast path: trusted workloads skip the build entirely (the gate is a no-op + // for them), preserving exact current behavior. + if !isUntrusted(mcpServer) { + return true + } + + builder, err := ctrlutil.NewPodTemplateSpecBuilder(mcpServer.Spec.PodTemplateSpec, mcpContainerName) + if err != nil { + // The malformed-raw-template case is already owned by + // validateAndUpdatePodTemplateStatus, which ran earlier in Reconcile and + // terminates the reconcile — reaching here means the template parsed. + ctxLogger.Error(err, "Failed to build PodTemplateSpec for untrusted env validation") + return true + } + podTemplate := builder.WithSecrets(mcpServer.Spec.Secrets).Build() + + if err := ctrlutil.ValidateNoSecretEnvForUntrusted(podTemplate, mcpContainerName, true); err != nil { + // Capture the transition before the status update so the Warning fires + // only when entering the invalid state, not on every re-observation. + wasInvalid := meta.IsStatusConditionFalse(mcpServer.Status.Conditions, mcpv1beta1.ConditionTypeValid) + + gateMessage := err.Error() + " " + untrustedGateSuffix + mcpServer.Status.Phase = mcpv1beta1.MCPServerPhaseFailed + mcpServer.Status.Message = gateMessage + meta.SetStatusCondition(&mcpServer.Status.Conditions, metav1.Condition{ + Type: mcpv1beta1.ConditionTypeValid, + Status: metav1.ConditionFalse, + Reason: mcpv1beta1.ConditionReasonSecretEnvRejected, + Message: gateMessage, + ObservedGeneration: mcpServer.Generation, + }) + setReadyCondition(mcpServer, metav1.ConditionFalse, mcpv1beta1.ConditionReasonNotReady, gateMessage) + + // legacy r.Status().Update call site (see .claude/rules/operator.md + // "Status Writes") — consistent with the other terminal writes in this + // reconciler; the equality guard keeps it idempotent. + var statusErr error + if !equality.Semantic.DeepEqual(statusBefore, &mcpServer.Status) { + statusErr = r.Status().Update(ctx, mcpServer) + if statusErr != nil { + ctxLogger.Error(statusErr, "Failed to update MCPServer status after untrusted env validation failure") + } + } + + // Emit the Warning only on the transition into the invalid state, and only + // once the condition persisted — a failing status write would otherwise + // re-fire the event on every reconcile. + if !wasInvalid && statusErr == nil && r.Recorder != nil { + r.Recorder.Eventf(mcpServer, nil, corev1.EventTypeWarning, + mcpv1beta1.ConditionReasonSecretEnvRejected, "ValidateUntrustedEnv", + "untrusted workload rejected: %s. Deployment blocked until the spec is fixed.", err.Error()) + } + + ctxLogger.Info("Untrusted workload rejected Secret/ConfigMap-sourced backend env or Secret volume", + "reason", err.Error()) + return false + } + + // Pass path: clear a previously latched Valid condition so a fixed spec + // recovers (and the one-shot Warning re-arms for any future rejection). + // Guarded by the statusBefore DeepEqual like the reject path: removing a + // condition that was never set mutates nothing (RemoveStatusCondition is a + // no-op on absent conditions), so the write only happens on the actual + // rejected→passing transition. + if meta.FindStatusCondition(mcpServer.Status.Conditions, mcpv1beta1.ConditionTypeValid) != nil { + meta.RemoveStatusCondition(&mcpServer.Status.Conditions, mcpv1beta1.ConditionTypeValid) + if !equality.Semantic.DeepEqual(statusBefore, &mcpServer.Status) { + if statusErr := r.Status().Update(ctx, mcpServer); statusErr != nil { + ctxLogger.Error(statusErr, "Failed to clear latched Valid condition after untrusted env validation passed") + } + } + } + + return true +} + // deploymentForMCPServer returns a MCPServer Deployment object // //nolint:gocyclo @@ -1070,6 +1237,18 @@ func (r *MCPServerReconciler) deploymentForMCPServer( WithServiceAccount(serviceAccount). WithSecrets(m.Spec.Secrets). Build() + // Reject Secret/ConfigMap-sourced backend env and Secret volumes for + // untrusted-flagged workloads. Reconcile gates terminally before this + // builder runs; this direct check is defense-in-depth for the + // deploymentForMCPServer path (both spec.secrets and raw podTemplateSpec + // converge in the built patch). Wave 1 adds a CEL admission rule mirroring + // this check (untrusted ⇒ literal-only backend env) — the Go check stays. + // The error is a typed *SpecValidationError (VirtualMCPServer pattern) so + // call sites peel it with errors.As into terminal treatment (condition, no + // requeue) instead of a transient forever-backoff. + if err := ctrlutil.ValidateNoSecretEnvForUntrusted(finalPodTemplateSpec, mcpContainerName, isUntrusted(m)); err != nil { + return nil, &SpecValidationError{Message: err.Error()} + } // Add pod template patch if we have one if finalPodTemplateSpec != nil { podTemplatePatch, err := json.Marshal(finalPodTemplateSpec) diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go new file mode 100644 index 0000000000..5cadd18fc1 --- /dev/null +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go @@ -0,0 +1,328 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" + "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1/v1beta1test" + "github.com/stacklok/toolhive/cmd/thv-operator/internal/testutil" + ctrlutil "github.com/stacklok/toolhive/cmd/thv-operator/pkg/controllerutil" +) + +// withUntrustedAnnotation marks the MCPServer fixture as untrusted via the +// interim Wave 0 opt-in annotation. +func withUntrustedAnnotation() v1beta1test.MCPServerOption { + return func(m *mcpv1beta1.MCPServer) { + if m.Annotations == nil { + m.Annotations = map[string]string{} + } + m.Annotations[untrustedAnnotationKey] = "true" + } +} + +func withSecrets(secrets ...mcpv1beta1.SecretRef) v1beta1test.MCPServerOption { + return func(m *mcpv1beta1.MCPServer) { + m.Spec.Secrets = secrets + } +} + +// setupUntrustedReconciler builds a fake-client-backed reconciler for the +// untrusted env gate tests and returns it with the event recorder. +func setupUntrustedReconciler(t *testing.T, objs ...client.Object) (*MCPServerReconciler, *events.FakeRecorder) { + t.Helper() + + s := testutil.NewScheme(t) + eventRecorder := events.NewFakeRecorder(10) + + fakeClient := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(objs...). + WithStatusSubresource(&mcpv1beta1.MCPServer{}). + Build() + + r := &MCPServerReconciler{ + Client: fakeClient, + Scheme: s, + Recorder: eventRecorder, + PlatformDetector: ctrlutil.NewSharedPlatformDetector(), + } + return r, eventRecorder +} + +// reconcileOnce runs a single Reconcile for the given MCPServer fixture. +func reconcileOnce(t *testing.T, r *MCPServerReconciler, mcpServer *mcpv1beta1.MCPServer) ctrl.Result { + t.Helper() + + ctx := log.IntoContext(t.Context(), log.Log) + req := ctrl.Request{NamespacedName: types.NamespacedName{ + Name: mcpServer.Name, + Namespace: mcpServer.Namespace, + }} + result, err := r.Reconcile(ctx, req) + require.NoError(t, err) + return result +} + +func TestMCPServerReconciler_UntrustedSecretEnvRejected(t *testing.T) { + t.Parallel() + + mcpServer := v1beta1test.NewMCPServer("untrusted-secrets", "default", + withUntrustedAnnotation(), + withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), + ) + + r, recorder := setupUntrustedReconciler(t, mcpServer) + + // Reconcile #11: annotation + spec.secrets → terminal rejection. + result := reconcileOnce(t, r, mcpServer) + assert.True(t, result.IsZero(), "terminal spec error must not requeue or schedule a retry") + + updated := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), updated)) + + condition := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionTypeValid) + require.NotNil(t, condition, "Valid condition should be set") + assert.Equal(t, metav1.ConditionFalse, condition.Status) + assert.Equal(t, mcpv1beta1.ConditionReasonSecretEnvRejected, condition.Reason) + assert.Equal(t, updated.Generation, condition.ObservedGeneration) + assert.Contains(t, condition.Message, `env var "API_TOKEN"`) + assert.Contains(t, condition.Message, `Secret "backend-creds"`) + + ready := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionTypeReady) + require.NotNil(t, ready, "Ready condition should be set") + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, mcpv1beta1.MCPServerPhaseFailed, updated.Status.Phase) + + // One-shot Warning event on the false-transition. + require.Eventually(t, func() bool { + return countContaining(drainEvents(recorder), mcpv1beta1.ConditionReasonSecretEnvRejected) == 1 + }, 2*time.Second, 10*time.Millisecond, "expected one Warning event on the invalid transition") + + // No Deployment may be created for the rejected workload. + dep := &appsv1.Deployment{} + err := r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), dep) + assert.True(t, apierrors.IsNotFound(err), "no Deployment should be created for a rejected workload") + + // Reconcile #14 (idempotency): the terminal branch itself must be a no-op on + // the second reconcile — the Valid/Ready conditions it owns must be + // byte-identical afterwards (proving no spurious write or churn from the + // gate), and no duplicate event must fire. Note: the pre-existing advisory + // validators (stdio replica cap, session storage, rate limit) perform their + // own legacy write-per-reconcile, so a whole-object ResourceVersion + // comparison is not meaningful here; the gate's idempotency is scoped to + // what it owns. + result = reconcileOnce(t, r, mcpServer) + assert.True(t, result.IsZero()) + + afterSecond := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), afterSecond)) + + conditionAfter := meta.FindStatusCondition(afterSecond.Status.Conditions, mcpv1beta1.ConditionTypeValid) + require.NotNil(t, conditionAfter) + assert.Equal(t, *condition, *conditionAfter, "gate-owned Valid condition must not churn on re-observe") + readyAfter := meta.FindStatusCondition(afterSecond.Status.Conditions, mcpv1beta1.ConditionTypeReady) + require.NotNil(t, readyAfter) + assert.Equal(t, *ready, *readyAfter, "gate-owned Ready condition must not churn on re-observe") + assert.Equal(t, updated.Status.Phase, afterSecond.Status.Phase) + assert.Equal(t, updated.Status.Message, afterSecond.Status.Message) + + time.Sleep(50 * time.Millisecond) // let any async event emission settle + assert.Empty(t, drainEvents(recorder), "no event expected while staying invalid") +} + +func TestMCPServerReconciler_TrustedSecretsDeployAsToday(t *testing.T) { + t.Parallel() + + // #12: identical spec.secrets but no annotation — no behavior change; the + // reconcile must proceed past the gate (it will requeue waiting on the + // runconfig ConfigMap, which is the normal pre-Deployment path). + mcpServer := v1beta1test.NewMCPServer("trusted-secrets", "default", + withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), + ) + + r, _ := setupUntrustedReconciler(t, mcpServer) + + result := reconcileOnce(t, r, mcpServer) + + updated := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), updated)) + + condition := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionTypeValid) + assert.Nil(t, condition, "no Valid condition should be recorded for trusted workloads") + assert.NotEqual(t, mcpv1beta1.MCPServerPhaseFailed, updated.Status.Phase) + assert.False(t, result.IsZero(), + "trusted reconcile should continue down the normal path (requeue waiting on runconfig)") +} + +func TestMCPServerReconciler_UntrustedRawTemplateSecretEnvRejected(t *testing.T) { + t.Parallel() + + // #13: annotation + raw podTemplateSpec smuggling secretKeyRef onto the mcp container. + mcpServer := v1beta1test.NewMCPServer("untrusted-raw-template", "default", + withUntrustedAnnotation(), + v1beta1test.WithPodTemplateSpec(&runtime.RawExtension{ + Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"API_TOKEN","valueFrom":{"secretKeyRef":{"name":"smuggled-secret","key":"token"}}}]}]}}`), + }), + ) + + r, recorder := setupUntrustedReconciler(t, mcpServer) + + result := reconcileOnce(t, r, mcpServer) + assert.True(t, result.IsZero()) + + updated := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), updated)) + + condition := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionTypeValid) + require.NotNil(t, condition, "Valid condition should be set") + assert.Equal(t, metav1.ConditionFalse, condition.Status) + assert.Equal(t, mcpv1beta1.ConditionReasonSecretEnvRejected, condition.Reason) + assert.Contains(t, condition.Message, `env var "API_TOKEN"`) + assert.Contains(t, condition.Message, `Secret "smuggled-secret"`) + + require.Eventually(t, func() bool { + return countContaining(drainEvents(recorder), mcpv1beta1.ConditionReasonSecretEnvRejected) == 1 + }, 2*time.Second, 10*time.Millisecond, "expected one Warning event on the invalid transition") +} + +func TestMCPServerReconciler_UntrustedCompliantDeploys(t *testing.T) { + t.Parallel() + + // #15: annotation present but workload compliant (literal env only) — + // the gate passes and the reconcile proceeds normally. + mcpServer := v1beta1test.NewMCPServer("untrusted-compliant", "default", + withUntrustedAnnotation(), + v1beta1test.WithEnv(mcpv1beta1.EnvVar{Name: "SENTINEL", Value: "literal-value"}), + v1beta1test.WithPodTemplateSpec(&runtime.RawExtension{ + Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"LITERAL","value":"ok"}]}]}}`), + }), + ) + + r, _ := setupUntrustedReconciler(t, mcpServer) + + result := reconcileOnce(t, r, mcpServer) + + updated := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), updated)) + + condition := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionTypeValid) + assert.Nil(t, condition, "compliant untrusted workload must not get a Valid=False condition") + assert.NotEqual(t, mcpv1beta1.MCPServerPhaseFailed, updated.Status.Phase) + assert.False(t, result.IsZero(), + "compliant untrusted reconcile should continue down the normal path") +} + +// TestMCPServerReconciler_UntrustedLatchClearsAndWarningReArms pins the +// rejected → fixed → rejected lifecycle: +// 1. A rejected spec latches Valid=False (one Warning). +// 2. Fixing the spec clears the Valid condition and the workload proceeds +// past the gate (requeue waiting on runconfig, like any valid spec). +// 3. Re-breaking the spec latches Valid=False again AND the one-shot Warning +// fires a second time — proving the pass path genuinely un-poisons the +// latch instead of leaving the workload permanently silenced. +func TestMCPServerReconciler_UntrustedLatchClearsAndWarningReArms(t *testing.T) { + t.Parallel() + + ctx := log.IntoContext(t.Context(), log.Log) + + // Start REJECTED: untrusted + spec.secrets. + mcpServer := v1beta1test.NewMCPServer("untrusted-latch", "default", + withUntrustedAnnotation(), + withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), + ) + r, recorder := setupUntrustedReconciler(t, mcpServer) + + result := reconcileOnce(t, r, mcpServer) + assert.True(t, result.IsZero(), "rejected spec is terminal: no requeue") + require.Eventually(t, func() bool { + return countContaining(drainEvents(recorder), mcpv1beta1.ConditionReasonSecretEnvRejected) == 1 + }, 2*time.Second, 10*time.Millisecond, "expected the first Warning on the invalid transition") + + current := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(mcpServer), current)) + require.NotNil(t, meta.FindStatusCondition(current.Status.Conditions, mcpv1beta1.ConditionTypeValid), + "Valid=False must be latched on rejection") + + // FIX the spec: drop spec.secrets. The gate must clear the latched Valid + // condition and let the reconcile proceed down the normal path. + current.Spec.Secrets = nil + require.NoError(t, r.Update(ctx, current)) + + result = reconcileOnce(t, r, mcpServer) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(mcpServer), current)) + assert.Nil(t, meta.FindStatusCondition(current.Status.Conditions, mcpv1beta1.ConditionTypeValid), + "passing the gate must clear the latched Valid condition") + assert.NotEqual(t, mcpv1beta1.ConditionReasonSecretEnvRejected, current.Status.Message, + "passing the gate must clear the latched gate-rejection message") + assert.False(t, result.IsZero(), + "fixed workload must proceed down the normal path (requeue waiting on runconfig)") + + // RE-BREAK the spec: the latch must re-engage and — critically — the + // Warning must fire again, which only happens when the pass path above + // removed the Valid=False latch (wasInvalid would otherwise stay true). + current.Spec.Secrets = []mcpv1beta1.SecretRef{{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}} + require.NoError(t, r.Update(ctx, current)) + + result = reconcileOnce(t, r, mcpServer) + assert.True(t, result.IsZero()) + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(mcpServer), current)) + condition := meta.FindStatusCondition(current.Status.Conditions, mcpv1beta1.ConditionTypeValid) + require.NotNil(t, condition, "Valid=False must latch again on re-rejection") + assert.Equal(t, metav1.ConditionFalse, condition.Status) + require.Eventually(t, func() bool { + return countContaining(drainEvents(recorder), mcpv1beta1.ConditionReasonSecretEnvRejected) == 1 + }, 2*time.Second, 10*time.Millisecond, + "the Warning must fire on the second invalid transition (latch-poisoning case)") +} + +func TestDeploymentForMCPServer_UntrustedSecretEnvRejectedAtBuildTime(t *testing.T) { + t.Parallel() + + // Defense-in-depth: deploymentForMCPServer itself rejects the built patch + // for untrusted workloads (spec.secrets seam)... + mcpServer := v1beta1test.NewMCPServer("untrusted-build-gate", "default", + withUntrustedAnnotation(), + withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), + ) + + r, _ := setupUntrustedReconciler(t, mcpServer) + + ctx := log.IntoContext(t.Context(), log.Log) + deployment, err := r.deploymentForMCPServer(ctx, mcpServer, "test-checksum") + require.Error(t, err) + assert.Nil(t, deployment) + assert.Contains(t, err.Error(), `env var "API_TOKEN"`) + assert.Contains(t, err.Error(), `Secret "backend-creds"`) + + // The defense-in-depth rejection is a typed SpecValidationError so the + // reconcile call sites peel it with errors.As into terminal treatment + // (condition, no requeue) instead of a transient forever-backoff. + var specErr *SpecValidationError + assert.ErrorAs(t, err, &specErr, "gate rejection must be a terminal typed error") + + // ...and for a trusted workload the same secrets pass the build gate. + trusted := v1beta1test.NewMCPServer("trusted-build-gate", "default", + withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), + ) + deployment, err = r.deploymentForMCPServer(ctx, trusted, "test-checksum") + require.NoError(t, err) + require.NotNil(t, deployment) +} diff --git a/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation.go b/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation.go new file mode 100644 index 0000000000..d1aed401c5 --- /dev/null +++ b/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation.go @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllerutil + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" +) + +// ValidateNoSecretEnvForUntrusted inspects a fully-built PodTemplateSpec patch for an +// untrusted-flagged workload and rejects every Secret- or ConfigMap-sourced data +// channel into the backend (mcp) container: +// - env ValueFrom SecretKeyRef / ConfigMapKeyRef on containers + initContainers, +// - envFrom secretRef / configMapRef (the whole-source twin of the per-key rules) +// on containers + initContainers, +// - Secret-, ConfigMap-, projected (Secret/ConfigMap source), and CSI +// secretProvider volumes (files are the same channel as env). +// +// It checks BOTH the generated entries (from spec.secrets via WithSecrets) and +// user-supplied entries from spec.podTemplateSpec, because both land in the same +// patch — validating pre-merge would miss the merged result. untrusted is passed +// as a parameter because the MCPServer.spec.untrusted field does not exist until +// Wave 1; the controller currently derives it from an interim opt-in annotation. +// +// This is a reconcile-time defense-in-depth gate; Wave 1 complements it with a +// CEL admission rule (untrusted ⇒ literal-only backend env) per +// .claude/rules/operator.md ("When a Go check intentionally mirrors a CEL rule +// for defense-in-depth, say so in a comment"). +// +// FieldRef/ResourceFieldRef env sources expose pod metadata, not credentials, and +// are allowed. Error messages name the env var and its source object (metadata +// only — never values), per .claude/rules/go-style.md "No sensitive data in errors". +// +// When untrusted is false this is a pure no-op and returns nil. +func ValidateNoSecretEnvForUntrusted(podTemplate *corev1.PodTemplateSpec, containerName string, untrusted bool) error { + if !untrusted || podTemplate == nil { + return nil + } + + for _, containers := range [][]corev1.Container{ + podTemplate.Spec.Containers, + podTemplate.Spec.InitContainers, + } { + for _, container := range containers { + if container.Name != containerName { + // The gate scopes to the backend container only; sidecars are + // trusted workloads managed by the operator. + continue + } + if err := validateBackendContainerEnv(containerName, &container); err != nil { + return err + } + } + } + + // Secrets-as-files is the same channel as secrets-as-env; leaving the volume + // twin open would make the env gate theater. ConfigMap volumes are rejected + // for parity with the ConfigMapKeyRef env rule, and the CSI secrets-store + // driver / projected-Secret sources close the remaining out-of-tree mounts. + for i := range podTemplate.Spec.Volumes { + if err := validateUntrustedVolume(&podTemplate.Spec.Volumes[i]); err != nil { + return err + } + } + + return nil +} + +// validateBackendContainerEnv rejects Secret- and ConfigMap-sourced env on one +// backend container: per-key ValueFrom refs and whole-source envFrom refs. +func validateBackendContainerEnv(containerName string, container *corev1.Container) error { + for _, env := range container.Env { + if env.ValueFrom == nil { + continue + } + if ref := env.ValueFrom.SecretKeyRef; ref != nil { + return fmt.Errorf( + "untrusted workload: env var %q on container %q sources its value from Secret %q (key %q), "+ + "which is not allowed; untrusted workloads must use literal env values", + env.Name, containerName, ref.Name, ref.Key) + } + if ref := env.ValueFrom.ConfigMapKeyRef; ref != nil { + return fmt.Errorf( + "untrusted workload: env var %q on container %q sources its value from ConfigMap %q (key %q), "+ + "which is not allowed; untrusted workloads must use literal env values", + env.Name, containerName, ref.Name, ref.Key) + } + } + for _, envFrom := range container.EnvFrom { + if ref := envFrom.SecretRef; ref != nil { + return fmt.Errorf( + "untrusted workload: envFrom on container %q sources values from Secret %q, "+ + "which is not allowed; untrusted workloads must use literal env values", + containerName, ref.Name) + } + if ref := envFrom.ConfigMapRef; ref != nil { + return fmt.Errorf( + "untrusted workload: envFrom on container %q sources values from ConfigMap %q, "+ + "which is not allowed; untrusted workloads must use literal env values", + containerName, ref.Name) + } + } + return nil +} + +// validateUntrustedVolume rejects one volume when it sources content from a +// Secret or ConfigMap by any channel: direct, CSI secrets-store, or projected. +func validateUntrustedVolume(volume *corev1.Volume) error { + if volume.Secret != nil { + return fmt.Errorf( + "untrusted workload: volume %q sources its content from Secret %q, "+ + "which is not allowed; untrusted workloads must not mount Secrets", + volume.Name, volume.Secret.SecretName) + } + if volume.ConfigMap != nil { + return fmt.Errorf( + "untrusted workload: volume %q sources its content from ConfigMap %q, "+ + "which is not allowed; untrusted workloads must not mount ConfigMaps", + volume.Name, volume.ConfigMap.Name) + } + // The CSI secrets-store driver has no typed PodSpec field: it resolves the + // SecretProviderClass from a driver-specific volume attribute. Treat any + // volume carrying that attribute as Secret-sourced regardless of driver. + if csi := volume.CSI; csi != nil && csi.VolumeAttributes["secretProviderClass"] != "" { + return fmt.Errorf( + "untrusted workload: volume %q sources its content from CSI SecretProviderClass %q, "+ + "which is not allowed; untrusted workloads must not mount Secrets", + volume.Name, csi.VolumeAttributes["secretProviderClass"]) + } + if projected := volume.Projected; projected != nil { + for _, source := range projected.Sources { + if source.Secret != nil { + return fmt.Errorf( + "untrusted workload: volume %q projects Secret %q, "+ + "which is not allowed; untrusted workloads must not mount Secrets", + volume.Name, source.Secret.Name) + } + if source.ConfigMap != nil { + return fmt.Errorf( + "untrusted workload: volume %q projects ConfigMap %q, "+ + "which is not allowed; untrusted workloads must not mount ConfigMaps", + volume.Name, source.ConfigMap.Name) + } + } + } + return nil +} diff --git a/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation_test.go b/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation_test.go new file mode 100644 index 0000000000..42ab821779 --- /dev/null +++ b/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation_test.go @@ -0,0 +1,372 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllerutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" +) + +// secretEnvVar builds a SecretKeyRef-sourced EnvVar for test fixtures. +func secretEnvVar(name, secretName, key string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + Key: key, + }, + }, + } +} + +// configMapEnvVar builds a ConfigMapKeyRef-sourced EnvVar for test fixtures. +func configMapEnvVar(name, configMapName, key string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: configMapName}, + Key: key, + }, + }, + } +} + +func TestValidateNoSecretEnvForUntrusted(t *testing.T) { + t.Parallel() + + const backend = "mcp" + + tests := []struct { + name string + podTemplate *corev1.PodTemplateSpec + untrusted bool + wantErr string // empty means no error; otherwise a substring of the error + }{ + { + name: "secretkeyref env on backend container is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: backend, + Env: []corev1.EnvVar{secretEnvVar("API_TOKEN", "backend-creds", "token")}, + }}, + }}, + untrusted: true, + wantErr: `env var "API_TOKEN" on container "mcp" sources its value from Secret "backend-creds"`, + }, + { + name: "secretkeyref env is allowed when workload is trusted", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: backend, + Env: []corev1.EnvVar{secretEnvVar("API_TOKEN", "backend-creds", "token")}, + }}, + }}, + untrusted: false, + }, + { + name: "configmapkeyref env on backend container is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: backend, + Env: []corev1.EnvVar{configMapEnvVar("API_TOKEN", "backend-config", "token")}, + }}, + }}, + untrusted: true, + wantErr: `env var "API_TOKEN" on container "mcp" sources its value from ConfigMap "backend-config"`, + }, + { + name: "fieldref env on backend container is allowed", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: backend, + Env: []corev1.EnvVar{{ + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }}, + }}, + }}, + untrusted: true, + }, + { + name: "secretkeyref env on backend init container is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{ + Name: backend, + Env: []corev1.EnvVar{secretEnvVar("INIT_TOKEN", "init-creds", "token")}, + }}, + }}, + untrusted: true, + wantErr: `env var "INIT_TOKEN" on container "mcp" sources its value from Secret "init-creds"`, + }, + { + name: "secretkeyref env on a sidecar container is allowed", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "sidecar", + Env: []corev1.EnvVar{secretEnvVar("SIDECAR_TOKEN", "sidecar-creds", "token")}, + }}, + }}, + untrusted: true, + }, + { + name: "literal-only env on backend container is allowed", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: backend, + Env: []corev1.EnvVar{{Name: "SENTINEL", Value: "literal-value"}}, + }}, + }}, + untrusted: true, + }, + { + name: "secret volume is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{{ + Name: "creds", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "backend-creds"}, + }, + }}, + }}, + untrusted: true, + wantErr: `volume "creds" sources its content from Secret "backend-creds"`, + }, + { + name: "envFrom secretRef on backend container is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: backend, + EnvFrom: []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "backend-creds"}, + }, + }}, + }}, + }}, + untrusted: true, + wantErr: `envFrom on container "mcp" sources values from Secret "backend-creds"`, + }, + { + name: "envFrom configMapRef on backend container is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: backend, + EnvFrom: []corev1.EnvFromSource{{ + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "backend-config"}, + }, + }}, + }}, + }}, + untrusted: true, + wantErr: `envFrom on container "mcp" sources values from ConfigMap "backend-config"`, + }, + { + name: "envFrom secretRef on backend init container is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{ + Name: backend, + EnvFrom: []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "init-creds"}, + }, + }}, + }}, + }}, + untrusted: true, + wantErr: `envFrom on container "mcp" sources values from Secret "init-creds"`, + }, + { + name: "envFrom secretRef on a sidecar container is allowed", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "sidecar", + EnvFrom: []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "sidecar-creds"}, + }, + }}, + }}, + }}, + untrusted: true, + }, + { + name: "configmap volume is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{{ + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "backend-config"}, + }, + }, + }}, + }}, + untrusted: true, + wantErr: `volume "config" sources its content from ConfigMap "backend-config"`, + }, + { + name: "projected volume with secret source is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{{ + Name: "projected", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "backend-creds"}, + }, + }}, + }, + }, + }}, + }}, + untrusted: true, + wantErr: `volume "projected" projects Secret "backend-creds"`, + }, + { + name: "projected volume with configmap source is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{{ + Name: "projected", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "backend-config"}, + }, + }}, + }, + }, + }}, + }}, + untrusted: true, + wantErr: `volume "projected" projects ConfigMap "backend-config"`, + }, + { + name: "projected volume with only downward-api sources is allowed", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{{ + Name: "projected", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{{ + DownwardAPI: &corev1.DownwardAPIProjection{ + Items: []corev1.DownwardAPIVolumeFile{{ + Path: "labels", + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.labels"}, + }}, + }, + }}, + }, + }, + }}, + }}, + untrusted: true, + }, + { + name: "csi secretproviderclass volume is rejected", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{{ + Name: "csi-creds", + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "secrets-store.csi.k8s.io", + VolumeAttributes: map[string]string{"secretProviderClass": "vault-creds"}, + }, + }, + }}, + }}, + untrusted: true, + wantErr: `volume "csi-creds" sources its content from CSI SecretProviderClass "vault-creds"`, + }, + { + name: "csi volume without secretproviderclass attribute is allowed", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{{ + Name: "csi-data", + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "ebs.csi.aws.com", + VolumeAttributes: map[string]string{"foo": "bar"}, + }, + }, + }}, + }}, + untrusted: true, + }, + { + name: "nil pod template is a no-op", + podTemplate: nil, + untrusted: true, + }, + { + name: "merged spec.secrets and raw template entries are both caught", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: backend, + Env: []corev1.EnvVar{ + // From spec.secrets via WithSecrets. + secretEnvVar("FROM_SPEC_SECRETS", "declared-secret", "key"), + // From a raw podTemplateSpec smuggled onto the same container. + secretEnvVar("FROM_RAW_TEMPLATE", "smuggled-secret", "key"), + }, + }}, + }}, + untrusted: true, + wantErr: `env var "FROM_SPEC_SECRETS" on container "mcp" sources its value from Secret "declared-secret"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := ValidateNoSecretEnvForUntrusted(tt.podTemplate, backend, tt.untrusted) + if tt.wantErr == "" { + require.NoError(t, err) + } else { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + } + }) + } +} + +// TestValidateNoSecretEnvForUntrusted_MergedBuilderOutput exercises the gate against +// the actual PodTemplateSpecBuilder output, proving that both seams (declarative +// spec.secrets and raw spec.podTemplateSpec) converge into a patch the gate rejects. +func TestValidateNoSecretEnvForUntrusted_MergedBuilderOutput(t *testing.T) { + t.Parallel() + + builder, err := NewPodTemplateSpecBuilder(nil, "mcp") + require.NoError(t, err) + + podTemplate := builder. + WithSecrets([]mcpv1beta1.SecretRef{ + {Name: "declared-secret", Key: "token", TargetEnvName: "FROM_SPEC_SECRETS"}, + }). + Build() + require.NotNil(t, podTemplate) + + // Simulate the merged case: a raw template entry on the same container would + // land alongside the WithSecrets entry in the same patch. + podTemplate.Spec.Containers[0].Env = append( + podTemplate.Spec.Containers[0].Env, + secretEnvVar("FROM_RAW_TEMPLATE", "smuggled-secret", "token"), + ) + + err = ValidateNoSecretEnvForUntrusted(podTemplate, "mcp", true) + require.Error(t, err) + assert.Contains(t, err.Error(), `env var "FROM_SPEC_SECRETS" on container "mcp" sources its value from Secret "declared-secret"`) +} diff --git a/deploy/charts/operator/templates/clusterrole/role.yaml b/deploy/charts/operator/templates/clusterrole/role.yaml index f4ddc88f92..8172b420b5 100644 --- a/deploy/charts/operator/templates/clusterrole/role.yaml +++ b/deploy/charts/operator/templates/clusterrole/role.yaml @@ -97,6 +97,18 @@ rules: - get - list - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/docs/arch/adr/0001-untrusted-mcp-egress-broker.md b/docs/arch/adr/0001-untrusted-mcp-egress-broker.md new file mode 100644 index 0000000000..a16c26ad7c --- /dev/null +++ b/docs/arch/adr/0001-untrusted-mcp-egress-broker.md @@ -0,0 +1,332 @@ +# ADR-0001: Untrusted MCP Server Egress Credential Broker + +- **Status**: Accepted +- **Date**: 2026-07-21 +- **Deciders**: ToolHive core team +- **Source**: Final consolidated plan, `.claude/scratch/untrusted-mcp-egress-broker-plan.md` §9 + (sections 1–8 are the design history and evidence trail; §9 supersedes them where they conflict). +- **Wave-0 implementation specs**: `.claude/scratch/wave0/01`–`04`, Wave-1 CRD handoff at + `.claude/scratch/wave0/wave1-handoff.md`. + +--- + +## 1. Context + +ToolHive fronts MCP servers written by third parties. Some deployments need to run servers the +operator **does not trust** — arbitrary community images whose code may be malicious or +compromised — while still letting those servers call upstream APIs (GitHub, Google, …) **as the +human driving the agent** (Case 1: user-delegated upstream OAuth). + +The design constraint that shapes everything else: + +> **A malicious or compromised MCP server must never possess a real credential** — not in an +> environment variable, not in a file, not in a header — **not even a short-lived one**. A +> malicious server replays what it receives immediately; TTL is irrelevant when the thief is the +> intended recipient. + +Consequence: a server that holds no credentials can make no authenticated call itself. A **trusted +broker must terminate every credentialed connection** and attach the credential at a boundary the +server cannot reach. + +### The exposure being closed (verified against code) + +Today the user's live upstream OAuth token is handed to the backend MCP server: + +- The vMCP `upstream_inject` strategy sets `Authorization: Bearer ` on the + HTTP request vMCP sends to the backend (`pkg/vmcp/auth/strategies/upstream_inject.go:85`), via + the auth RoundTripper in `pkg/vmcp/client/client.go` and + `pkg/vmcp/session/internal/backend/mcp_session.go`. +- The proxy-runner `upstreamswap` middleware does the same replacement on the gateway path + (`pkg/auth/upstreamswap/middleware.go:179-186`). +- A server's own static upstream credential is delivered as a K8s Secret env var on the backend + container (`WithSecrets(m.Spec.Secrets)` at + `cmd/thv-operator/controllers/mcpserver_controller.go:1071`, materialized as + `ValueFrom.SecretKeyRef` in + `cmd/thv-operator/pkg/controllerutil/podtemplatespec_builder.go:61-85`). + +What already exists and is **reused, not rebuilt**: upstream consent flow with PKCE +(`pkg/authserver/server/handlers/authorize.go`, `callback.go`), the per-session upstream token +store keyed `{prefix}upstream:{sessionID}:{provider}` (`pkg/authserver/storage/redis_keys.go`), +`tsid` minted into the ToolHive JWT (`pkg/authserver/server/session/session.go`), per-request +loading of upstream tokens into `Identity` (`pkg/auth/token.go:1169-1192`), refresh-on-read +(`pkg/auth/upstreamtoken/service.go`), and inbound `StripAuthMiddleware` +(`pkg/transport/middleware/strip_auth.go`). + +### The two structural blockers that forced the design + +Verified during plan scrutiny (plan §8.2): + +- **Blocker A — per-call dispatch IDs cannot reach the server's egress call.** vMCP can attach a + value to the vMCP→backend hop (headers, MCP `_meta`), but no MCP mechanism — by spec or + convention — makes a server copy an inbound value into the outbound HTTPS request *it* sends to + `api.github.com`. An untrusted server cannot be relied on to forward it faithfully, to the + intended host only, or without exfiltrating it. Any design that requires server cooperation is + void as a security boundary for unmodified servers. +- **Blocker B — backend pods are multi-tenant, so egress cannot be attributed to a user.** A single + MCPServer is one shared replica-based StatefulSet (`buildStatefulSetSpec`, + `pkg/container/kubernetes/client.go:557`); vMCP routes by capability to a shared Service URL with + no principal in the addressing. When a shared pod dials out, nothing on the egress path says + which user's tool call triggered it. + +Together: **for an unmodified untrusted server in a shared pod there is no reliable per-call +credential carriage and no reliable per-user egress attribution.** One of the two constraints had +to break. We break tenancy, not the "unmodified server" constraint — breaking the latter defeats +the point of fronting arbitrary third-party servers. + +--- + +## 2. Decisions (ratified — plan §9.2) + +| # | Decision | One-line rationale | +|---|----------|--------------------| +| D1 | **No per-call dispatch records** (retired) | Token-selection input never crosses the untrusted server in any form | +| D2 | **Single-tenant backends**: one backend pod per `(user, MCPServer)` | Pod identity *is* user identity; attribution needs no server cooperation | +| D3 | **Data plane = Envoy sidecar per backend pod + Go `ext_authz` injector** | Sidecar identity == pod identity; ext_authz header mutation is exactly the injection job | +| D4 | **Trusted identity-aware front mandatory (vMCP), stated as a requirement not a brand** | The requirement is per-session lifecycle ownership + per-user routing; vMCP is the natural home | +| D5 | **Credential-to-destination binding is a hard invariant** | Provider P's credential is emitted only for hosts in P's allowlist, checked before any header is written | +| D6 | **Response-side leakage control** | No redirect following; v1 allowlist restricted to known API hosts; token-value response scanning in Wave 5 | +| D7 | **Broker-side lockdown** | Sidecar egress NetworkPolicy-restricted to allowlisted destinations; per-dial resolved-IP validation (DNS-rebinding protection) | +| D8 | **Consent-on-demand in both subsystems** | Proxy path and vMCP path are independent patches; both gain a consent URL | +| D9 | **CA trust distribution is a compatibility matrix** | Bump CA via ConfigMap volume + runtime-specific env (`SSL_CERT_FILE`, `NODE_EXTRA_CA_CERTS`, …) | +| D10 | **stdio servers that ignore proxy env fail closed** | NetworkPolicy blocks direct egress; compatibility limit, documented — not a security gap | +| D11 | **Audit cardinality discipline** | User sub / pod name go to structured audit logs, never metrics labels | + +### D1 — No per-call dispatch records (retired Option A) + +**Decision.** The token-selection key (`tsid`) never travels in anything the untrusted server +emits. No `dispatch-id` header, `_meta` field, or tool-arg convention. The per-call dispatch-record +model (original plan D1/Option A) is removed from the primary design; a record-format sketch is +retained in the plan appendix solely as a future optimization for **first-party servers built with +a ToolHive SDK** that opt in to echoing a dispatch token upstream — and even then it is never the +security boundary. + +**Rationale.** Blocker A: there is no way to make an unmodified server propagate a per-call +identifier into its own egress, and a malicious server that *can* see the identifier can replay or +exfiltrate it. This also aligns with `.claude/rules/security.md` ("Prefer Stateless Routing Over +Stored Routing"): the pod→user mapping is a stable fact established once at pod creation, not a +per-call record written to a shared control API. + +**Alternatives considered.** Per-call dispatch records with a broker control API (finer-grained but +void for unmodified servers — Blocker A); per-session `(mcp-session → tsid)` push records (same +propagation problem, plus shared-state recovery burden). + +### D2 — Single-tenant backends + +**Decision.** Untrusted mode runs the backend MCP server **single-tenant: one backend pod per +`(user, MCPServer)` pair** — not per provider, not per tool. The pod→user binding is established +once at pod creation; the Envoy sidecar (D3) resolves identity locally from its own pod's binding. + +**Rationale.** Blocker B: this is the only break that preserves unmodified-server compatibility. +With 1:1 pod↔user mapping, the broker attributes egress by the pod's stable network identity — +no server cooperation, no propagated identifiers. Per-`(user, MCPServer)` granularity (rather than +per-user shared across servers) matches the existing session model and avoids cross-server +coupling; revisit only if pod fan-out proves untenable (Wave 2 measures this). + +**Alternatives considered.** Multi-tenant pods + per-call attribution (impossible, Blockers A+B); +per-user pods shared across MCPServers (rejected: unsupported by the session model, couples +unrelated servers' lifecycles). + +**Cost (owned explicitly).** Untrusted mode needs a per-session backend lifecycle: spin-up on +first use, idle-TTL teardown, cold-start latency, pod fan-out with DoS controls. This is the +largest single work item in the program (Wave 2) and interacts with the per-pod session caps in +`docs/arch/13-vmcp-scalability.md`. Session-restore (`pkg/vmcp/session/factory.go:400`) and its +15 s bound (`pkg/vmcp/server/sessionmanager/session_manager.go:211`) must tolerate pod creation. + +### D3 — Data plane: Envoy sidecar per backend pod + Go `ext_authz` service + +**Decision.** Each single-tenant backend pod carries an **Envoy sidecar** that terminates and +re-originates TLS ("bump"). Credential injection lives in a small **Go service invoked by Envoy as +an `ext_authz` filter** — not `ext_proc`. The Go service resolves pod→user, calls +`GetAllUpstreamCredentials` (refresh-on-read already exists, +`pkg/auth/upstreamtoken/service.go:88`), enforces D5, and returns the header mutation. `ext_proc` +is added **only** if response-body inspection (D6c, Wave 5) is built — the two filters are not +interchangeable and "pick later" was explicitly rejected during scrutiny. + +**Rationale.** ext_authz's job is exactly "mutate request headers on allow"; ext_proc is a +stream-processing filter whose cost and complexity are unjustified until response inspection +exists. The sidecar topology means the sidecar's identity *is* the user's identity — no +SPIFFE/mTLS or other workload-identity infrastructure is required, and TLS-bump stays +loopback-local to the pod. Envoy is also the standard egress-gateway substrate operators already +trust, unlike a bespoke Squid-bump image. + +**Alternatives considered.** +- **Squid + ssl_bump (rejected for K8s)**: cannot run our Go injectors; Squid remains the + Docker-mode egress reference only (`pkg/container/docker/squid.go` already enforces + `OutboundNetworkPermissions` there — this ADR extends an already-shipping Docker pattern into + K8s, where the same CRD fields are currently dead). +- **Shared Envoy egress gateway per namespace (rejected for v1)**: cheaper per-connection, but + sound only with workload-identity infrastructure (SPIFFE/mTLS) to supply pod→user; the marginal + cost of a sidecar inside an already-per-user pod is small. Recorded as a possible future + optimization once workload identity exists; composes with static egress IP (Cilium + EgressGateway / cloud NAT) for IdP-side allowlisting. +- **Go forward proxy with a goproxy-style core (rejected)**: a bespoke TLS-bump core is a + high-value security component; Envoy's maturity and operator familiarity win. + +### D4 — Trusted identity-aware front mandatory + +**Decision.** Untrusted-mode workloads MUST be fronted by a trusted, identity-aware component that +owns the per-session backend lifecycle and per-user routing. **vMCP is that component**; +enforcement is CEL validation requiring untrusted MCPServers to be members of an MCPGroup fronted +by a VirtualMCPServer. The requirement is lifecycle ownership, not vMCP per se — the upstream-token +machinery is not vMCP-exclusive (`pkg/auth/upstreamswap` is proxy-runner middleware) — but vMCP is +the natural and only supported home in v1. + +**Rationale.** Only a component holding per-request `Identity` and the `(iss,sub)` session binding +(`pkg/vmcp/session/internal/security/security.go:137-192`) can establish the pod→user binding at +pod creation and route subsequent calls to the right pod. + +**Alternatives considered.** Proxy-runner-only untrusted mode (rejected: no session lifecycle +ownership, no per-user routing dimension). + +### D5 — Credential-to-destination binding is a hard invariant + +**Decision.** The injector emits provider P's credential **only** when the destination host is in +P's allowlist, evaluated **before any header is written**. The binding is encoded in the egress +policy CRD as `provider → {allowedHosts, allowedMethods, allowedPathPrefixes}` (extending the +currently K8s-dead `OutboundNetworkPermissions`, +`cmd/thv-operator/api/v1beta1/mcpserver_types.go:645-661`). + +**Rationale.** Without it, a malicious server points its egress at `attacker.com` under TLS-bump +and the broker — having resolved pod→user→real GitHub token — injects that token into a request +to `attacker.com`. Pod resolution selects the token; only destination-binding stops it going to +the wrong place. This is a security invariant, not an ACL nicety. + +### D6 — Response-side leakage control + +**Decision.** The broker must never let an injected credential come back to the server: +(a) **never follow redirects** — return 3xx to the server untouched; (b) **v1: destination +allowlists are restricted to known API hosts** (config-level, e.g. `api.github.com` — not +arbitrary echo services); (c) **Wave 5: response scanning** for the injected token value with +redact/block (`ext_proc`). + +**Rationale.** The canonical attack is an httpbin-style echo endpoint: server asks the broker to +call a host that reflects the `Authorization` header back in the response body. (a)+(b) close it +for v1 within the known-API-host constraint; (c) closes it generally. + +### D7 — Broker-side lockdown + +**Decision.** The sidecar's own egress is NetworkPolicy-restricted to allowlisted destination +hosts only, and resolved destination IPs are validated **per dial** against the allowlist — +the same pattern as `WithDialControl` (`pkg/vmcp/client/client.go:99-103`). The broker is never a +general SSRF proxy. + +**Rationale.** A TLS-terminating proxy that will dial anything is an SSRF oracle into both the +cluster and the internet; per-dial IP validation closes DNS-rebinding (name resolves to an +allowlisted host at policy-check time, to a link-local/internal IP at dial time). + +### D8 — Consent-on-demand in both subsystems + +**Decision.** Missing/expired upstream consent produces an actionable response in **both** +independent subsystems (they must not be conflated): +- **Proxy path** (`pkg/auth/upstreamswap/middleware.go:112-119`): keep the 401 + RFC 6750 + `WWW-Authenticate` challenge, add the consent URL in `error_description` or a structured body. +- **vMCP path** (`pkg/vmcp/auth/strategies/upstream_inject.go:82` → `pkg/vmcp/client/client.go`): + a structured MCP error carrying the authorize URL + provider name. + +**Rationale.** The broker never talks to agents; consent must be driven by the trusted front. +Today the vMCP path surfaces a bare authentication failure with no remediation path. + +### D9 — CA trust distribution is a compatibility matrix + +**Decision.** The bump CA is delivered via ConfigMap volume mount through the `--k8s-pod-patch` +seam (`cmd/thv-operator/controllers/mcpserver_controller.go:1079` → `applyPodTemplatePatch`, +`pkg/container/kubernetes/client.go:1251-1292`) plus runtime-specific env: `SSL_CERT_FILE` (Go), +`NODE_EXTRA_CA_CERTS` (Node), `REQUESTS_CA_BUNDLE` (Python/requests), `CURL_CA_BUNDLE`. Distroless +images: volume mount only. Per-tenant CA where feasible; a cluster CA is acceptable for v1 with a +documented rotation procedure. + +**Rationale.** The bump CA in the pod trust store is an intentional, scoped MITM; if any runtime +in the image doesn't trust it, TLS-bump fails closed (availability loss, not confidentiality). + +### D10 — stdio servers fail closed, documented + +**Decision.** A stdio server that ignores `HTTP_PROXY`/`HTTPS_PROXY` env makes direct egress +attempts → NetworkPolicy blocks them → the server cannot call upstreams. This is a compatibility +limit, not a security gap, and is documented in user-facing docs. + +### D11 — Audit cardinality + +**Decision.** Every injected call is audit-logged with workload, user sub, pod name, destination +host, method, path, and status — in **structured audit logs only**. Metrics carry injection +counts and ACL denials per workload+provider **only**; user sub, pod name, and dispatch details +never appear in metric labels (cardinality explosion + PII in metrics). + +--- + +## 3. Rejected alternatives (recorded so they are not re-litigated) + +1. **Mediated tool-call model.** ToolHive's trusted layer makes the upstream call itself; the + server is reduced to a schema/transformer, or declares egress that the trusted layer executes. + This is the *only* model that removes the server from directing the credential's authority + (it would also close the §5 residual). **Rejected** because it breaks the + arbitrary-third-party-server model that is the entire point of the feature. +2. **Per-call dispatch propagation for unmodified servers** (original Option A). Void: no MCP + mechanism carries an inbound value into the server's own egress, and an untrusted server cannot + be trusted to (Blocker A). Demoted to a first-party-SDK opt-in optimization, never the security + boundary. +3. **Shared-gateway topology for v1.** Requires workload-identity infrastructure to attribute + connections to users; sidecar-in-pod gets the same property for free in an already-per-user + pod. Revisit once SPIFFE/mTLS exists. +4. **Squid data plane for Kubernetes.** Cannot host the per-user Go injection logic; bespoke + bump images are a worse operational and security posture than Envoy. Squid stays Docker-only. +5. **DNS pinning as a load-bearing control.** With default-deny egress that permits only the + sidecar, and the broker re-resolving names itself, the server's own resolver cannot help it + reach a forbidden host. Kept only as optional defense-in-depth; no work is sequenced behind it. + +--- + +## 4. Wave plan (delivery roadmap — plan §9.3) + +Each wave is one dev-pipeline run; nothing is deferred out of a wave. + +| Wave | Contents | Critical path? | +|------|----------|----------------| +| **W0 — ADR + hardening PRs** | This ADR. **0.1** read-side binding check (`ErrInvalidBinding` is documented at `pkg/authserver/storage/types.go:565` but never returned by `memory.go:791-823` or `redis.go:1199-1209` — documented-vs-actual drift, hard gate for untrusted mode). **0.2** reject `valueFrom.secretKeyRef` in backend container env for untrusted workloads. **0.3** encrypt upstream tokens at rest in Redis (plaintext JSON today, `pkg/authserver/storage/redis.go:843`). **0.4** operator RBAC for `networkpolicies`. | Independent, merge first | +| **W1 — CRD surface + sentinel machinery** | `MCPServer.spec.untrusted`, `EgressPolicy` (`provider → allowedHosts/methods/pathPrefixes`), CEL validation (`untrusted ⇒ group membership + single-tenant + no secretKeyRef + provider allowlist`), sentinel env injection for token-requiring servers. Spec: `.claude/scratch/wave0/wave1-handoff.md`. | | +| **W2 — Session-scoped single-tenant backend lifecycle** | Spin-up on first use, idle-TTL teardown, pod→(user, MCPServer) registry (Redis, TTL'd), session-restore integration (`factory.go:400`, 15 s `restoreSessionTimeout` at `session_manager.go:211` is insufficient for pod creation), mandatory session affinity for untrusted groups, DoS controls (per-user quota, creation rate limit, capacity admission vs the 1,000-session LRU cap, `docs/arch/13-vmcp-scalability.md`), backend-pod survival across vMCP restarts. | **CRITICAL PATH — largest wave** | +| **W3 — Envoy sidecar + ext_authz injector** | `pkg/egressbroker` Go ext_authz service (pod→user from W2 registry, D5 precondition, inject Authorization), Envoy sidecar config (TLS-bump, per-workload routes/ACLs, no-redirect D6a, per-dial IP validation D7), per-tenant bump CA + ConfigMap distribution, NetworkPolicy generation (greenfield — the `client.go:360` TODO becomes real), e2e proving the token never appears in pod env/dumps/responses. | | +| **W4 — Consent-on-demand** | Both subsystems per D8; client/CLI rendering + post-consent retry. | | +| **W5 — Hardening + ops** | Response-side token scanning (ext_proc, D6c), audit pipeline + dashboards + alerting on ACL denials/leakage hits (D11), sidecar HA + upgrade story + CA rotation runbook, docs (new broker arch doc; update 09, 05, 13; user-facing untrusted-mode doc stating §5 loudly), first-party-SDK dispatch-echo spike (appendix only). | | + +--- + +## 5. The security boundary, stated loudly + +**Untrusted mode defeats credential theft, replay, cross-user use, and post-hoc use.** The server +never holds a token it can carry off, replay later, or use as another user. + +**It does NOT defeat:** + +1. **Authority abuse within scope for the current call.** The server shapes every outbound request + (path, method, body), so it directs the current user's credential within granted scopes for the + current session. A compromised GitHub MCP server can still call `DELETE /repos/{user}/{repo}` + as the user. **Mitigations (these are the actual boundary):** least-privilege upstream consent + scopes; per-tool method+path ACLs enforced in Envoy route config and the injector; full audit of + every injected call (D11) as the primary detection surface. +2. **Data exfiltration via MCP responses.** Anything the upstream API returns to the server can be + embedded in tool responses to the agent. The credential never leaks; the data it fetches can. + +Operators must be told both limits — here and in the Wave-5 user-facing doc — so "untrusted mode" +is not over-trusted. + +## 6. Consequences + +- **Positive**: unmodified third-party servers can call upstream APIs as the consenting user while + never possessing a credential; the exposure at `upstream_inject.go:85` is interposed on; every + credentialed call is ACL'd and audited at a point the server cannot reach. +- **Negative / costs**: per-session pod fan-out (capacity, cold start, DoS surface — Wave 2 owns + this); Envoy sidecar per pod; TLS-bump means the broker sees plaintext of every upstream request + *and* response — the sidecar is a high-value target and a data-privacy consideration; + cert-pinning servers and proxy-ignoring stdio servers fail closed (compatibility limits); + purely request-driven servers are the only kind that function (no background polling — enforced + by default-deny egress, documented). +- **Operational**: bump-CA rotation runbook required; per-tenant CA preferred; NetworkPolicy + support in the CNI becomes a hard dependency for untrusted mode. + +## 7. Remaining open questions (owned by Wave gates, not blocking W0/W1) + +1. Exact lifecycle-owner split for single-tenant backends (vMCP component vs new controller) — + Wave 2 design gate. +2. Per-tenant vs cluster bump CA — default per-tenant where feasible (D9). +3. Idle-TTL default and cold-start SLO for session-scoped pods — Wave 2. +4. Non-HTTP upstreams (DBs, gRPC, SSH-git): deny by default; revisit per demand. diff --git a/pkg/auth/token.go b/pkg/auth/token.go index 85c11848fc..6f783aa244 100644 --- a/pkg/auth/token.go +++ b/pkg/auth/token.go @@ -26,6 +26,7 @@ import ( "github.com/stacklok/toolhive-core/env" "github.com/stacklok/toolhive/pkg/auth/upstreamtoken" "github.com/stacklok/toolhive/pkg/authserver/server/keys" + "github.com/stacklok/toolhive/pkg/authserver/storage" "github.com/stacklok/toolhive/pkg/networking" "github.com/stacklok/toolhive/pkg/oauthproto" ) @@ -1181,7 +1182,24 @@ func (v *TokenValidator) loadUpstreamTokens( return nil, nil, nil } - creds, failed, err := v.upstreamTokenReader.GetAllUpstreamCredentials(ctx, tsid) + // Assert the caller's binding against the stored rows: the row's owning user + // must match the JWT subject and the consenting OAuth client must match the + // JWT's client. Storage enforces this per row (mismatched rows are excluded); + // the ctx already carries the Identity, so the explicit UserID is + // redundant-but-consistent — the client check is the new teeth. + // + // The ToolHive auth server mints "client_id" as a top-level JWT claim + // (pkg/authserver/server/session Session.New writes it into JWTClaims.Extra + // via ClientIDClaimKey, and fosite's JWTClaims.ToMap copies Extra into the + // top-level claim set), so this lookup fires for ToolHive-issued tokens. + // Third-party JWTs without the claim simply skip the client dimension — + // the user binding still applies. + expected := &storage.ExpectedBinding{UserID: identitySubject(claims)} + if clientID, ok := claims["client_id"].(string); ok { + expected.ClientID = clientID + } + + creds, failed, err := v.upstreamTokenReader.GetAllUpstreamCredentials(ctx, tsid, expected) if err != nil { // Log tsid at DEBUG only — it is credential-adjacent and must not // leak into WARN-level logs or returned error strings. @@ -1191,6 +1209,15 @@ func (v *TokenValidator) loadUpstreamTokens( return creds, failed, nil } +// identitySubject extracts the sub claim as the expected binding user, mirroring +// claimsToIdentity (which requires sub per OIDC Core 1.0 §5.1). Kept as a +// separate read because loadUpstreamTokens receives the raw claims map, not the +// converted Identity. +func identitySubject(claims jwt.MapClaims) string { + sub, _ := claims["sub"].(string) + return sub +} + // Middleware creates an HTTP middleware that validates JWT tokens and creates Identity. func (v *TokenValidator) Middleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1230,6 +1257,10 @@ func (v *TokenValidator) Middleware(next http.Handler) http.Handler { // never mutated afterwards (see the UpstreamTokens doc comment in identity.go). if v.upstreamTokenReader != nil { loadCtx := WithIdentity(r.Context(), identity) + // Carry the canonical user in the storage binding key as well so + // read-side binding validation enforces it without the storage + // package importing pkg/auth. + loadCtx = storage.ContextWithBindingUser(loadCtx, identity.PlatformUserID) creds, failed, loadErr := v.loadUpstreamTokens(loadCtx, claims) if loadErr != nil { slog.WarnContext(loadCtx, "upstream token storage unavailable", diff --git a/pkg/auth/token_test.go b/pkg/auth/token_test.go index b19ea3b9f7..2549aca760 100644 --- a/pkg/auth/token_test.go +++ b/pkg/auth/token_test.go @@ -11,6 +11,7 @@ import ( "encoding/pem" "errors" "fmt" + "maps" "net/http" "net/http/httptest" "os" @@ -20,6 +21,7 @@ import ( "github.com/golang-jwt/jwt/v5" "github.com/lestrrat-go/jwx/v3/jwk" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -28,6 +30,7 @@ import ( upstreamtokenmocks "github.com/stacklok/toolhive/pkg/auth/upstreamtoken/mocks" "github.com/stacklok/toolhive/pkg/authserver/server/keys" keysmocks "github.com/stacklok/toolhive/pkg/authserver/server/keys/mocks" + "github.com/stacklok/toolhive/pkg/authserver/storage" "github.com/stacklok/toolhive/pkg/networking" "github.com/stacklok/toolhive/pkg/oauthproto" ) @@ -2235,7 +2238,7 @@ func TestLoadUpstreamTokens(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) reader := upstreamtokenmocks.NewMockTokenReader(ctrl) - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-abc"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{ "github": {AccessToken: "gh-token", IDToken: "gh-id-token"}, "atlassian": {AccessToken: "atl-token"}, @@ -2301,7 +2304,7 @@ func TestLoadUpstreamTokens(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) reader := upstreamtokenmocks.NewMockTokenReader(ctrl) - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-abc"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). Return(nil, nil, errors.New("storage unavailable")) v := &TokenValidator{upstreamTokenReader: reader} @@ -2318,7 +2321,7 @@ func TestLoadUpstreamTokens(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) reader := upstreamtokenmocks.NewMockTokenReader(ctrl) - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-abc"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{}, []string{"github"}, nil) v := &TokenValidator{upstreamTokenReader: reader} @@ -2335,7 +2338,7 @@ func TestLoadUpstreamTokens(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) reader := upstreamtokenmocks.NewMockTokenReader(ctrl) - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-abc"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{}, []string(nil), nil) v := &TokenValidator{upstreamTokenReader: reader} @@ -2410,11 +2413,109 @@ func TestMiddleware_UpstreamTokenEnrichment(t *testing.T) { upstreamtoken.TokenSessionIDClaimKey: "session-xyz", } + t.Run("passes expected binding built from sub and client_id claims", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := upstreamtokenmocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-xyz", + &storage.ExpectedBinding{UserID: "test-user", ClientID: "client-A"}). + Return(map[string]upstreamtoken.UpstreamCredential{ + "github": {AccessToken: "gh-tok"}, + }, []string(nil), nil) + v := makeValidator(t, WithUpstreamTokenReader(reader)) + + handler := v.Middleware(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + req := httptest.NewRequest("GET", "/", nil) + claims := maps.Clone(claimsWithTsid) + claims["client_id"] = "client-A" + req.Header.Set("Authorization", "Bearer "+signToken(claims)) + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + }) + + t.Run("carries the canonical user in the storage binding ctx key", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := upstreamtokenmocks.NewMockTokenReader(ctrl) + + // Prove the ctx the reader receives carries the binding user: perform a + // nil-expected storage read against a row owned by a different user from + // inside the mock. It must fail binding — which only happens when the + // middleware placed the identity's PlatformUserID into the ctx key. + stor := storage.NewMemoryStorage() + t.Cleanup(func() { _ = stor.Close() }) + require.NoError(t, stor.StoreUpstreamTokens(context.Background(), "probe-session", "github", + &storage.UpstreamTokens{ + ProviderID: "github", AccessToken: "probe", + ExpiresAt: time.Now().Add(time.Hour), UserID: "some-other-user", + })) + + var bindingErr error + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz", gomock.Any()). + DoAndReturn(func(ctx context.Context, _ string, _ *storage.ExpectedBinding, + ) (map[string]upstreamtoken.UpstreamCredential, []string, error) { + _, bindingErr = stor.GetUpstreamTokens(ctx, "probe-session", "github", nil) + return map[string]upstreamtoken.UpstreamCredential{}, nil, nil + }) + v := makeValidator(t, WithUpstreamTokenReader(reader)) + + handler := v.Middleware(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + req := httptest.NewRequest("GET", "/", nil) + req.Header.Set("Authorization", "Bearer "+signToken(claimsWithTsid)) + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + require.ErrorIs(t, bindingErr, storage.ErrInvalidBinding, + "nil-expected storage read from the load ctx must resolve the JWT user's binding") + }) + + t.Run("scenario 15: storage excludes rows bound to a different client, request proceeds with empty maps", func(t *testing.T) { + t.Parallel() + // Full middleware → service → memory-storage path: the JWT's client_id + // does not match the stored row's consenting client, so storage excludes + // the row and the request proceeds with non-nil empty upstream maps — + // distinguishing "checked, nothing released" from "no tsid". + stor := storage.NewMemoryStorage() + t.Cleanup(func() { _ = stor.Close() }) + require.NoError(t, stor.StoreUpstreamTokens(context.Background(), "session-xyz", "github", + &storage.UpstreamTokens{ + ProviderID: "github", + AccessToken: "victim-access-token", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "test-user", + ClientID: "client-B", // consented by a different OAuth client + })) + + svc := upstreamtoken.NewInProcessService(stor, nil) + v := makeValidator(t, WithUpstreamTokenReader(svc)) + + var captured *Identity + handler := v.Middleware(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + captured, _ = IdentityFromContext(r.Context()) + })) + req := httptest.NewRequest("GET", "/", nil) + claims := maps.Clone(claimsWithTsid) + claims["client_id"] = "client-A" + req.Header.Set("Authorization", "Bearer "+signToken(claims)) + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + require.Equal(t, http.StatusOK, rr.Code) + require.NotNil(t, captured) + require.NotNil(t, captured.UpstreamTokens, "tsid was present: map must be non-nil") + assert.Empty(t, captured.UpstreamTokens, "rows bound to a different client must not be released") + assert.NotContains(t, captured.UpstreamTokens, "github") + }) + t.Run("enriches identity with upstream tokens", func(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) reader := upstreamtokenmocks.NewMockTokenReader(ctrl) - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{ "github": {AccessToken: "gh-tok", IDToken: "gh-id-tok"}, }, []string(nil), nil) @@ -2443,7 +2544,7 @@ func TestMiddleware_UpstreamTokenEnrichment(t *testing.T) { // The service layer preserves the empty IDToken field (see // TestInProcessService_GetAllUpstreamCredentials); the projection that // drops providers whose IDToken is empty happens in Middleware. - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{ "github": {AccessToken: "gh-tok", IDToken: ""}, "atlassian": {AccessToken: "atl-tok", IDToken: ""}, @@ -2483,7 +2584,7 @@ func TestMiddleware_UpstreamTokenEnrichment(t *testing.T) { // github has an ID token; atlassian has only an access token. The // projection in Middleware must key UpstreamTokens on both providers // but UpstreamIDTokens only on github. - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{ "github": {AccessToken: "gh-tok", IDToken: "gh-id-tok"}, "atlassian": {AccessToken: "atl-tok", IDToken: ""}, @@ -2524,8 +2625,9 @@ func TestMiddleware_UpstreamTokenEnrichment(t *testing.T) { var loadCtxHadIdentity bool var loadCtxCanonicalUser string var loadCtxHadCanonicalUser bool - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz"). - DoAndReturn(func(ctx context.Context, _ string) (map[string]upstreamtoken.UpstreamCredential, []string, error) { + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz", gomock.Any()). + DoAndReturn(func(ctx context.Context, _ string, _ *storage.ExpectedBinding, + ) (map[string]upstreamtoken.UpstreamCredential, []string, error) { loadCtxIdentity, loadCtxHadIdentity = IdentityFromContext(ctx) loadCtxCanonicalUser, loadCtxHadCanonicalUser = CanonicalUserFromContext(ctx) return map[string]upstreamtoken.UpstreamCredential{"github": {AccessToken: "gh-tok"}}, nil, nil @@ -2554,7 +2656,7 @@ func TestMiddleware_UpstreamTokenEnrichment(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) reader := upstreamtokenmocks.NewMockTokenReader(ctrl) - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz", gomock.Any()). Return(nil, nil, errors.New("redis down")) v := makeValidator(t, WithUpstreamTokenReader(reader)) @@ -2576,7 +2678,7 @@ func TestMiddleware_UpstreamTokenEnrichment(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) reader := upstreamtokenmocks.NewMockTokenReader(ctrl) - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{}, []string{"github"}, nil) v := makeValidator(t, WithUpstreamTokenReader(reader)) @@ -2601,7 +2703,7 @@ func TestMiddleware_UpstreamTokenEnrichment(t *testing.T) { ctrl := gomock.NewController(t) reader := upstreamtokenmocks.NewMockTokenReader(ctrl) // atlassian succeeded, github failed — the middleware must still reject the request - reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz"). + reader.EXPECT().GetAllUpstreamCredentials(gomock.Any(), "session-xyz", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{"atlassian": {AccessToken: "atl-tok"}}, []string{"github"}, nil) v := makeValidator(t, WithUpstreamTokenReader(reader)) diff --git a/pkg/auth/upstreamtoken/mocks/mock_token_reader.go b/pkg/auth/upstreamtoken/mocks/mock_token_reader.go index b613070321..abaf9797ad 100644 --- a/pkg/auth/upstreamtoken/mocks/mock_token_reader.go +++ b/pkg/auth/upstreamtoken/mocks/mock_token_reader.go @@ -14,6 +14,7 @@ import ( reflect "reflect" upstreamtoken "github.com/stacklok/toolhive/pkg/auth/upstreamtoken" + storage "github.com/stacklok/toolhive/pkg/authserver/storage" gomock "go.uber.org/mock/gomock" ) @@ -42,9 +43,9 @@ func (m *MockTokenReader) EXPECT() *MockTokenReaderMockRecorder { } // GetAllUpstreamCredentials mocks base method. -func (m *MockTokenReader) GetAllUpstreamCredentials(ctx context.Context, sessionID string) (map[string]upstreamtoken.UpstreamCredential, []string, error) { +func (m *MockTokenReader) GetAllUpstreamCredentials(ctx context.Context, sessionID string, expected *storage.ExpectedBinding) (map[string]upstreamtoken.UpstreamCredential, []string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllUpstreamCredentials", ctx, sessionID) + ret := m.ctrl.Call(m, "GetAllUpstreamCredentials", ctx, sessionID, expected) ret0, _ := ret[0].(map[string]upstreamtoken.UpstreamCredential) ret1, _ := ret[1].([]string) ret2, _ := ret[2].(error) @@ -52,7 +53,7 @@ func (m *MockTokenReader) GetAllUpstreamCredentials(ctx context.Context, session } // GetAllUpstreamCredentials indicates an expected call of GetAllUpstreamCredentials. -func (mr *MockTokenReaderMockRecorder) GetAllUpstreamCredentials(ctx, sessionID any) *gomock.Call { +func (mr *MockTokenReaderMockRecorder) GetAllUpstreamCredentials(ctx, sessionID, expected any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllUpstreamCredentials", reflect.TypeOf((*MockTokenReader)(nil).GetAllUpstreamCredentials), ctx, sessionID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllUpstreamCredentials", reflect.TypeOf((*MockTokenReader)(nil).GetAllUpstreamCredentials), ctx, sessionID, expected) } diff --git a/pkg/auth/upstreamtoken/service.go b/pkg/auth/upstreamtoken/service.go index 503e15ab6d..ce04705101 100644 --- a/pkg/auth/upstreamtoken/service.go +++ b/pkg/auth/upstreamtoken/service.go @@ -44,8 +44,10 @@ func NewInProcessService( // GetValidTokens returns a valid upstream credential for a session and provider. // It transparently refreshes expired access tokens using the refresh token. -func (s *InProcessService) GetValidTokens(ctx context.Context, sessionID, providerName string) (*UpstreamCredential, error) { - tokens, err := s.storage.GetUpstreamTokens(ctx, sessionID, providerName) +func (s *InProcessService) GetValidTokens( + ctx context.Context, sessionID, providerName string, expected *storage.ExpectedBinding, +) (*UpstreamCredential, error) { + tokens, err := s.storage.GetUpstreamTokens(ctx, sessionID, providerName, expected) if err != nil { // ErrExpired returns tokens (including refresh token) alongside the error. // Attempt a refresh before giving up. @@ -86,9 +88,9 @@ func (s *InProcessService) GetValidTokens(ctx context.Context, sessionID, provid // // Returns an empty map and nil failed slice (not error) for unknown sessions. func (s *InProcessService) GetAllUpstreamCredentials( - ctx context.Context, sessionID string, + ctx context.Context, sessionID string, expected *storage.ExpectedBinding, ) (map[string]UpstreamCredential, []string, error) { - allTokens, err := s.storage.GetAllUpstreamTokens(ctx, sessionID) + allTokens, err := s.storage.GetAllUpstreamTokens(ctx, sessionID, expected) if err != nil { return nil, nil, fmt.Errorf("bulk read upstream tokens: %w", err) } diff --git a/pkg/auth/upstreamtoken/service_test.go b/pkg/auth/upstreamtoken/service_test.go index de3b11dfce..9085451ac3 100644 --- a/pkg/auth/upstreamtoken/service_test.go +++ b/pkg/auth/upstreamtoken/service_test.go @@ -76,7 +76,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { name: "valid tokens returned directly", sessionID: "session-1", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-1", "default"). + s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-1", "default", gomock.Any()). Return(validTokens, nil) }, setupRefresher: func(_ *storagemocks.MockUpstreamTokenRefresher) {}, @@ -89,7 +89,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { name: "expired tokens refreshed via storage ErrExpired", sessionID: "session-2", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-2", "default"). + s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-2", "default", gomock.Any()). Return(expiredTokens, storage.ErrExpired) }, setupRefresher: func(r *storagemocks.MockUpstreamTokenRefresher) { @@ -104,7 +104,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { sessionID: "session-3", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { // Storage returns expired tokens without error (defense in depth path) - s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-3", "default"). + s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-3", "default", gomock.Any()). Return(expiredTokens, nil) }, setupRefresher: func(r *storagemocks.MockUpstreamTokenRefresher) { @@ -118,7 +118,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { name: "session not found", sessionID: "session-4", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-4", "default"). + s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-4", "default", gomock.Any()). Return(nil, storage.ErrNotFound) }, setupRefresher: func(_ *storagemocks.MockUpstreamTokenRefresher) {}, @@ -128,7 +128,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { name: "expired with no refresh token", sessionID: "session-5", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-5", "default"). + s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-5", "default", gomock.Any()). Return(expiredNoRefresh, storage.ErrExpired) }, setupRefresher: func(_ *storagemocks.MockUpstreamTokenRefresher) {}, @@ -138,7 +138,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { name: "refresh fails", sessionID: "session-6", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-6", "default"). + s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-6", "default", gomock.Any()). Return(expiredTokens, storage.ErrExpired) }, setupRefresher: func(r *storagemocks.MockUpstreamTokenRefresher) { @@ -151,7 +151,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { name: "storage error propagated", sessionID: "session-7", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-7", "default"). + s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-7", "default", gomock.Any()). Return(nil, errors.New("redis connection lost")) }, setupRefresher: func(_ *storagemocks.MockUpstreamTokenRefresher) {}, @@ -161,7 +161,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { name: "ErrExpired with nil tokens returns ErrNoRefreshToken", sessionID: "session-8", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-8", "default"). + s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-8", "default", gomock.Any()). Return(nil, storage.ErrExpired) }, setupRefresher: func(_ *storagemocks.MockUpstreamTokenRefresher) {}, @@ -171,7 +171,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { name: "invalid binding returns ErrInvalidBinding", sessionID: "session-9", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-9", "default"). + s.EXPECT().GetUpstreamTokens(gomock.Any(), "session-9", "default", gomock.Any()). Return(nil, storage.ErrInvalidBinding) }, setupRefresher: func(_ *storagemocks.MockUpstreamTokenRefresher) {}, @@ -193,7 +193,7 @@ func TestInProcessService_GetValidTokens(t *testing.T) { svc := NewInProcessService(mockStorage, mockRefresher) - cred, err := svc.GetValidTokens(context.Background(), tt.sessionID, "default") + cred, err := svc.GetValidTokens(context.Background(), tt.sessionID, "default", nil) if tt.wantErr != nil { require.Error(t, err) @@ -231,12 +231,12 @@ func TestInProcessService_NilRefresher(t *testing.T) { mockStorage := storagemocks.NewMockUpstreamTokenStorage(ctrl) mockStorage.EXPECT(). - GetUpstreamTokens(gomock.Any(), "session-1", "default"). + GetUpstreamTokens(gomock.Any(), "session-1", "default", gomock.Any()). Return(expiredTokens, storage.ErrExpired) svc := NewInProcessService(mockStorage, nil) - cred, err := svc.GetValidTokens(context.Background(), "session-1", "default") + cred, err := svc.GetValidTokens(context.Background(), "session-1", "default", nil) require.Error(t, err) assert.ErrorIs(t, err, ErrNoRefreshToken) @@ -296,7 +296,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "all fresh tokens returned directly with IDs", sessionID: "session-1", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-1"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-1", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": freshTokens, "atlassian": freshTokens2, @@ -312,7 +312,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "mixed fresh and expired with successful refresh carries ID token through", sessionID: "session-2", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-2"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-2", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "atlassian": freshTokens2, "github": expiredTokens, @@ -333,7 +333,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "expired refresh that rotates id_token prefers refreshed ID token", sessionID: "session-11", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-11"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-11", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": expiredTokens, }, nil) @@ -346,11 +346,31 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { "github": {AccessToken: "new-github-token", IDToken: "rotated-github-id-token"}, }, }, + { + // Scenario 14: a row storage excluded by binding validation is simply + // absent from the bulk map. It must NOT appear in failed — no refresh + // is ever attempted for a row that isn't the caller's. + name: "binding-excluded provider absent from result and failed", + sessionID: "session-binding", + setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { + // Storage excluded the "gitlab" row (its stored user/client did + // not match the expected binding); only github comes back. + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-binding", gomock.Any()). + Return(map[string]*storage.UpstreamTokens{ + "github": freshTokens, + }, nil) + }, + setupRefresher: func(_ *storagemocks.MockUpstreamTokenRefresher) {}, + wantResult: map[string]UpstreamCredential{ + "github": {AccessToken: "github-access-token", IDToken: "github-id-token"}, + }, + wantFailed: nil, + }, { name: "expired refresh fails reports provider in failed slice", sessionID: "session-3", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-3"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-3", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": expiredTokens, }, nil) @@ -366,7 +386,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "multiple providers both fail refresh — all reported in failed slice", sessionID: "session-3b", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-3b"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-3b", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": expiredTokens, "atlassian": expiredTokensAtlassian, @@ -385,7 +405,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "empty session returns empty map", sessionID: "session-4", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-4"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-4", gomock.Any()). Return(map[string]*storage.UpstreamTokens{}, nil) }, setupRefresher: func(_ *storagemocks.MockUpstreamTokenRefresher) {}, @@ -395,7 +415,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "storage error propagated", sessionID: "session-5", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-5"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-5", gomock.Any()). Return(nil, errors.New("redis connection lost")) }, setupRefresher: func(_ *storagemocks.MockUpstreamTokenRefresher) {}, @@ -405,7 +425,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "nil tokens entry skipped", sessionID: "session-6", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-6"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-6", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": freshTokens, "atlassian": nil, @@ -420,7 +440,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "expired with no refresh token reports provider in failed slice", sessionID: "session-7", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-7"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-7", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": { ProviderID: "github", @@ -439,7 +459,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "zero ExpiresAt treated as non-expiring", sessionID: "session-8", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-8"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-8", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": { ProviderID: "github", @@ -458,7 +478,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "missing ID token carried through as empty string", sessionID: "session-9", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-9"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-9", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": { ProviderID: "github", @@ -482,7 +502,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { name: "all access tokens present with all empty ID tokens returns every provider", sessionID: "session-10", setupStorage: func(s *storagemocks.MockUpstreamTokenStorage) { - s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-10"). + s.EXPECT().GetAllUpstreamTokens(gomock.Any(), "session-10", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": { ProviderID: "github", @@ -520,7 +540,7 @@ func TestInProcessService_GetAllUpstreamCredentials(t *testing.T) { svc := NewInProcessService(mockStorage, mockRefresher) - result, failed, err := svc.GetAllUpstreamCredentials(context.Background(), tt.sessionID) + result, failed, err := svc.GetAllUpstreamCredentials(context.Background(), tt.sessionID, nil) if tt.wantErr { require.Error(t, err) @@ -545,7 +565,7 @@ func TestInProcessService_GetAllUpstreamCredentials_NilRefresher(t *testing.T) { mockStorage := storagemocks.NewMockUpstreamTokenStorage(ctrl) mockStorage.EXPECT(). - GetAllUpstreamTokens(gomock.Any(), "session-1"). + GetAllUpstreamTokens(gomock.Any(), "session-1", gomock.Any()). Return(map[string]*storage.UpstreamTokens{ "github": { ProviderID: "github", @@ -558,7 +578,7 @@ func TestInProcessService_GetAllUpstreamCredentials_NilRefresher(t *testing.T) { svc := NewInProcessService(mockStorage, nil) - result, failed, err := svc.GetAllUpstreamCredentials(context.Background(), "session-1") + result, failed, err := svc.GetAllUpstreamCredentials(context.Background(), "session-1", nil) require.NoError(t, err) assert.Equal(t, map[string]UpstreamCredential{}, result) diff --git a/pkg/auth/upstreamtoken/types.go b/pkg/auth/upstreamtoken/types.go index df26f3fc2d..373cdae49a 100644 --- a/pkg/auth/upstreamtoken/types.go +++ b/pkg/auth/upstreamtoken/types.go @@ -7,7 +7,11 @@ package upstreamtoken //go:generate go run go.uber.org/mock/mockgen -destination=mocks/mock_token_reader.go -package=mocks github.com/stacklok/toolhive/pkg/auth/upstreamtoken TokenReader -import "context" +import ( + "context" + + "github.com/stacklok/toolhive/pkg/authserver/storage" +) // TokenSessionIDClaimKey is the JWT claim key for the token session ID. // This links JWT access tokens to stored upstream IDP tokens. @@ -62,7 +66,12 @@ type TokenReader interface { // be expired. // // Returns an empty map and nil failed slice (not error) for unknown sessions. - GetAllUpstreamCredentials(ctx context.Context, sessionID string) ( + // + // The expected binding is forwarded to storage, which excludes any row whose + // stored binding (user / OAuth client / upstream subject) does not match; + // excluded providers are simply absent from the creds map. A nil expected + // still enforces the user binding whenever the ctx carries a platform user. + GetAllUpstreamCredentials(ctx context.Context, sessionID string, expected *storage.ExpectedBinding) ( creds map[string]UpstreamCredential, failed []string, err error) } @@ -81,5 +90,12 @@ type Service interface { // The returned UpstreamCredential.IDToken may be empty or expired; callers // MUST check its `exp` claim before using it (e.g. as the subject_token of // an RFC 8693 token exchange). See UpstreamCredential for details. - GetValidTokens(ctx context.Context, sessionID, providerName string) (*UpstreamCredential, error) + // + // The expected binding is forwarded to storage; a row whose stored binding + // does not match yields ErrInvalidBinding and is never refreshed. A nil + // expected still enforces the user binding whenever the ctx carries a + // platform user. + GetValidTokens( + ctx context.Context, sessionID, providerName string, expected *storage.ExpectedBinding, + ) (*UpstreamCredential, error) } diff --git a/pkg/authserver/integration_test.go b/pkg/authserver/integration_test.go index 0c5f20e214..aba4facc9b 100644 --- a/pkg/authserver/integration_test.go +++ b/pkg/authserver/integration_test.go @@ -1559,7 +1559,7 @@ func TestIntegration_UpstreamTokenService_GetValidTokens(t *testing.T) { ) // The service should return the upstream access token stored during callback. - cred, err := svc.GetValidTokens(context.Background(), tsid, "default") + cred, err := svc.GetValidTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) require.NotNil(t, cred) assert.NotEmpty(t, cred.AccessToken, "upstream access token should be present") @@ -1595,7 +1595,7 @@ func TestIntegration_UpstreamTokenService_RefreshExpiredTokens(t *testing.T) { stor := ts.authServer.IDPTokenStorage() // Read the stored tokens, then overwrite them with an expired ExpiresAt. - original, err := stor.GetUpstreamTokens(context.Background(), tsid, "default") + original, err := stor.GetUpstreamTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) require.NotNil(t, original) originalAccessToken := original.AccessToken @@ -1622,13 +1622,13 @@ func TestIntegration_UpstreamTokenService_RefreshExpiredTokens(t *testing.T) { // The service should transparently refresh the expired tokens. svc := upstreamtoken.NewInProcessService(stor, ts.authServer.UpstreamTokenRefresher()) - cred, err := svc.GetValidTokens(context.Background(), tsid, "default") + cred, err := svc.GetValidTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) require.NotNil(t, cred) assert.NotEmpty(t, cred.AccessToken, "refreshed upstream access token should be present") // Verify storage was updated with non-expired tokens after refresh. - refreshed, err := stor.GetUpstreamTokens(context.Background(), tsid, "default") + refreshed, err := stor.GetUpstreamTokens(context.Background(), tsid, "default", nil) require.NoError(t, err, "refreshed tokens should be retrievable without ErrExpired") assert.True(t, refreshed.ExpiresAt.After(time.Now()), "refreshed tokens should have a future expiry, got %v", refreshed.ExpiresAt) @@ -1667,7 +1667,7 @@ func TestIntegration_UpstreamTokenService_NonExpiringToken(t *testing.T) { stor := ts.authServer.IDPTokenStorage() // Read the tokens stored during the OAuth callback. - original, err := stor.GetUpstreamTokens(context.Background(), tsid, "default") + original, err := stor.GetUpstreamTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) require.NotNil(t, original) @@ -1687,13 +1687,13 @@ func TestIntegration_UpstreamTokenService_NonExpiringToken(t *testing.T) { svc := upstreamtoken.NewInProcessService(stor, ts.authServer.UpstreamTokenRefresher()) - cred, err := svc.GetValidTokens(context.Background(), tsid, "default") + cred, err := svc.GetValidTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) require.NotNil(t, cred) assert.NotEmpty(t, cred.AccessToken) // Confirm the token in storage still has a zero ExpiresAt — no refresh occurred. - refreshed, err := stor.GetUpstreamTokens(context.Background(), tsid, "default") + refreshed, err := stor.GetUpstreamTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) assert.True(t, refreshed.ExpiresAt.IsZero(), "non-expiring token must not gain an ExpiresAt after GetValidTokens") @@ -1714,7 +1714,7 @@ func TestIntegration_UpstreamTokenService_SessionNotFound(t *testing.T) { ts.authServer.UpstreamTokenRefresher(), ) - cred, err := svc.GetValidTokens(context.Background(), "non-existent-session-id", "default") + cred, err := svc.GetValidTokens(context.Background(), "non-existent-session-id", "default", nil) require.Error(t, err) assert.ErrorIs(t, err, upstreamtoken.ErrSessionNotFound) assert.Nil(t, cred) @@ -1745,7 +1745,7 @@ func TestIntegration_UpstreamTokenService_NoRefreshToken(t *testing.T) { svc := upstreamtoken.NewInProcessService(stor, ts.authServer.UpstreamTokenRefresher()) - cred, err := svc.GetValidTokens(context.Background(), sessionID, "test") + cred, err := svc.GetValidTokens(context.Background(), sessionID, "test", nil) require.Error(t, err) assert.ErrorIs(t, err, upstreamtoken.ErrNoRefreshToken) assert.Nil(t, cred) @@ -1816,7 +1816,7 @@ func TestIntegration_UpstreamTokenStorage(t *testing.T) { t.Run("tokens_retrievable_by_provider_name", func(t *testing.T) { t.Parallel() - tokens, err := ts.storage.GetUpstreamTokens(ctx, tsid, "default") + tokens, err := ts.storage.GetUpstreamTokens(ctx, tsid, "default", nil) require.NoError(t, err, "GetUpstreamTokens should not return error") require.NotNil(t, tokens, "tokens should not be nil") assert.NotEmpty(t, tokens.AccessToken, "upstream access token should not be empty") @@ -1824,14 +1824,14 @@ func TestIntegration_UpstreamTokenStorage(t *testing.T) { t.Run("provider_id_is_logical_name", func(t *testing.T) { t.Parallel() - tokens, err := ts.storage.GetUpstreamTokens(ctx, tsid, "default") + tokens, err := ts.storage.GetUpstreamTokens(ctx, tsid, "default", nil) require.NoError(t, err) assert.Equal(t, "default", tokens.ProviderID, "ProviderID should be the logical name 'default', not 'oidc' or 'oauth2'") }) t.Run("binding_fields_populated", func(t *testing.T) { t.Parallel() - tokens, err := ts.storage.GetUpstreamTokens(ctx, tsid, "default") + tokens, err := ts.storage.GetUpstreamTokens(ctx, tsid, "default", nil) require.NoError(t, err) assert.NotEmpty(t, tokens.UserID, "UserID should not be empty") assert.NotEmpty(t, tokens.UpstreamSubject, "UpstreamSubject should not be empty") @@ -1894,7 +1894,7 @@ func TestIntegration_RefreshPreservesUpstreamTokenBinding(t *testing.T) { ctx := context.Background() // Verify upstream tokens exist before refresh - tokens, err := ts.storage.GetUpstreamTokens(ctx, originalTSID, "default") + tokens, err := ts.storage.GetUpstreamTokens(ctx, originalTSID, "default", nil) require.NoError(t, err, "upstream tokens should exist before refresh") require.NotNil(t, tokens, "upstream tokens should not be nil before refresh") @@ -1924,7 +1924,7 @@ func TestIntegration_RefreshPreservesUpstreamTokenBinding(t *testing.T) { assert.Equal(t, originalTSID, newTSID, "tsid should be preserved across token refresh") // Verify upstream tokens are still retrievable at (tsid, "default") - tokensAfterRefresh, err := ts.storage.GetUpstreamTokens(ctx, newTSID, "default") + tokensAfterRefresh, err := ts.storage.GetUpstreamTokens(ctx, newTSID, "default", nil) require.NoError(t, err, "upstream tokens should still be retrievable after refresh") require.NotNil(t, tokensAfterRefresh, "upstream tokens should not be nil after refresh") assert.Equal(t, "default", tokensAfterRefresh.ProviderID, "ProviderID should still be 'default' after refresh") @@ -2151,7 +2151,7 @@ func TestIntegration_MultiUpstreamSequentialChain(t *testing.T) { ctx := context.Background() // Provider-1 tokens should be stored - tokens1, err := ts.storage.GetUpstreamTokens(ctx, tsid, "provider-1") + tokens1, err := ts.storage.GetUpstreamTokens(ctx, tsid, "provider-1", nil) require.NoError(t, err, "provider-1 tokens should be retrievable") require.NotNil(t, tokens1, "provider-1 tokens should not be nil") assert.NotEmpty(t, tokens1.AccessToken, "provider-1 access token should not be empty") @@ -2160,7 +2160,7 @@ func TestIntegration_MultiUpstreamSequentialChain(t *testing.T) { assert.Equal(t, sub, tokens1.UserID, "provider-1 UserID should match JWT sub claim") // Provider-2 tokens should be stored - tokens2, err := ts.storage.GetUpstreamTokens(ctx, tsid, "provider-2") + tokens2, err := ts.storage.GetUpstreamTokens(ctx, tsid, "provider-2", nil) require.NoError(t, err, "provider-2 tokens should be retrievable") require.NotNil(t, tokens2, "provider-2 tokens should not be nil") assert.NotEmpty(t, tokens2.AccessToken, "provider-2 access token should not be empty") @@ -2260,11 +2260,11 @@ func TestIntegration_MultiUpstreamChain_ConfigUpstreamFilter(t *testing.T) { ctx := context.Background() - tokens1, err := ts.storage.GetUpstreamTokens(ctx, tsid, "provider-1") + tokens1, err := ts.storage.GetUpstreamTokens(ctx, tsid, "provider-1", nil) require.NoError(t, err, "provider-1 tokens should be stored") require.NotNil(t, tokens1) - _, err = ts.storage.GetUpstreamTokens(ctx, tsid, "provider-2") + _, err = ts.storage.GetUpstreamTokens(ctx, tsid, "provider-2", nil) require.Error(t, err, "provider-2 should have been dropped from the chain by the filter") assert.ErrorIs(t, err, storage.ErrNotFound) } @@ -2409,7 +2409,7 @@ func TestIntegration_FullFlow_NonExpiringUpstreamToken(t *testing.T) { // The upstream tokens written during /callback must carry a zero ExpiresAt: // the upstream response had no expires_in, so convertOAuth2Token must // preserve the zero value all the way into storage. - original, err := stor.GetUpstreamTokens(context.Background(), tsid, "default") + original, err := stor.GetUpstreamTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) require.NotNil(t, original) require.NotEmpty(t, original.AccessToken) @@ -2421,7 +2421,7 @@ func TestIntegration_FullFlow_NonExpiringUpstreamToken(t *testing.T) { // token unchanged. No refresh user is queued — a refresh attempt would // cause mockoidc to return an error and fail the assertion below. svc := upstreamtoken.NewInProcessService(stor, ts.authServer.UpstreamTokenRefresher()) - cred, err := svc.GetValidTokens(context.Background(), tsid, "default") + cred, err := svc.GetValidTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) require.NotNil(t, cred) assert.Equal(t, original.AccessToken, cred.AccessToken, @@ -2429,7 +2429,7 @@ func TestIntegration_FullFlow_NonExpiringUpstreamToken(t *testing.T) { // Re-read storage: ExpiresAt must still be zero, confirming no refresh // side effect rewrote the row. - after, err := stor.GetUpstreamTokens(context.Background(), tsid, "default") + after, err := stor.GetUpstreamTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) assert.True(t, after.ExpiresAt.IsZero(), "non-expiring token must keep zero ExpiresAt after GetValidTokens (got %v)", @@ -2603,10 +2603,10 @@ func TestIntegration_MultiUpstreamChain_MixedExpiryOrderings(t *testing.T) { // Both providers' tokens must be in storage. The non-expiring // one must have a zero ExpiresAt; the expiring one must have a // future ExpiresAt. This pins the per-leg storage write path. - tokens1, err := stor.GetUpstreamTokens(ctx, tsid, "provider-1") + tokens1, err := stor.GetUpstreamTokens(ctx, tsid, "provider-1", nil) require.NoError(t, err, "provider-1 tokens must be retrievable") require.NotNil(t, tokens1) - tokens2, err := stor.GetUpstreamTokens(ctx, tsid, "provider-2") + tokens2, err := stor.GetUpstreamTokens(ctx, tsid, "provider-2", nil) require.NoError(t, err, "provider-2 tokens must be retrievable") require.NotNil(t, tokens2) @@ -2628,7 +2628,7 @@ func TestIntegration_MultiUpstreamChain_MixedExpiryOrderings(t *testing.T) { // would have returned an empty or incomplete map for one // ordering. svc := upstreamtoken.NewInProcessService(stor, ts.authServer.UpstreamTokenRefresher()) - all, _, err := svc.GetAllUpstreamCredentials(ctx, tsid) + all, _, err := svc.GetAllUpstreamCredentials(ctx, tsid, nil) require.NoError(t, err) require.Len(t, all, 2, "GetAllUpstreamCredentials must return both providers regardless of expiry ordering") assert.NotEmpty(t, all["provider-1"].AccessToken, "provider-1 access token must be present") @@ -2703,7 +2703,7 @@ func TestIntegration_FullFlow_NonExpiringUpstreamToken_Redis(t *testing.T) { // Same invariants as the memory-backed twin: zero ExpiresAt in storage, // no refresh on read, no rewrite of the stored row. - original, err := stor.GetUpstreamTokens(context.Background(), tsid, "default") + original, err := stor.GetUpstreamTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) require.NotNil(t, original) require.NotEmpty(t, original.AccessToken) @@ -2712,13 +2712,13 @@ func TestIntegration_FullFlow_NonExpiringUpstreamToken_Redis(t *testing.T) { original.ExpiresAt) svc := upstreamtoken.NewInProcessService(stor, ts.authServer.UpstreamTokenRefresher()) - cred, err := svc.GetValidTokens(context.Background(), tsid, "default") + cred, err := svc.GetValidTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) require.NotNil(t, cred) assert.Equal(t, original.AccessToken, cred.AccessToken, "non-expiring access token must be returned unchanged (no refresh)") - after, err := stor.GetUpstreamTokens(context.Background(), tsid, "default") + after, err := stor.GetUpstreamTokens(context.Background(), tsid, "default", nil) require.NoError(t, err) assert.True(t, after.ExpiresAt.IsZero(), "non-expiring token must keep zero ExpiresAt after GetValidTokens (got %v)", @@ -2798,10 +2798,10 @@ func TestIntegration_MultiUpstreamChain_MixedExpiryOrderings_Redis(t *testing.T) // the same per-leg storage invariant the memory-backed twin // asserts; replicated here so a Redis-only divergence in the // chain handler's storage write would surface. - tokens1, err := stor.GetUpstreamTokens(ctx, tsid, "provider-1") + tokens1, err := stor.GetUpstreamTokens(ctx, tsid, "provider-1", nil) require.NoError(t, err, "provider-1 tokens must be retrievable") require.NotNil(t, tokens1) - tokens2, err := stor.GetUpstreamTokens(ctx, tsid, "provider-2") + tokens2, err := stor.GetUpstreamTokens(ctx, tsid, "provider-2", nil) require.NoError(t, err, "provider-2 tokens must be retrievable") require.NotNil(t, tokens2) @@ -2831,7 +2831,7 @@ func TestIntegration_MultiUpstreamChain_MixedExpiryOrderings_Redis(t *testing.T) // (commit 1b3bc81e2), the integration flow always produces ttlMs > 0 and // the buggy Lua branch is unreachable from a real auth chain. The Lua // invariant is exercised at unit level by pkg/authserver/storage/redis_test.go. - tokensMap, _, err := svc.GetAllUpstreamCredentials(ctx, tsid) + tokensMap, _, err := svc.GetAllUpstreamCredentials(ctx, tsid, nil) require.NoError(t, err) require.Len(t, tokensMap, 2, "GetAllUpstreamCredentials must return both providers after chain") assert.NotEmpty(t, tokensMap["provider-1"].AccessToken, "provider-1 access token must be present") @@ -3017,7 +3017,7 @@ func TestIntegration_Callback_PreservesRefreshTokenOnReauth(t *testing.T) { sessionID1 := extractTSID(t, accessToken1, ts.PrivateKey.Public()) // Verify the first leg stored a non-empty RT (mockoidc always issues one). - tokens1, err := ts.storage.GetUpstreamTokens(ctx, sessionID1, providerName) + tokens1, err := ts.storage.GetUpstreamTokens(ctx, sessionID1, providerName, nil) require.NoError(t, err, "leg 1: upstream tokens should be stored") require.NotEmpty(t, tokens1.RefreshToken, "leg 1: RT must be non-empty (sanity check)") @@ -3062,7 +3062,7 @@ func TestIntegration_Callback_PreservesRefreshTokenOnReauth(t *testing.T) { // Canonical regression assertion: the new row's RT was carried forward from // the prior session even though the IdP omitted it in the token response. - tokens2, err := ts.storage.GetUpstreamTokens(ctx, sessionID2, providerName) + tokens2, err := ts.storage.GetUpstreamTokens(ctx, sessionID2, providerName, nil) require.NoError(t, err, "leg 2: upstream tokens should be stored") assert.Equal(t, priorRT, tokens2.RefreshToken, "leg 2: RT must be carried forward from the prior session (regression assertion)") @@ -3110,8 +3110,175 @@ func TestIntegration_Callback_PreservesRefreshTokenOnReauth(t *testing.T) { // empty when we enter maybeCarryForwardRefreshToken. The new user has no prior // row either, so GetLatestUpstreamTokensForUser returns ErrNotFound and the // guard returns early — the stored RT stays empty (ErrNotFound branch). - tokens3, err := ts.storage.GetUpstreamTokens(ctx, sessionID3, providerName) + tokens3, err := ts.storage.GetUpstreamTokens(ctx, sessionID3, providerName, nil) require.NoError(t, err, "leg 3: upstream tokens should be stored") assert.Empty(t, tokens3.RefreshToken, "leg 3: no RT carry-forward for a new user with no prior row (ErrNotFound path)") } + +// TestIntegration_UpstreamBinding_CrossUserReadRejected runs the full OAuth +// consent flow as user A, then attempts to read the stored upstream tokens +// with user B's context. Read-side binding validation must refuse the read +// (single read) and exclude the row (bulk read) on both storage backends. +func TestIntegration_UpstreamBinding_CrossUserReadRejected(t *testing.T) { + t.Parallel() + + for _, backend := range []struct { + name string + opts []testServerOption + }{ + {name: "memory"}, + {name: "redis", opts: []testServerOption{withRedisBackedStorage()}}, + } { + t.Run(backend.name, func(t *testing.T) { + t.Parallel() + + m := startMockOIDC(t) + ts := setupTestServerWithMockOIDC(t, m, backend.opts...) + + verifier := servercrypto.GeneratePKCEVerifier() + challenge := servercrypto.ComputePKCEChallenge(verifier) + + // Consent as user A (the mock IDP's default user). + authCode, _ := completeAuthorizationFlow(t, ts.Server.URL, authorizationParams{ + ClientID: testClientID, + RedirectURI: testRedirectURI, + State: "binding-cross-user-" + backend.name, + Challenge: challenge, + Scope: "openid profile", + ResponseType: "code", + }) + tokenData := exchangeCodeForTokens(t, ts.Server.URL, authCode, verifier, testAudience) + accessToken, ok := tokenData["access_token"].(string) + require.True(t, ok) + tsid := extractTSID(t, accessToken, ts.PrivateKey.Public()) + + stor := ts.authServer.IDPTokenStorage() + + // Sanity: user A's own binding ctx reads the row back. + userA, err := stor.GetUpstreamTokens(context.Background(), tsid, "default", nil) + require.NoError(t, err) + require.NotNil(t, userA) + require.NotEmpty(t, userA.UserID, "consent flow must persist the row's owning user") + + ctxA := storage.ContextWithBindingUser(context.Background(), userA.UserID) + got, err := stor.GetUpstreamTokens(ctxA, tsid, "default", nil) + require.NoError(t, err) + require.NotNil(t, got) + + // User B's ctx: single read is refused, nil tokens released. + ctxB := storage.ContextWithBindingUser(context.Background(), "attacker-user") + got, err = stor.GetUpstreamTokens(ctxB, tsid, "default", nil) + require.Error(t, err) + assert.ErrorIs(t, err, storage.ErrInvalidBinding) + assert.Nil(t, got, "cross-user read must not release any token material") + + // User B's ctx: bulk read excludes the row (empty map, nil error). + all, err := stor.GetAllUpstreamTokens(ctxB, tsid, nil) + require.NoError(t, err) + assert.NotContains(t, all, "default", "cross-user row must be excluded from bulk reads") + + // Explicit mismatched expected binding is refused as well. + _, err = stor.GetUpstreamTokens(context.Background(), tsid, "default", + &storage.ExpectedBinding{UserID: "attacker-user"}) + assert.ErrorIs(t, err, storage.ErrInvalidBinding) + }) + } +} + +// TestIntegration_UpstreamBinding_ClientClaimMinted pins the JWT contract the +// middleware's client binding relies on (pkg/auth/token.go loadUpstreamTokens): +// the ToolHive auth server must mint "client_id" as a top-level claim, equal +// to the consenting OAuth client, so storage-side client binding has a real +// claim to assert. A regression that drops the claim (e.g. fosite stops +// copying Extra into the claim map, or the session stops setting it) fails +// this test. +func TestIntegration_UpstreamBinding_ClientClaimMinted(t *testing.T) { + t.Parallel() + + for _, backend := range []struct { + name string + opts []testServerOption + }{ + {name: "memory"}, + {name: "redis", opts: []testServerOption{withRedisBackedStorage()}}, + } { + t.Run(backend.name, func(t *testing.T) { + t.Parallel() + + m := startMockOIDC(t) + ts := setupTestServerWithMockOIDC(t, m, backend.opts...) + + verifier := servercrypto.GeneratePKCEVerifier() + challenge := servercrypto.ComputePKCEChallenge(verifier) + + authCode, _ := completeAuthorizationFlow(t, ts.Server.URL, authorizationParams{ + ClientID: testClientID, + RedirectURI: testRedirectURI, + State: "binding-client-claim-" + backend.name, + Challenge: challenge, + Scope: "openid profile", + ResponseType: "code", + }) + tokenData := exchangeCodeForTokens(t, ts.Server.URL, authCode, verifier, testAudience) + accessToken, ok := tokenData["access_token"].(string) + require.True(t, ok) + + parsedToken, err := jwt.ParseSigned(accessToken, []jose.SignatureAlgorithm{jose.RS256}) + require.NoError(t, err) + var claims map[string]interface{} + require.NoError(t, parsedToken.Claims(ts.PrivateKey.Public(), &claims)) + + clientID, ok := claims["client_id"].(string) + require.True(t, ok, "the minted access token must carry a top-level client_id claim") + assert.Equal(t, testClientID, clientID) + + // And the stored row is bound to that same client, so the middleware's + // ExpectedBinding{ClientID: claims["client_id"]} assertion is a real check. + tsid, ok := claims["tsid"].(string) + require.True(t, ok) + row, err := ts.storage.GetUpstreamTokens(context.Background(), tsid, "default", nil) + require.NoError(t, err) + assert.Equal(t, testClientID, row.ClientID) + }) + } +} + +// TestIntegration_UpstreamBinding_ReauthChainUnaffected walks a two-upstream +// chain end to end: the callback chain-consistency reads run with the legit +// re-authenticating user's binding, so no leg is excluded and the flow +// completes with an authorization code. +func TestIntegration_UpstreamBinding_ReauthChainUnaffected(t *testing.T) { + t.Parallel() + + m1 := startMockOIDC(t) + m2 := startMockOIDC(t) + ts := setupTestServerWithTwoUpstreams(t, m1, m2) + + verifier := servercrypto.GeneratePKCEVerifier() + challenge := servercrypto.ComputePKCEChallenge(verifier) + + // runChainFlow drives both upstream legs; each leg's chain-consistency read + // runs with the legit re-authenticating user's binding, so no leg is + // excluded and the flow completes with an authorization code. + authCode := runChainFlow(t, ts.Server.URL, challenge, "binding-chain-walk") + require.NotEmpty(t, authCode, "chain with matching binding must complete and issue a code") + + tokenData := exchangeCodeForTokens(t, ts.Server.URL, authCode, verifier, testAudience) + accessToken, ok := tokenData["access_token"].(string) + require.True(t, ok) + tsid := extractTSID(t, accessToken, ts.PrivateKey.Public()) + + // Both legs' rows are stored and readable with the flow's own user binding. + parsedToken, err := jwt.ParseSigned(accessToken, []jose.SignatureAlgorithm{jose.RS256}) + require.NoError(t, err) + var claims map[string]interface{} + require.NoError(t, parsedToken.Claims(ts.PrivateKey.Public(), &claims)) + sub, ok := claims["sub"].(string) + require.True(t, ok) + + ctx := storage.ContextWithBindingUser(context.Background(), sub) + all, err := ts.storage.GetAllUpstreamTokens(ctx, tsid, nil) + require.NoError(t, err) + assert.Len(t, all, 2, "both upstream legs must remain readable for the chain user") +} diff --git a/pkg/authserver/runner/embeddedauthserver.go b/pkg/authserver/runner/embeddedauthserver.go index 4431ad0796..763ec81824 100644 --- a/pkg/authserver/runner/embeddedauthserver.go +++ b/pkg/authserver/runner/embeddedauthserver.go @@ -7,6 +7,7 @@ package runner import ( "bytes" "context" + "encoding/base64" "fmt" "log/slog" "net/http" @@ -22,6 +23,7 @@ import ( "github.com/stacklok/toolhive/pkg/authserver/server/handlers" "github.com/stacklok/toolhive/pkg/authserver/server/keys" "github.com/stacklok/toolhive/pkg/authserver/storage" + "github.com/stacklok/toolhive/pkg/authserver/tokenenc" "github.com/stacklok/toolhive/pkg/authserver/upstream" "github.com/stacklok/toolhive/pkg/bodylimit" ) @@ -711,11 +713,53 @@ func createStorage(ctx context.Context, cfg *storage.RunConfig) (storage.Storage if err != nil { return nil, fmt.Errorf("invalid Redis config: %w", err) } - return storage.NewRedisStorage(ctx, redisCfg, cfg.RedisConfig.KeyPrefix) + kr, err := resolveTokenEncryptionKeyring(cfg.RedisConfig.TokenEncryption) + if err != nil { + return nil, fmt.Errorf("invalid Redis token encryption config: %w", err) + } + var opts []storage.RedisStorageOption + if kr != nil { + opts = append(opts, storage.WithTokenEncryption(kr)) + } + return storage.NewRedisStorage(ctx, redisCfg, cfg.RedisConfig.KeyPrefix, opts...) } return nil, fmt.Errorf("unsupported storage type: %s", cfg.Type) } +// resolveTokenEncryptionKeyring builds the upstream-token encryption keyring +// from the serializable config. A nil config returns a nil keyring (encryption +// disabled). Otherwise every referenced env var is resolved and decoded, and +// the keyring is validated — any failure is fatal to startup so a +// misconfigured deployment can never silently degrade to plaintext. +// Key material is never logged. +func resolveTokenEncryptionKeyring(rc *storage.TokenEncryptionRunConfig) (tokenenc.Keyring, error) { + if rc == nil { + return nil, nil + } + + kekByID := make(map[string][]byte, len(rc.Keys)) + for id, envVar := range rc.Keys { + if envVar == "" { + return nil, fmt.Errorf("token encryption: key %q has no environment variable name", id) + } + value := os.Getenv(envVar) + if value == "" { + return nil, fmt.Errorf("token encryption: environment variable %q for key %q is not set", envVar, id) + } + key, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return nil, fmt.Errorf("token encryption: key %q is not valid base64: %w", id, err) + } + kekByID[id] = key + } + + kr, err := tokenenc.NewStaticKeyring(rc.ActiveKeyID, kekByID) + if err != nil { + return nil, err + } + return kr, nil +} + // convertRedisRunConfig converts a serializable RedisRunConfig to a runtime // tcredis.Config. It resolves ACL credentials from environment variables and // parses duration strings. Connection-mode topology and defaulting are handled diff --git a/pkg/authserver/runner/embeddedauthserver_test.go b/pkg/authserver/runner/embeddedauthserver_test.go index f4aa34064e..030f6b4503 100644 --- a/pkg/authserver/runner/embeddedauthserver_test.go +++ b/pkg/authserver/runner/embeddedauthserver_test.go @@ -10,6 +10,7 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/x509" + "encoding/base64" "encoding/json" "encoding/pem" "fmt" @@ -23,6 +24,7 @@ import ( "testing" "time" + "github.com/alicebob/miniredis/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,6 +32,7 @@ import ( servercrypto "github.com/stacklok/toolhive/pkg/authserver/server/crypto" "github.com/stacklok/toolhive/pkg/authserver/server/keys" "github.com/stacklok/toolhive/pkg/authserver/storage" + "github.com/stacklok/toolhive/pkg/authserver/tokenenc" "github.com/stacklok/toolhive/pkg/oauthproto" ) @@ -1229,6 +1232,225 @@ func TestCreateStorage(t *testing.T) { } +// encTestKeyB64 returns a base64-encoded deterministic 32-byte KEK for tests. +func encTestKeyB64(seed byte) string { + key := make([]byte, 32) + for i := range key { + key[i] = seed + byte(i) + } + return base64.StdEncoding.EncodeToString(key) +} + +// TestResolveTokenEncryptionKeyring covers matrix items 19 and 20: keyring +// construction from RunConfig via env-var indirection, and startup-fatal +// validation — misconfigured encryption must never silently degrade to +// plaintext. These subtests use t.Setenv which is incompatible with +// t.Parallel. +// +//nolint:paralleltest // t.Setenv requires sequential execution +func TestResolveTokenEncryptionKeyring(t *testing.T) { + // Matrix item 19: valid config + env vars set → keyring constructed. + t.Run("valid config resolves keyring", func(t *testing.T) { + t.Setenv("TEST_TE_KEY_K1", encTestKeyB64(1)) + t.Setenv("TEST_TE_KEY_K2", encTestKeyB64(2)) + + kr, err := resolveTokenEncryptionKeyring(&storage.TokenEncryptionRunConfig{ + ActiveKeyID: "k2", + Keys: map[string]string{ + "k1": "TEST_TE_KEY_K1", + "k2": "TEST_TE_KEY_K2", + }, + }) + require.NoError(t, err) + require.NotNil(t, kr) + + // Round-trip through the keyring proves the keys resolved correctly: + // new writes seal under the active ID k2. + sealed, err := tokenenc.Seal(kr, "probe-key", []byte("probe")) + require.NoError(t, err) + assert.Contains(t, string(sealed), `"kid":"k2"`) + opened, legacy, err := tokenenc.Open(kr, "probe-key", sealed) + require.NoError(t, err) + assert.False(t, legacy) + assert.Equal(t, []byte("probe"), opened) + }) + + t.Run("nil config disables encryption", func(t *testing.T) { + kr, err := resolveTokenEncryptionKeyring(nil) + require.NoError(t, err) + assert.Nil(t, kr) + }) + + // Matrix item 20: every misconfiguration is a startup error. + tests := []struct { + name string + cfg *storage.TokenEncryptionRunConfig + setup func(t *testing.T) + wantErr string + }{ + { + name: "env var missing", + cfg: &storage.TokenEncryptionRunConfig{ + ActiveKeyID: "k1", + Keys: map[string]string{"k1": "TEST_TE_MISSING_VAR_12345"}, + }, + wantErr: `environment variable "TEST_TE_MISSING_VAR_12345" for key "k1" is not set`, + }, + { + name: "bad base64", + cfg: &storage.TokenEncryptionRunConfig{ + ActiveKeyID: "k1", + Keys: map[string]string{"k1": "TEST_TE_BAD_B64"}, + }, + setup: func(t *testing.T) { + t.Helper() + t.Setenv("TEST_TE_BAD_B64", "not!base64!") + }, + wantErr: `key "k1" is not valid base64`, + }, + { + name: "wrong key length", + cfg: &storage.TokenEncryptionRunConfig{ + ActiveKeyID: "k1", + Keys: map[string]string{"k1": "TEST_TE_SHORT_KEY"}, + }, + setup: func(t *testing.T) { + t.Helper() + t.Setenv("TEST_TE_SHORT_KEY", base64.StdEncoding.EncodeToString(make([]byte, 16))) + }, + wantErr: `key "k1" must be 32 bytes, got 16`, + }, + { + name: "active key ID absent from map", + cfg: &storage.TokenEncryptionRunConfig{ + ActiveKeyID: "k2", + Keys: map[string]string{"k1": "TEST_TE_ONLY_K1"}, + }, + setup: func(t *testing.T) { + t.Helper() + t.Setenv("TEST_TE_ONLY_K1", encTestKeyB64(1)) + }, + wantErr: `active key ID "k2" not present`, + }, + { + name: "empty keys map", + cfg: &storage.TokenEncryptionRunConfig{ + ActiveKeyID: "k1", + Keys: map[string]string{}, + }, + wantErr: "at least one key is required", + }, + { + name: "empty env var name", + cfg: &storage.TokenEncryptionRunConfig{ + ActiveKeyID: "k1", + Keys: map[string]string{"k1": ""}, + }, + wantErr: `key "k1" has no environment variable name`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup(t) + } + kr, err := resolveTokenEncryptionKeyring(tt.cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Nil(t, kr, "no keyring on validation failure — never silent plaintext") + }) + } +} + +// TestCreateStorage_TokenEncryptionValidation covers the createStorage wiring: +// token-encryption config errors surface before any Redis connection attempt +// (fail fast at startup). +func TestCreateStorage_TokenEncryptionValidation(t *testing.T) { + t.Setenv("TEST_TE_CS_USER", "user") + t.Setenv("TEST_TE_CS_PASS", "pass") + + _, err := createStorage(context.Background(), &storage.RunConfig{ + Type: string(storage.TypeRedis), + RedisConfig: &storage.RedisRunConfig{ + KeyPrefix: "test:", + Addr: "localhost:6399", // unreachable; validation must fire first + ACLUserConfig: &storage.ACLUserRunConfig{ + UsernameEnvVar: "TEST_TE_CS_USER", + PasswordEnvVar: "TEST_TE_CS_PASS", + }, + TokenEncryption: &storage.TokenEncryptionRunConfig{ + ActiveKeyID: "k1", + Keys: map[string]string{"k1": "TEST_TE_CS_MISSING_VAR_12345"}, + }, + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid Redis token encryption config") + assert.Contains(t, err.Error(), "not set") +} + +// TestCreateStorage_TokenEncryptionEndToEnd closes the option hand-off gap: +// unit tests of resolveTokenEncryptionKeyring and of WithTokenEncryption leave +// room for createStorage to silently drop the keyring while every test stays +// green. This drives the full createStorage path — valid TokenEncryption +// RunConfig + env vars → miniredis-backed storage — and asserts the produced +// backend actually seals upstream-token rows (envelope on the wire, no token +// plaintext) and reads them back. +// +//nolint:paralleltest // t.Setenv requires sequential execution +func TestCreateStorage_TokenEncryptionEndToEnd(t *testing.T) { + t.Setenv("TEST_TE_E2E_KEK", encTestKeyB64(1)) + t.Setenv("TEST_TE_E2E_REDIS_USER", "testuser") + t.Setenv("TEST_TE_E2E_REDIS_PASS", "testpass") + + mr := miniredis.RunT(t) + // convertRedisRunConfig requires ACL credentials via env indirection. + mr.RequireUserAuth("testuser", "testpass") + + stor, err := createStorage(context.Background(), &storage.RunConfig{ + Type: string(storage.TypeRedis), + RedisConfig: &storage.RedisRunConfig{ + Addr: mr.Addr(), + KeyPrefix: "test:auth:", + ACLUserConfig: &storage.ACLUserRunConfig{ + UsernameEnvVar: "TEST_TE_E2E_REDIS_USER", + PasswordEnvVar: "TEST_TE_E2E_REDIS_PASS", + }, + TokenEncryption: &storage.TokenEncryptionRunConfig{ + ActiveKeyID: "k1", + Keys: map[string]string{"k1": "TEST_TE_E2E_KEK"}, + }, + }, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = stor.Close() }) + + ctx := context.Background() + tokens := &storage.UpstreamTokens{ + ProviderID: "github", + AccessToken: "e2e-access-token-SECRET", + RefreshToken: "e2e-refresh-token-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-e2e", + } + require.NoError(t, stor.StoreUpstreamTokens(ctx, "e2e-session", "github", tokens)) + + // The raw Redis value must be an envelope: no token substring anywhere. + raw, err := mr.Get("test:auth:upstream:e2e-session:github") + require.NoError(t, err) + assert.Contains(t, raw, `"kid":"k1"`, "createStorage must hand the keyring to the storage backend") + assert.Contains(t, raw, `"edek"`) + assert.NotContains(t, raw, tokens.AccessToken) + assert.NotContains(t, raw, tokens.RefreshToken) + + // And the round-trip through the produced storage decrypts correctly. + got, err := stor.GetUpstreamTokens(ctx, "e2e-session", "github", nil) + require.NoError(t, err) + assert.Equal(t, tokens.AccessToken, got.AccessToken) + assert.Equal(t, tokens.RefreshToken, got.RefreshToken) +} + // TestConvertRedisRunConfig covers the runner-owned conversion steps: nil // guard and ACL credential resolution. Connection-mode topology validation is // owned by the shared toolhive-core redis package and exercised in its tests. diff --git a/pkg/authserver/server/handlers/callback.go b/pkg/authserver/server/handlers/callback.go index 7f7114c33b..90e03d0d5d 100644 --- a/pkg/authserver/server/handlers/callback.go +++ b/pkg/authserver/server/handlers/callback.go @@ -148,6 +148,10 @@ func (h *Handler) CallbackHandler(w http.ResponseWriter, req *http.Request) { // only the canonical user for storage keying. (StoreUpstreamTokens does not need // this; it keys off tokens.UserID below.) ctx = auth.WithPlatformUser(ctx, subject) + // Also carry the user in the storage binding key so read-side binding + // validation (checkUpstreamBinding) enforces it on nil-expected reads + // without the storage package importing pkg/auth. + ctx = storage.ContextWithBindingUser(ctx, subject) // Convert IDP tokens to storage tokens with binding fields. // SessionExpiresAt is set unconditionally as the Fosite session bound. Storage @@ -577,7 +581,19 @@ func (h *Handler) verifyChainIdentity(ctx context.Context, sessionID string, cha if len(chain) <= 1 { return nil } - allTokens, err := h.storage.GetAllUpstreamTokens(ctx, sessionID) + // Storage enforces the user binding per row: a first-leg row owned by a + // different user is excluded here, which the check below surfaces as the + // same identity-mismatch failure (absent row on a chain that reached this + // point means the row failed binding or was never stored). + // + // UserID is the only asserted dimension: the first leg's UpstreamSubject + // cannot be asserted here because the stored row IS where the upstream + // subject was learned (self-referential), and per the doc comment above + // later legs legitimately carry different upstream identities. The consenting + // client is the one driving this flow, so a ClientID assertion would be + // tautological; client binding is enforced on the request-serving path + // (pkg/auth/token.go loadUpstreamTokens). + allTokens, err := h.storage.GetAllUpstreamTokens(ctx, sessionID, &storage.ExpectedBinding{UserID: subject}) if err != nil { slog.Error("failed to load upstream tokens for chain identity check", "error", err) return fmt.Errorf("failed to load upstream tokens for identity check: %w", err) diff --git a/pkg/authserver/server/handlers/handler.go b/pkg/authserver/server/handlers/handler.go index 8a54204bb6..013ce5e05c 100644 --- a/pkg/authserver/server/handlers/handler.go +++ b/pkg/authserver/server/handlers/handler.go @@ -223,7 +223,23 @@ func (h *Handler) WellKnownRoutes(r chi.Router) { // the user is re-prompted up front, rather than the stale token surfacing as a // runtime auth error later at MCP-request token-swap time. func (h *Handler) nextMissingUpstream(ctx context.Context, sessionID string, chain []string) (string, error) { - stored, err := h.storage.GetAllUpstreamTokens(ctx, sessionID) + // Assert the resolved canonical user against every stored row: a row written + // for a different user is excluded by storage and therefore appears missing, + // prompting re-consent instead of treating a cross-user row as satisfied. + // + // UserID is the only dimension independently known here. UpstreamSubject is + // NOT assertable: for chain[0] the row IS the source of the upstream subject + // (self-referential — comparing it to itself proves nothing), and later legs + // are connect-this-backend flows whose upstream identity legitimately + // differs from the first leg's. ClientID is not asserted here either: this + // read runs mid-flow under the consenting client itself, so a client check + // could only pass; the client binding has teeth on the request-serving path + // (pkg/auth/token.go loadUpstreamTokens), not here. + expected := &storage.ExpectedBinding{} + if userID, ok := auth.CanonicalUserFromContext(ctx); ok { + expected.UserID = userID + } + stored, err := h.storage.GetAllUpstreamTokens(ctx, sessionID, expected) if err != nil { return "", fmt.Errorf("failed to check upstream token state: %w", err) } diff --git a/pkg/authserver/server/handlers/handler_chain_test.go b/pkg/authserver/server/handlers/handler_chain_test.go index ba8abf59ff..01e3200878 100644 --- a/pkg/authserver/server/handlers/handler_chain_test.go +++ b/pkg/authserver/server/handlers/handler_chain_test.go @@ -430,7 +430,7 @@ func TestNextMissingUpstream_StorageError(t *testing.T) { stor := mocks.NewMockStorage(ctrl) storageErr := errors.New("connection refused") - stor.EXPECT().GetAllUpstreamTokens(gomock.Any(), gomock.Any()).Return(nil, storageErr).Times(1) + stor.EXPECT().GetAllUpstreamTokens(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, storageErr).Times(1) jwtStrategy := compose.NewOAuth2JWTStrategy( func(_ context.Context) (any, error) { diff --git a/pkg/authserver/server/handlers/helpers_test.go b/pkg/authserver/server/handlers/helpers_test.go index 2ee8ede7b2..5bb5ec879e 100644 --- a/pkg/authserver/server/handlers/helpers_test.go +++ b/pkg/authserver/server/handlers/helpers_test.go @@ -362,8 +362,8 @@ func baseTestSetup(t *testing.T, opts ...baseTestSetupOption) (fosite.OAuth2Prov return nil }).AnyTimes() - stor.EXPECT().GetAllUpstreamTokens(gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, sessionID string) (map[string]*storage.UpstreamTokens, error) { + stor.EXPECT().GetAllUpstreamTokens(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, sessionID string, _ *storage.ExpectedBinding) (map[string]*storage.UpstreamTokens, error) { // GetAllUpstreamTokens takes only (ctx, sessionID) — no tokens argument to // carry the user — so a user-keyed storage decorator can resolve the user // only from ctx. Capture the ctx here so a test can assert the callback diff --git a/pkg/authserver/storage/binding.go b/pkg/authserver/storage/binding.go new file mode 100644 index 0000000000..7ec02e7bf3 --- /dev/null +++ b/pkg/authserver/storage/binding.go @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "crypto/subtle" + "errors" + + "go.uber.org/mock/gomock" +) + +// bindingCtxKey is the context key carrying the canonical platform user for +// read-side binding validation. It is unexported: the value is placed by +// ContextWithBindingUser and consumed only by checkUpstreamBinding, keeping the +// storage package free of any dependency on the platform auth package (import +// cycle) while still letting storage resolve the caller's user on +// request-serving paths. +type bindingCtxKey struct{} + +// ContextWithBindingUser returns a context carrying the canonical platform +// user that read-side binding validation must enforce when the caller passes +// a nil *ExpectedBinding (or an empty ExpectedBinding.UserID). Callers obtain +// the user from their identity layer (e.g. auth.CanonicalUserFromContext) and +// attach it with this helper; an empty userID returns the context unchanged. +func ContextWithBindingUser(ctx context.Context, userID string) context.Context { + if userID == "" { + return ctx + } + return context.WithValue(ctx, bindingCtxKey{}, userID) +} + +// checkUpstreamBinding compares a stored upstream-token row against the +// identity the caller expects it to be bound to. A nil stored row passes (the +// caller's own not-found / tombstone handling applies). A nil expected means +// "resolve the user from ctx only" (placed by ContextWithBindingUser); the +// client/subject checks are skipped in that case. +// +// A dimension is only enforced when BOTH the expected value and the stored +// value are non-empty; a stored row with an empty field (a legacy row predating +// binding fields) is released. Returns ErrInvalidBinding on mismatch. +// +// Strict mode (ExpectedBinding.Strict) fails closed on the user dimension: a +// row with an empty stored UserID is rejected outright, so a legacy row that +// cannot prove its owner is never released to a strict caller. +// +// Comparisons use subtle.ConstantTimeCompare (matching the session-binding +// convention in pkg/vmcp/session/internal/security) so a stolen session ID +// cannot be used to probe stored binding values via timing. +func checkUpstreamBinding(ctx context.Context, stored *UpstreamTokens, expected *ExpectedBinding) error { + if stored == nil { + return nil + } + + if expected != nil && expected.Strict && stored.UserID == "" { + // Fail closed: the strict caller demands an owned row; a legacy row + // with no recorded owner is indistinguishable from a stolen one. + return &bindingMismatchError{dimension: "user_id"} + } + + expectedUser := "" + if expected != nil { + expectedUser = expected.UserID + } + if expectedUser == "" { + expectedUser, _ = ctx.Value(bindingCtxKey{}).(string) + } + if err := compareBindingDimension(expectedUser, stored.UserID, "user_id"); err != nil { + return err + } + + if expected == nil { + return nil + } + if err := compareBindingDimension(expected.ClientID, stored.ClientID, "client_id"); err != nil { + return err + } + return compareBindingDimension(expected.UpstreamSubject, stored.UpstreamSubject, "upstream_subject") +} + +// compareBindingDimension enforces one binding dimension. A mismatch returns an +// error whose message carries only the dimension name — never the compared +// values — while errors.Is finds the ErrInvalidBinding sentinel. Both empty +// and one-side-empty comparisons pass per the legacy-row rule. +func compareBindingDimension(expected, stored, dimension string) error { + if expected == "" || stored == "" { + return nil + } + // ConstantTimeCompare short-circuits on length mismatch; leaking binding + // length is acceptable (user/client IDs are non-secret identifiers). + if subtle.ConstantTimeCompare([]byte(expected), []byte(stored)) != 1 { + return &bindingMismatchError{dimension: dimension} + } + return nil +} + +// bindingMismatchError carries the binding dimension that failed validation so +// logs can name it (metadata only — never token material) while error identity +// stays the ErrInvalidBinding sentinel. The gomock matcher below treats any +// ErrInvalidBinding as satisfying the sentinel, which both keeps errors.Is +// working for callers and lets gomock EXPECT(...).Return(storage.ErrInvalidBinding) +// match the wrapped error produced here. +type bindingMismatchError struct{ dimension string } + +func (e *bindingMismatchError) Error() string { + return ErrInvalidBinding.Error() + ": " + e.dimension +} + +// Is reports whether target is the ErrInvalidBinding sentinel. It also lets +// gomock's Eq matcher (which prefers gomock.Matcher implementations on the +// actual value) match the sentinel against this error. +func (*bindingMismatchError) Is(target error) bool { + return target == ErrInvalidBinding +} + +var _ gomock.Matcher = (*bindingMismatchError)(nil) + +// Matches implements gomock.Matcher: an expected ErrInvalidBinding matches any +// binding mismatch, regardless of dimension. +func (*bindingMismatchError) Matches(x any) bool { + target, ok := x.(error) + return ok && target == ErrInvalidBinding +} + +func (*bindingMismatchError) String() string { + return "is " + ErrInvalidBinding.Error() +} + +// bindingMismatchDimension extracts the failed dimension from a binding error +// for log metadata; "unknown" when the error is not a binding mismatch. +func bindingMismatchDimension(err error) string { + var mismatch *bindingMismatchError + if errors.As(err, &mismatch) { + return mismatch.dimension + } + return "unknown" +} diff --git a/pkg/authserver/storage/binding_test.go b/pkg/authserver/storage/binding_test.go new file mode 100644 index 0000000000..037ee95be9 --- /dev/null +++ b/pkg/authserver/storage/binding_test.go @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "bytes" + "context" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// captureWarnLogs swaps in a buffered slog handler at warn level for the +// duration of the test and returns the captured output. Process-global, not +// safe for t.Parallel(). +func captureWarnLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + handler := slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}) + orig := slog.Default() + slog.SetDefault(slog.New(handler)) + t.Cleanup(func() { slog.SetDefault(orig) }) + return &buf +} + +// assertBindingExclusionWarn checks that the bulk-read binding exclusion WARN +// names the session, provider, and mismatch dimension — metadata only. +func assertBindingExclusionWarn(t *testing.T, out, sessionID, provider, dimension string) { + t.Helper() + assert.Contains(t, out, "excluding upstream token row: binding validation failed", + "expected binding exclusion WARN; got: %s", out) + assert.Contains(t, out, sessionID, "WARN should name the session") + assert.Contains(t, out, provider, "WARN should name the provider") + assert.Contains(t, out, dimension, "WARN should name the mismatch dimension") +} + +// TestCheckUpstreamBinding covers the pure comparison logic of the binding +// helper: per-dimension empty-field rule and constant-time mismatch behavior. +func TestCheckUpstreamBinding(t *testing.T) { + t.Parallel() + + stored := &UpstreamTokens{ + ProviderID: "github", + UserID: "user-A", + ClientID: "client-A", + UpstreamSubject: "subject-A", + } + + tests := []struct { + name string + stored *UpstreamTokens + expected *ExpectedBinding + wantErr bool + }{ + {name: "nil stored row passes", stored: nil, expected: &ExpectedBinding{UserID: "user-B"}}, + {name: "nil expected with no resolver makes no assertion", stored: stored, expected: nil}, + {name: "expected UserID match", stored: stored, expected: &ExpectedBinding{UserID: "user-A"}}, + {name: "expected UserID mismatch", stored: stored, expected: &ExpectedBinding{UserID: "user-B"}, wantErr: true}, + {name: "client ID match", stored: stored, expected: &ExpectedBinding{ClientID: "client-A"}}, + {name: "client ID mismatch", stored: stored, expected: &ExpectedBinding{ClientID: "client-B"}, wantErr: true}, + {name: "upstream subject match", stored: stored, expected: &ExpectedBinding{UpstreamSubject: "subject-A"}}, + {name: "upstream subject mismatch", stored: stored, expected: &ExpectedBinding{UpstreamSubject: "subject-B"}, wantErr: true}, + { + name: "legacy row: empty stored fields pass with asserted expected", + stored: &UpstreamTokens{ProviderID: "github"}, + expected: &ExpectedBinding{UserID: "user-A", ClientID: "client-A", UpstreamSubject: "subject-A"}, + }, + {name: "empty expected fields skip the dimension", stored: stored, expected: &ExpectedBinding{}}, + { + name: "strict: legacy row (empty stored UserID) is rejected", + stored: &UpstreamTokens{ProviderID: "github"}, + expected: &ExpectedBinding{UserID: "user-A", Strict: true}, + wantErr: true, + }, + { + name: "strict: legacy row rejected even without asserted user", + stored: &UpstreamTokens{ProviderID: "github"}, + expected: &ExpectedBinding{Strict: true}, + wantErr: true, + }, + { + name: "strict: fully-owned row matching expected passes", + stored: stored, + expected: &ExpectedBinding{UserID: "user-A", ClientID: "client-A", UpstreamSubject: "subject-A", Strict: true}, + }, + { + name: "strict: owned row with mismatched user still fails", + stored: stored, + expected: &ExpectedBinding{UserID: "user-B", Strict: true}, + wantErr: true, + }, + { + name: "strict: row with UserID but empty other fields passes", + stored: &UpstreamTokens{ProviderID: "github", UserID: "user-A"}, + expected: &ExpectedBinding{UserID: "user-A", ClientID: "client-B", Strict: true}, + }, + {name: "equal-length user mismatch fails", stored: stored, expected: &ExpectedBinding{UserID: "user-B"}, wantErr: true}, + { + name: "differing-length user mismatch fails", + stored: stored, + expected: &ExpectedBinding{UserID: "user-A-longer"}, + wantErr: true, + }, + { + name: "differing-length client mismatch fails", + stored: stored, + expected: &ExpectedBinding{ClientID: "client-A-longer"}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := checkUpstreamBinding(context.Background(), tt.stored, tt.expected) + if tt.wantErr { + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + return + } + require.NoError(t, err) + }) + } +} + +// TestCheckUpstreamBinding_CtxUser covers the ctx-fallback user resolution +// (scenarios 1 and 2 of the read-side matrix at the helper level). +func TestCheckUpstreamBinding_CtxUser(t *testing.T) { + t.Parallel() + + stored := &UpstreamTokens{ProviderID: "github", UserID: "user-A"} + + t.Run("nil expected, ctx user matches: row passes", func(t *testing.T) { + t.Parallel() + ctx := ContextWithBindingUser(context.Background(), "user-A") + require.NoError(t, checkUpstreamBinding(ctx, stored, nil)) + }) + + t.Run("nil expected, ctx user mismatches: ErrInvalidBinding", func(t *testing.T) { + t.Parallel() + ctx := ContextWithBindingUser(context.Background(), "user-B") + err := checkUpstreamBinding(ctx, stored, nil) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + }) + + t.Run("explicit expected UserID wins over ctx user", func(t *testing.T) { + t.Parallel() + ctx := ContextWithBindingUser(context.Background(), "user-B") + require.NoError(t, checkUpstreamBinding(ctx, stored, &ExpectedBinding{UserID: "user-A"})) + }) + + t.Run("ContextWithBindingUser with empty user leaves ctx unchanged", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + require.Equal(t, ctx, ContextWithBindingUser(ctx, "")) + }) +} diff --git a/pkg/authserver/storage/config.go b/pkg/authserver/storage/config.go index 281be69866..9fbf9c9b5b 100644 --- a/pkg/authserver/storage/config.go +++ b/pkg/authserver/storage/config.go @@ -104,6 +104,29 @@ type RedisRunConfig struct { // SentinelTLS configures TLS for Sentinel connections. Only applies when SentinelConfig is set. SentinelTLS *RedisTLSRunConfig `json:"sentinel_tls,omitempty" yaml:"sentinel_tls,omitempty"` + + // TokenEncryption configures envelope encryption for upstream OAuth tokens + // at rest. Presence enables encryption; absence preserves the current + // plaintext behavior. + TokenEncryption *TokenEncryptionRunConfig `json:"token_encryption,omitempty" yaml:"token_encryption,omitempty"` +} + +// TokenEncryptionRunConfig configures AES-256-GCM envelope encryption of +// upstream OAuth token values stored in Redis. Key material is referenced by +// environment variable name — never inline — following the established +// credential indirection convention (see ACLUserRunConfig). Key resolution, +// validation, and keyring construction happen in the runner when the storage +// backend is created; misconfiguration is startup-fatal (never silently +// degrades to plaintext). +type TokenEncryptionRunConfig struct { + // ActiveKeyID identifies the KEK used to encrypt new writes. Other IDs in + // Keys remain decrypt-only (retired), supporting lazy read-old/write-new + // rotation. + ActiveKeyID string `json:"active_key_id" yaml:"active_key_id"` + + // Keys maps key ID → name of the environment variable holding the + // base64-encoded 32-byte key-encryption key. + Keys map[string]string `json:"keys" yaml:"keys"` } // SentinelRunConfig contains Redis Sentinel configuration. diff --git a/pkg/authserver/storage/memory.go b/pkg/authserver/storage/memory.go index 0e79a584c5..ccf29ae002 100644 --- a/pkg/authserver/storage/memory.go +++ b/pkg/authserver/storage/memory.go @@ -788,7 +788,11 @@ func cloneUpstreamTokens(t *UpstreamTokens) *UpstreamTokens { // GetUpstreamTokens retrieves the upstream IDP tokens for a session and provider. // Returns a defensive copy to prevent aliasing issues. -func (s *MemoryStorage) GetUpstreamTokens(_ context.Context, sessionID, providerName string) (*UpstreamTokens, error) { +// Binding validation (see ExpectedBinding) runs before expiry: a mismatched row +// returns (nil, ErrInvalidBinding) and never enters the refresh path. +func (s *MemoryStorage) GetUpstreamTokens( + ctx context.Context, sessionID, providerName string, expected *ExpectedBinding, +) (*UpstreamTokens, error) { if sessionID == "" { return nil, fosite.ErrInvalidRequest.WithHint("session ID cannot be empty") } @@ -811,6 +815,13 @@ func (s *MemoryStorage) GetUpstreamTokens(_ context.Context, sessionID, provider return nil, nil } + if err := checkUpstreamBinding(ctx, result, expected); err != nil { + // Unlike ErrExpired, no legitimate use exists for a mismatched row — + // release nothing. + return nil, err + } + warnIfAssertedUserOnLegacyRow(result, expected, sessionID, providerName) + // Check the token's own ExpiresAt (access token expiry), not the entry's expiresAt // (storage TTL which includes DefaultRefreshTokenTTL buffer for refresh token survival). // Return tokens along with ErrExpired so callers can use the refresh token. @@ -826,7 +837,12 @@ func (s *MemoryStorage) GetUpstreamTokens(_ context.Context, sessionID, provider // Returns a map of providerName -> tokens with defensive copies. // Returns an empty map (not error) for unknown sessions. // Includes expired tokens (no expiry filtering at bulk-read level). -func (s *MemoryStorage) GetAllUpstreamTokens(_ context.Context, sessionID string) (map[string]*UpstreamTokens, error) { +// Binding validation applies per row: a mismatched row is excluded from the +// result (with a WARN) rather than failing the whole read — callers treat a +// missing provider as "needs consent", the safe degradation. +func (s *MemoryStorage) GetAllUpstreamTokens( + ctx context.Context, sessionID string, expected *ExpectedBinding, +) (map[string]*UpstreamTokens, error) { s.mu.RLock() defer s.mu.RUnlock() @@ -836,12 +852,41 @@ func (s *MemoryStorage) GetAllUpstreamTokens(_ context.Context, sessionID string continue } // Defensive copy (cloneUpstreamTokens handles nil) - result[key.providerName] = cloneUpstreamTokens(entry.value) + tokens := cloneUpstreamTokens(entry.value) + if err := checkUpstreamBinding(ctx, tokens, expected); err != nil { + slog.Warn("excluding upstream token row: binding validation failed", + "session_id", sessionID, + "provider", key.providerName, + "dimension", bindingMismatchDimension(err), + ) + continue + } + warnIfAssertedUserOnLegacyRow(tokens, expected, sessionID, key.providerName) + result[key.providerName] = tokens } return result, nil } +// warnIfAssertedUserOnLegacyRow emits a single WARN for the suspicious case: +// the caller explicitly asserted an owning user (or Strict mode) but the row +// carries no recorded owner (a legacy row predating binding fields), so the +// user dimension could not actually be verified. Plain legacy passthrough +// (nil expected, no ctx user) logs nothing — warning there would be spam on +// hot paths. +func warnIfAssertedUserOnLegacyRow(stored *UpstreamTokens, expected *ExpectedBinding, sessionID, providerName string) { + if stored == nil || stored.UserID != "" || expected == nil { + return + } + if expected.UserID == "" && !expected.Strict { + return + } + slog.Warn("upstream token row has no recorded owner; user binding could not be verified", + "session_id", sessionID, + "provider", providerName, + ) +} + // DeleteUpstreamTokens removes all upstream IDP tokens for a session (all providers). func (s *MemoryStorage) DeleteUpstreamTokens(_ context.Context, sessionID string) error { s.mu.Lock() diff --git a/pkg/authserver/storage/memory_test.go b/pkg/authserver/storage/memory_test.go index 3497dc3aa4..197b191cf8 100644 --- a/pkg/authserver/storage/memory_test.go +++ b/pkg/authserver/storage/memory_test.go @@ -450,7 +450,7 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { } require.NoError(t, s.StoreUpstreamTokens(ctx, "session-123", "provider-a", tokens)) - retrieved, err := s.GetUpstreamTokens(ctx, "session-123", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "session-123", "provider-a", nil) require.NoError(t, err) assert.Equal(t, tokens.AccessToken, retrieved.AccessToken) assert.Equal(t, tokens.RefreshToken, retrieved.RefreshToken) @@ -461,7 +461,7 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { t.Run("get non-existent", func(t *testing.T) { withStorage(t, func(ctx context.Context, s *MemoryStorage) { - _, err := s.GetUpstreamTokens(ctx, "non-existent", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "non-existent", "provider-a", nil) requireNotFoundError(t, err) }) }) @@ -470,7 +470,7 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { withStorage(t, func(ctx context.Context, s *MemoryStorage) { require.NoError(t, s.StoreUpstreamTokens(ctx, "to-delete", "provider-a", &UpstreamTokens{AccessToken: "test"})) require.NoError(t, s.DeleteUpstreamTokens(ctx, "to-delete")) - _, err := s.GetUpstreamTokens(ctx, "to-delete", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "to-delete", "provider-a", nil) requireNotFoundError(t, err) }) }) @@ -480,7 +480,7 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.StoreUpstreamTokens(ctx, "session", "provider-a", &UpstreamTokens{AccessToken: "token-1", UserID: "user1"})) require.NoError(t, s.StoreUpstreamTokens(ctx, "session", "provider-a", &UpstreamTokens{AccessToken: "token-2", UserID: "user2"})) - retrieved, err := s.GetUpstreamTokens(ctx, "session", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "session", "provider-a", nil) require.NoError(t, err) assert.Equal(t, "token-2", retrieved.AccessToken) assert.Equal(t, "user2", retrieved.UserID) @@ -496,7 +496,7 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { })) assert.Equal(t, 1, s.Stats().UpstreamTokens) - retrieved, err := s.GetUpstreamTokens(ctx, "expired", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "expired", "provider-a", nil) require.Error(t, err) assert.ErrorIs(t, err, ErrExpired) // Tokens should be returned alongside ErrExpired for refresh purposes @@ -514,11 +514,11 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.StoreUpstreamTokens(ctx, "session-1", "provider-a", tokensA)) require.NoError(t, s.StoreUpstreamTokens(ctx, "session-1", "provider-b", tokensB)) - retrievedA, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a") + retrievedA, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a", nil) require.NoError(t, err) assert.Equal(t, "access-a", retrievedA.AccessToken) - retrievedB, err := s.GetUpstreamTokens(ctx, "session-1", "provider-b") + retrievedB, err := s.GetUpstreamTokens(ctx, "session-1", "provider-b", nil) require.NoError(t, err) assert.Equal(t, "access-b", retrievedB.AccessToken) }) @@ -530,14 +530,14 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.StoreUpstreamTokens(ctx, "session-1", "provider-a", tokensA)) // Provider B should not be affected by provider A's data - _, err := s.GetUpstreamTokens(ctx, "session-1", "provider-b") + _, err := s.GetUpstreamTokens(ctx, "session-1", "provider-b", nil) requireNotFoundError(t, err) // Store provider B and verify provider A is unchanged tokensB := &UpstreamTokens{AccessToken: "access-b", ExpiresAt: time.Now().Add(time.Hour)} require.NoError(t, s.StoreUpstreamTokens(ctx, "session-1", "provider-b", tokensB)) - retrievedA, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a") + retrievedA, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a", nil) require.NoError(t, err) assert.Equal(t, "access-a", retrievedA.AccessToken) }) @@ -551,7 +551,7 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.StoreUpstreamTokens(ctx, "session-1", "provider-a", tokensA)) require.NoError(t, s.StoreUpstreamTokens(ctx, "session-1", "provider-b", tokensB)) - all, err := s.GetAllUpstreamTokens(ctx, "session-1") + all, err := s.GetAllUpstreamTokens(ctx, "session-1", nil) require.NoError(t, err) assert.Len(t, all, 2) assert.Equal(t, "access-a", all["provider-a"].AccessToken) @@ -561,7 +561,7 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { t.Run("GetAllUpstreamTokens unknown session", func(t *testing.T) { withStorage(t, func(ctx context.Context, s *MemoryStorage) { - all, err := s.GetAllUpstreamTokens(ctx, "unknown-session") + all, err := s.GetAllUpstreamTokens(ctx, "unknown-session", nil) require.NoError(t, err) assert.Empty(t, all) }) @@ -575,7 +575,7 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { ExpiresAt: time.Now().Add(-time.Hour), })) - all, err := s.GetAllUpstreamTokens(ctx, "session-1") + all, err := s.GetAllUpstreamTokens(ctx, "session-1", nil) require.NoError(t, err) require.Len(t, all, 1) assert.Equal(t, "expired-access", all["provider-a"].AccessToken) @@ -591,9 +591,9 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.DeleteUpstreamTokens(ctx, "session-1")) - _, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a", nil) requireNotFoundError(t, err) - _, err = s.GetUpstreamTokens(ctx, "session-1", "provider-b") + _, err = s.GetUpstreamTokens(ctx, "session-1", "provider-b", nil) requireNotFoundError(t, err) assert.Equal(t, 0, s.Stats().UpstreamTokens) }) @@ -609,7 +609,7 @@ func TestMemoryStorage_UpstreamTokens(t *testing.T) { t.Run("empty providerName returns error for Get", func(t *testing.T) { withStorage(t, func(ctx context.Context, s *MemoryStorage) { - _, err := s.GetUpstreamTokens(ctx, "session-1", "") + _, err := s.GetUpstreamTokens(ctx, "session-1", "", nil) require.Error(t, err) assert.ErrorIs(t, err, fosite.ErrInvalidRequest) }) @@ -841,14 +841,14 @@ func TestMemoryStorage_GetLatestUpstreamTokensForUser(t *testing.T) { require.NoError(t, s.DeleteUpstreamTokensForProvider(ctx, "session-1", "provider-a")) - _, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a", nil) requireNotFoundError(t, err) - got, err := s.GetUpstreamTokens(ctx, "session-1", "provider-b") + got, err := s.GetUpstreamTokens(ctx, "session-1", "provider-b", nil) require.NoError(t, err) assert.Equal(t, "b", got.AccessToken) - all, err := s.GetAllUpstreamTokens(ctx, "session-1") + all, err := s.GetAllUpstreamTokens(ctx, "session-1", nil) require.NoError(t, err) assert.Len(t, all, 1) }) @@ -1020,11 +1020,11 @@ func TestMemoryStorage_CleanupExpired(t *testing.T) { }, getStats: func(st Stats) int { return st.UpstreamTokens }, verifyGone: func(ctx context.Context, s *MemoryStorage) error { - _, err := s.GetUpstreamTokens(ctx, "expired", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "expired", "provider-a", nil) return err }, verifyKeep: func(ctx context.Context, s *MemoryStorage) error { - _, err := s.GetUpstreamTokens(ctx, "valid", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "valid", "provider-a", nil) return err }, }, @@ -1082,7 +1082,7 @@ func TestMemoryStorage_CleanupExpired(t *testing.T) { assert.Equal(t, 1, s.Stats().UpstreamTokens, "only the expiring token should be removed") - tokens, err := s.GetUpstreamTokens(ctx, "never-expiring", "provider-a") + tokens, err := s.GetUpstreamTokens(ctx, "never-expiring", "provider-a", nil) require.NoError(t, err) require.NotNil(t, tokens) assert.Equal(t, "non-expiring-token", tokens.AccessToken) @@ -1102,7 +1102,7 @@ func TestMemoryStorage_CleanupExpired(t *testing.T) { s.cleanupExpired() assert.Equal(t, 0, s.Stats().UpstreamTokens) - _, getErr := s.GetUpstreamTokens(ctx, "sess-1", "github") + _, getErr := s.GetUpstreamTokens(ctx, "sess-1", "github", nil) requireNotFoundError(t, getErr) }) }) @@ -1322,7 +1322,7 @@ func TestMemoryStorage_InputValidation(t *testing.T) { t.Run("StoreUpstreamTokens nil tokens is valid", func(t *testing.T) { withStorage(t, func(ctx context.Context, s *MemoryStorage) { require.NoError(t, s.StoreUpstreamTokens(ctx, "session-id", "provider-a", nil)) - retrieved, err := s.GetUpstreamTokens(ctx, "session-id", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "session-id", "provider-a", nil) require.NoError(t, err) assert.Nil(t, retrieved) }) @@ -1620,7 +1620,7 @@ func TestMemoryStorage_DeleteUser_CascadesAssociatedData(t *testing.T) { assert.ErrorIs(t, err, ErrNotFound) // Verify the user's upstream tokens are gone - _, err = s.GetUpstreamTokens(ctx, "session-user", "google") + _, err = s.GetUpstreamTokens(ctx, "session-user", "google", nil) assert.ErrorIs(t, err, ErrNotFound) // Verify the other user's identity is still there @@ -1629,7 +1629,7 @@ func TestMemoryStorage_DeleteUser_CascadesAssociatedData(t *testing.T) { assert.Equal(t, "other-user", retrieved.UserID) // Verify the other user's upstream tokens are still there - otherRetrieved, err := s.GetUpstreamTokens(ctx, "session-other", "google") + otherRetrieved, err := s.GetUpstreamTokens(ctx, "session-other", "google", nil) require.NoError(t, err) assert.Equal(t, "other-token", otherRetrieved.AccessToken) }) @@ -2190,3 +2190,310 @@ func TestScopesHash_DistinctForDistinctScopes(t *testing.T) { assert.NotEqual(t, a, d) assert.Equal(t, d, e) } + +// seedBoundRow stores a live, fully-bound upstream token row for +// (sessionID, provider). +func seedBoundRow(ctx context.Context, t *testing.T, s *MemoryStorage, sessionID, provider string) { + t.Helper() + row := &UpstreamTokens{ + ProviderID: provider, + AccessToken: "access-" + provider, + RefreshToken: "refresh-" + provider, + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-A", + ClientID: "client-A", + UpstreamSubject: "subject-A", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, sessionID, provider, row)) +} + +// TestMemoryStorage_UpstreamBinding covers the read-side binding matrix +// (scenarios 1-12) for the in-memory backend. Log-capturing cases live in +// TestMemoryStorage_UpstreamBinding_WarnLogs because slog.SetDefault is +// process-global. +func TestMemoryStorage_UpstreamBinding(t *testing.T) { + t.Parallel() + + t.Run("scenario 1: matching ctx user, nil expected, row returned", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + seedBoundRow(ctx, t, s, "sess-1", "github") + + readCtx := ContextWithBindingUser(ctx, "user-A") + got, err := s.GetUpstreamTokens(readCtx, "sess-1", "github", nil) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "user-A", got.UserID) + }) + }) + + t.Run("scenario 2: mismatched ctx user, nil expected, ErrInvalidBinding with nil tokens", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + seedBoundRow(ctx, t, s, "sess-2", "github") + + readCtx := ContextWithBindingUser(ctx, "user-B") + got, err := s.GetUpstreamTokens(readCtx, "sess-2", "github", nil) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got, "mismatched row must not be released, even for refresh") + }) + }) + + t.Run("scenario 3: no identity in ctx, nil expected, row returned", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + seedBoundRow(ctx, t, s, "sess-3", "github") + + got, err := s.GetUpstreamTokens(ctx, "sess-3", "github", nil) + require.NoError(t, err) + require.NotNil(t, got) + }) + }) + + t.Run("scenario 4: expected UserID match and mismatch", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + seedBoundRow(ctx, t, s, "sess-4", "github") + + got, err := s.GetUpstreamTokens(ctx, "sess-4", "github", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.NotNil(t, got) + + got, err = s.GetUpstreamTokens(ctx, "sess-4", "github", &ExpectedBinding{UserID: "user-B"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got) + }) + }) + + t.Run("scenario 5: client ID binding variants", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + seedBoundRow(ctx, t, s, "sess-5", "github") + + // match + got, err := s.GetUpstreamTokens(ctx, "sess-5", "github", &ExpectedBinding{ClientID: "client-A"}) + require.NoError(t, err) + require.NotNil(t, got) + + // mismatch + got, err = s.GetUpstreamTokens(ctx, "sess-5", "github", &ExpectedBinding{ClientID: "client-B"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got) + + // expected-empty skips the dimension + got, err = s.GetUpstreamTokens(ctx, "sess-5", "github", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.NotNil(t, got) + + // stored-empty legacy row passes with an asserted client + legacy := &UpstreamTokens{ + ProviderID: "github", AccessToken: "legacy-access", + ExpiresAt: time.Now().Add(time.Hour), UserID: "user-A", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-5b", "github", legacy)) + got, err = s.GetUpstreamTokens(ctx, "sess-5b", "github", + &ExpectedBinding{UserID: "user-A", ClientID: "client-B"}) + require.NoError(t, err) + require.NotNil(t, got) + }) + }) + + t.Run("scenario 6: upstream subject mismatch", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + seedBoundRow(ctx, t, s, "sess-6", "github") + + got, err := s.GetUpstreamTokens(ctx, "sess-6", "github", &ExpectedBinding{UpstreamSubject: "subject-B"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got) + }) + }) + + t.Run("scenario 7: expired + mismatched row yields ErrInvalidBinding, not ErrExpired", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + row := &UpstreamTokens{ + ProviderID: "github", AccessToken: "stale-access", RefreshToken: "stale-refresh", + ExpiresAt: time.Now().Add(-time.Hour), UserID: "user-A", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-7", "github", row)) + + got, err := s.GetUpstreamTokens(ctx, "sess-7", "github", &ExpectedBinding{UserID: "user-B"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.NotErrorIs(t, err, ErrExpired, "binding must win over expiry") + assert.Nil(t, got, "no refresh material released for a mismatched row") + }) + }) + + t.Run("scenario 8: bulk read excludes only the mismatched provider", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + seedBoundRow(ctx, t, s, "sess-8", "github") + other := &UpstreamTokens{ + ProviderID: "gitlab", AccessToken: "other-access", + ExpiresAt: time.Now().Add(time.Hour), UserID: "user-B", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-8", "gitlab", other)) + + all, err := s.GetAllUpstreamTokens(ctx, "sess-8", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.Len(t, all, 1) + assert.Contains(t, all, "github") + assert.NotContains(t, all, "gitlab") + }) + }) + + t.Run("scenario 9: bulk read with all rows mismatched returns empty map, nil error", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + seedBoundRow(ctx, t, s, "sess-9", "github") + seedBoundRow(ctx, t, s, "sess-9", "gitlab") + + all, err := s.GetAllUpstreamTokens(ctx, "sess-9", &ExpectedBinding{UserID: "user-B"}) + require.NoError(t, err) + require.NotNil(t, all, "unknown-session semantics preserved: empty map, not nil") + assert.Empty(t, all) + }) + }) + + t.Run("scenario 12: tombstone passes through the binding path", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + // Store nil tokens: the write path persists a deletion tombstone. + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-12", "github", nil)) + + got, err := s.GetUpstreamTokens(ctx, "sess-12", "github", &ExpectedBinding{UserID: "user-B"}) + require.NoError(t, err) + assert.Nil(t, got, "tombstone semantics unchanged: (nil, nil), no panic") + + // Strict mode must not reject a tombstone either — there is no row + // to bind, so the caller's tombstone handling applies. + got, err = s.GetUpstreamTokens(ctx, "sess-12", "github", &ExpectedBinding{UserID: "user-B", Strict: true}) + require.NoError(t, err) + assert.Nil(t, got) + }) + }) + + t.Run("strict mode: legacy row (empty stored UserID) fails closed, owned row passes", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + legacy := &UpstreamTokens{ + ProviderID: "github", AccessToken: "legacy-access", RefreshToken: "legacy-refresh", + ExpiresAt: time.Now().Add(time.Hour), + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-strict", "github", legacy)) + + // Permissive default: the legacy row is released (pre-Strict behavior). + got, err := s.GetUpstreamTokens(ctx, "sess-strict", "github", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.NotNil(t, got) + + // Strict: the same row fails closed — no token material released. + got, err = s.GetUpstreamTokens(ctx, "sess-strict", "github", &ExpectedBinding{UserID: "user-A", Strict: true}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got, "strict mode must not release a row that cannot prove its owner") + + // Strict bulk read excludes the legacy row. + all, err := s.GetAllUpstreamTokens(ctx, "sess-strict", &ExpectedBinding{UserID: "user-A", Strict: true}) + require.NoError(t, err) + assert.NotContains(t, all, "github") + + // Strict with an owned row passes normally. + seedBoundRow(ctx, t, s, "sess-strict-owned", "github") + got, err = s.GetUpstreamTokens(ctx, "sess-strict-owned", "github", + &ExpectedBinding{UserID: "user-A", Strict: true}) + require.NoError(t, err) + require.NotNil(t, got) + }) + }) + + t.Run("scenario 11: equal-length and differing-length mismatches both fail", func(t *testing.T) { + withStorage(t, func(ctx context.Context, s *MemoryStorage) { + seedBoundRow(ctx, t, s, "sess-11", "github") + + _, err := s.GetUpstreamTokens(ctx, "sess-11", "github", &ExpectedBinding{UserID: "user-B"}) + assert.ErrorIs(t, err, ErrInvalidBinding, "equal-length mismatch") + + _, err = s.GetUpstreamTokens(ctx, "sess-11", "github", &ExpectedBinding{UserID: "user-A-plus-extra"}) + assert.ErrorIs(t, err, ErrInvalidBinding, "differing-length mismatch") + }) + }) +} + +// TestMemoryStorage_UpstreamBinding_WarnLogs covers the log-asserting binding +// scenarios. It uses slog.SetDefault (process-global) and therefore does not +// run in parallel. +// +//nolint:paralleltest // captures slog default +func TestMemoryStorage_UpstreamBinding_WarnLogs(t *testing.T) { + //nolint:paralleltest // captures slog default + t.Run("scenario 8: bulk exclusion emits WARN with session, provider, dimension", func(t *testing.T) { + s := NewMemoryStorage() + t.Cleanup(func() { _ = s.Close() }) + ctx := context.Background() + seedBoundRow(ctx, t, s, "sess-w8", "github") + other := &UpstreamTokens{ + ProviderID: "gitlab", AccessToken: "other-access", + ExpiresAt: time.Now().Add(time.Hour), UserID: "user-B", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-w8", "gitlab", other)) + + out := captureWarnLogs(t) + all, err := s.GetAllUpstreamTokens(ctx, "sess-w8", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.Len(t, all, 1) + assertBindingExclusionWarn(t, out.String(), "sess-w8", "gitlab", "user_id") + }) + + //nolint:paralleltest // captures slog default + t.Run("scenario 10: legacy row returned with asserted user logs the unverified-owner WARN", func(t *testing.T) { + s := NewMemoryStorage() + t.Cleanup(func() { _ = s.Close() }) + ctx := context.Background() + legacy := &UpstreamTokens{ + ProviderID: "github", AccessToken: "legacy-access", + ExpiresAt: time.Now().Add(time.Hour), + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-w10", "github", legacy)) + + out := captureWarnLogs(t) + got, err := s.GetUpstreamTokens(ctx, "sess-w10", "github", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.NotNil(t, got, "legacy rows are released per the empty-field rule") + // The caller asserted an owner the row cannot prove: exactly one WARN, + // naming session and provider — never the exclusion WARN (nothing was + // excluded) and never token material. + assert.Contains(t, out.String(), "no recorded owner") + assert.Contains(t, out.String(), "sess-w10") + assert.Contains(t, out.String(), "github") + assert.NotContains(t, out.String(), "binding validation failed") + + out.Reset() + all, err := s.GetAllUpstreamTokens(ctx, "sess-w10", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + assert.Contains(t, all, "github") + assert.Contains(t, out.String(), "no recorded owner") + assert.NotContains(t, out.String(), "binding validation failed") + }) + + //nolint:paralleltest // captures slog default + t.Run("scenario 10b: plain legacy passthrough (nil expected, no ctx user) logs nothing", func(t *testing.T) { + s := NewMemoryStorage() + t.Cleanup(func() { _ = s.Close() }) + ctx := context.Background() + legacy := &UpstreamTokens{ + ProviderID: "github", AccessToken: "legacy-access", + ExpiresAt: time.Now().Add(time.Hour), + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-w10b", "github", legacy)) + + out := captureWarnLogs(t) + got, err := s.GetUpstreamTokens(ctx, "sess-w10b", "github", nil) + require.NoError(t, err) + require.NotNil(t, got) + assert.Empty(t, out.String(), "legacy passthrough without an asserted user must not log (hot-path spam)") + + // An owned row with an asserted user is fully verified — no WARN either. + seedBoundRow(ctx, t, s, "sess-w10c", "github") + out.Reset() + got, err = s.GetUpstreamTokens(ctx, "sess-w10c", "github", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.NotNil(t, got) + assert.Empty(t, out.String(), "a fully-owned row matching the asserted user must not log") + }) +} diff --git a/pkg/authserver/storage/mocks/mock_storage.go b/pkg/authserver/storage/mocks/mock_storage.go index 467bd2e784..9cf5ab3307 100644 --- a/pkg/authserver/storage/mocks/mock_storage.go +++ b/pkg/authserver/storage/mocks/mock_storage.go @@ -287,18 +287,18 @@ func (mr *MockUpstreamTokenStorageMockRecorder) DeleteUpstreamTokensForProvider( } // GetAllUpstreamTokens mocks base method. -func (m *MockUpstreamTokenStorage) GetAllUpstreamTokens(ctx context.Context, sessionID string) (map[string]*storage.UpstreamTokens, error) { +func (m *MockUpstreamTokenStorage) GetAllUpstreamTokens(ctx context.Context, sessionID string, expected *storage.ExpectedBinding) (map[string]*storage.UpstreamTokens, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllUpstreamTokens", ctx, sessionID) + ret := m.ctrl.Call(m, "GetAllUpstreamTokens", ctx, sessionID, expected) ret0, _ := ret[0].(map[string]*storage.UpstreamTokens) ret1, _ := ret[1].(error) return ret0, ret1 } // GetAllUpstreamTokens indicates an expected call of GetAllUpstreamTokens. -func (mr *MockUpstreamTokenStorageMockRecorder) GetAllUpstreamTokens(ctx, sessionID any) *gomock.Call { +func (mr *MockUpstreamTokenStorageMockRecorder) GetAllUpstreamTokens(ctx, sessionID, expected any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllUpstreamTokens", reflect.TypeOf((*MockUpstreamTokenStorage)(nil).GetAllUpstreamTokens), ctx, sessionID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllUpstreamTokens", reflect.TypeOf((*MockUpstreamTokenStorage)(nil).GetAllUpstreamTokens), ctx, sessionID, expected) } // GetLatestUpstreamTokensForUser mocks base method. @@ -317,18 +317,18 @@ func (mr *MockUpstreamTokenStorageMockRecorder) GetLatestUpstreamTokensForUser(c } // GetUpstreamTokens mocks base method. -func (m *MockUpstreamTokenStorage) GetUpstreamTokens(ctx context.Context, sessionID, providerName string) (*storage.UpstreamTokens, error) { +func (m *MockUpstreamTokenStorage) GetUpstreamTokens(ctx context.Context, sessionID, providerName string, expected *storage.ExpectedBinding) (*storage.UpstreamTokens, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetUpstreamTokens", ctx, sessionID, providerName) + ret := m.ctrl.Call(m, "GetUpstreamTokens", ctx, sessionID, providerName, expected) ret0, _ := ret[0].(*storage.UpstreamTokens) ret1, _ := ret[1].(error) return ret0, ret1 } // GetUpstreamTokens indicates an expected call of GetUpstreamTokens. -func (mr *MockUpstreamTokenStorageMockRecorder) GetUpstreamTokens(ctx, sessionID, providerName any) *gomock.Call { +func (mr *MockUpstreamTokenStorageMockRecorder) GetUpstreamTokens(ctx, sessionID, providerName, expected any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUpstreamTokens", reflect.TypeOf((*MockUpstreamTokenStorage)(nil).GetUpstreamTokens), ctx, sessionID, providerName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUpstreamTokens", reflect.TypeOf((*MockUpstreamTokenStorage)(nil).GetUpstreamTokens), ctx, sessionID, providerName, expected) } // StoreUpstreamTokens mocks base method. @@ -759,18 +759,18 @@ func (mr *MockStorageMockRecorder) GetAccessTokenSession(ctx, signature, session } // GetAllUpstreamTokens mocks base method. -func (m *MockStorage) GetAllUpstreamTokens(ctx context.Context, sessionID string) (map[string]*storage.UpstreamTokens, error) { +func (m *MockStorage) GetAllUpstreamTokens(ctx context.Context, sessionID string, expected *storage.ExpectedBinding) (map[string]*storage.UpstreamTokens, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllUpstreamTokens", ctx, sessionID) + ret := m.ctrl.Call(m, "GetAllUpstreamTokens", ctx, sessionID, expected) ret0, _ := ret[0].(map[string]*storage.UpstreamTokens) ret1, _ := ret[1].(error) return ret0, ret1 } // GetAllUpstreamTokens indicates an expected call of GetAllUpstreamTokens. -func (mr *MockStorageMockRecorder) GetAllUpstreamTokens(ctx, sessionID any) *gomock.Call { +func (mr *MockStorageMockRecorder) GetAllUpstreamTokens(ctx, sessionID, expected any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllUpstreamTokens", reflect.TypeOf((*MockStorage)(nil).GetAllUpstreamTokens), ctx, sessionID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllUpstreamTokens", reflect.TypeOf((*MockStorage)(nil).GetAllUpstreamTokens), ctx, sessionID, expected) } // GetAuthorizeCodeSession mocks base method. @@ -864,18 +864,18 @@ func (mr *MockStorageMockRecorder) GetRefreshTokenSession(ctx, signature, sessio } // GetUpstreamTokens mocks base method. -func (m *MockStorage) GetUpstreamTokens(ctx context.Context, sessionID, providerName string) (*storage.UpstreamTokens, error) { +func (m *MockStorage) GetUpstreamTokens(ctx context.Context, sessionID, providerName string, expected *storage.ExpectedBinding) (*storage.UpstreamTokens, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetUpstreamTokens", ctx, sessionID, providerName) + ret := m.ctrl.Call(m, "GetUpstreamTokens", ctx, sessionID, providerName, expected) ret0, _ := ret[0].(*storage.UpstreamTokens) ret1, _ := ret[1].(error) return ret0, ret1 } // GetUpstreamTokens indicates an expected call of GetUpstreamTokens. -func (mr *MockStorageMockRecorder) GetUpstreamTokens(ctx, sessionID, providerName any) *gomock.Call { +func (mr *MockStorageMockRecorder) GetUpstreamTokens(ctx, sessionID, providerName, expected any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUpstreamTokens", reflect.TypeOf((*MockStorage)(nil).GetUpstreamTokens), ctx, sessionID, providerName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUpstreamTokens", reflect.TypeOf((*MockStorage)(nil).GetUpstreamTokens), ctx, sessionID, providerName, expected) } // GetUser mocks base method. diff --git a/pkg/authserver/storage/redis.go b/pkg/authserver/storage/redis.go index dcd7090a5c..86fa6ab8ef 100644 --- a/pkg/authserver/storage/redis.go +++ b/pkg/authserver/storage/redis.go @@ -19,6 +19,7 @@ import ( tcredis "github.com/stacklok/toolhive-core/redis" "github.com/stacklok/toolhive/pkg/authserver/server/session" + "github.com/stacklok/toolhive/pkg/authserver/tokenenc" ) // nullMarker is used to store nil upstream tokens in Redis. @@ -89,6 +90,24 @@ func warnDroppedIndexMembers(operation, indexKey, expectedPrefix string, dropped type RedisStorage struct { client redis.UniversalClient keyPrefix string + // tokenEnc, when non-nil, enables AES-256-GCM envelope encryption for + // upstream OAuth token values at rest. When nil, writes are plaintext + // (current behavior), reads of plaintext pass through, and reads of + // envelope values fail loudly rather than returning corrupt tokens. + tokenEnc tokenenc.Keyring +} + +// RedisStorageOption configures optional RedisStorage behavior at construction. +type RedisStorageOption func(*RedisStorage) + +// WithTokenEncryption enables envelope encryption of upstream OAuth token +// values at rest using kr as the key-encryption-key source. The keyring is +// validated by tokenenc.NewStaticKeyring before it reaches this option; a nil +// keyring leaves encryption disabled. +func WithTokenEncryption(kr tokenenc.Keyring) RedisStorageOption { + return func(s *RedisStorage) { + s.tokenEnc = kr + } } // storedSession is a serializable wrapper for fosite.Requester. @@ -114,7 +133,9 @@ type storedSession struct { // delegated to the shared toolhive-core redis package. cfg.Password may be // empty when the Redis server does not require authentication (the auth server // does not mandate ACL auth); the keyPrefix is storage-specific and required. -func NewRedisStorage(ctx context.Context, cfg tcredis.Config, keyPrefix string) (*RedisStorage, error) { +func NewRedisStorage( + ctx context.Context, cfg tcredis.Config, keyPrefix string, opts ...RedisStorageOption, +) (*RedisStorage, error) { if keyPrefix == "" { return nil, errors.New("invalid redis configuration: key prefix is required") } @@ -124,19 +145,29 @@ func NewRedisStorage(ctx context.Context, cfg tcredis.Config, keyPrefix string) return nil, err } - return &RedisStorage{ + s := &RedisStorage{ client: client, keyPrefix: keyPrefix, - }, nil + } + for _, opt := range opts { + opt(s) + } + return s, nil } // NewRedisStorageWithClient creates a RedisStorage with a pre-configured client. // This is useful for testing with miniredis. -func NewRedisStorageWithClient(client redis.UniversalClient, keyPrefix string) *RedisStorage { - return &RedisStorage{ +func NewRedisStorageWithClient( + client redis.UniversalClient, keyPrefix string, opts ...RedisStorageOption, +) *RedisStorage { + s := &RedisStorage{ client: client, keyPrefix: keyPrefix, } + for _, opt := range opts { + opt(s) + } + return s } // defaultSessionFactory creates session prototypes for deserialization. @@ -709,38 +740,40 @@ func (s *storedUpstreamTokens) toUpstreamTokens() *UpstreamTokens { } } -// storeUpstreamTokensScript atomically reads the existing UserID, writes new token -// data, updates the session index set, and updates user reverse-index sets. -// This prevents a race condition where concurrent writes for the same session -// could leave orphaned entries in user sets. +// storeUpstreamTokensScript atomically writes new token data, updates the +// session index set, and updates user reverse-index sets, preventing a race +// condition where concurrent writes for the same session could leave orphaned +// entries in user sets. +// +// UserID bookkeeping is Go-side: the Go write path issues one best-effort GET +// immediately before this script and passes the previous row's UserID as +// ARGV[3]. This indirection exists because token values may be encrypted +// envelopes (see WithTokenEncryption), which cjson cannot decode; the GO side +// holds the keyring and extracts the UserID in the clear. The extra read is +// non-atomic with the script, so a concurrent writer changing a row's UserID +// between the GET and the script can leave a stale member in the old user's +// reverse-index set. That staleness is bounded by the member key's TTL and +// tolerated by readers (they skip missing rows); it matches the best-effort +// reverse-index cleanup posture used throughout this file. // // KEYS[1] = per-provider token key (e.g. "thv:auth:{ns:name}:upstream:{sessionID}:{providerName}") // KEYS[2] = session index set key (e.g. "thv:auth:{ns:name}:upstream:idx:{sessionID}") -// ARGV[1] = new token data (JSON or "null" marker) +// ARGV[1] = new token data (JSON or "null" marker; an encrypted envelope when encryption is enabled) // ARGV[2] = TTL in milliseconds -// ARGV[3] = new UserID ("" if no user) -// ARGV[4] = user upstream set key prefix (e.g. "thv:auth:{ns:name}:user:upstream:") +// ARGV[3] = previous row's UserID, from the Go-side GET ("" if none/unknown) +// ARGV[4] = new UserID ("" if no user) +// ARGV[5] = user upstream set key prefix (e.g. "thv:auth:{ns:name}:user:upstream:") // -// Cluster slot invariant: this script reads oldUserID from KEYS[1] inside the -// script body (atomic with the rest of the work), so the user-set keys are -// constructed dynamically as `ARGV[4] .. userID` rather than being passed as -// declared KEYS. Every dynamically-built key MUST therefore inherit the -// `{ns:name}` hash tag that is baked into ARGV[4] (s.keyPrefix). All callers -// derive ARGV[4] from s.keyPrefix, which DeriveKeyPrefix builds with the -// `{ns:name}` hash tag, so user-set keys land on the same Redis Cluster slot -// as KEYS[1] and KEYS[2]. A future refactor that strips the hash tag — or -// rebuilds setPrefix from raw inputs without the tag — will silently pass on -// standalone Redis and fail with CROSSSLOT under Cluster. +// Cluster slot invariant: all set keys are constructed inside the script as +// `ARGV[5] .. userID` rather than passed as declared KEYS. Every +// dynamically-built key MUST therefore inherit the `{ns:name}` hash tag that +// is baked into ARGV[5] (s.keyPrefix). All callers derive ARGV[5] from +// s.keyPrefix, which DeriveKeyPrefix builds with the `{ns:name}` hash tag, so +// user-set keys land on the same Redis Cluster slot as KEYS[1] and KEYS[2]. A +// future refactor that strips the hash tag — or rebuilds setPrefix from raw +// inputs without the tag — will silently pass on standalone Redis and fail +// with CROSSSLOT under Cluster. var storeUpstreamTokensScript = redis.NewScript(` -local oldUserID = "" -local existing = redis.call('GET', KEYS[1]) -if existing and existing ~= "null" then - local ok, decoded = pcall(cjson.decode, existing) - if ok and type(decoded) == "table" and decoded.user_id and decoded.user_id ~= "" then - oldUserID = decoded.user_id - end -end - local ttlMs = tonumber(ARGV[2]) if ttlMs > 0 then redis.call('SET', KEYS[1], ARGV[1], 'PX', ttlMs) @@ -796,8 +829,9 @@ else -- else: idxTTL >= ttlMs, index already outlives this member. end -local newUserID = ARGV[3] -local setPrefix = ARGV[4] +local oldUserID = ARGV[3] +local newUserID = ARGV[4] +local setPrefix = ARGV[5] if oldUserID ~= "" and oldUserID ~= newUserID then redis.call('SREM', setPrefix .. oldUserID, KEYS[1]) @@ -810,8 +844,13 @@ end return 1 `) -// marshalUpstreamTokensWithTTL marshals tokens and calculates TTL. -func marshalUpstreamTokensWithTTL(tokens *UpstreamTokens) ([]byte, time.Duration, error) { +// marshalUpstreamTokensWithTTL marshals tokens and calculates TTL. When +// envelope encryption is enabled (WithTokenEncryption), the marshaled JSON is +// sealed with redisKey as AAD; the nil-token tombstone is never sealed (it +// carries no secret). +func (s *RedisStorage) marshalUpstreamTokensWithTTL( + tokens *UpstreamTokens, redisKey string, +) ([]byte, time.Duration, error) { if tokens == nil { return []byte(nullMarker), DefaultAccessTokenTTL, nil } @@ -845,6 +884,13 @@ func marshalUpstreamTokensWithTTL(tokens *UpstreamTokens) ([]byte, time.Duration return nil, 0, fmt.Errorf("failed to marshal upstream tokens: %w", err) } + if s.tokenEnc != nil { + data, err = tokenenc.Seal(s.tokenEnc, redisKey, data) + if err != nil { + return nil, 0, fmt.Errorf("failed to encrypt upstream tokens: %w", err) + } + } + // Add DefaultRefreshTokenTTL beyond access token expiry so the refresh token // survives in storage for transparent token refresh by the middleware. // Zero ExpiresAt means the token never expires; return ttl=0 so the Lua script @@ -881,11 +927,17 @@ func (s *RedisStorage) StoreUpstreamTokens(ctx context.Context, sessionID, provi key := redisUpstreamKey(s.keyPrefix, sessionID, providerName) idxKey := redisSetKey(s.keyPrefix, KeyTypeUpstreamIdx, sessionID) - data, ttl, err := marshalUpstreamTokensWithTTL(tokens) + data, ttl, err := s.marshalUpstreamTokensWithTTL(tokens, key) if err != nil { return err } + // Read the previous row's UserID for reverse-index maintenance. The Lua + // script cannot do this itself because the stored value may be an + // encrypted envelope. Best-effort: on any failure pass "" — the stale + // reverse-index member is TTL-bounded and tolerated by readers. + oldUserID := s.readUserIDForCleanup(ctx, key) + newUserID := "" if tokens != nil { newUserID = tokens.UserID @@ -897,6 +949,7 @@ func (s *RedisStorage) StoreUpstreamTokens(ctx context.Context, sessionID, provi []string{key, idxKey}, string(data), ttl.Milliseconds(), + oldUserID, newUserID, userSetKeyPrefix, ).Result() @@ -910,7 +963,11 @@ func (s *RedisStorage) StoreUpstreamTokens(ctx context.Context, sessionID, provi // GetUpstreamTokens retrieves the upstream IDP tokens for a session and provider. // Returns a new UpstreamTokens struct deserialized from Redis, which acts as // a defensive copy - callers cannot modify the stored data by mutating the return value. -func (s *RedisStorage) GetUpstreamTokens(ctx context.Context, sessionID, providerName string) (*UpstreamTokens, error) { +// Binding validation (see ExpectedBinding) runs after deserialization and before +// expiry is surfaced: a mismatched row returns (nil, ErrInvalidBinding). +func (s *RedisStorage) GetUpstreamTokens( + ctx context.Context, sessionID, providerName string, expected *ExpectedBinding, +) (*UpstreamTokens, error) { if sessionID == "" { return nil, fosite.ErrInvalidRequest.WithHint("session ID cannot be empty") } @@ -919,13 +976,27 @@ func (s *RedisStorage) GetUpstreamTokens(ctx context.Context, sessionID, provide } key := redisUpstreamKey(s.keyPrefix, sessionID, providerName) - return s.getUpstreamTokensFromKey(ctx, key) + tokens, err := s.getUpstreamTokensFromKey(ctx, key) + if err != nil && !errors.Is(err, ErrExpired) { + return nil, err + } + + // Binding is checked before the ErrExpired carried by the unmarshal is + // surfaced: a mismatched row must never release refresh material. + if bindErr := checkUpstreamBinding(ctx, tokens, expected); bindErr != nil { + return nil, bindErr + } + warnIfAssertedUserOnLegacyRow(tokens, expected, sessionID, providerName) + + return tokens, err } // GetAllUpstreamTokens retrieves all upstream IDP tokens for a session across all providers. // Uses SMEMBERS on the session index set to find all provider keys, then MGET to fetch them. // Returns a map of providerName -> tokens. Returns an empty map for unknown sessions. -func (s *RedisStorage) GetAllUpstreamTokens(ctx context.Context, sessionID string) (map[string]*UpstreamTokens, error) { +func (s *RedisStorage) GetAllUpstreamTokens( + ctx context.Context, sessionID string, expected *ExpectedBinding, +) (map[string]*UpstreamTokens, error) { idxKey := redisSetKey(s.keyPrefix, KeyTypeUpstreamIdx, sessionID) result := make(map[string]*UpstreamTokens) @@ -956,37 +1027,68 @@ func (s *RedisStorage) GetAllUpstreamTokens(ctx context.Context, sessionID strin keyPrefix := fmt.Sprintf("%s%s:%s:", s.keyPrefix, KeyTypeUpstream, sessionID) for i, val := range values { - if val == nil { - continue + providerName, tokens := s.parseBulkUpstreamRow(ctx, providerKeys[i], keyPrefix, val, sessionID, expected) + if providerName != "" { + result[providerName] = tokens } + } - data, ok := val.(string) - if !ok { - slog.Warn("skipping upstream token entry: unexpected type", "key", providerKeys[i]) - continue - } + return result, nil +} - tokens, parseErr := unmarshalUpstreamTokens([]byte(data)) - if parseErr != nil && !errors.Is(parseErr, ErrExpired) { - slog.Warn("skipping corrupt upstream token entry", "key", providerKeys[i], "error", parseErr) - continue - } +// parseBulkUpstreamRow decrypts and decodes one MGET value of a +// GetAllUpstreamTokens bulk read into its provider name and tokens. Rows that +// are missing, corrupt, undecryptable, or fail per-row binding validation are +// skipped (empty provider name returned); exclusions are logged at WARN with +// metadata only. +func (s *RedisStorage) parseBulkUpstreamRow( + ctx context.Context, + key, keyPrefix string, + val any, + sessionID string, + expected *ExpectedBinding, +) (string, *UpstreamTokens) { + if val == nil { + return "", nil + } - // Extract provider name from the key - providerName := "" - if len(providerKeys[i]) > len(keyPrefix) { - providerName = providerKeys[i][len(keyPrefix):] - } - if providerName == "" && tokens != nil { - providerName = tokens.ProviderID - } + data, ok := val.(string) + if !ok { + slog.Warn("skipping upstream token entry: unexpected type", "key", key) + return "", nil + } - if providerName != "" { - result[providerName] = tokens - } + tokens, parseErr := s.decodeUpstreamTokens(ctx, key, []byte(data)) + if parseErr != nil && !errors.Is(parseErr, ErrExpired) { + slog.Warn("skipping corrupt upstream token entry", "key", key, "error", parseErr) + return "", nil } - return result, nil + // Extract provider name from the key + providerName := "" + if len(key) > len(keyPrefix) { + providerName = key[len(keyPrefix):] + } + if providerName == "" && tokens != nil { + providerName = tokens.ProviderID + } + if providerName == "" { + return "", nil + } + + // Binding validation applies per row: a mismatched row is excluded + // rather than failing the whole read. + if bindErr := checkUpstreamBinding(ctx, tokens, expected); bindErr != nil { + slog.Warn("excluding upstream token row: binding validation failed", + "session_id", sessionID, + "provider", providerName, + "dimension", bindingMismatchDimension(bindErr), + ) + return "", nil + } + warnIfAssertedUserOnLegacyRow(tokens, expected, sessionID, providerName) + + return providerName, tokens } // DeleteUpstreamTokens removes all upstream IDP tokens for a session (all providers). @@ -1010,18 +1112,12 @@ func (s *RedisStorage) DeleteUpstreamTokens(ctx context.Context, sessionID strin return fmt.Errorf("%w: %w", ErrNotFound, fosite.ErrNotFound.WithHint("Upstream tokens not found")) } - // Collect UserIDs for reverse-index cleanup before deleting + // Collect UserIDs for reverse-index cleanup before deleting. Decryption + // failures skip that row's cleanup (best-effort) without failing the delete. var userIDs []string for _, providerKey := range providerKeys { - data, getErr := s.client.Get(ctx, providerKey).Bytes() - if getErr != nil { - continue - } - if string(data) != nullMarker { - var stored storedUpstreamTokens - if unmarshalErr := json.Unmarshal(data, &stored); unmarshalErr == nil && stored.UserID != "" { - userIDs = append(userIDs, stored.UserID) - } + if userID := s.readUserIDForCleanup(ctx, providerKey); userID != "" { + userIDs = append(userIDs, userID) } } @@ -1031,12 +1127,15 @@ func (s *RedisStorage) DeleteUpstreamTokens(ctx context.Context, sessionID strin return fmt.Errorf("failed to delete upstream tokens: %w", err) } - // Best-effort secondary index cleanup for user:upstream sets + // Best-effort secondary index cleanup for user:upstream sets — one variadic + // SRem per user instead of a round-trip per provider. for _, userID := range userIDs { userUpstreamSetKey := redisSetKey(s.keyPrefix, KeyTypeUserUpstream, userID) - for _, providerKey := range providerKeys { - warnOnCleanupErr(s.client.SRem(ctx, userUpstreamSetKey, providerKey).Err(), "SRem", userUpstreamSetKey) + members := make([]any, len(providerKeys)) + for i, providerKey := range providerKeys { + members[i] = providerKey } + warnOnCleanupErr(s.client.SRem(ctx, userUpstreamSetKey, members...).Err(), "SRem", userUpstreamSetKey) } return nil @@ -1056,15 +1155,9 @@ func (s *RedisStorage) DeleteUpstreamTokensForProvider(ctx context.Context, sess idxKey := redisSetKey(s.keyPrefix, KeyTypeUpstreamIdx, sessionID) // Best-effort: read UserID for reverse-index cleanup before deleting. - var userID string - data, err := s.client.Get(ctx, key).Bytes() - if err == nil && string(data) != nullMarker { - var stored storedUpstreamTokens - if unmarshalErr := json.Unmarshal(data, &stored); unmarshalErr == nil { - userID = stored.UserID - } - } - // redis.Nil (key absent) or any error on GET: skip user-index cleanup gracefully. + // Absent rows, tombstones, and decrypt/unmarshal failures all yield "" + // and skip user-index cleanup gracefully. + userID := s.readUserIDForCleanup(ctx, key) // Del returns count 0 on absent key — treat as nil (not ErrNotFound). if delErr := s.client.Del(ctx, key).Err(); delErr != nil { @@ -1119,7 +1212,7 @@ func (s *RedisStorage) GetLatestUpstreamTokensForUser(ctx context.Context, userI var winner *storedUpstreamTokens for i, val := range values { - stored, ok := parseUserUpstreamEntry(val, providerID, members[i]) + stored, ok := s.parseUserUpstreamEntry(ctx, val, providerID, members[i]) if !ok { continue } @@ -1162,9 +1255,12 @@ func compareExpiryInt64(a, b int64) int { // parseUserUpstreamEntry parses one raw Redis value from the user-upstream index // and returns the decoded storedUpstreamTokens together with a match flag. // It returns (nil, false) for nil values, type mismatches, deletion tombstones, -// JSON decode errors, and rows whose ProviderID does not match providerID. -// keyName is used only for warning log messages. -func parseUserUpstreamEntry(val any, providerID, keyName string) (*storedUpstreamTokens, bool) { +// decryption or JSON decode errors, and rows whose ProviderID does not match +// providerID. keyName is the member key the value was fetched under — it is +// both the envelope AAD and the identifier used in warning log messages. +func (s *RedisStorage) parseUserUpstreamEntry( + ctx context.Context, val any, providerID, keyName string, +) (*storedUpstreamTokens, bool) { if val == nil { // Dangling set member: the per-provider key has been TTL-evicted. // Skip it; the next write will clean up the index entry (best-effort). @@ -1182,8 +1278,17 @@ func parseUserUpstreamEntry(val any, providerID, keyName string) (*storedUpstrea return nil, false } + plaintext, legacy, err := tokenenc.Open(s.tokenEnc, keyName, []byte(data)) + if err != nil { + slog.Warn("skipping upstream token entry: decryption failed", "key", keyName, "error", err) + return nil, false + } + if !legacy && tokenenc.NeedsRotation(s.tokenEnc, []byte(data)) { + s.lazyReseal(ctx, keyName, plaintext) + } + var stored storedUpstreamTokens - if unmarshalErr := json.Unmarshal([]byte(data), &stored); unmarshalErr != nil { + if unmarshalErr := json.Unmarshal(plaintext, &stored); unmarshalErr != nil { slog.Warn("skipping corrupt upstream token entry", "key", keyName, "error", unmarshalErr) return nil, false } @@ -1205,7 +1310,90 @@ func (s *RedisStorage) getUpstreamTokensFromKey(ctx context.Context, key string) return nil, fmt.Errorf("failed to get upstream tokens: %w", err) } - return unmarshalUpstreamTokens(data) + return s.decodeUpstreamTokens(ctx, key, data) +} + +// decodeUpstreamTokens decrypts (when the value is an envelope) and +// deserializes one upstream-token value stored under redisKey. The redisKey +// is the AAD the envelope was sealed against. The nullMarker tombstone check +// happens inside unmarshalUpstreamTokens, before decryption would matter +// (tombstones are never sealed). +func (s *RedisStorage) decodeUpstreamTokens(ctx context.Context, redisKey string, data []byte) (*UpstreamTokens, error) { + if string(data) == nullMarker { + return unmarshalUpstreamTokens(data) + } + + plaintext, legacy, err := tokenenc.Open(s.tokenEnc, redisKey, data) + if err != nil { + return nil, fmt.Errorf("failed to decrypt upstream tokens: %w", err) + } + if legacy && s.tokenEnc != nil { + slog.Debug("read legacy plaintext upstream token row with encryption enabled", + "key", redisKey) + } + + // Lazy re-encryption: a readable envelope sealed under a retired key is + // re-sealed with the active key so rotation converges without a sweep. + if !legacy && tokenenc.NeedsRotation(s.tokenEnc, data) { + s.lazyReseal(ctx, redisKey, plaintext) + } + + return unmarshalUpstreamTokens(plaintext) +} + +// readUserIDForCleanup best-effort extracts the UserID from the row stored +// under key, decrypting envelope values first. It returns "" on any failure: +// callers treat UserID extraction for reverse-index cleanup as optional and +// never fail their primary operation over it. +func (s *RedisStorage) readUserIDForCleanup(ctx context.Context, key string) string { + data, err := s.client.Get(ctx, key).Bytes() + if err != nil || string(data) == nullMarker { + return "" + } + plaintext, _, err := tokenenc.Open(s.tokenEnc, key, data) + if err != nil { + slog.Warn("skipping user-index cleanup: failed to decrypt upstream token row", + "key", key, "error", err) + return "" + } + var stored storedUpstreamTokens + if err := json.Unmarshal(plaintext, &stored); err != nil { + return "" + } + return stored.UserID +} + +// lazyReseal re-seals plaintext with the active key, overwriting the row at +// redisKey while preserving its remaining TTL. Best-effort: failures are +// logged and the read that triggered this still succeeds. The re-seal is a +// plain SET, so a concurrent StoreUpstreamTokens can be clobbered back to a +// retired-kid envelope — harmless, since the next read re-seals again. +func (s *RedisStorage) lazyReseal(ctx context.Context, redisKey string, plaintext []byte) { + sealed, err := tokenenc.Seal(s.tokenEnc, redisKey, plaintext) + if err != nil { + slog.Warn("lazy token re-encryption failed", "key", redisKey, "error", err) + return + } + pttl, err := s.client.PTTL(ctx, redisKey).Result() + if err != nil { + slog.Warn("lazy token re-encryption failed", "key", redisKey, "error", err) + return + } + // PTTL returns -1ns for "no expiry" and -2ns for "key missing" (go-redis + // passes the sentinel integers through as durations). + switch { + case pttl > 0: + err = s.client.Set(ctx, redisKey, sealed, pttl).Err() + case pttl == -1: + // Key exists with no expiry; preserve that. + err = s.client.Set(ctx, redisKey, sealed, 0).Err() + default: + // Key vanished between read and re-seal; nothing to do. + return + } + if err != nil { + slog.Warn("lazy token re-encryption failed", "key", redisKey, "error", err) + } } // unmarshalUpstreamTokens deserializes upstream tokens from JSON bytes. diff --git a/pkg/authserver/storage/redis_integration_test.go b/pkg/authserver/storage/redis_integration_test.go index ab43f00a58..38f7132e30 100644 --- a/pkg/authserver/storage/redis_integration_test.go +++ b/pkg/authserver/storage/redis_integration_test.go @@ -627,7 +627,7 @@ func TestIntegration_UpstreamTokens(t *testing.T) { } require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-up-1", "provider-a", tokens)) - retrieved, err := s.GetUpstreamTokens(ctx, "sess-up-1", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "sess-up-1", "provider-a", nil) require.NoError(t, err) assert.Equal(t, "upstream-access", retrieved.AccessToken) assert.Equal(t, "user-up-1", retrieved.UserID) @@ -638,7 +638,7 @@ func TestIntegration_UpstreamTokens(t *testing.T) { t.Run("nil tokens stored and retrieved", func(t *testing.T) { withIntegrationStorage(t, func(ctx context.Context, s *RedisStorage) { require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-nil", "provider-a", nil)) - retrieved, err := s.GetUpstreamTokens(ctx, "sess-nil", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "sess-nil", "provider-a", nil) require.NoError(t, err) assert.Nil(t, retrieved) }) @@ -652,7 +652,7 @@ func TestIntegration_UpstreamTokens(t *testing.T) { require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-ow", "provider-a", &UpstreamTokens{ AccessToken: "new", UserID: "user2", ExpiresAt: time.Now().Add(time.Hour), })) - retrieved, err := s.GetUpstreamTokens(ctx, "sess-ow", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "sess-ow", "provider-a", nil) require.NoError(t, err) assert.Equal(t, "new", retrieved.AccessToken) assert.Equal(t, "user2", retrieved.UserID) @@ -666,7 +666,7 @@ func TestIntegration_UpstreamTokens(t *testing.T) { RefreshToken: "expired-refresh", ExpiresAt: time.Now().Add(-time.Hour), })) - tokens, err := s.GetUpstreamTokens(ctx, "sess-exp", "provider-a") + tokens, err := s.GetUpstreamTokens(ctx, "sess-exp", "provider-a", nil) assert.ErrorIs(t, err, ErrExpired) // Expired tokens should still return the data (needed for refresh) require.NotNil(t, tokens, "expired tokens should return data for refresh") @@ -681,7 +681,7 @@ func TestIntegration_UpstreamTokens(t *testing.T) { AccessToken: "del-me", ExpiresAt: time.Now().Add(time.Hour), })) require.NoError(t, s.DeleteUpstreamTokens(ctx, "sess-del")) - _, err := s.GetUpstreamTokens(ctx, "sess-del", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "sess-del", "provider-a", nil) requireRedisNotFoundError(t, err) }) }) @@ -834,7 +834,7 @@ func TestIntegration_ProviderIdentity(t *testing.T) { assert.ErrorIs(t, err, ErrNotFound) _, err = s.GetProviderIdentity(ctx, "google", "cascade-sub") assert.ErrorIs(t, err, ErrNotFound) - _, err = s.GetUpstreamTokens(ctx, "cascade-sess", "provider-a") + _, err = s.GetUpstreamTokens(ctx, "cascade-sess", "provider-a", nil) assert.ErrorIs(t, err, ErrNotFound) }) }) @@ -1269,7 +1269,7 @@ func TestIntegration_MigrateLegacyUpstreamData(t *testing.T) { assert.Equal(t, int64(0), exists, "legacy token key should be deleted after migration") // Token should be readable under the new key format. - tokens, err := s.GetUpstreamTokens(ctx, sessionID, "default") + tokens, err := s.GetUpstreamTokens(ctx, sessionID, "default", nil) require.NoError(t, err) require.NotNil(t, tokens) assert.Equal(t, "legacy-at", tokens.AccessToken) @@ -1308,7 +1308,7 @@ func TestIntegration_MigrateLegacyUpstreamData(t *testing.T) { // --- Verify DeleteUser cascade includes migrated token --- require.NoError(t, s.DeleteUser(ctx, userID)) - _, err = s.GetUpstreamTokens(ctx, sessionID, "default") + _, err = s.GetUpstreamTokens(ctx, sessionID, "default", nil) assert.ErrorIs(t, err, ErrNotFound, "migrated token should be removed by DeleteUser cascade") _, err = s.GetUser(ctx, userID) @@ -1326,7 +1326,7 @@ func TestIntegration_MigrateLegacyUpstreamData(t *testing.T) { require.NoError(t, s.MigrateLegacyUpstreamData(ctx, "default", "oidc")) // Verify migrated. - tokens, err := s.GetUpstreamTokens(ctx, "idem-sess", "default") + tokens, err := s.GetUpstreamTokens(ctx, "idem-sess", "default", nil) require.NoError(t, err) assert.Equal(t, "idem-at", tokens.AccessToken) @@ -1334,7 +1334,7 @@ func TestIntegration_MigrateLegacyUpstreamData(t *testing.T) { require.NoError(t, s.MigrateLegacyUpstreamData(ctx, "default", "oidc")) // Token should still be there, unchanged. - tokens, err = s.GetUpstreamTokens(ctx, "idem-sess", "default") + tokens, err = s.GetUpstreamTokens(ctx, "idem-sess", "default", nil) require.NoError(t, err) assert.Equal(t, "idem-at", tokens.AccessToken) }) @@ -1379,7 +1379,7 @@ func TestIntegration_MigrateLegacyUpstreamData(t *testing.T) { // Every legacy key should have been migrated. for i := 0; i < keyCount; i++ { - tokens, err := s.GetUpstreamTokens(ctx, fmt.Sprintf("page-sess-%d", i), "default") + tokens, err := s.GetUpstreamTokens(ctx, fmt.Sprintf("page-sess-%d", i), "default", nil) require.NoError(t, err, "key %d should be migrated", i) assert.Equal(t, fmt.Sprintf("at-%d", i), tokens.AccessToken) assert.Equal(t, "default", tokens.ProviderID) @@ -1406,7 +1406,7 @@ func TestIntegration_MigrateLegacyUpstreamData(t *testing.T) { require.NoError(t, s.MigrateLegacyUpstreamData(ctx, "default", "oidc")) // The new-format token should be unchanged. - tokens, err := s.GetUpstreamTokens(ctx, "new-sess", "github") + tokens, err := s.GetUpstreamTokens(ctx, "new-sess", "github", nil) require.NoError(t, err) assert.Equal(t, "new-at", tokens.AccessToken) assert.Equal(t, "github", tokens.ProviderID, "new-format token ProviderID should be untouched") diff --git a/pkg/authserver/storage/redis_migrate.go b/pkg/authserver/storage/redis_migrate.go index 4a7ea1f435..cb08f905bb 100644 --- a/pkg/authserver/storage/redis_migrate.go +++ b/pkg/authserver/storage/redis_migrate.go @@ -13,6 +13,8 @@ import ( "time" "github.com/redis/go-redis/v9" + + "github.com/stacklok/toolhive/pkg/authserver/tokenenc" ) // MigrationResult holds counts of items migrated during bulk migration. @@ -133,6 +135,8 @@ func (s *RedisStorage) migrateUpstreamTokenKeys(ctx context.Context, providerNam } // migrateSingleUpstreamToken migrates one legacy upstream token key to the new format. +// +//nolint:gocyclo // sequential guard clauses over one row; splitting hurts readability func (s *RedisStorage) migrateSingleUpstreamToken( ctx context.Context, legacyKey, sessionID, providerName string, @@ -167,6 +171,16 @@ func (s *RedisStorage) migrateSingleUpstreamToken( return nil } + // Legacy keys predate the per-provider key format and therefore predate + // envelope encryption; their values are always plaintext JSON. If an + // envelope ever shows up here (e.g. an operator manually renamed a sealed + // row to a legacy key name), decryption would fail on the AAD mismatch — + // surface that as a migration failure for the row rather than migrating + // garbage. + if !tokenenc.IsLegacyValue(data) { + return fmt.Errorf("unexpected encrypted value at legacy key %s", legacyKey) + } + // Deserialize to check ProviderID var stored storedUpstreamTokens if err := json.Unmarshal(data, &stored); err != nil { @@ -188,6 +202,15 @@ func (s *RedisStorage) migrateSingleUpstreamToken( return fmt.Errorf("marshal failed: %w", err) } + // Seal the rewritten row: migration doubles as a free encryption pass for + // pre-existing plaintext rows. The AAD is the NEW key the value lands under. + if s.tokenEnc != nil { + newData, err = tokenenc.Seal(s.tokenEnc, newKey, newData) + if err != nil { + return fmt.Errorf("failed to encrypt migrated upstream token: %w", err) + } + } + // Preserve TTL from the legacy key ttl, err := s.client.PTTL(ctx, legacyKey).Result() if err != nil { @@ -229,6 +252,118 @@ func (s *RedisStorage) migrateSingleUpstreamToken( return nil } +// MigrateUpstreamTokenEncryption performs a one-shot sweep that converges +// upstream token rows onto the current encryption posture: plaintext rows are +// sealed, and envelopes sealed under a retired key ID are re-sealed with the +// active key. It is optional — reads already lazy-re-encrypt stale envelopes +// and writes always seal — and intended for operators who want immediate +// convergence after enabling encryption or rotating keys. +// +// It is a no-op when encryption is not configured (keyring nil): plaintext +// rows are the desired end state in that posture. Tombstones and rows that +// fail to decrypt (unknown key ID, corruption) are skipped with a WARN and do +// not fail the sweep. The sweep is idempotent and safe to re-run; it mirrors +// the SCAN pattern of MigrateLegacyUpstreamData. +func (s *RedisStorage) MigrateUpstreamTokenEncryption(ctx context.Context) error { + if s.tokenEnc == nil { + return nil + } + + pattern := s.keyPrefix + KeyTypeUpstream + ":*" + upstreamPrefixLen := len(s.keyPrefix) + len(KeyTypeUpstream) + 1 + + var cursor uint64 + var resealed, skipped, failed int + for { + keys, nextCursor, err := s.client.Scan(ctx, cursor, pattern, 100).Result() + if err != nil { + return fmt.Errorf("token encryption sweep: SCAN failed: %w", err) + } + + for _, key := range keys { + if len(key) <= upstreamPrefixLen { + continue + } + // Skip index sets (upstream:idx:...) — only token rows hold values. + if strings.HasPrefix(key[upstreamPrefixLen:], "idx:") { + continue + } + + done, err := s.resealUpstreamRow(ctx, key) + switch { + case err != nil: + slog.Warn("token encryption sweep: failed to re-seal row", "key", key, "error", err) + failed++ + case done: + resealed++ + default: + skipped++ + } + } + + cursor = nextCursor + if cursor == 0 { + break + } + } + + if failed > 0 { + return fmt.Errorf("token encryption sweep: %d row(s) failed to re-seal", failed) + } + if resealed > 0 { + slog.Info("token encryption sweep complete", "resealed", resealed, "skipped", skipped) + } + return nil +} + +// resealUpstreamRow re-seals one upstream token row when it is plaintext or +// sealed under a retired key ID. It reports whether the row was rewritten. +// Tombstones, missing keys, and already-current envelopes are no-ops. +func (s *RedisStorage) resealUpstreamRow(ctx context.Context, key string) (bool, error) { + data, err := s.client.Get(ctx, key).Bytes() + if err != nil { + if errors.Is(err, redis.Nil) { + return false, nil + } + return false, fmt.Errorf("GET failed for %s: %w", key, err) + } + if string(data) == nullMarker { + return false, nil + } + if !tokenenc.IsLegacyValue(data) && !tokenenc.NeedsRotation(s.tokenEnc, data) { + return false, nil // already sealed under the active key + } + + plaintext, _, err := tokenenc.Open(s.tokenEnc, key, data) + if err != nil { + return false, fmt.Errorf("decrypt failed for %s: %w", key, err) + } + + sealed, err := tokenenc.Seal(s.tokenEnc, key, plaintext) + if err != nil { + return false, fmt.Errorf("seal failed for %s: %w", key, err) + } + + // Preserve the row's TTL. PTTL returns -1ns (no expiry) or -2ns (vanished) + // — go-redis passes the sentinel integers through as durations. + pttl, err := s.client.PTTL(ctx, key).Result() + if err != nil { + return false, fmt.Errorf("PTTL failed for %s: %w", key, err) + } + switch { + case pttl > 0: + err = s.client.Set(ctx, key, sealed, pttl).Err() + case pttl == -1: + err = s.client.Set(ctx, key, sealed, 0).Err() + default: + return false, nil // vanished between GET and re-seal + } + if err != nil { + return false, fmt.Errorf("SET failed for %s: %w", key, err) + } + return true, nil +} + // migrateProviderIdentityKeys scans for provider identity keys stored under the // legacy protocol-type ID and duplicates them under the new logical provider name. func (s *RedisStorage) migrateProviderIdentityKeys( diff --git a/pkg/authserver/storage/redis_test.go b/pkg/authserver/storage/redis_test.go index cb9d562811..372a1a2b24 100644 --- a/pkg/authserver/storage/redis_test.go +++ b/pkg/authserver/storage/redis_test.go @@ -8,10 +8,8 @@ package storage import ( - "bytes" "context" "fmt" - "log/slog" "net/url" "strings" "sync" @@ -692,7 +690,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { } require.NoError(t, s.StoreUpstreamTokens(ctx, "session-123", "provider-a", tokens)) - retrieved, err := s.GetUpstreamTokens(ctx, "session-123", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "session-123", "provider-a", nil) require.NoError(t, err) assert.Equal(t, tokens.AccessToken, retrieved.AccessToken) assert.Equal(t, tokens.RefreshToken, retrieved.RefreshToken) @@ -703,7 +701,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { t.Run("get non-existent", func(t *testing.T) { withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { - _, err := s.GetUpstreamTokens(ctx, "non-existent", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "non-existent", "provider-a", nil) requireRedisNotFoundError(t, err) }) }) @@ -712,7 +710,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { require.NoError(t, s.StoreUpstreamTokens(ctx, "to-delete", "provider-a", &UpstreamTokens{AccessToken: "test", ExpiresAt: time.Now().Add(time.Hour)})) require.NoError(t, s.DeleteUpstreamTokens(ctx, "to-delete")) - _, err := s.GetUpstreamTokens(ctx, "to-delete", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "to-delete", "provider-a", nil) requireRedisNotFoundError(t, err) }) }) @@ -722,7 +720,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.StoreUpstreamTokens(ctx, "session", "provider-a", &UpstreamTokens{AccessToken: "token-1", UserID: "user1", ExpiresAt: time.Now().Add(time.Hour)})) require.NoError(t, s.StoreUpstreamTokens(ctx, "session", "provider-a", &UpstreamTokens{AccessToken: "token-2", UserID: "user2", ExpiresAt: time.Now().Add(time.Hour)})) - retrieved, err := s.GetUpstreamTokens(ctx, "session", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "session", "provider-a", nil) require.NoError(t, err) assert.Equal(t, "token-2", retrieved.AccessToken) assert.Equal(t, "user2", retrieved.UserID) @@ -740,7 +738,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { ExpiresAt: time.Now().Add(-time.Hour), })) - retrieved, err := s.GetUpstreamTokens(ctx, "expired", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "expired", "provider-a", nil) require.Error(t, err) assert.ErrorIs(t, err, ErrExpired) // Tokens should be returned alongside ErrExpired for refresh purposes @@ -759,7 +757,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { ProviderID: "test-provider", })) - retrieved, err := s.GetUpstreamTokens(ctx, "no-expiry", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "no-expiry", "provider-a", nil) require.NoError(t, err) require.NotNil(t, retrieved) assert.Equal(t, "no-expiry-token", retrieved.AccessToken) @@ -827,7 +825,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { // key evicts; the non-expiring one remains; the index stays intact. mr.FastForward(time.Hour + DefaultRefreshTokenTTL + time.Second) - all, err := s.GetAllUpstreamTokens(ctx, "inverse-session") + all, err := s.GetAllUpstreamTokens(ctx, "inverse-session", nil) require.NoError(t, err) require.Contains(t, all, "provider-nonexpiring", "non-expiring token must remain reachable through the index") @@ -935,7 +933,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { "index TTL is left alone on same-provider rewrite (acceptable limitation)") // The new value is reachable. - retrieved, err := s.GetUpstreamTokens(ctx, "rewrite-session", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "rewrite-session", "provider-a", nil) require.NoError(t, err) require.NotNil(t, retrieved) assert.Equal(t, "now-expiring", retrieved.AccessToken) @@ -951,7 +949,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { SessionExpiresAt: sessionExpiry, })) - retrieved, err := s.GetUpstreamTokens(ctx, "sess-bound", "github") + retrieved, err := s.GetUpstreamTokens(ctx, "sess-bound", "github", nil) require.NoError(t, err) require.NotNil(t, retrieved) assert.Equal(t, "pat-token", retrieved.AccessToken) @@ -965,7 +963,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { // Fast-forward past SessionExpiresAt + DefaultRefreshTokenTTL mr.FastForward(time.Hour + DefaultRefreshTokenTTL + time.Second) - _, err = s.GetUpstreamTokens(ctx, "sess-bound", "github") + _, err = s.GetUpstreamTokens(ctx, "sess-bound", "github", nil) requireRedisNotFoundError(t, err) }) }) @@ -1017,7 +1015,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { t.Run("nil tokens is valid", func(t *testing.T) { withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { require.NoError(t, s.StoreUpstreamTokens(ctx, "session-id", "provider-a", nil)) - retrieved, err := s.GetUpstreamTokens(ctx, "session-id", "provider-a") + retrieved, err := s.GetUpstreamTokens(ctx, "session-id", "provider-a", nil) require.NoError(t, err) assert.Nil(t, retrieved) }) @@ -1040,11 +1038,11 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.StoreUpstreamTokens(ctx, "session-multi", "github", tokensA)) require.NoError(t, s.StoreUpstreamTokens(ctx, "session-multi", "google", tokensB)) - retrievedA, err := s.GetUpstreamTokens(ctx, "session-multi", "github") + retrievedA, err := s.GetUpstreamTokens(ctx, "session-multi", "github", nil) require.NoError(t, err) assert.Equal(t, "github-access", retrievedA.AccessToken) - retrievedB, err := s.GetUpstreamTokens(ctx, "session-multi", "google") + retrievedB, err := s.GetUpstreamTokens(ctx, "session-multi", "google", nil) require.NoError(t, err) assert.Equal(t, "google-access", retrievedB.AccessToken) }) @@ -1067,7 +1065,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.StoreUpstreamTokens(ctx, "session-all", "github", tokensA)) require.NoError(t, s.StoreUpstreamTokens(ctx, "session-all", "google", tokensB)) - allTokens, err := s.GetAllUpstreamTokens(ctx, "session-all") + allTokens, err := s.GetAllUpstreamTokens(ctx, "session-all", nil) require.NoError(t, err) require.Len(t, allTokens, 2) @@ -1078,7 +1076,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { t.Run("GetAllUpstreamTokens unknown session", func(t *testing.T) { withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { - allTokens, err := s.GetAllUpstreamTokens(ctx, "unknown-session") + allTokens, err := s.GetAllUpstreamTokens(ctx, "unknown-session", nil) require.NoError(t, err) assert.Empty(t, allTokens) }) @@ -1095,13 +1093,13 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.DeleteUpstreamTokens(ctx, "session-wipe")) - _, err := s.GetUpstreamTokens(ctx, "session-wipe", "github") + _, err := s.GetUpstreamTokens(ctx, "session-wipe", "github", nil) requireRedisNotFoundError(t, err) - _, err = s.GetUpstreamTokens(ctx, "session-wipe", "google") + _, err = s.GetUpstreamTokens(ctx, "session-wipe", "google", nil) requireRedisNotFoundError(t, err) - allTokens, err := s.GetAllUpstreamTokens(ctx, "session-wipe") + allTokens, err := s.GetAllUpstreamTokens(ctx, "session-wipe", nil) require.NoError(t, err) assert.Empty(t, allTokens) }) @@ -1113,7 +1111,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { require.Error(t, err) require.ErrorIs(t, err, fosite.ErrInvalidRequest) - _, err = s.GetUpstreamTokens(ctx, "session-ep", "") + _, err = s.GetUpstreamTokens(ctx, "session-ep", "", nil) require.Error(t, err) require.ErrorIs(t, err, fosite.ErrInvalidRequest) }) @@ -1142,7 +1140,7 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { key := redisUpstreamKey(s.keyPrefix, "legacy-session", "github") require.NoError(t, mr.Set(key, legacyJSON)) - retrieved, err := s.GetUpstreamTokens(ctx, "legacy-session", "github") + retrieved, err := s.GetUpstreamTokens(ctx, "legacy-session", "github", nil) require.NoError(t, err) require.NotNil(t, retrieved) @@ -1170,14 +1168,14 @@ func TestRedisStorage_UpstreamTokens(t *testing.T) { require.NoError(t, s.DeleteUpstreamTokensForProvider(ctx, "session-1", "provider-a")) - _, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a") + _, err := s.GetUpstreamTokens(ctx, "session-1", "provider-a", nil) requireRedisNotFoundError(t, err) - got, err := s.GetUpstreamTokens(ctx, "session-1", "provider-b") + got, err := s.GetUpstreamTokens(ctx, "session-1", "provider-b", nil) require.NoError(t, err) assert.Equal(t, "b", got.AccessToken) - all, err := s.GetAllUpstreamTokens(ctx, "session-1") + all, err := s.GetAllUpstreamTokens(ctx, "session-1", nil) require.NoError(t, err) assert.Len(t, all, 1) }) @@ -1211,7 +1209,7 @@ func TestRedisStorage_MigrateLegacyUpstreamData(t *testing.T) { assert.Equal(t, int64(0), exists, "legacy key should be deleted after migration") // New key should be readable - tokens, err := s.GetUpstreamTokens(ctx, "legacy-session", "default") + tokens, err := s.GetUpstreamTokens(ctx, "legacy-session", "default", nil) require.NoError(t, err) require.NotNil(t, tokens) assert.Equal(t, "legacy-at", tokens.AccessToken) @@ -1236,7 +1234,7 @@ func TestRedisStorage_MigrateLegacyUpstreamData(t *testing.T) { assert.Equal(t, int64(1), exists, "legacy key should not be deleted for non-legacy provider ID") // New key should not exist - _, err = s.GetUpstreamTokens(ctx, "logical-session", "default") + _, err = s.GetUpstreamTokens(ctx, "logical-session", "default", nil) require.Error(t, err) assert.ErrorIs(t, err, ErrNotFound) }) @@ -1256,7 +1254,7 @@ func TestRedisStorage_MigrateLegacyUpstreamData(t *testing.T) { require.NoError(t, s.MigrateLegacyUpstreamData(ctx, "default", "oidc")) // New key should have the original new-format data, not the legacy data - tokens, err := s.GetUpstreamTokens(ctx, "both-keys", "default") + tokens, err := s.GetUpstreamTokens(ctx, "both-keys", "default", nil) require.NoError(t, err) assert.Equal(t, "new-token", tokens.AccessToken) }) @@ -1331,7 +1329,7 @@ func TestRedisStorage_MigrateLegacyUpstreamData(t *testing.T) { // All should be readable under new format for i := 0; i < 3; i++ { - tokens, err := s.GetUpstreamTokens(ctx, fmt.Sprintf("session-%d", i), "default") + tokens, err := s.GetUpstreamTokens(ctx, fmt.Sprintf("session-%d", i), "default", nil) require.NoError(t, err) assert.Equal(t, fmt.Sprintf("at-%d", i), tokens.AccessToken) assert.Equal(t, "default", tokens.ProviderID) @@ -1350,7 +1348,7 @@ func TestRedisStorage_MigrateLegacyUpstreamData(t *testing.T) { require.NoError(t, s.MigrateLegacyUpstreamData(ctx, "default", "oidc")) // New key should be unchanged - tokens, err := s.GetUpstreamTokens(ctx, "new-session", "default") + tokens, err := s.GetUpstreamTokens(ctx, "new-session", "default", nil) require.NoError(t, err) assert.Equal(t, "new-at", tokens.AccessToken) }) @@ -1376,7 +1374,7 @@ func TestRedisStorage_MigrateLegacyUpstreamData(t *testing.T) { require.NoError(t, s.MigrateLegacyUpstreamData(ctx, "default", "oidc")) // Sanity: token is reachable under the new key. - tokens, err := s.GetUpstreamTokens(ctx, "del-session", "default") + tokens, err := s.GetUpstreamTokens(ctx, "del-session", "default", nil) require.NoError(t, err) assert.Equal(t, "del-at", tokens.AccessToken) @@ -1384,7 +1382,7 @@ func TestRedisStorage_MigrateLegacyUpstreamData(t *testing.T) { require.NoError(t, s.DeleteUser(ctx, userID)) // The upstream token must be gone. - _, err = s.GetUpstreamTokens(ctx, "del-session", "default") + _, err = s.GetUpstreamTokens(ctx, "del-session", "default", nil) require.Error(t, err) assert.ErrorIs(t, err, ErrNotFound, "migrated token should be removed by DeleteUser cascade") }) @@ -1587,7 +1585,7 @@ func TestRedisStorage_DeleteUser_CascadesAssociatedData(t *testing.T) { assert.ErrorIs(t, err, ErrNotFound) // Verify the user's upstream tokens are gone - _, err = s.GetUpstreamTokens(ctx, "session-user", "provider-a") + _, err = s.GetUpstreamTokens(ctx, "session-user", "provider-a", nil) assert.ErrorIs(t, err, ErrNotFound) // Verify the other user still exists @@ -1601,7 +1599,7 @@ func TestRedisStorage_DeleteUser_CascadesAssociatedData(t *testing.T) { assert.Equal(t, "other-user", retrieved.UserID) // Verify the other user's upstream tokens are still there - otherRetrieved, err := s.GetUpstreamTokens(ctx, "session-other", "provider-a") + otherRetrieved, err := s.GetUpstreamTokens(ctx, "session-other", "provider-a", nil) require.NoError(t, err) assert.Equal(t, "other-token", otherRetrieved.AccessToken) }) @@ -2845,19 +2843,6 @@ func runDCRConcurrentAccess( assert.Zero(t, atomic.LoadInt32(&getErrCount), "no concurrent Get should have errored") } -// captureWarnLogs swaps in a buffered slog handler at warn level for the -// duration of the test and returns the captured output. Process-global, not -// safe for t.Parallel(). -func captureWarnLogs(t *testing.T) *bytes.Buffer { - t.Helper() - var buf bytes.Buffer - handler := slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}) - orig := slog.Default() - slog.SetDefault(slog.New(handler)) - t.Cleanup(func() { slog.SetDefault(orig) }) - return &buf -} - // assertForeignDropWarn checks that the warn log emitted for a filtered // CROSSSLOT-defending op names the operation, the dropped member, and the // expected key prefix. @@ -2904,7 +2889,7 @@ func TestForeignMembersFilteredFromIndexOps(t *testing.T) { //nolint:paralleltes t.Run("GetAllUpstreamTokens", func(t *testing.T) { buf := captureWarnLogs(t) - got, err := storage.GetAllUpstreamTokens(ctx, "session-X") + got, err := storage.GetAllUpstreamTokens(ctx, "session-X", nil) require.NoError(t, err) require.Len(t, got, 1) require.Contains(t, got, "github") @@ -2985,3 +2970,295 @@ func TestFilterIndexMembersByPrefix(t *testing.T) { }) } } + +// seedRedisBoundRow stores a live, fully-bound upstream token row for +// (sessionID, provider). +func seedRedisBoundRow(ctx context.Context, t *testing.T, s *RedisStorage, sessionID, provider string) { + t.Helper() + row := &UpstreamTokens{ + ProviderID: provider, + AccessToken: "access-" + provider, + RefreshToken: "refresh-" + provider, + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-A", + ClientID: "client-A", + UpstreamSubject: "subject-A", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, sessionID, provider, row)) +} + +// TestRedisStorage_UpstreamBinding covers the read-side binding matrix +// (scenarios 1-12) for the Redis backend. Log-capturing cases live in +// TestRedisStorage_UpstreamBinding_WarnLogs because slog.SetDefault is +// process-global. +func TestRedisStorage_UpstreamBinding(t *testing.T) { + t.Parallel() + + t.Run("scenario 1: matching ctx user, nil expected, row returned", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + seedRedisBoundRow(ctx, t, s, "sess-1", "github") + + readCtx := ContextWithBindingUser(ctx, "user-A") + got, err := s.GetUpstreamTokens(readCtx, "sess-1", "github", nil) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "user-A", got.UserID) + }) + }) + + t.Run("scenario 2: mismatched ctx user, nil expected, ErrInvalidBinding with nil tokens", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + seedRedisBoundRow(ctx, t, s, "sess-2", "github") + + readCtx := ContextWithBindingUser(ctx, "user-B") + got, err := s.GetUpstreamTokens(readCtx, "sess-2", "github", nil) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got, "mismatched row must not be released, even for refresh") + }) + }) + + t.Run("scenario 3: no identity in ctx, nil expected, row returned", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + seedRedisBoundRow(ctx, t, s, "sess-3", "github") + + got, err := s.GetUpstreamTokens(ctx, "sess-3", "github", nil) + require.NoError(t, err) + require.NotNil(t, got) + }) + }) + + t.Run("scenario 4: expected UserID match and mismatch", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + seedRedisBoundRow(ctx, t, s, "sess-4", "github") + + got, err := s.GetUpstreamTokens(ctx, "sess-4", "github", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.NotNil(t, got) + + got, err = s.GetUpstreamTokens(ctx, "sess-4", "github", &ExpectedBinding{UserID: "user-B"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got) + }) + }) + + t.Run("scenario 5: client ID binding variants", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + seedRedisBoundRow(ctx, t, s, "sess-5", "github") + + got, err := s.GetUpstreamTokens(ctx, "sess-5", "github", &ExpectedBinding{ClientID: "client-A"}) + require.NoError(t, err) + require.NotNil(t, got) + + got, err = s.GetUpstreamTokens(ctx, "sess-5", "github", &ExpectedBinding{ClientID: "client-B"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got) + + got, err = s.GetUpstreamTokens(ctx, "sess-5", "github", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.NotNil(t, got) + + legacy := &UpstreamTokens{ + ProviderID: "github", AccessToken: "legacy-access", + ExpiresAt: time.Now().Add(time.Hour), UserID: "user-A", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-5b", "github", legacy)) + got, err = s.GetUpstreamTokens(ctx, "sess-5b", "github", + &ExpectedBinding{UserID: "user-A", ClientID: "client-B"}) + require.NoError(t, err) + require.NotNil(t, got) + }) + }) + + t.Run("scenario 6: upstream subject mismatch", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + seedRedisBoundRow(ctx, t, s, "sess-6", "github") + + got, err := s.GetUpstreamTokens(ctx, "sess-6", "github", &ExpectedBinding{UpstreamSubject: "subject-B"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got) + }) + }) + + t.Run("scenario 7: expired + mismatched row yields ErrInvalidBinding, not ErrExpired", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + row := &UpstreamTokens{ + ProviderID: "github", AccessToken: "stale-access", RefreshToken: "stale-refresh", + ExpiresAt: time.Now().Add(-time.Hour), UserID: "user-A", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-7", "github", row)) + + got, err := s.GetUpstreamTokens(ctx, "sess-7", "github", &ExpectedBinding{UserID: "user-B"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.NotErrorIs(t, err, ErrExpired, "binding must win over expiry") + assert.Nil(t, got, "no refresh material released for a mismatched row") + }) + }) + + t.Run("scenario 8: bulk read excludes only the mismatched provider", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + seedRedisBoundRow(ctx, t, s, "sess-8", "github") + other := &UpstreamTokens{ + ProviderID: "gitlab", AccessToken: "other-access", + ExpiresAt: time.Now().Add(time.Hour), UserID: "user-B", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-8", "gitlab", other)) + + all, err := s.GetAllUpstreamTokens(ctx, "sess-8", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.Len(t, all, 1) + assert.Contains(t, all, "github") + assert.NotContains(t, all, "gitlab") + }) + }) + + t.Run("scenario 9: bulk read with all rows mismatched returns empty map, nil error", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + seedRedisBoundRow(ctx, t, s, "sess-9", "github") + seedRedisBoundRow(ctx, t, s, "sess-9", "gitlab") + + all, err := s.GetAllUpstreamTokens(ctx, "sess-9", &ExpectedBinding{UserID: "user-B"}) + require.NoError(t, err) + require.NotNil(t, all, "unknown-session semantics preserved: empty map, not nil") + assert.Empty(t, all) + }) + }) + + t.Run("strict mode: legacy row (empty stored UserID) fails closed, owned row passes", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + legacy := &UpstreamTokens{ + ProviderID: "github", AccessToken: "legacy-access", RefreshToken: "legacy-refresh", + ExpiresAt: time.Now().Add(time.Hour), + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-strict", "github", legacy)) + + // Permissive default: the legacy row is released (pre-Strict behavior). + got, err := s.GetUpstreamTokens(ctx, "sess-strict", "github", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.NotNil(t, got) + + // Strict: the same row fails closed — no token material released. + got, err = s.GetUpstreamTokens(ctx, "sess-strict", "github", &ExpectedBinding{UserID: "user-A", Strict: true}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.Nil(t, got, "strict mode must not release a row that cannot prove its owner") + + // Strict bulk read excludes the legacy row. + all, err := s.GetAllUpstreamTokens(ctx, "sess-strict", &ExpectedBinding{UserID: "user-A", Strict: true}) + require.NoError(t, err) + assert.NotContains(t, all, "github") + + // Strict with an owned row passes normally. + seedRedisBoundRow(ctx, t, s, "sess-strict-owned", "github") + got, err = s.GetUpstreamTokens(ctx, "sess-strict-owned", "github", + &ExpectedBinding{UserID: "user-A", Strict: true}) + require.NoError(t, err) + require.NotNil(t, got) + }) + }) + + t.Run("scenario 11: equal-length and differing-length mismatches both fail", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + seedRedisBoundRow(ctx, t, s, "sess-11", "github") + + _, err := s.GetUpstreamTokens(ctx, "sess-11", "github", &ExpectedBinding{UserID: "user-B"}) + assert.ErrorIs(t, err, ErrInvalidBinding, "equal-length mismatch") + + _, err = s.GetUpstreamTokens(ctx, "sess-11", "github", &ExpectedBinding{UserID: "user-A-plus-extra"}) + assert.ErrorIs(t, err, ErrInvalidBinding, "differing-length mismatch") + }) + }) + + t.Run("scenario 12: nullMarker tombstone passes through binding path", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + // Store nil tokens: the write path persists a deletion tombstone. + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-12", "github", nil)) + + got, err := s.GetUpstreamTokens(ctx, "sess-12", "github", &ExpectedBinding{UserID: "user-B"}) + require.NoError(t, err) + assert.Nil(t, got, "tombstone semantics unchanged: (nil, nil), no panic") + }) + }) +} + +// TestRedisStorage_UpstreamBinding_WarnLogs covers the log-asserting binding +// scenarios. It uses slog.SetDefault (process-global) and therefore does not +// run in parallel. +func TestRedisStorage_UpstreamBinding_WarnLogs(t *testing.T) { //nolint:paralleltest // captures slog default + t.Run("scenario 8: bulk exclusion emits WARN with session, provider, dimension", func(t *testing.T) { //nolint:paralleltest // captures slog default + s, mr := newTestRedisStorage(t) + t.Cleanup(func() { + _ = s.Close() + mr.Close() + }) + ctx := context.Background() + seedRedisBoundRow(ctx, t, s, "sess-w8", "github") + other := &UpstreamTokens{ + ProviderID: "gitlab", AccessToken: "other-access", + ExpiresAt: time.Now().Add(time.Hour), UserID: "user-B", + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-w8", "gitlab", other)) + + out := captureWarnLogs(t) + all, err := s.GetAllUpstreamTokens(ctx, "sess-w8", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.Len(t, all, 1) + assertBindingExclusionWarn(t, out.String(), "sess-w8", "gitlab", "user_id") + }) + + t.Run("scenario 10: legacy row returned with asserted user logs the unverified-owner WARN", func(t *testing.T) { //nolint:paralleltest // captures slog default + s, mr := newTestRedisStorage(t) + t.Cleanup(func() { + _ = s.Close() + mr.Close() + }) + ctx := context.Background() + legacy := &UpstreamTokens{ + ProviderID: "github", AccessToken: "legacy-access", + ExpiresAt: time.Now().Add(time.Hour), + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-w10", "github", legacy)) + + out := captureWarnLogs(t) + got, err := s.GetUpstreamTokens(ctx, "sess-w10", "github", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + require.NotNil(t, got, "legacy rows are released per the empty-field rule") + // The caller asserted an owner the row cannot prove: the WARN names + // session and provider; it is NOT the exclusion WARN (nothing excluded). + assert.Contains(t, out.String(), "no recorded owner") + assert.Contains(t, out.String(), "sess-w10") + assert.Contains(t, out.String(), "github") + assert.NotContains(t, out.String(), "binding validation failed") + + out.Reset() + all, err := s.GetAllUpstreamTokens(ctx, "sess-w10", &ExpectedBinding{UserID: "user-A"}) + require.NoError(t, err) + assert.Contains(t, all, "github") + assert.Contains(t, out.String(), "no recorded owner") + assert.NotContains(t, out.String(), "binding validation failed") + }) + + t.Run("scenario 10b: plain legacy passthrough (nil expected, no ctx user) logs nothing", func(t *testing.T) { //nolint:paralleltest // captures slog default + s, mr := newTestRedisStorage(t) + t.Cleanup(func() { + _ = s.Close() + mr.Close() + }) + ctx := context.Background() + legacy := &UpstreamTokens{ + ProviderID: "github", AccessToken: "legacy-access", + ExpiresAt: time.Now().Add(time.Hour), + } + require.NoError(t, s.StoreUpstreamTokens(ctx, "sess-w10b", "github", legacy)) + + out := captureWarnLogs(t) + got, err := s.GetUpstreamTokens(ctx, "sess-w10b", "github", nil) + require.NoError(t, err) + require.NotNil(t, got) + assert.Empty(t, out.String(), "legacy passthrough without an asserted user must not log (hot-path spam)") + }) +} diff --git a/pkg/authserver/storage/redis_tokenenc_test.go b/pkg/authserver/storage/redis_tokenenc_test.go new file mode 100644 index 0000000000..63979a03c7 --- /dev/null +++ b/pkg/authserver/storage/redis_tokenenc_test.go @@ -0,0 +1,713 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Tests for envelope encryption of upstream tokens at rest +// (WithTokenEncryption). Security assertions here are real: raw Redis values +// are inspected for plaintext leakage, AAD binding, rotation convergence, and +// fail-loud behavior with encryption disabled. + +//nolint:paralleltest // parallel execution handled by withRedisStorage helper +package storage + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/alicebob/miniredis/v2/server" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/authserver/tokenenc" +) + +// encTestKey returns a deterministic 32-byte KEK for tests. +func encTestKey(seed byte) []byte { + key := make([]byte, 32) + for i := range key { + key[i] = seed + byte(i) + } + return key +} + +func newEncTestKeyring(t *testing.T, activeID string, keys map[string][]byte) tokenenc.Keyring { + t.Helper() + kr, err := tokenenc.NewStaticKeyring(activeID, keys) + require.NoError(t, err) + return kr +} + +// newEncryptedTestRedisStorage returns miniredis-backed storage with envelope +// encryption enabled under a single key "k1", plus the raw client for direct +// value inspection. +func newEncryptedTestRedisStorage(t *testing.T) (*RedisStorage, *miniredis.Miniredis, *redis.Client) { + t.Helper() + mr := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + kr := newEncTestKeyring(t, "k1", map[string][]byte{"k1": encTestKey(1)}) + s := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(kr)) + return s, mr, client +} + +func withEncryptedRedisStorage(t *testing.T, fn func(context.Context, *RedisStorage, *miniredis.Miniredis, *redis.Client)) { + t.Helper() + t.Parallel() + s, mr, client := newEncryptedTestRedisStorage(t) + t.Cleanup(func() { + _ = s.Close() + mr.Close() + }) + fn(context.Background(), s, mr, client) +} + +// encTestTokens builds a token fixture with recognizable secret material. +func encTestTokens() *UpstreamTokens { + return &UpstreamTokens{ + ProviderID: "provider-a", + AccessToken: "enc-access-token-SECRET", + RefreshToken: "enc-refresh-token-SECRET", + IDToken: "enc-id-token-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-enc", + UpstreamSubject: "upstream-subject-enc", + ClientID: "client-enc", + } +} + +// rawUpstreamValue fetches the raw stored value for direct inspection. +func rawUpstreamValue(t *testing.T, ctx context.Context, client *redis.Client, key string) string { //nolint:revive // ctx after t is the test-helper convention used across this file + t.Helper() + raw, err := client.Get(ctx, key).Result() + require.NoError(t, err) + return raw +} + +func TestRedisStorage_TokenEncryption(t *testing.T) { + t.Parallel() + + // Matrix item 9: stored value is an envelope containing no token plaintext. + t.Run("raw value is envelope with no token substrings", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + tokens := encTestTokens() + require.NoError(t, s.StoreUpstreamTokens(ctx, "enc-session", "provider-a", tokens)) + + key := redisUpstreamKey(s.keyPrefix, "enc-session", "provider-a") + raw := rawUpstreamValue(t, ctx, client, key) + + assert.True(t, json.Valid([]byte(raw))) + assert.Contains(t, raw, `"v":1`) + assert.Contains(t, raw, `"kid":"k1"`) + assert.Contains(t, raw, `"edek"`) + assert.Contains(t, raw, `"ct"`) + assert.NotContains(t, raw, tokens.AccessToken) + assert.NotContains(t, raw, tokens.RefreshToken) + assert.NotContains(t, raw, tokens.IDToken) + assert.NotContains(t, raw, "access_token", "field names must not leak either") + assert.NotContains(t, raw, "refresh_token") + }) + }) + + // Matrix item 10: full round-trip through every read path. + t.Run("round-trip via all read paths", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, _ *redis.Client) { + tokens := encTestTokens() + require.NoError(t, s.StoreUpstreamTokens(ctx, "enc-session", "provider-a", tokens)) + require.NoError(t, s.StoreUpstreamTokens(ctx, "enc-session", "provider-b", &UpstreamTokens{ + ProviderID: "provider-b", + AccessToken: "enc-access-b-SECRET", + RefreshToken: "enc-refresh-b-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-enc", + })) + + got, err := s.GetUpstreamTokens(ctx, "enc-session", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, tokens.AccessToken, got.AccessToken) + assert.Equal(t, tokens.RefreshToken, got.RefreshToken) + assert.Equal(t, tokens.IDToken, got.IDToken) + assert.Equal(t, tokens.UserID, got.UserID) + assert.Equal(t, tokens.UpstreamSubject, got.UpstreamSubject) + assert.Equal(t, tokens.ClientID, got.ClientID) + + all, err := s.GetAllUpstreamTokens(ctx, "enc-session", nil) + require.NoError(t, err) + require.Len(t, all, 2) + assert.Equal(t, tokens.AccessToken, all["provider-a"].AccessToken) + assert.Equal(t, "enc-access-b-SECRET", all["provider-b"].AccessToken) + + latest, err := s.GetLatestUpstreamTokensForUser(ctx, "user-enc", "provider-a") + require.NoError(t, err) + assert.Equal(t, tokens.AccessToken, latest.AccessToken) + assert.Equal(t, tokens.RefreshToken, latest.RefreshToken) + }) + }) + + // Matrix item 10 (binding interplay): decrypt → unmarshal → binding check. + t.Run("binding validation applies to encrypted rows", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, _ *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "enc-session", "provider-a", encTestTokens())) + + _, err := s.GetUpstreamTokens(ctx, "enc-session", "provider-a", + &ExpectedBinding{UserID: "different-user"}) + assert.ErrorIs(t, err, ErrInvalidBinding) + + got, err := s.GetUpstreamTokens(ctx, "enc-session", "provider-a", + &ExpectedBinding{UserID: "user-enc", ClientID: "client-enc"}) + require.NoError(t, err) + assert.Equal(t, "enc-access-token-SECRET", got.AccessToken) + + // Mismatched rows are excluded from bulk reads, not fatal. + all, err := s.GetAllUpstreamTokens(ctx, "enc-session", + &ExpectedBinding{UserID: "different-user"}) + require.NoError(t, err) + assert.Empty(t, all) + }) + }) + + // Matrix item 11: legacy plaintext rows remain readable with encryption on. + t.Run("legacy plaintext row readable with encryption on", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + // Inject a plaintext row directly, bypassing the sealing write path. + key := redisUpstreamKey(s.keyPrefix, "legacy-session", "provider-a") + legacy := `{"provider_id":"provider-a","access_token":"legacy-access-SECRET",` + + `"refresh_token":"legacy-refresh-SECRET","id_token":"","expires_at":0,` + + `"session_expires_at":0,"user_id":"legacy-user","upstream_subject":"","client_id":""}` + require.NoError(t, client.Set(ctx, key, legacy, 0).Err()) + + got, err := s.GetUpstreamTokens(ctx, "legacy-session", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, "legacy-access-SECRET", got.AccessToken) + assert.Equal(t, "legacy-refresh-SECRET", got.RefreshToken) + assert.Equal(t, "legacy-user", got.UserID) + + // The next write seals the row. + require.NoError(t, s.StoreUpstreamTokens(ctx, "legacy-session", "provider-a", &UpstreamTokens{ + ProviderID: "provider-a", + AccessToken: "rewritten-SECRET", + RefreshToken: "rewritten-refresh-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "legacy-user", + })) + raw := rawUpstreamValue(t, ctx, client, key) + assert.Contains(t, raw, `"kid":"k1"`) + assert.NotContains(t, raw, "rewritten-SECRET") + }) + }) + + // Matrix item 12: lazy re-encryption on read after rotation. + t.Run("rotation: read of stale kid row re-seals with active key", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, mr *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "rot-session", "provider-a", encTestTokens())) + key := redisUpstreamKey(s.keyPrefix, "rot-session", "provider-a") + assert.Contains(t, rawUpstreamValue(t, ctx, client, key), `"kid":"k1"`) + + // Rotate: k2 active, k1 retired. + kr2 := newEncTestKeyring(t, "k2", map[string][]byte{"k1": encTestKey(1), "k2": encTestKey(2)}) + rotated := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(kr2)) + + got, err := rotated.GetUpstreamTokens(ctx, "rot-session", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, "enc-access-token-SECRET", got.AccessToken) + + // The row was lazily re-sealed under k2, TTL preserved. + raw := rawUpstreamValue(t, ctx, client, key) + assert.Contains(t, raw, `"kid":"k2"`) + assert.NotContains(t, raw, "enc-access-token-SECRET") + assert.Greater(t, mr.TTL(key), time.Duration(0), "lazy re-seal must preserve the row TTL") + + // A keyring that only knows k2 can now read the row. + kr2only := newEncTestKeyring(t, "k2", map[string][]byte{"k2": encTestKey(2)}) + converged := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(kr2only)) + got, err = converged.GetUpstreamTokens(ctx, "rot-session", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, "enc-refresh-token-SECRET", got.RefreshToken) + }) + }) + + // Matrix item 12 (bulk path): lazy re-encryption also fires on bulk and user reads. + t.Run("rotation converges via bulk and user reads", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "rot-bulk", "provider-a", encTestTokens())) + key := redisUpstreamKey(s.keyPrefix, "rot-bulk", "provider-a") + + kr2 := newEncTestKeyring(t, "k2", map[string][]byte{"k1": encTestKey(1), "k2": encTestKey(2)}) + rotated := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(kr2)) + + all, err := rotated.GetAllUpstreamTokens(ctx, "rot-bulk", nil) + require.NoError(t, err) + require.Len(t, all, 1) + assert.Contains(t, rawUpstreamValue(t, ctx, client, key), `"kid":"k2"`) + }) + }) + + // Matrix item 13: encryption disabled + plaintext fleet = bit-for-bit current behavior. + t.Run("encryption disabled stores and reads plaintext", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, mr *miniredis.Miniredis) { + tokens := encTestTokens() + require.NoError(t, s.StoreUpstreamTokens(ctx, "plain-session", "provider-a", tokens)) + + key := redisUpstreamKey(s.keyPrefix, "plain-session", "provider-a") + raw, err := mr.Get(key) + require.NoError(t, err) + assert.Contains(t, raw, tokens.AccessToken, "disabled encryption must keep plaintext behavior") + assert.NotContains(t, raw, `"edek"`) + + got, err := s.GetUpstreamTokens(ctx, "plain-session", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, tokens.AccessToken, got.AccessToken) + }) + }) + + // Matrix item 14: encryption disabled + envelope value = hard error. + t.Run("encryption disabled with envelope value fails loudly", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "enc-session", "provider-a", encTestTokens())) + + // Same fleet, storage without a keyring. + plain := NewRedisStorageWithClient(client, "test:auth:") + + _, err := plain.GetUpstreamTokens(ctx, "enc-session", "provider-a", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "no keyring configured") + + // Bulk and user-index reads skip the row rather than returning garbage. + all, err := plain.GetAllUpstreamTokens(ctx, "enc-session", nil) + require.NoError(t, err) + assert.Empty(t, all) + + _, err = plain.GetLatestUpstreamTokensForUser(ctx, "user-enc", "provider-a") + requireRedisNotFoundError(t, err) + }) + }) + + // Matrix item 15: reverse-index cleanup extracts UserID from encrypted rows. + t.Run("deletes clean reverse index from encrypted rows", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "del-session", "provider-a", encTestTokens())) + require.NoError(t, s.StoreUpstreamTokens(ctx, "del-session", "provider-b", &UpstreamTokens{ + ProviderID: "provider-b", + AccessToken: "enc-b-SECRET", + RefreshToken: "enc-b-refresh-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-enc", + })) + + // DeleteUpstreamTokensForProvider cleans the user reverse index. + require.NoError(t, s.DeleteUpstreamTokensForProvider(ctx, "del-session", "provider-b")) + _, err := s.GetLatestUpstreamTokensForUser(ctx, "user-enc", "provider-b") + requireRedisNotFoundError(t, err) + + // DeleteUpstreamTokens (whole session) cleans it too. + require.NoError(t, s.DeleteUpstreamTokens(ctx, "del-session")) + _, err = s.GetLatestUpstreamTokensForUser(ctx, "user-enc", "provider-a") + requireRedisNotFoundError(t, err) + + userSetKey := redisSetKey(s.keyPrefix, KeyTypeUserUpstream, "user-enc") + members, err := client.SMembers(ctx, userSetKey).Result() + require.NoError(t, err) + assert.Empty(t, members) + }) + }) + + // Matrix item 16: corrupt ciphertext in a bulk read skips the row, keeps siblings. + t.Run("corrupt ciphertext skipped in bulk read", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "corrupt-session", "provider-a", encTestTokens())) + require.NoError(t, s.StoreUpstreamTokens(ctx, "corrupt-session", "provider-b", &UpstreamTokens{ + ProviderID: "provider-b", + AccessToken: "enc-b-SECRET", + RefreshToken: "enc-b-refresh-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-enc", + })) + + // Corrupt provider-a's envelope value in place. + badKey := redisUpstreamKey(s.keyPrefix, "corrupt-session", "provider-a") + raw := rawUpstreamValue(t, ctx, client, badKey) + corrupted := strings.Replace(raw, `"ct":"`, `"ct":"AAAA`, 1) + require.NotEqual(t, raw, corrupted) + require.NoError(t, client.Set(ctx, badKey, corrupted, 0).Err()) + + all, err := s.GetAllUpstreamTokens(ctx, "corrupt-session", nil) + require.NoError(t, err) + require.Len(t, all, 1, "corrupt row skipped, sibling returned") + assert.Equal(t, "enc-b-SECRET", all["provider-b"].AccessToken) + + // Point read of the corrupt row errors. + _, err = s.GetUpstreamTokens(ctx, "corrupt-session", "provider-a", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to decrypt upstream tokens") + + // User-index read skips the corrupt row, finds the sibling. + latest, err := s.GetLatestUpstreamTokensForUser(ctx, "user-enc", "provider-b") + require.NoError(t, err) + assert.Equal(t, "enc-b-SECRET", latest.AccessToken) + }) + }) + + // Matrix items 4 + 16 at storage level: AAD binds ciphertext to its key. + t.Run("row copied to another key fails decryption", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "victim-session", "provider-a", encTestTokens())) + srcKey := redisUpstreamKey(s.keyPrefix, "victim-session", "provider-a") + raw := rawUpstreamValue(t, ctx, client, srcKey) + + // Attacker with Redis write access copies the row to their own key. + attackerKey := redisUpstreamKey(s.keyPrefix, "attacker-session", "provider-a") + require.NoError(t, client.Set(ctx, attackerKey, raw, 0).Err()) + + _, err := s.GetUpstreamTokens(ctx, "attacker-session", "provider-a", nil) + require.Error(t, err, "AAD binding must defeat cut-and-paste row copies") + assert.Contains(t, err.Error(), "failed to decrypt upstream tokens") + }) + }) + + // Matrix items 6 + 17: tombstones stay verbatim "null" with encryption on. + t.Run("tombstone written verbatim and read as nil with encryption on", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "tomb-session", "provider-a", nil)) + + key := redisUpstreamKey(s.keyPrefix, "tomb-session", "provider-a") + raw := rawUpstreamValue(t, ctx, client, key) + assert.Equal(t, nullMarker, raw, "tombstone must be written verbatim, never sealed") + + got, err := s.GetUpstreamTokens(ctx, "tomb-session", "provider-a", nil) + require.NoError(t, err) + assert.Nil(t, got) + + // Tombstone in a bulk read yields a nil-tokens entry and no error, + // matching the plaintext-fleet behavior. + all, err := s.GetAllUpstreamTokens(ctx, "tomb-session", nil) + require.NoError(t, err) + require.Contains(t, all, "provider-a") + assert.Nil(t, all["provider-a"]) + }) + }) + + // Matrix item 18: legacy key migration re-seals plaintext rows. + t.Run("legacy migration re-seals plaintext rows", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + // Plant a legacy-format key (no provider suffix) with a plaintext value. + legacyKey := redisKey(s.keyPrefix, KeyTypeUpstream, "migrate-session") + legacy := `{"provider_id":"oidc","access_token":"migrate-access-SECRET",` + + `"refresh_token":"migrate-refresh-SECRET","id_token":"","expires_at":0,` + + `"session_expires_at":0,"user_id":"migrate-user","upstream_subject":"","client_id":""}` + require.NoError(t, client.Set(ctx, legacyKey, legacy, 0).Err()) + + require.NoError(t, s.MigrateLegacyUpstreamData(ctx, "provider-a", "oidc")) + + newKey := redisUpstreamKey(s.keyPrefix, "migrate-session", "provider-a") + raw := rawUpstreamValue(t, ctx, client, newKey) + assert.Contains(t, raw, `"kid":"k1"`, "migrated row must be sealed") + assert.NotContains(t, raw, "migrate-access-SECRET") + assert.NotContains(t, raw, "migrate-refresh-SECRET") + + // The migrated row reads back through the decrypting path. + got, err := s.GetUpstreamTokens(ctx, "migrate-session", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, "migrate-access-SECRET", got.AccessToken) + assert.Equal(t, "migrate-refresh-SECRET", got.RefreshToken) + assert.Equal(t, "provider-a", got.ProviderID) + assert.Equal(t, "migrate-user", got.UserID) + }) + }) + + // The rewrite path (old-user reverse-index cleanup) works across encrypted rows. + t.Run("rewrite from user1 to user2 cleans old reverse index", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, _ *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "rewrite-session", "provider-a", &UpstreamTokens{ + ProviderID: "provider-a", + AccessToken: "first-SECRET", + RefreshToken: "first-refresh-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-old", + })) + require.NoError(t, s.StoreUpstreamTokens(ctx, "rewrite-session", "provider-a", &UpstreamTokens{ + ProviderID: "provider-a", + AccessToken: "second-SECRET", + RefreshToken: "second-refresh-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-new", + })) + + // The old user's reverse index no longer references the row. + _, err := s.GetLatestUpstreamTokensForUser(ctx, "user-old", "provider-a") + requireRedisNotFoundError(t, err) + + latest, err := s.GetLatestUpstreamTokensForUser(ctx, "user-new", "provider-a") + require.NoError(t, err) + assert.Equal(t, "second-SECRET", latest.AccessToken) + }) + }) +} + +// TestRedisStorage_MigrateUpstreamTokenEncryption covers the optional +// operator sweep that converges plaintext and stale-kid rows immediately. +func TestRedisStorage_MigrateUpstreamTokenEncryption(t *testing.T) { + t.Parallel() + + t.Run("sweep seals plaintext and re-seals stale kid rows", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + // One encrypted row (k1) and one plaintext row. + require.NoError(t, s.StoreUpstreamTokens(ctx, "sweep-a", "provider-a", encTestTokens())) + plainKey := redisUpstreamKey(s.keyPrefix, "sweep-b", "provider-a") + legacy := `{"provider_id":"provider-a","access_token":"sweep-plain-SECRET",` + + `"refresh_token":"sweep-plain-refresh-SECRET","id_token":"","expires_at":0,` + + `"session_expires_at":0,"user_id":"sweep-user","upstream_subject":"","client_id":""}` + require.NoError(t, client.Set(ctx, plainKey, legacy, 0).Err()) + + // Rotate to k2, then sweep. + kr2 := newEncTestKeyring(t, "k2", map[string][]byte{"k1": encTestKey(1), "k2": encTestKey(2)}) + rotated := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(kr2)) + require.NoError(t, rotated.MigrateUpstreamTokenEncryption(ctx)) + + for _, key := range []string{ + redisUpstreamKey(s.keyPrefix, "sweep-a", "provider-a"), + plainKey, + } { + raw := rawUpstreamValue(t, ctx, client, key) + assert.Contains(t, raw, `"kid":"k2"`, "key %s must converge to k2", key) + } + raw := rawUpstreamValue(t, ctx, client, plainKey) + assert.NotContains(t, raw, "sweep-plain-SECRET") + + // Both rows read back correctly. + got, err := rotated.GetUpstreamTokens(ctx, "sweep-a", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, "enc-access-token-SECRET", got.AccessToken) + got, err = rotated.GetUpstreamTokens(ctx, "sweep-b", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, "sweep-plain-SECRET", got.AccessToken) + + // Sweep is idempotent. + require.NoError(t, rotated.MigrateUpstreamTokenEncryption(ctx)) + }) + }) + + t.Run("sweep is a no-op without a keyring", func(t *testing.T) { + withRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "noop", "provider-a", encTestTokens())) + require.NoError(t, s.MigrateUpstreamTokenEncryption(ctx)) + + key := redisUpstreamKey(s.keyPrefix, "noop", "provider-a") + require.NoError(t, s.MigrateUpstreamTokenEncryption(ctx)) + raw, err := s.client.Get(ctx, key).Result() + require.NoError(t, err) + assert.Contains(t, raw, "enc-access-token-SECRET", "plaintext fleet untouched") + }) + }) + + t.Run("sweep skips tombstones and unknown-kid rows without failing", func(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "sweep-tomb", "provider-a", nil)) + + // Row sealed under a key the sweep's keyring does not know. + foreignKR := newEncTestKeyring(t, "kz", map[string][]byte{"kz": encTestKey(9)}) + foreign := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(foreignKR)) + require.NoError(t, foreign.StoreUpstreamTokens(ctx, "sweep-foreign", "provider-a", encTestTokens())) + + err := s.MigrateUpstreamTokenEncryption(ctx) + require.Error(t, err, "undecryptable rows are reported") + + // Tombstone survived verbatim. + raw := rawUpstreamValue(t, ctx, client, redisUpstreamKey(s.keyPrefix, "sweep-tomb", "provider-a")) + assert.Equal(t, nullMarker, raw) + }) + }) +} + +// TestRedisStorage_TokenEncryption_UserIDCleanupWarns documents that a row +// whose UserID cannot be extracted (undecryptable under the current keyring) +// still deletes cleanly — the reverse-index cleanup is skipped, not fatal. +func TestRedisStorage_TokenEncryption_UserIDCleanupWarns(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "warn-session", "provider-a", encTestTokens())) + + // Storage that cannot decrypt the row (different key). + otherKR := newEncTestKeyring(t, "kx", map[string][]byte{"kx": encTestKey(8)}) + other := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(otherKR)) + + require.NoError(t, other.DeleteUpstreamTokensForProvider(ctx, "warn-session", "provider-a"), + "delete must succeed even when UserID extraction fails") + + _, err := s.GetUpstreamTokens(ctx, "warn-session", "provider-a", nil) + requireRedisNotFoundError(t, err) + }) +} + +// TestRedisStorage_TokenEncryption_IndexTTLInvariant pins that the extra +// read in the encrypted write path does not disturb the Lua script's index +// TTL bookkeeping. +func TestRedisStorage_TokenEncryption_IndexTTLInvariant(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, mr *miniredis.Miniredis, _ *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "ttl-session", "provider-expiring", &UpstreamTokens{ + ProviderID: "provider-expiring", + AccessToken: "ttl-expiring-SECRET", + RefreshToken: "ttl-refresh-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + })) + require.NoError(t, s.StoreUpstreamTokens(ctx, "ttl-session", "provider-nonexpiring", &UpstreamTokens{ + ProviderID: "provider-nonexpiring", + AccessToken: "ttl-nonexpiring-SECRET", + })) + + idxKey := redisSetKey(s.keyPrefix, KeyTypeUpstreamIdx, "ttl-session") + assert.Equal(t, time.Duration(0), mr.TTL(idxKey), + "index set must go persistent when a non-expiring member is added") + + memberKey := redisUpstreamKey(s.keyPrefix, "ttl-session", "provider-expiring") + assert.Greater(t, mr.TTL(memberKey), time.Duration(0)) + }) +} + +// TestRedisStorage_TokenEncryption_ExpiredTokens checks ErrExpired still +// surfaces with decrypted rows (refresh path must keep working). +func TestRedisStorage_TokenEncryption_ExpiredTokens(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, _ *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "expired-enc", "provider-a", &UpstreamTokens{ + ProviderID: "provider-a", + AccessToken: "expired-enc-access-SECRET", + RefreshToken: "expired-enc-refresh-SECRET", + ExpiresAt: time.Now().Add(-time.Hour), + })) + + got, err := s.GetUpstreamTokens(ctx, "expired-enc", "provider-a", nil) + assert.ErrorIs(t, err, ErrExpired) + require.NotNil(t, got) + assert.Equal(t, "expired-enc-refresh-SECRET", got.RefreshToken) + }) +} + +// TestRedisStorage_TokenEncryption_LazyResealPreservesNoExpiry pins the +// PTTL == -1 branch: non-expiring rows stay non-expiring after lazy re-seal. +func TestRedisStorage_TokenEncryption_LazyResealPreservesNoExpiry(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, mr *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "noexp-rot", "provider-a", &UpstreamTokens{ + ProviderID: "provider-a", + AccessToken: "noexp-access-SECRET", + RefreshToken: "noexp-refresh-SECRET", + })) + key := redisUpstreamKey(s.keyPrefix, "noexp-rot", "provider-a") + require.Equal(t, time.Duration(0), mr.TTL(key)) + + kr2 := newEncTestKeyring(t, "k2", map[string][]byte{"k1": encTestKey(1), "k2": encTestKey(2)}) + rotated := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(kr2)) + _, err := rotated.GetUpstreamTokens(ctx, "noexp-rot", "provider-a", nil) + require.NoError(t, err) + + assert.Contains(t, rawUpstreamValue(t, ctx, client, key), `"kid":"k2"`) + assert.Equal(t, time.Duration(0), mr.TTL(key), "non-expiring row must stay non-expiring after re-seal") + }) +} + +// TestRedisStorage_TokenEncryption_BindingBeforeExpiry pins the ordering in an +// encrypted fleet: an expired AND binding-mismatched row must surface +// ErrInvalidBinding (not ErrExpired) and release no refresh material — the +// refresh path must never be entered for a row that isn't the caller's. +func TestRedisStorage_TokenEncryption_BindingBeforeExpiry(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, _ *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "bind-exp", "provider-a", &UpstreamTokens{ + ProviderID: "provider-a", + AccessToken: "stale-enc-access-SECRET", + RefreshToken: "stale-enc-refresh-SECRET", + ExpiresAt: time.Now().Add(-time.Hour), + UserID: "user-enc", + ClientID: "client-enc", + })) + + got, err := s.GetUpstreamTokens(ctx, "bind-exp", "provider-a", + &ExpectedBinding{UserID: "different-user"}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidBinding) + assert.NotErrorIs(t, err, ErrExpired, "binding must win over expiry") + assert.Nil(t, got, "no refresh material may be released for a mismatched row") + + // Control: the same row with the matching binding surfaces ErrExpired + // and its refresh material, proving the ordering above is what refused + // the mismatched read. + got, err = s.GetUpstreamTokens(ctx, "bind-exp", "provider-a", + &ExpectedBinding{UserID: "user-enc", ClientID: "client-enc"}) + assert.ErrorIs(t, err, ErrExpired) + require.NotNil(t, got) + assert.Equal(t, "stale-enc-refresh-SECRET", got.RefreshToken) + }) +} + +// TestRedisStorage_TokenEncryption_LazyResealFailure makes the post-read +// re-SET fail and pins the degraded behavior: the WARN fires but the read +// still returns the decrypted tokens (lazy re-seal is best-effort; the next +// read retries it). +// +//nolint:paralleltest // captures slog default +func TestRedisStorage_TokenEncryption_LazyResealFailure(t *testing.T) { + mr := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { + _ = client.Close() + mr.Close() + }) + ctx := context.Background() + + kr1 := newEncTestKeyring(t, "k1", map[string][]byte{"k1": encTestKey(1)}) + s := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(kr1)) + require.NoError(t, s.StoreUpstreamTokens(ctx, "reseal-fail", "provider-a", encTestTokens())) + key := redisUpstreamKey(s.keyPrefix, "reseal-fail", "provider-a") + + // Rotate: reads through this instance want to re-seal k1 rows under k2. + kr2 := newEncTestKeyring(t, "k2", map[string][]byte{"k1": encTestKey(1), "k2": encTestKey(2)}) + rotated := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(kr2)) + + // Fail only the post-read re-SET: PTTL must keep working so lazyReseal + // reaches the SET branch. + out := captureWarnLogs(t) + mr.Server().SetPreHook(func(c *server.Peer, cmd string, _ ...string) bool { + if strings.EqualFold(cmd, "set") { + c.WriteError("write disabled") + return true + } + return false + }) + t.Cleanup(func() { mr.Server().SetPreHook(nil) }) + + got, err := rotated.GetUpstreamTokens(ctx, "reseal-fail", "provider-a", nil) + require.NoError(t, err, "a failed lazy re-seal must not fail the read") + assert.Equal(t, "enc-access-token-SECRET", got.AccessToken) + assert.Contains(t, out.String(), "lazy token re-encryption failed") + assert.Contains(t, out.String(), key) + + // The row was NOT re-sealed (still under k1) — the next read retries. + assert.Contains(t, rawUpstreamValue(t, ctx, client, key), `"kid":"k1"`) +} + +// TestRedisStorage_TokenEncryption_SecondInstanceSameKeyring verifies two +// storage instances sharing a fleet and keyring interoperate (multi-replica +// auth-server deployments). +func TestRedisStorage_TokenEncryption_SecondInstanceSameKeyring(t *testing.T) { + withEncryptedRedisStorage(t, func(ctx context.Context, s *RedisStorage, _ *miniredis.Miniredis, client *redis.Client) { + require.NoError(t, s.StoreUpstreamTokens(ctx, "shared-session", "provider-a", encTestTokens())) + + replicaKR := newEncTestKeyring(t, "k1", map[string][]byte{"k1": encTestKey(1)}) + replica := NewRedisStorageWithClient(client, "test:auth:", WithTokenEncryption(replicaKR)) + + got, err := replica.GetUpstreamTokens(ctx, "shared-session", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, "enc-access-token-SECRET", got.AccessToken) + + // Writes from the replica are readable by the first instance. + require.NoError(t, replica.StoreUpstreamTokens(ctx, "shared-session", "provider-a", &UpstreamTokens{ + ProviderID: "provider-a", + AccessToken: fmt.Sprintf("replica-written-%s", "SECRET"), + RefreshToken: "replica-refresh-SECRET", + ExpiresAt: time.Now().Add(time.Hour), + UserID: "user-enc", + })) + got, err = s.GetUpstreamTokens(ctx, "shared-session", "provider-a", nil) + require.NoError(t, err) + assert.Equal(t, "replica-written-SECRET", got.AccessToken) + }) +} diff --git a/pkg/authserver/storage/types.go b/pkg/authserver/storage/types.go index 7b236d6cd6..8475aefc07 100644 --- a/pkg/authserver/storage/types.go +++ b/pkg/authserver/storage/types.go @@ -52,6 +52,29 @@ var ( // DefaultPendingAuthorizationTTL is the default TTL for pending authorization requests. const DefaultPendingAuthorizationTTL = 10 * time.Minute +// ExpectedBinding carries the caller-asserted identity a stored upstream-token +// row must match before it is released. A nil *ExpectedBinding means "resolve +// the user from ctx only" (see checkUpstreamBinding); nil/empty fields skip +// that dimension for backward compatibility with rows written before binding +// fields existed. +type ExpectedBinding struct { + // UserID is the internal ToolHive user the row must belong to. When empty, + // the canonical platform user resolved from ctx is used instead (if any). + UserID string + // ClientID is the OAuth client that must have consented the row. + ClientID string + // UpstreamSubject is the upstream IdP subject the row must carry. Asserted + // only where the caller knows the expected subject (e.g. the chain-consistency + // read of the first leg). + UpstreamSubject string + // Strict fails closed on legacy rows: when true, a stored row with an empty + // UserID fails with ErrInvalidBinding instead of being released under the + // permissive empty-field rule. Wave 1 sets this for untrusted workloads, + // where a row that cannot prove its owner must never be released. + // Default false preserves the legacy permissive behavior. + Strict bool +} + // UpstreamTokens represents tokens obtained from an upstream Identity Provider. // These tokens are stored with binding fields for security validation and // ProviderID for multi-IDP support. @@ -559,16 +582,26 @@ type UpstreamTokenStorage interface { // GetUpstreamTokens retrieves the upstream IDP tokens for a session and provider. // Returns ErrNotFound if the session/provider combination does not exist. + // Returns ErrInvalidBinding (with nil tokens) if the stored row's binding does + // not match the expected binding — see ExpectedBinding for the per-dimension + // empty-field rule and the ctx fallback. Binding is checked before expiry, so + // a mismatched row never surfaces ErrExpired: the refresh path must not be + // entered for a row that isn't the caller's. // Returns ErrExpired if the tokens have expired. When ErrExpired is returned, // the token data (including refresh token) is also returned to allow callers // to attempt a token refresh. - // Returns ErrInvalidBinding if binding validation fails. - GetUpstreamTokens(ctx context.Context, sessionID, providerName string) (*UpstreamTokens, error) + GetUpstreamTokens( + ctx context.Context, sessionID, providerName string, expected *ExpectedBinding, + ) (*UpstreamTokens, error) // GetAllUpstreamTokens retrieves all upstream IDP tokens for a session across all providers. // Returns a map of providerName -> tokens. Returns an empty map (not error) for unknown sessions. // Includes expired tokens (no expiry filtering at bulk-read level). - GetAllUpstreamTokens(ctx context.Context, sessionID string) (map[string]*UpstreamTokens, error) + // Binding validation (see ExpectedBinding) applies per row: a row whose binding + // fails is excluded from the result map (a WARN is logged) rather than failing + // the whole read — callers treat a missing provider as "needs consent", which + // is the safe degradation. + GetAllUpstreamTokens(ctx context.Context, sessionID string, expected *ExpectedBinding) (map[string]*UpstreamTokens, error) // DeleteUpstreamTokens removes all upstream IDP tokens for a session (all providers). // Returns ErrNotFound if the session does not exist. diff --git a/pkg/authserver/tokenenc/tokenenc.go b/pkg/authserver/tokenenc/tokenenc.go new file mode 100644 index 0000000000..1922787c2b --- /dev/null +++ b/pkg/authserver/tokenenc/tokenenc.go @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package tokenenc implements envelope encryption for upstream OAuth tokens +// stored at rest in Redis. +// +// It lives at pkg/authserver/tokenenc (not under storage/internal) because two +// sibling packages must reference the Keyring type: pkg/authserver/storage +// (seal/open on the Redis value path) and pkg/authserver/runner (keyring +// construction from serializable config). Callers should treat this package +// as an implementation detail of the auth-server storage layer, not a +// general-purpose encryption API. +// +// Scheme: a fresh random 256-bit data encryption key (DEK) is generated per +// sealed record. The plaintext is encrypted with AES-256-GCM under the DEK, +// and the DEK is itself encrypted ("wrapped") under a key-encryption key (KEK) +// resolved from a Keyring. The stored value is a JSON envelope: +// +// {"v":1,"kid":"k1","edek":"","ct":""} +// +// The Redis key under which the envelope is stored is fed as AES-GCM +// additional authenticated data on the payload ciphertext, cryptographically +// binding each ciphertext to its key: copying a value to another key fails +// decryption, defeating cut-and-paste row swaps. +// +// Values that predate encryption (legacy plaintext JSON) are detected by +// envelope-shape inspection and passed through untouched, enabling +// read-old/write-new migration. +package tokenenc + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" +) + +const ( + // envelopeVersion is the only supported envelope format version. + envelopeVersion = 1 + + // keySize is the required KEK and DEK size in bytes (AES-256). + keySize = 32 +) + +// envelope is the on-the-wire representation of an encrypted token record. +// Nonce layout follows the pkg/secrets/aes convention: nonce|ciphertext|tag. +type envelope struct { + V int `json:"v"` + KID string `json:"kid"` + EDEK string `json:"edek"` + CT string `json:"ct"` +} + +// Keyring resolves key-encryption keys by ID. Exactly one key is active and +// encrypts new writes; retired keys remain available for decryption only, +// supporting lazy read-old/write-new rotation. +type Keyring interface { + // Active returns the ID and key used to encrypt new writes. + Active() (id string, key []byte) + // ByID resolves a key (active or retired) for decryption. The second + // return value reports whether the ID is known. + ByID(id string) (key []byte, ok bool) +} + +// staticKeyring is a Keyring backed by a fixed, startup-validated key set. +type staticKeyring struct { + activeID string + keys map[string][]byte +} + +// NewStaticKeyring builds a Keyring from configuration. It fails loudly on +// any misconfiguration: no keys, an empty or unknown active ID, duplicate +// handling, or any key that is not exactly 32 bytes. Key material is cloned +// so later mutation of the caller's slices or map cannot corrupt the keyring. +func NewStaticKeyring(activeID string, keys map[string][]byte) (Keyring, error) { + if len(keys) == 0 { + return nil, errors.New("token encryption: at least one key is required") + } + if activeID == "" { + return nil, errors.New("token encryption: active key ID is required") + } + if _, ok := keys[activeID]; !ok { + return nil, fmt.Errorf("token encryption: active key ID %q not present in key map", activeID) + } + + cloned := make(map[string][]byte, len(keys)) + for id, key := range keys { + if id == "" { + return nil, errors.New("token encryption: key ID cannot be empty") + } + if len(key) != keySize { + return nil, fmt.Errorf("token encryption: key %q must be %d bytes, got %d", id, keySize, len(key)) + } + // An all-zero KEK silently encrypts every row with a public constant — + // worse than plaintext because it looks protected. Reject it. + allZero := true + for _, b := range key { + if b != 0 { + allZero = false + break + } + } + if allZero { + return nil, fmt.Errorf("token encryption: key %q must not be all zero bytes", id) + } + cloned[id] = append([]byte(nil), key...) + } + + return &staticKeyring{activeID: activeID, keys: cloned}, nil +} + +// Active returns the active key ID and key. +func (k *staticKeyring) Active() (string, []byte) { + return k.activeID, k.keys[k.activeID] +} + +// ByID resolves a key by ID (active or retired). +func (k *staticKeyring) ByID(id string) ([]byte, bool) { + key, ok := k.keys[id] + return key, ok +} + +// Seal encrypts plaintext for storage under redisKey. The redisKey is bound +// to the ciphertext as AES-GCM additional authenticated data. The returned +// value is a JSON envelope safe to store as the Redis value. +// +// The Keyring must be non-nil; a nil keyring is a programming error and +// returns an error rather than silently storing plaintext. +func Seal(kr Keyring, redisKey string, plaintext []byte) ([]byte, error) { + if kr == nil { + return nil, errors.New("token encryption: keyring is required") + } + + // Fresh random DEK per record. + dek := make([]byte, keySize) + if _, err := io.ReadFull(rand.Reader, dek); err != nil { + return nil, fmt.Errorf("token encryption: failed to generate data key: %w", err) + } + + activeID, kek := kr.Active() + + edek, err := gcmSeal(kek, dek, nil) + if err != nil { + return nil, fmt.Errorf("token encryption: failed to wrap data key: %w", err) + } + + ct, err := gcmSeal(dek, plaintext, []byte(redisKey)) + if err != nil { + return nil, fmt.Errorf("token encryption: failed to encrypt record: %w", err) + } + + env := envelope{ + V: envelopeVersion, + KID: activeID, + EDEK: base64.StdEncoding.EncodeToString(edek), + CT: base64.StdEncoding.EncodeToString(ct), + } + data, err := json.Marshal(env) + if err != nil { + return nil, fmt.Errorf("token encryption: failed to marshal envelope: %w", err) + } + return data, nil +} + +// Open decrypts an envelope previously produced by Seal. redisKey must be the +// key the value is stored under — it is verified as AAD, so a value copied to +// a different key fails decryption. +// +// If value is not an envelope (it is legacy plaintext JSON from before +// encryption was enabled), Open returns it unchanged with legacy=true so the +// caller can unmarshal it directly. The "null" deletion tombstone carries no +// secret and is never sealed; callers must check for it before calling Open. +// +// A nil keyring means encryption is disabled: plaintext passes through +// (legacy=true), but an actual envelope is a hard error — "disabling" +// encryption on an encrypted fleet must fail loudly, never return corrupt +// tokens. +func Open(kr Keyring, redisKey string, value []byte) (plaintext []byte, legacy bool, err error) { + var env envelope + if jsonErr := json.Unmarshal(value, &env); jsonErr != nil || + env.V != envelopeVersion || env.KID == "" || env.EDEK == "" || env.CT == "" { + // Not an envelope: legacy plaintext JSON. + return value, true, nil + } + + if kr == nil { + return nil, false, errors.New("token encryption: encrypted value found but no keyring configured") + } + + kek, ok := kr.ByID(env.KID) + if !ok { + return nil, false, fmt.Errorf("token encryption: unknown key ID %q", env.KID) + } + + edek, err := base64.StdEncoding.DecodeString(env.EDEK) + if err != nil { + return nil, false, fmt.Errorf("token encryption: malformed wrapped data key: %w", err) + } + dek, err := gcmOpen(kek, edek, nil) + if err != nil { + return nil, false, fmt.Errorf("token encryption: failed to unwrap data key: %w", err) + } + + ct, err := base64.StdEncoding.DecodeString(env.CT) + if err != nil { + return nil, false, fmt.Errorf("token encryption: malformed ciphertext: %w", err) + } + plaintext, err = gcmOpen(dek, ct, []byte(redisKey)) + if err != nil { + return nil, false, fmt.Errorf("token encryption: failed to decrypt record: %w", err) + } + + return plaintext, false, nil +} + +// IsLegacyValue reports whether value predates envelope encryption (i.e. Open +// would return it with legacy=true). Callers use it for migration decisions — +// e.g. sealing a plaintext row on rewrite — without attempting decryption. +func IsLegacyValue(value []byte) bool { + var env envelope + if jsonErr := json.Unmarshal(value, &env); jsonErr != nil || + env.V != envelopeVersion || env.KID == "" || env.EDEK == "" || env.CT == "" { + return true + } + return false +} + +// NeedsRotation reports whether value is an envelope sealed under a key ID +// other than the active one. Legacy plaintext and non-envelope values return +// false: their migration path is a full re-seal, not a rotation. A nil +// keyring reports false (rotation is meaningless when encryption is off). +func NeedsRotation(kr Keyring, value []byte) bool { + if kr == nil { + return false + } + var env envelope + if jsonErr := json.Unmarshal(value, &env); jsonErr != nil || + env.V != envelopeVersion || env.KID == "" || env.EDEK == "" || env.CT == "" { + return false + } + activeID, _ := kr.Active() + return env.KID != activeID +} + +// EnvelopeKeyID extracts the key ID from an envelope value for observability +// (never for security decisions). ok is false for non-envelope values. +func EnvelopeKeyID(value []byte) (kid string, ok bool) { + var env envelope + if jsonErr := json.Unmarshal(value, &env); jsonErr != nil || + env.V != envelopeVersion || env.KID == "" || env.EDEK == "" || env.CT == "" { + return "", false + } + return env.KID, true +} + +// gcmSeal encrypts plaintext with AES-256-GCM under key, returning +// nonce|ciphertext|tag. aad is additional authenticated data (may be nil). +func gcmSeal(key, plaintext, aad []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + return gcm.Seal(nonce, nonce, plaintext, aad), nil +} + +// gcmOpen reverses gcmSeal. +func gcmOpen(key, ciphertext, aad []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + if len(ciphertext) < gcm.NonceSize() { + return nil, errors.New("malformed ciphertext") + } + return gcm.Open(nil, ciphertext[:gcm.NonceSize()], ciphertext[gcm.NonceSize():], aad) +} diff --git a/pkg/authserver/tokenenc/tokenenc_test.go b/pkg/authserver/tokenenc/tokenenc_test.go new file mode 100644 index 0000000000..dcc463744f --- /dev/null +++ b/pkg/authserver/tokenenc/tokenenc_test.go @@ -0,0 +1,358 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package tokenenc + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testKey returns a deterministic 32-byte key derived from seed. +func testKey(seed byte) []byte { + key := make([]byte, 32) + for i := range key { + key[i] = seed + byte(i) + } + return key +} + +func newTestKeyring(t *testing.T, activeID string, keys map[string][]byte) Keyring { + t.Helper() + kr, err := NewStaticKeyring(activeID, keys) + require.NoError(t, err) + return kr +} + +func TestSealOpen_RoundTrip(t *testing.T) { + t.Parallel() + + kr := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + plaintext := []byte(`{"access_token":"secret-token-value","refresh_token":"refresh-secret"}`) + + sealed, err := Seal(kr, "test:auth:upstream:sess1:providerA", plaintext) + require.NoError(t, err) + + // The sealed value is an envelope and contains no plaintext material. + assert.Contains(t, string(sealed), `"v":1`) + assert.Contains(t, string(sealed), `"kid":"k1"`) + assert.NotContains(t, string(sealed), "secret-token-value") + assert.NotContains(t, string(sealed), "refresh-secret") + + opened, legacy, err := Open(kr, "test:auth:upstream:sess1:providerA", sealed) + require.NoError(t, err) + assert.False(t, legacy) + assert.Equal(t, plaintext, opened) +} + +func TestOpen_UnknownKeyID(t *testing.T) { + t.Parallel() + + sealKR := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + openKR := newTestKeyring(t, "k2", map[string][]byte{"k2": testKey(2)}) + + sealed, err := Seal(sealKR, "key1", []byte("payload")) + require.NoError(t, err) + + opened, legacy, err := Open(openKR, "key1", sealed) + require.Error(t, err) + assert.Contains(t, err.Error(), `unknown key ID "k1"`) + assert.Nil(t, opened, "no partial plaintext on failure") + assert.False(t, legacy) +} + +func TestOpen_TamperedCiphertext(t *testing.T) { + t.Parallel() + + kr := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + sealed, err := Seal(kr, "key1", []byte("payload")) + require.NoError(t, err) + + // Flip a byte inside the base64-decoded ct, re-encode. + var env map[string]any + require.NoError(t, json.Unmarshal(sealed, &env)) + ct, err := base64.StdEncoding.DecodeString(env["ct"].(string)) + require.NoError(t, err) + ct[len(ct)-1] ^= 0xff + env["ct"] = base64.StdEncoding.EncodeToString(ct) + tampered, err := json.Marshal(env) + require.NoError(t, err) + + opened, legacy, err := Open(kr, "key1", tampered) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to decrypt record") + assert.Nil(t, opened) + assert.False(t, legacy) +} + +func TestOpen_AADMismatch(t *testing.T) { + t.Parallel() + + kr := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + sealed, err := Seal(kr, "test:auth:upstream:sess1:providerA", []byte("payload")) + require.NoError(t, err) + + // Copying the value to another (session, provider) key must fail. + opened, legacy, err := Open(kr, "test:auth:upstream:sess2:providerB", sealed) + require.Error(t, err, "cut-and-paste to a different redis key must fail decryption") + assert.Nil(t, opened) + assert.False(t, legacy) +} + +func TestOpen_LegacyPlaintext(t *testing.T) { + t.Parallel() + + kr := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + legacy := []byte(`{"access_token":"tok","refresh_token":"ref","user_id":"u1"}`) + + opened, isLegacy, err := Open(kr, "key1", legacy) + require.NoError(t, err) + assert.True(t, isLegacy) + assert.Equal(t, legacy, opened, "legacy plaintext passes through untouched") + + // Envelope-lookalike JSON missing required fields is also legacy. + partial := []byte(`{"v":1,"kid":"k1"}`) + opened, isLegacy, err = Open(kr, "key1", partial) + require.NoError(t, err) + assert.True(t, isLegacy) + assert.Equal(t, partial, opened) + + // Non-JSON garbage is legacy (caller's unmarshal will fail downstream). + garbage := []byte("not json at all") + opened, isLegacy, err = Open(kr, "key1", garbage) + require.NoError(t, err) + assert.True(t, isLegacy) + assert.Equal(t, garbage, opened) +} + +func TestOpen_NilKeyring(t *testing.T) { + t.Parallel() + + kr := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + sealed, err := Seal(kr, "key1", []byte("payload")) + require.NoError(t, err) + + // Envelope with no keyring: hard error, never corrupt tokens. + _, _, err = Open(nil, "key1", sealed) + require.Error(t, err) + assert.Contains(t, err.Error(), "no keyring configured") + + // Plaintext with no keyring: passthrough (encryption disabled on plaintext fleet). + legacy := []byte(`{"access_token":"tok"}`) + opened, isLegacy, err := Open(nil, "key1", legacy) + require.NoError(t, err) + assert.True(t, isLegacy) + assert.Equal(t, legacy, opened) + + // Seal with no keyring: error. + _, err = Seal(nil, "key1", []byte("payload")) + require.Error(t, err) + assert.Contains(t, err.Error(), "keyring is required") +} + +func TestNewStaticKeyring_Validation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + activeID string + keys map[string][]byte + wantErr string + }{ + { + name: "zero keys", + activeID: "k1", + keys: map[string][]byte{}, + wantErr: "at least one key is required", + }, + { + name: "nil keys", + activeID: "k1", + keys: nil, + wantErr: "at least one key is required", + }, + { + name: "empty active ID", + activeID: "", + keys: map[string][]byte{"k1": testKey(1)}, + wantErr: "active key ID is required", + }, + { + name: "active ID not in map", + activeID: "k2", + keys: map[string][]byte{"k1": testKey(1)}, + wantErr: `active key ID "k2" not present`, + }, + { + name: "16-byte key rejected", + activeID: "k1", + keys: map[string][]byte{"k1": testKey(1)[:16]}, + wantErr: "must be 32 bytes, got 16", + }, + { + name: "empty key ID rejected", + activeID: "k1", + keys: map[string][]byte{"k1": testKey(1), "": testKey(2)}, + wantErr: "key ID cannot be empty", + }, + { + name: "all-zero KEK rejected", + activeID: "k1", + keys: map[string][]byte{"k1": make([]byte, 32)}, + wantErr: `key "k1" must not be all zero bytes`, + }, + { + name: "single non-zero byte is enough", + activeID: "k1", + keys: map[string][]byte{"k1": append(make([]byte, 31), 1)}, + wantErr: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + kr, err := NewStaticKeyring(tt.activeID, tt.keys) + if tt.wantErr == "" { + require.NoError(t, err) + require.NotNil(t, kr) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Nil(t, kr) + }) + } +} + +func TestNewStaticKeyring_ClonesInput(t *testing.T) { + t.Parallel() + + key := testKey(1) + keys := map[string][]byte{"k1": key} + kr := newTestKeyring(t, "k1", keys) + + // Mutating the caller's map/slice after construction must not corrupt the keyring. + keys["k1"][0] ^= 0xff + keys["k2"] = testKey(2) + + _, active := kr.Active() + assert.Equal(t, testKey(1), active) + _, ok := kr.ByID("k2") + assert.False(t, ok) +} + +func TestKeyring_RetiredKeyDecryptsNeverEncrypts(t *testing.T) { + t.Parallel() + + oldKey := testKey(1) + newKey := testKey(2) + + // Seal under k1 while it is active. + kr1 := newTestKeyring(t, "k1", map[string][]byte{"k1": oldKey}) + sealed, err := Seal(kr1, "key1", []byte("payload")) + require.NoError(t, err) + assert.Contains(t, string(sealed), `"kid":"k1"`) + + // Rotate: k2 active, k1 retired. Retired key still decrypts... + kr2 := newTestKeyring(t, "k2", map[string][]byte{"k1": oldKey, "k2": newKey}) + opened, legacy, err := Open(kr2, "key1", sealed) + require.NoError(t, err) + assert.False(t, legacy) + assert.Equal(t, []byte("payload"), opened) + + // ...but new writes use only the active key. + resealed, err := Seal(kr2, "key1", []byte("payload")) + require.NoError(t, err) + assert.Contains(t, string(resealed), `"kid":"k2"`) + assert.NotContains(t, string(resealed), `"kid":"k1"`) + + // k1 alone cannot open k2 envelopes (rotation actually re-keys). + kr1only := newTestKeyring(t, "k1", map[string][]byte{"k1": oldKey}) + _, _, err = Open(kr1only, "key1", resealed) + require.Error(t, err) +} + +func TestSeal_FreshDEKPerRecord(t *testing.T) { + t.Parallel() + + kr := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + plaintext := []byte("same plaintext") + + a, err := Seal(kr, "key1", plaintext) + require.NoError(t, err) + b, err := Seal(kr, "key1", plaintext) + require.NoError(t, err) + + assert.False(t, bytes.Equal(a, b), "identical plaintexts must not produce identical envelopes") +} + +func TestNeedsRotation(t *testing.T) { + t.Parallel() + + oldKR := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + sealedK1, err := Seal(oldKR, "key1", []byte("payload")) + require.NoError(t, err) + + rotatedKR := newTestKeyring(t, "k2", map[string][]byte{"k1": testKey(1), "k2": testKey(2)}) + sealedK2, err := Seal(rotatedKR, "key1", []byte("payload")) + require.NoError(t, err) + + assert.True(t, NeedsRotation(rotatedKR, sealedK1), "k1 envelope needs rotation when k2 is active") + assert.False(t, NeedsRotation(rotatedKR, sealedK2), "k2 envelope is current") + assert.False(t, NeedsRotation(rotatedKR, []byte(`{"access_token":"tok"}`)), "legacy plaintext is not a rotation case") + assert.False(t, NeedsRotation(nil, sealedK1), "nil keyring: rotation meaningless") +} + +func TestIsLegacyValue(t *testing.T) { + t.Parallel() + + kr := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + sealed, err := Seal(kr, "key1", []byte("payload")) + require.NoError(t, err) + + assert.False(t, IsLegacyValue(sealed)) + assert.True(t, IsLegacyValue([]byte(`{"access_token":"tok"}`))) + assert.True(t, IsLegacyValue([]byte("null"))) + assert.True(t, IsLegacyValue([]byte("garbage"))) +} + +func TestEnvelopeKeyID(t *testing.T) { + t.Parallel() + + kr := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) + sealed, err := Seal(kr, "key1", []byte("payload")) + require.NoError(t, err) + + kid, ok := EnvelopeKeyID(sealed) + assert.True(t, ok) + assert.Equal(t, "k1", kid) + + _, ok = EnvelopeKeyID([]byte(`{"access_token":"tok"}`)) + assert.False(t, ok) +} + +// TestSealOpen_RandomKeys guards against accidental dependence on the +// deterministic testKey helper: real random KEKs must round-trip too. +func TestSealOpen_RandomKeys(t *testing.T) { + t.Parallel() + + kek := make([]byte, 32) + _, err := rand.Read(kek) + require.NoError(t, err) + + kr := newTestKeyring(t, "k1", map[string][]byte{"k1": kek}) + sealed, err := Seal(kr, "key1", []byte("payload")) + require.NoError(t, err) + + opened, legacy, err := Open(kr, "key1", sealed) + require.NoError(t, err) + assert.False(t, legacy) + assert.Equal(t, []byte("payload"), opened) +} diff --git a/pkg/vmcp/auth/factory/incoming_upstream_test.go b/pkg/vmcp/auth/factory/incoming_upstream_test.go index 79e25c13c8..7844bdf535 100644 --- a/pkg/vmcp/auth/factory/incoming_upstream_test.go +++ b/pkg/vmcp/auth/factory/incoming_upstream_test.go @@ -111,7 +111,7 @@ func TestNewOIDCAuthMiddleware_UpstreamTokenReaderWiring(t *testing.T) { ctrl := gomock.NewController(t) reader := upstreamtokenmocks.NewMockTokenReader(ctrl) reader.EXPECT(). - GetAllUpstreamCredentials(gomock.Any(), "session-abc"). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{ "google": {AccessToken: "gcp-access-token", IDToken: "gcp-id-token"}, }, []string(nil), nil) From c08106886f2cb4eef9618f43c090ca349ba80c1d Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 22 Jul 2026 09:10:44 +0300 Subject: [PATCH 02/12] Add untrusted mode: CRD, single-tenant pods, egress broker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the untrusted-MCP-server architecture (ADR-0001) across three waves: an MCP server flagged untrusted never possesses a credential — the operator forbids secret-sourced env, injects sentinel literals, and every upstream call egresses through a per-pod Envoy sidecar that terminates TLS and injects the user's own OAuth credential at the boundary. Wave 1 - CRD surface: MCPServer.spec.untrusted and spec.egressPolicy (provider-keyed host/method/path constraints) with CEL admission rules: untrusted requires egress policy, group membership, no secrets, no podTemplateSpec, ClientIP affinity, no backendReplicas. Sentinel env injection with forgery/collision validation. Wave 2 - single-tenant lifecycle: vMCP session layer provisions one backend pod per (user, MCPServer) via a PodLifecycle that clones the operator-built StatefulSet template (fail-closed re-verification of secret material), deterministic pod names for cross-replica restore, Redis-ledger DoS admission (per-user quota, per-server and global caps, fail closed), and a reaper for idle/timeout/zombie GC. Session restore recreates missing pods with an extended budget. Wave 3 - egress broker: Envoy sidecar per untrusted pod with SDS TLS-bump (per-tenant ECDSA CA, generation-named rotation) and a Go ext_authz service that resolves pod-to-user from annotations and injects credentials only after destination binding (D5), never following redirects, with per-dial IP allowlisting (D7). Operator manages CA Secrets, policy ConfigMap, and default-deny NetworkPolicy (loopback + cluster DNS + resolved destinations only). Review-hardened: quota ledger verified, bootstrap SDS wiring proven, CA rotation made race-free, DNS egress pinned to cluster DNS, transient/terminal error split, contract package shared across the pod boundary. --- .github/workflows/operator-ci.yml | 2 + Taskfile.yml | 11 +- cmd/thv-egressbroker/main.go | 178 ++++ cmd/thv-operator/Taskfile.yml | 2 + .../api/v1beta1/mcpserver_types.go | 87 ++ .../api/v1beta1/zz_generated.deepcopy.go | 57 ++ .../controllers/mcpserver_controller.go | 91 +- .../controllers/mcpserver_runconfig.go | 11 + .../mcpserver_untrusted_env_test.go | 330 +++++++- .../mcpserver_untrusted_resources.go | 778 ++++++++++++++++++ .../mcpserver_untrusted_resources_test.go | 582 +++++++++++++ .../virtualmcpserver_deployment.go | 56 ++ .../virtualmcpserver_rbac_untrusted_test.go | 153 ++++ .../pkg/controllerutil/untrusted_sentinel.go | 176 ++++ .../controllerutil/untrusted_sentinel_test.go | 315 +++++++ ...cpserver_untrusted_cel_integration_test.go | 178 ++++ ...erver_untrusted_egress_integration_test.go | 250 ++++++ ...ver_untrusted_sentinel_integration_test.go | 191 +++++ .../mcp-server-untrusted/suite_test.go | 68 ++ .../test-integration/testutil/envtest.go | 2 + .../toolhive.stacklok.dev_mcpservers.yaml | 212 +++++ .../toolhive.stacklok.dev_mcpservers.yaml | 212 +++++ docs/operator/crd-api.md | 42 + go.mod | 14 +- go.sum | 8 + pkg/auth/upstreamtoken/strict_reader.go | 40 + pkg/auth/upstreamtoken/strict_reader_test.go | 131 +++ pkg/authserver/tokenenc/tokenenc.go | 11 - pkg/authserver/tokenenc/tokenenc_test.go | 15 - pkg/egressbroker/ca.go | 191 +++++ pkg/egressbroker/ca_test.go | 179 ++++ pkg/egressbroker/config.go | 153 ++++ pkg/egressbroker/config_test.go | 112 +++ pkg/egressbroker/dialcontrol.go | 100 +++ pkg/egressbroker/dialcontrol_test.go | 90 ++ pkg/egressbroker/envoyconfig.go | 243 ++++++ pkg/egressbroker/envoyconfig_test.go | 301 +++++++ pkg/egressbroker/identity.go | 220 +++++ pkg/egressbroker/identity_test.go | 118 +++ pkg/egressbroker/injector.go | 125 +++ pkg/egressbroker/injector_test.go | 221 +++++ pkg/egressbroker/policy.go | 273 ++++++ pkg/egressbroker/policy_test.go | 160 ++++ pkg/egressbroker/resolve.go | 112 +++ pkg/egressbroker/resolve_test.go | 134 +++ pkg/egressbroker/server.go | 266 ++++++ pkg/egressbroker/server_test.go | 231 ++++++ .../session/session_data_storage_redis.go | 17 + .../session/session_data_storage_test.go | 21 + pkg/vmcp/cli/serve.go | 42 +- pkg/vmcp/cli/untrusted.go | 217 +++++ pkg/vmcp/cli/untrusted_test.go | 105 +++ pkg/vmcp/server/serve.go | 2 + pkg/vmcp/server/serve_test.go | 1 + pkg/vmcp/server/server.go | 21 +- pkg/vmcp/server/session_manager_interface.go | 6 + pkg/vmcp/server/sessionmanager/factory.go | 22 + .../server/sessionmanager/session_manager.go | 122 ++- .../session_manager_untrusted_test.go | 306 +++++++ pkg/vmcp/session/factory.go | 90 +- pkg/vmcp/session/factory_untrusted_test.go | 199 +++++ pkg/vmcp/session/untrusted/addressing.go | 239 ++++++ .../untrusted/addressing_external_test.go | 226 +++++ pkg/vmcp/session/untrusted/admission.go | 169 ++++ pkg/vmcp/session/untrusted/admission_test.go | 150 ++++ pkg/vmcp/session/untrusted/egress.go | 281 +++++++ pkg/vmcp/session/untrusted/egress_test.go | 271 ++++++ pkg/vmcp/session/untrusted/lifecycle.go | 593 +++++++++++++ .../untrusted/lifecycle_integration_test.go | 265 ++++++ pkg/vmcp/session/untrusted/lifecycle_test.go | 384 +++++++++ pkg/vmcp/session/untrusted/metrics.go | 70 ++ pkg/vmcp/session/untrusted/metrics_test.go | 172 ++++ .../session/untrusted/mocks/mock_lifecycle.go | 126 +++ pkg/vmcp/session/untrusted/naming.go | 144 ++++ pkg/vmcp/session/untrusted/naming_test.go | 99 +++ pkg/vmcp/session/untrusted/reaper.go | 343 ++++++++ pkg/vmcp/session/untrusted/reaper_test.go | 231 ++++++ pkg/vmcp/session/untrusted/rediskeys.go | 48 ++ pkg/vmcp/session/untrusted/template_clone.go | 199 +++++ .../session/untrusted/template_clone_test.go | 280 +++++++ pkg/vmcp/session/untrusted/tokenstore.go | 64 ++ pkg/vmcp/session/untrusted/wiring.go | 134 +++ pkg/vmcp/workloads/k8s.go | 14 + pkg/vmcp/workloads/k8s_test.go | 48 ++ .../untrusted-egress/chainsaw-test.yaml | 196 +++++ .../mcpserver-untrusted/chainsaw-test.yaml | 238 ++++++ 86 files changed, 13519 insertions(+), 68 deletions(-) create mode 100644 cmd/thv-egressbroker/main.go create mode 100644 cmd/thv-operator/controllers/mcpserver_untrusted_resources.go create mode 100644 cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go create mode 100644 cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go create mode 100644 cmd/thv-operator/pkg/controllerutil/untrusted_sentinel.go create mode 100644 cmd/thv-operator/pkg/controllerutil/untrusted_sentinel_test.go create mode 100644 cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_cel_integration_test.go create mode 100644 cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_egress_integration_test.go create mode 100644 cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_sentinel_integration_test.go create mode 100644 cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go create mode 100644 pkg/auth/upstreamtoken/strict_reader.go create mode 100644 pkg/auth/upstreamtoken/strict_reader_test.go create mode 100644 pkg/egressbroker/ca.go create mode 100644 pkg/egressbroker/ca_test.go create mode 100644 pkg/egressbroker/config.go create mode 100644 pkg/egressbroker/config_test.go create mode 100644 pkg/egressbroker/dialcontrol.go create mode 100644 pkg/egressbroker/dialcontrol_test.go create mode 100644 pkg/egressbroker/envoyconfig.go create mode 100644 pkg/egressbroker/envoyconfig_test.go create mode 100644 pkg/egressbroker/identity.go create mode 100644 pkg/egressbroker/identity_test.go create mode 100644 pkg/egressbroker/injector.go create mode 100644 pkg/egressbroker/injector_test.go create mode 100644 pkg/egressbroker/policy.go create mode 100644 pkg/egressbroker/policy_test.go create mode 100644 pkg/egressbroker/resolve.go create mode 100644 pkg/egressbroker/resolve_test.go create mode 100644 pkg/egressbroker/server.go create mode 100644 pkg/egressbroker/server_test.go create mode 100644 pkg/vmcp/cli/untrusted.go create mode 100644 pkg/vmcp/cli/untrusted_test.go create mode 100644 pkg/vmcp/server/sessionmanager/session_manager_untrusted_test.go create mode 100644 pkg/vmcp/session/factory_untrusted_test.go create mode 100644 pkg/vmcp/session/untrusted/addressing.go create mode 100644 pkg/vmcp/session/untrusted/addressing_external_test.go create mode 100644 pkg/vmcp/session/untrusted/admission.go create mode 100644 pkg/vmcp/session/untrusted/admission_test.go create mode 100644 pkg/vmcp/session/untrusted/egress.go create mode 100644 pkg/vmcp/session/untrusted/egress_test.go create mode 100644 pkg/vmcp/session/untrusted/lifecycle.go create mode 100644 pkg/vmcp/session/untrusted/lifecycle_integration_test.go create mode 100644 pkg/vmcp/session/untrusted/lifecycle_test.go create mode 100644 pkg/vmcp/session/untrusted/metrics.go create mode 100644 pkg/vmcp/session/untrusted/metrics_test.go create mode 100644 pkg/vmcp/session/untrusted/mocks/mock_lifecycle.go create mode 100644 pkg/vmcp/session/untrusted/naming.go create mode 100644 pkg/vmcp/session/untrusted/naming_test.go create mode 100644 pkg/vmcp/session/untrusted/reaper.go create mode 100644 pkg/vmcp/session/untrusted/reaper_test.go create mode 100644 pkg/vmcp/session/untrusted/rediskeys.go create mode 100644 pkg/vmcp/session/untrusted/template_clone.go create mode 100644 pkg/vmcp/session/untrusted/template_clone_test.go create mode 100644 pkg/vmcp/session/untrusted/tokenstore.go create mode 100644 pkg/vmcp/session/untrusted/wiring.go create mode 100644 test/e2e/chainsaw/operator/untrusted-egress/chainsaw-test.yaml create mode 100644 test/e2e/chainsaw/operator/validation/mcpserver-untrusted/chainsaw-test.yaml diff --git a/.github/workflows/operator-ci.yml b/.github/workflows/operator-ci.yml index 9dd4142719..7a8e4361a5 100644 --- a/.github/workflows/operator-ci.yml +++ b/.github/workflows/operator-ci.yml @@ -222,3 +222,5 @@ jobs: chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/setup --config .chainsaw.yaml chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/test-scenarios --parallel 10 --config .chainsaw.yaml chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/cleanup --config .chainsaw.yaml + chainsaw test --test-dir test/e2e/chainsaw/operator/validation/mcpserver-untrusted --config .chainsaw.yaml + chainsaw test --test-dir test/e2e/chainsaw/operator/untrusted-egress --config .chainsaw.yaml diff --git a/Taskfile.yml b/Taskfile.yml index 58760ad42a..809b2db027 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -326,6 +326,13 @@ tasks: cmds: - docker build --load -t ghcr.io/stacklok/toolhive/egress-proxy:local containers/egress-proxy/ + build-egressbroker-image: + desc: Build the egress broker sidecar image with ko + env: + KO_DOCKER_REPO: ghcr.io/stacklok/toolhive/egressbroker + cmds: + - ko build --local --bare ./cmd/thv-egressbroker + build-all-images: - desc: Build all container images (main app, vmcp, and egress proxy) - deps: [build-image, build-vmcp-image, build-egress-proxy] + desc: Build all container images (main app, vmcp, egress proxy, and egress broker) + deps: [build-image, build-vmcp-image, build-egress-proxy, build-egressbroker-image] diff --git a/cmd/thv-egressbroker/main.go b/cmd/thv-egressbroker/main.go new file mode 100644 index 0000000000..194d396d27 --- /dev/null +++ b/cmd/thv-egressbroker/main.go @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Command thv-egressbroker is the per-pod credential-broker sidecar for +// untrusted MCP servers (ADR-0001): it serves Envoy ext_authz (destination +// binding + Authorization injection) and SDS (per-SNI bump-cert minting) on +// a loopback-only gRPC socket. It is the only component in the untrusted pod +// that ever holds an upstream credential. +package main + +import ( + "context" + "encoding/base64" + "fmt" + "log/slog" + "os" + "os/signal" + "strings" + "syscall" + + goredis "github.com/redis/go-redis/v9" + + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken" + "github.com/stacklok/toolhive/pkg/authserver/storage" + "github.com/stacklok/toolhive/pkg/authserver/tokenenc" + "github.com/stacklok/toolhive/pkg/egressbroker" + vmcpconfig "github.com/stacklok/toolhive/pkg/vmcp/config" +) + +// Environment contract for the token store (injected at clone time by the +// Wave-3 operator wiring; the KEK comes from a sidecar-only Secret env). +const ( + // envRedisAddr is the session/upstream-token Redis address (host:port). + envRedisAddr = "THV_EGRESSBROKER_REDIS_ADDR" + // envRedisKeyPrefix is the auth-server per-tenant key prefix the upstream + // token rows live under (e.g. "thv:auth:{ns}:{name}:"). + envRedisKeyPrefix = "THV_EGRESSBROKER_REDIS_KEY_PREFIX" + // envKEKBase64 is the base64 token-encryption KEK (32 bytes decoded). + // When unset, token rows are read unencrypted (legacy plaintext storage); + // encrypted rows then fail closed (Open rejects non-legacy values). + envKEKBase64 = "THV_EGRESSBROKER_KEK" + // envEnvoyBootstrapOut is the path the broker renders the Envoy bootstrap + // into (a shared emptyDir the Envoy container mounts read-only). When + // unset, bootstrap rendering is skipped (the Envoy config is managed + // externally). + envEnvoyBootstrapOut = "THV_EGRESSBROKER_ENVOY_BOOTSTRAP_OUT" + // defaultKEKID is the key ID for the single static KEK. + defaultKEKID = "kek-1" +) + +func main() { + if err := run(); err != nil { + // Startup failures are loud and carry no credential material. + slog.Error("egressbroker failed to start", "error", err) + os.Exit(1) + } +} + +func run() error { + cfg, err := egressbroker.LoadConfig(os.Getenv) + if err != nil { + return err + } + resolver, err := egressbroker.NewPodIdentityResolver(os.Getenv) + if err != nil { + return err + } + policy, err := cfg.ReadPolicyFile() + if err != nil { + return err + } + ca, err := cfg.ReadBumpCA() + if err != nil { + return err + } + dialCIDRs, err := cfg.ResolveDialAllowlist(policy) + if err != nil { + return err + } + dialGuard, err := egressbroker.ParseIPAllowlist(dialCIDRs) + if err != nil { + return err + } + if err := renderEnvoyBootstrap(cfg, policy); err != nil { + return err + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + tokenReader, err := buildTokenReader(ctx, dialGuard) + if err != nil { + return err + } + + injector, err := egressbroker.NewCredentialInjector(resolver.PodIdentity(), policy, tokenReader) + if err != nil { + return err + } + authz, err := egressbroker.NewAuthorizationServer(injector) + if err != nil { + return err + } + sds, err := egressbroker.NewSecretDiscoveryServer(ca, policy) + if err != nil { + return err + } + server, err := egressbroker.NewServer(cfg.ListenAddress, cfg.ListenPort, authz, sds) + if err != nil { + return err + } + return server.Run(ctx) +} + +// renderEnvoyBootstrap writes the Envoy bootstrap for this policy into the +// shared config emptyDir, when configured. The bootstrap's route table +// carries exactly the policy's host allowlist (D6b); the ext_authz cluster +// points at this process (loopback); no redirect-following filter exists +// (D6a). +func renderEnvoyBootstrap(cfg *egressbroker.Config, policy *egressbroker.EgressPolicy) error { + out := strings.TrimSpace(os.Getenv(envEnvoyBootstrapOut)) + if out == "" { + return nil + } + return egressbroker.WriteEnvoyBootstrap(out, egressbroker.EnvoyConfigParams{ + ExtAuthzAddress: cfg.ListenAddress, + ExtAuthzPort: cfg.ListenPort, + ProxyPort: egressbroker.DefaultProxyPort, + AllowedHosts: policy.HostAllowlist(), + }) +} + +// buildTokenReader constructs the upstream-token reader against the vMCP's +// session-storage Redis. The Redis dials go through the D7 guard: the Redis +// address is operator-injected config, and the guard keeps the broker from +// ever dialing outside the policy's destination set. No refresh is wired (the +// broker holds no OAuth client material): expired rows surface on the failed +// list and deny with "re-consent required" (Wave 4 consent UX). +func buildTokenReader(_ context.Context, dialGuard *egressbroker.IPAllowlist) (upstreamtoken.TokenReader, error) { + addr := strings.TrimSpace(os.Getenv(envRedisAddr)) + keyPrefix := strings.TrimSpace(os.Getenv(envRedisKeyPrefix)) + if addr == "" || keyPrefix == "" { + return nil, fmt.Errorf("egressbroker: %s and %s must be set (upstream token store)", + envRedisAddr, envRedisKeyPrefix) + } + + opts, err := tokenEncOption() + if err != nil { + return nil, err + } + + client := goredis.NewClient(&goredis.Options{ + Addr: addr, + Password: os.Getenv(vmcpconfig.RedisPasswordEnvVar), + Dialer: dialGuard.DialContext, + }) + stor := storage.NewRedisStorageWithClient(client, keyPrefix, opts...) + return upstreamtoken.NewInProcessService(stor, nil), nil +} + +// tokenEncOption builds the token-encryption option from the KEK env. A +// malformed KEK is a startup error (fail closed); an absent KEK means +// plaintext legacy rows only. +func tokenEncOption() ([]storage.RedisStorageOption, error) { + raw := strings.TrimSpace(os.Getenv(envKEKBase64)) + if raw == "" { + return nil, nil + } + kek, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, fmt.Errorf("egressbroker: %s is not valid base64", envKEKBase64) + } + keyring, err := tokenenc.NewStaticKeyring(defaultKEKID, map[string][]byte{defaultKEKID: kek}) + if err != nil { + return nil, fmt.Errorf("egressbroker: invalid token-encryption key: %w", err) + } + return []storage.RedisStorageOption{storage.WithTokenEncryption(keyring)}, nil +} diff --git a/cmd/thv-operator/Taskfile.yml b/cmd/thv-operator/Taskfile.yml index 759494f56a..eff9dd2159 100644 --- a/cmd/thv-operator/Taskfile.yml +++ b/cmd/thv-operator/Taskfile.yml @@ -231,6 +231,8 @@ tasks: - chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/setup - chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/test-scenarios - chainsaw test --test-dir test/e2e/chainsaw/operator/single-tenancy/cleanup + - chainsaw test --test-dir test/e2e/chainsaw/operator/validation/mcpserver-untrusted + - chainsaw test --test-dir test/e2e/chainsaw/operator/untrusted-egress thv-operator-e2e-test: desc: | diff --git a/cmd/thv-operator/api/v1beta1/mcpserver_types.go b/cmd/thv-operator/api/v1beta1/mcpserver_types.go index 48784fddcb..244c03fa86 100644 --- a/cmd/thv-operator/api/v1beta1/mcpserver_types.go +++ b/cmd/thv-operator/api/v1beta1/mcpserver_types.go @@ -38,6 +38,11 @@ const ( // ConditionReasonGroupRefNotReady indicates the referenced MCPGroup is not in the Ready state ConditionReasonGroupRefNotReady = "GroupRefNotReady" + + // ConditionReasonGroupRefNotVMCPFronted indicates an untrusted workload's + // MCPGroup is not fronted by any VirtualMCPServer (ADR-0001 D4 requires the + // trusted identity-aware front). + ConditionReasonGroupRefNotVMCPFronted = "GroupRefNotVMCPFronted" ) const ( @@ -53,6 +58,13 @@ const ( // attempted to source backend (mcp) container env from a Secret or ConfigMap, // or mount a Secret volume — rejected by the untrusted env gate. ConditionReasonSecretEnvRejected = "SecretEnvRejected" + + // ConditionReasonUntrustedPolicyInvalid indicates a terminal rejection of an + // untrusted workload's egress policy (missing/invalid spec.egressPolicy, or + // destinations that resolve to no usable addresses). Distinct from + // ConditionReasonNotReady so operators can tell "fix the spec" apart from + // "wait for convergence". + ConditionReasonUntrustedPolicyInvalid = "UntrustedEgressPolicyInvalid" ) // Condition type for CA bundle validation @@ -228,6 +240,12 @@ const SessionStorageProviderRedis = "redis" // +kubebuilder:validation:XValidation:rule="!has(self.rateLimiting) || (has(self.sessionStorage) && self.sessionStorage.provider == 'redis')",message="rateLimiting requires sessionStorage with provider 'redis'" // +kubebuilder:validation:XValidation:rule="!(has(self.rateLimiting) && has(self.rateLimiting.perUser)) || has(self.oidcConfigRef) || has(self.externalAuthConfigRef)",message="rateLimiting.perUser requires authentication (oidcConfigRef or externalAuthConfigRef)" // +kubebuilder:validation:XValidation:rule="!has(self.rateLimiting) || !has(self.rateLimiting.tools) || self.rateLimiting.tools.all(t, !has(t.perUser)) || has(self.oidcConfigRef) || has(self.externalAuthConfigRef)",message="per-tool perUser rate limiting requires authentication (oidcConfigRef or externalAuthConfigRef)" +// +kubebuilder:validation:XValidation:rule="!self.untrusted || (has(self.egressPolicy) && size(self.egressPolicy.providers) > 0)",message="egressPolicy with at least one provider is required when untrusted is true" +// +kubebuilder:validation:XValidation:rule="!self.untrusted || has(self.groupRef)",message="untrusted workloads must belong to an MCPGroup fronted by a VirtualMCPServer" +// +kubebuilder:validation:XValidation:rule="!self.untrusted || !has(self.secrets) || size(self.secrets) == 0",message="spec.secrets is forbidden when untrusted is true; declare providers in egressPolicy and use sentinel env" +// +kubebuilder:validation:XValidation:rule="!self.untrusted || !has(self.podTemplateSpec)",message="podTemplateSpec is forbidden when untrusted is true" +// +kubebuilder:validation:XValidation:rule="!self.untrusted || self.sessionAffinity == 'ClientIP'",message="untrusted workloads require sessionAffinity ClientIP" +// +kubebuilder:validation:XValidation:rule="!self.untrusted || !has(self.backendReplicas)",message="backendReplicas is managed by the untrusted-mode session lifecycle and must not be set" // //nolint:lll // CEL validation rules exceed line length limit type MCPServerSpec struct { @@ -384,6 +402,24 @@ type MCPServerSpec struct { // +optional GroupRef *MCPGroupRef `json:"groupRef,omitempty"` + // Untrusted marks this MCP server as running untrusted code. When true, the operator + // enforces the untrusted-mode invariants (ADR-0001): no Secret/ConfigMap-sourced env on + // the backend container, single-tenant session-scoped backend pods, mandatory MCPGroup + // membership behind a VirtualMCPServer, and egress only through the credential-broker + // sidecar per EgressPolicy. K8s-only; ignored (and inert) in CLI/Docker mode. + // +kubebuilder:default=false + // +optional + Untrusted bool `json:"untrusted,omitempty"` + + // EgressPolicy declares which upstream providers this server may call and where the + // broker may inject each provider's credential. Required when Untrusted is true. + // When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend + // pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar + // + vMCP). Do not merge the two vocabularies: OutboundNetworkPermissions is the + // Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + // +optional + EgressPolicy *EgressPolicy `json:"egressPolicy,omitempty"` + // SessionAffinity controls whether the Service routes repeated client connections to the same pod. // MCP protocols (SSE, streamable-http) are stateful, so ClientIP is the default. // Set to "None" for stateless servers or when using an external load balancer with its own affinity. @@ -667,6 +703,57 @@ type OutboundNetworkPermissions struct { AllowPort []int32 `json:"allowPort,omitempty"` } +// EgressPolicy binds upstream providers to the exact destinations their credentials +// may be injected for. The broker emits provider P's credential ONLY to hosts in +// P's AllowedHosts, with method/path further constrained by the entry (ADR-0001 D5). +type EgressPolicy struct { + // Providers maps an upstream provider name (as configured in the embedded auth + // server's upstream chain / upstream_inject strategy) to its egress constraints. + // +listType=map + // +listMapKey=provider + // +kubebuilder:validation:MinItems=1 + Providers []ProviderEgress `json:"providers"` +} + +// ProviderEgress is the per-provider egress constraint. Credential-to-destination +// binding (ADR-0001 D5): the broker injects this provider's credential only to +// AllowedHosts, only on AllowedMethods, only under AllowedPathPrefixes. +type ProviderEgress struct { + // Provider is the logical upstream provider name (e.g. "github", "google"). + // Must match the provider name used by the auth server and the vMCP + // upstream_inject strategy config for this workload. + // +kubebuilder:validation:MinLength=1 + Provider string `json:"provider"` + + // AllowedHosts is the exact set of destination hosts the credential may be + // injected for. Exact hostnames or one-label wildcards ("*.githubusercontent.com"); + // no ports, no schemes. + // +listType=set + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:items:Pattern=`^(\*\.)?[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$` + AllowedHosts []string `json:"allowedHosts"` + + // AllowedMethods constrains HTTP methods the broker will inject on. + // Empty means GET/HEAD/OPTIONS only (safe default — read-only). + // +listType=set + // +kubebuilder:validation:items:Enum=GET;HEAD;OPTIONS;POST;PUT;PATCH;DELETE + // +optional + AllowedMethods []string `json:"allowedMethods,omitempty"` + + // AllowedPathPrefixes constrains URL paths the broker will inject on + // (prefix match, e.g. "/repos/"). Empty means "/" (all paths on the allowed hosts). + // +listType=set + // +optional + AllowedPathPrefixes []string `json:"allowedPathPrefixes,omitempty"` + + // CredentialEnvName names the backend env var that would normally carry this + // provider's token (e.g. "GITHUB_TOKEN"). The operator injects a SENTINEL literal + // here so servers that refuse to start tokenless still boot; the broker ignores + // the sentinel value. Sentinel literal format: "thv-untrusted-sentinel:". + // +optional + CredentialEnvName string `json:"credentialEnvName,omitempty"` +} + // CABundleSource defines a source for CA certificate bundles. type CABundleSource struct { // ConfigMapRef references a ConfigMap containing the CA certificate bundle. diff --git a/cmd/thv-operator/api/v1beta1/zz_generated.deepcopy.go b/cmd/thv-operator/api/v1beta1/zz_generated.deepcopy.go index ccb82ed571..e954ec39cc 100644 --- a/cmd/thv-operator/api/v1beta1/zz_generated.deepcopy.go +++ b/cmd/thv-operator/api/v1beta1/zz_generated.deepcopy.go @@ -225,6 +225,28 @@ func (in *DCRUpstreamConfig) DeepCopy() *DCRUpstreamConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressPolicy) DeepCopyInto(out *EgressPolicy) { + *out = *in + if in.Providers != nil { + in, out := &in.Providers, &out.Providers + *out = make([]ProviderEgress, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressPolicy. +func (in *EgressPolicy) DeepCopy() *EgressPolicy { + if in == nil { + return nil + } + out := new(EgressPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EmbeddedAuthServerCIMDConfig) DeepCopyInto(out *EmbeddedAuthServerCIMDConfig) { *out = *in @@ -1780,6 +1802,11 @@ func (in *MCPServerSpec) DeepCopyInto(out *MCPServerSpec) { *out = new(MCPGroupRef) **out = **in } + if in.EgressPolicy != nil { + in, out := &in.EgressPolicy, &out.EgressPolicy + *out = new(EgressPolicy) + (*in).DeepCopyInto(*out) + } if in.Replicas != nil { in, out := &in.Replicas, &out.Replicas *out = new(int32) @@ -2479,6 +2506,36 @@ func (in *PrometheusConfig) DeepCopy() *PrometheusConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProviderEgress) DeepCopyInto(out *ProviderEgress) { + *out = *in + if in.AllowedHosts != nil { + in, out := &in.AllowedHosts, &out.AllowedHosts + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AllowedMethods != nil { + in, out := &in.AllowedMethods, &out.AllowedMethods + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AllowedPathPrefixes != nil { + in, out := &in.AllowedPathPrefixes, &out.AllowedPathPrefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderEgress. +func (in *ProviderEgress) DeepCopy() *ProviderEgress { + if in == nil { + return nil + } + out := new(ProviderEgress) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxyDeploymentOverrides) DeepCopyInto(out *ProxyDeploymentOverrides) { *out = *in diff --git a/cmd/thv-operator/controllers/mcpserver_controller.go b/cmd/thv-operator/controllers/mcpserver_controller.go index 3b6e75d1e9..69ca38a23e 100644 --- a/cmd/thv-operator/controllers/mcpserver_controller.go +++ b/cmd/thv-operator/controllers/mcpserver_controller.go @@ -372,6 +372,41 @@ func (r *MCPServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, err } + // Ensure untrusted-mode data-plane resources (bump CA, egress-policy + // ConfigMap, egress NetworkPolicy), or delete them on the untrusted→trusted + // flip. Terminal spec errors (invalid/unresolvable EgressPolicy) surface as + // a condition and stop without requeue, per the gate pattern. + if err := r.ensureUntrustedResources(ctx, mcpServer); err != nil { + var specErr *SpecValidationError + if stderrors.As(err, &specErr) { + ctxLogger.Error(err, "Untrusted egress policy rejected") + mcpServer.Status.Phase = mcpv1beta1.MCPServerPhaseFailed + mcpServer.Status.Message = fmt.Sprintf("Invalid untrusted egress policy: %s", specErr.Error()) + // Terminal: a distinct reason (not ConditionReasonNotReady) and the + // observed generation, so the rejection is unambiguous and stale + // conditions from a newer spec are detectable. + meta.SetStatusCondition(&mcpServer.Status.Conditions, metav1.Condition{ + Type: mcpv1beta1.ConditionTypeReady, + Status: metav1.ConditionFalse, + Reason: mcpv1beta1.ConditionReasonUntrustedPolicyInvalid, + Message: mcpServer.Status.Message, + ObservedGeneration: mcpServer.Generation, + }) + if statusErr := r.Status().Update(ctx, mcpServer); statusErr != nil { + ctxLogger.Error(statusErr, "Failed to update MCPServer status after untrusted policy validation failure") + } + return ctrl.Result{}, nil + } + ctxLogger.Error(err, "Failed to ensure untrusted resources") + mcpServer.Status.Phase = mcpv1beta1.MCPServerPhaseFailed + mcpServer.Status.Message = fmt.Sprintf("Failed to ensure untrusted resources: %s", err.Error()) + setReadyCondition(mcpServer, metav1.ConditionFalse, mcpv1beta1.ConditionReasonNotReady, mcpServer.Status.Message) + if statusErr := r.Status().Update(ctx, mcpServer); statusErr != nil { + ctxLogger.Error(statusErr, "Failed to update MCPServer status after untrusted resources error") + } + return ctrl.Result{}, err + } + // Ensure RunConfig ConfigMap exists and is up to date if err := r.ensureRunConfigConfigMap(ctx, mcpServer); err != nil { ctxLogger.Error(err, "Failed to ensure RunConfig ConfigMap") @@ -612,6 +647,21 @@ func (r *MCPServerReconciler) validateGroupRef(ctx context.Context, mcpServer *m Message: fmt.Sprintf("MCPGroup '%s' is not ready (current phase: %s)", groupName, group.Status.Phase), ObservedGeneration: mcpServer.Generation, }) + } else if isUntrusted(mcpServer) && !r.groupIsVMCPFronted(ctx, mcpServer.Namespace, groupName) { + // Untrusted workloads are served only through a VirtualMCPServer + // (ADR-0001 D4: the trusted identity-aware front that provisions the + // per-session pods). A group with no fronting vMCP leaves the workload + // undiscoverable as an untrusted backend — surface it instead of + // reporting the group valid. + meta.SetStatusCondition(&mcpServer.Status.Conditions, metav1.Condition{ + Type: mcpv1beta1.ConditionGroupRefValidated, + Status: metav1.ConditionFalse, + Reason: mcpv1beta1.ConditionReasonGroupRefNotVMCPFronted, + Message: fmt.Sprintf( + "MCPGroup '%s' is not fronted by a VirtualMCPServer; untrusted workloads require one", + groupName), + ObservedGeneration: mcpServer.Generation, + }) } else { meta.SetStatusCondition(&mcpServer.Status.Conditions, metav1.Condition{ Type: mcpv1beta1.ConditionGroupRefValidated, @@ -628,6 +678,24 @@ func (r *MCPServerReconciler) validateGroupRef(ctx context.Context, mcpServer *m } +// groupIsVMCPFronted reports whether at least one VirtualMCPServer in the +// namespace fronts the named MCPGroup. List failures fail closed (reported +// as not-fronted) — a transient API error must never silently attest that an +// untrusted workload's group is vMCP-fronted. +func (r *MCPServerReconciler) groupIsVMCPFronted(ctx context.Context, namespace, groupName string) bool { + vmcps := &mcpv1beta1.VirtualMCPServerList{} + if err := r.List(ctx, vmcps, client.InNamespace(namespace)); err != nil { + log.FromContext(ctx).Error(err, "Failed to list VirtualMCPServers for untrusted groupRef validation") + return false + } + for i := range vmcps.Items { + if vmcps.Items[i].Spec.GroupRef.GetName() == groupName { + return true + } + } + return false +} + // setCABundleRefCondition sets the CA bundle validation status condition func setCABundleRefCondition(mcpServer *mcpv1beta1.MCPServer, status metav1.ConditionStatus, reason, message string) { meta.SetStatusCondition(&mcpServer.Status.Conditions, metav1.Condition{ @@ -1080,17 +1148,10 @@ func logOBOSecretEnvVarError(ctx context.Context, err error) { "see the referenced MCPExternalAuthConfig status for details") } -// untrustedAnnotationKey is the interim (Wave 0) opt-in annotation that marks an -// MCPServer as untrusted. Wave 1 replaces this with the spec.untrusted field and -// deletes the annotation. -const untrustedAnnotationKey = "toolhive.stacklok.dev/untrusted" - // isUntrusted reports whether the workload must be treated as untrusted. -// Wave 1 replaces the body with `return m.Spec.Untrusted`. -// Interim (Wave 0): annotation opt-in so the gate is exercisable and e2e-testable -// before the CRD field exists. The annotation is not a documented user feature. +// Wave 1: reads the spec field (the interim Wave-0 annotation is removed). func isUntrusted(m *mcpv1beta1.MCPServer) bool { - return m.Annotations[untrustedAnnotationKey] == "true" + return m.Spec.Untrusted } // untrustedGateSuffix documents the gate's semantics on every rejection @@ -1249,6 +1310,18 @@ func (r *MCPServerReconciler) deploymentForMCPServer( if err := ctrlutil.ValidateNoSecretEnvForUntrusted(finalPodTemplateSpec, mcpContainerName, isUntrusted(m)); err != nil { return nil, &SpecValidationError{Message: err.Error()} } + // Wave 1: for untrusted workloads, inject sentinel literals for each declared + // credentialEnvName so token-requiring servers boot (ADR-0001). Literal values + // pass the Wave-0 gate above by construction. Sentinel collision and forgery + // rejections are terminal SpecValidationErrors, same pattern as the gate. + if m.Spec.Untrusted && finalPodTemplateSpec != nil { + if err := ctrlutil.InjectUntrustedSentinels(finalPodTemplateSpec, mcpContainerName, m.Spec.EgressPolicy); err != nil { + return nil, &SpecValidationError{Message: err.Error()} + } + if err := ctrlutil.ValidateUntrustedSentinelForgery(finalPodTemplateSpec, mcpContainerName, m.Spec.EgressPolicy); err != nil { + return nil, &SpecValidationError{Message: err.Error()} + } + } // Add pod template patch if we have one if finalPodTemplateSpec != nil { podTemplatePatch, err := json.Marshal(finalPodTemplateSpec) diff --git a/cmd/thv-operator/controllers/mcpserver_runconfig.go b/cmd/thv-operator/controllers/mcpserver_runconfig.go index 373fd0e0db..b2fd0bfe3a 100644 --- a/cmd/thv-operator/controllers/mcpserver_runconfig.go +++ b/cmd/thv-operator/controllers/mcpserver_runconfig.go @@ -277,6 +277,17 @@ func (r *MCPServerReconciler) createRunConfigFromMCPServer(m *mcpv1beta1.MCPServ options = append(options, runner.WithRateLimitConfig(m.Namespace, m.Spec.RateLimiting)) } + // Untrusted workloads: stamp the MCPServer UID as a container label so the + // backend StatefulSet (built from these labels in DeployWorkload) carries the + // rename-safe selector the vMCP pod lifecycle resolves its clone template by + // (LabelMCPServerUID + toolhive=true). Without it the lifecycle finds zero + // StatefulSets and untrusted session provisioning always soft-fails. + if isUntrusted(m) { + options = append(options, runner.WithLabels([]string{ + untrustedMCPServerUIDLabel + "=" + string(m.UID), + })) + } + // Use the RunConfigBuilder for operator context with full builder pattern runConfig, err := runner.NewOperatorRunConfigBuilder( context.Background(), diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go index 5cadd18fc1..b9bfbc5cd5 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go @@ -4,12 +4,16 @@ package controllers import ( + "encoding/json" + "net" + "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,23 +31,51 @@ import ( ctrlutil "github.com/stacklok/toolhive/cmd/thv-operator/pkg/controllerutil" ) -// withUntrustedAnnotation marks the MCPServer fixture as untrusted via the -// interim Wave 0 opt-in annotation. -func withUntrustedAnnotation() v1beta1test.MCPServerOption { +// withUntrustedSpec marks the MCPServer fixture as untrusted via the +// Wave-1 spec.untrusted field. +func withUntrustedSpec() v1beta1test.MCPServerOption { return func(m *mcpv1beta1.MCPServer) { - if m.Annotations == nil { - m.Annotations = map[string]string{} - } - m.Annotations[untrustedAnnotationKey] = "true" + m.Spec.Untrusted = true } } +// withUntrustedCompliantPolicy adds a minimal valid EgressPolicy (hostname +// destination; reconcile-path tests stub untrustedDNSLookup so no real DNS +// resolution happens). Wave 3: untrusted servers without an EgressPolicy +// terminate at the egress-resource gate, so fixtures that must proceed down +// the normal reconcile path need this. +func withUntrustedCompliantPolicy() v1beta1test.MCPServerOption { + return v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.Untrusted = true + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{ + Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + }}, + } + }) +} + func withSecrets(secrets ...mcpv1beta1.SecretRef) v1beta1test.MCPServerOption { return func(m *mcpv1beta1.MCPServer) { m.Spec.Secrets = secrets } } +// TestIsUntrusted pins the Wave-1 body of isUntrusted: it reads spec.untrusted +// and nothing else — in particular the interim Wave-0 annotation is inert. +func TestIsUntrusted(t *testing.T) { + t.Parallel() + + assert.False(t, isUntrusted(v1beta1test.NewMCPServer("plain", "default"))) + assert.True(t, isUntrusted(v1beta1test.NewMCPServer("flagged", "default", withUntrustedSpec()))) + + annotated := v1beta1test.NewMCPServer("annotated", "default", v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Annotations = map[string]string{"toolhive.stacklok.dev/untrusted": "true"} + })) + assert.False(t, isUntrusted(annotated), "the interim Wave-0 annotation must have no effect in Wave 1") +} + // setupUntrustedReconciler builds a fake-client-backed reconciler for the // untrusted env gate tests and returns it with the event recorder. func setupUntrustedReconciler(t *testing.T, objs ...client.Object) (*MCPServerReconciler, *events.FakeRecorder) { @@ -67,10 +99,18 @@ func setupUntrustedReconciler(t *testing.T, objs ...client.Object) (*MCPServerRe return r, eventRecorder } -// reconcileOnce runs a single Reconcile for the given MCPServer fixture. +// reconcileOnce runs a single Reconcile for the given MCPServer fixture. The +// untrusted egress gate resolves policy destinations through the package-level +// untrustedDNSLookup stub; this helper installs the fixture stub for the whole +// test lifetime (t.Cleanup restores it) so a mid-test restore never leaves a +// parallel sibling's Reconcile hitting real DNS. func reconcileOnce(t *testing.T, r *MCPServerReconciler, mcpServer *mcpv1beta1.MCPServer) ctrl.Result { t.Helper() + untrustedDNSTestLock(t, func(string) ([]net.IP, error) { + return []net.IP{net.ParseIP("140.82.114.26")}, nil + }) + ctx := log.IntoContext(t.Context(), log.Log) req := ctrl.Request{NamespacedName: types.NamespacedName{ Name: mcpServer.Name, @@ -85,13 +125,13 @@ func TestMCPServerReconciler_UntrustedSecretEnvRejected(t *testing.T) { t.Parallel() mcpServer := v1beta1test.NewMCPServer("untrusted-secrets", "default", - withUntrustedAnnotation(), + withUntrustedSpec(), withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), ) r, recorder := setupUntrustedReconciler(t, mcpServer) - // Reconcile #11: annotation + spec.secrets → terminal rejection. + // Reconcile #11: spec.untrusted + spec.secrets → terminal rejection. result := reconcileOnce(t, r, mcpServer) assert.True(t, result.IsZero(), "terminal spec error must not requeue or schedule a retry") @@ -151,7 +191,7 @@ func TestMCPServerReconciler_UntrustedSecretEnvRejected(t *testing.T) { func TestMCPServerReconciler_TrustedSecretsDeployAsToday(t *testing.T) { t.Parallel() - // #12: identical spec.secrets but no annotation — no behavior change; the + // #12: identical spec.secrets but spec.untrusted=false — no behavior change; the // reconcile must proceed past the gate (it will requeue waiting on the // runconfig ConfigMap, which is the normal pre-Deployment path). mcpServer := v1beta1test.NewMCPServer("trusted-secrets", "default", @@ -172,12 +212,39 @@ func TestMCPServerReconciler_TrustedSecretsDeployAsToday(t *testing.T) { "trusted reconcile should continue down the normal path (requeue waiting on runconfig)") } +// TestMCPServerReconciler_InterimAnnotationIsInert pins the Wave-1 removal of the +// interim Wave-0 opt-in annotation: annotation set + spec.untrusted=false + secrets +// must sail past the gate exactly like a trusted workload. +func TestMCPServerReconciler_InterimAnnotationIsInert(t *testing.T) { + t.Parallel() + + mcpServer := v1beta1test.NewMCPServer("annotated-but-trusted", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Annotations = map[string]string{"toolhive.stacklok.dev/untrusted": "true"} + }), + withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), + ) + + r, _ := setupUntrustedReconciler(t, mcpServer) + + result := reconcileOnce(t, r, mcpServer) + + updated := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), updated)) + + condition := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionTypeValid) + assert.Nil(t, condition, "the interim Wave-0 annotation must no longer arm the gate") + assert.NotEqual(t, mcpv1beta1.MCPServerPhaseFailed, updated.Status.Phase) + assert.False(t, result.IsZero(), + "annotated-but-trusted reconcile should continue down the normal path") +} + func TestMCPServerReconciler_UntrustedRawTemplateSecretEnvRejected(t *testing.T) { t.Parallel() - // #13: annotation + raw podTemplateSpec smuggling secretKeyRef onto the mcp container. + // #13: spec.untrusted + raw podTemplateSpec smuggling secretKeyRef onto the mcp container. mcpServer := v1beta1test.NewMCPServer("untrusted-raw-template", "default", - withUntrustedAnnotation(), + withUntrustedSpec(), v1beta1test.WithPodTemplateSpec(&runtime.RawExtension{ Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"API_TOKEN","valueFrom":{"secretKeyRef":{"name":"smuggled-secret","key":"token"}}}]}]}}`), }), @@ -206,10 +273,10 @@ func TestMCPServerReconciler_UntrustedRawTemplateSecretEnvRejected(t *testing.T) func TestMCPServerReconciler_UntrustedCompliantDeploys(t *testing.T) { t.Parallel() - // #15: annotation present but workload compliant (literal env only) — + // #15: untrusted but compliant (literal env only) — // the gate passes and the reconcile proceeds normally. mcpServer := v1beta1test.NewMCPServer("untrusted-compliant", "default", - withUntrustedAnnotation(), + withUntrustedCompliantPolicy(), v1beta1test.WithEnv(mcpv1beta1.EnvVar{Name: "SENTINEL", Value: "literal-value"}), v1beta1test.WithPodTemplateSpec(&runtime.RawExtension{ Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"LITERAL","value":"ok"}]}]}}`), @@ -245,7 +312,7 @@ func TestMCPServerReconciler_UntrustedLatchClearsAndWarningReArms(t *testing.T) // Start REJECTED: untrusted + spec.secrets. mcpServer := v1beta1test.NewMCPServer("untrusted-latch", "default", - withUntrustedAnnotation(), + withUntrustedCompliantPolicy(), withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), ) r, recorder := setupUntrustedReconciler(t, mcpServer) @@ -299,7 +366,7 @@ func TestDeploymentForMCPServer_UntrustedSecretEnvRejectedAtBuildTime(t *testing // Defense-in-depth: deploymentForMCPServer itself rejects the built patch // for untrusted workloads (spec.secrets seam)... mcpServer := v1beta1test.NewMCPServer("untrusted-build-gate", "default", - withUntrustedAnnotation(), + withUntrustedSpec(), withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), ) @@ -326,3 +393,232 @@ func TestDeploymentForMCPServer_UntrustedSecretEnvRejectedAtBuildTime(t *testing require.NoError(t, err) require.NotNil(t, deployment) } + +// TestDeploymentForMCPServer_UntrustedSentinelInjection pins the Wave-1 sentinel +// seam in deploymentForMCPServer: for an untrusted workload with a declared +// credentialEnvName, the --k8s-pod-patch argument carries the literal sentinel +// env var on the mcp container, and the pod patch still passes the Wave-0 gate. +func TestDeploymentForMCPServer_UntrustedSentinelInjection(t *testing.T) { + t.Parallel() + + mcpServer := v1beta1test.NewMCPServer("untrusted-sentinel", "default", + withUntrustedSpec(), + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + CredentialEnvName: "GITHUB_TOKEN", + }}} + }), + ) + + r, _ := setupUntrustedReconciler(t, mcpServer) + ctx := log.IntoContext(t.Context(), log.Log) + + deployment, err := r.deploymentForMCPServer(ctx, mcpServer, "test-checksum") + require.NoError(t, err) + require.NotNil(t, deployment) + + patch := podPatchFromDeployment(t, deployment) + require.Len(t, patch.Spec.Containers, 1) + env := patch.Spec.Containers[0].Env + require.Len(t, env, 1, "exactly one sentinel env var expected") + assert.Equal(t, "GITHUB_TOKEN", env[0].Name) + assert.Equal(t, "thv-untrusted-sentinel:github", env[0].Value) + assert.Nil(t, env[0].ValueFrom, "sentinel env must be a literal Value, never ValueFrom") + + // The injected patch composes with the Wave-0 gate. + require.NoError(t, ctrlutil.ValidateNoSecretEnvForUntrusted(patch, "mcp", true)) +} + +// TestDeploymentForMCPServer_UntrustedSentinelCollision pins the terminal rejection +// when a declared credentialEnvName collides with user-declared env. +func TestDeploymentForMCPServer_UntrustedSentinelCollision(t *testing.T) { + t.Parallel() + + mcpServer := v1beta1test.NewMCPServer("untrusted-collision", "default", + withUntrustedSpec(), + v1beta1test.WithEnv(mcpv1beta1.EnvVar{Name: "GITHUB_TOKEN", Value: "user-value"}), + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.PodTemplateSpec = &runtime.RawExtension{ + Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"GITHUB_TOKEN","value":"user-value"}]}]}}`), + } + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + CredentialEnvName: "GITHUB_TOKEN", + }}} + }), + ) + + r, _ := setupUntrustedReconciler(t, mcpServer) + ctx := log.IntoContext(t.Context(), log.Log) + + deployment, err := r.deploymentForMCPServer(ctx, mcpServer, "test-checksum") + require.Error(t, err) + assert.Nil(t, deployment) + assert.Contains(t, err.Error(), `env var "GITHUB_TOKEN" on container "mcp" already exists with a different value`) + + var specErr *SpecValidationError + assert.ErrorAs(t, err, &specErr, "sentinel collision must be a terminal typed error") +} + +// TestDeploymentForMCPServer_UntrustedSentinelForgery pins the terminal rejection +// of a user-forged sentinel literal in the raw pod template. +func TestDeploymentForMCPServer_UntrustedSentinelForgery(t *testing.T) { + t.Parallel() + + mcpServer := v1beta1test.NewMCPServer("untrusted-forgery", "default", + withUntrustedSpec(), + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.PodTemplateSpec = &runtime.RawExtension{ + Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"FORGED","value":"thv-untrusted-sentinel:attacker"}]}]}}`), + } + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + }}} + }), + ) + + r, _ := setupUntrustedReconciler(t, mcpServer) + ctx := log.IntoContext(t.Context(), log.Log) + + deployment, err := r.deploymentForMCPServer(ctx, mcpServer, "test-checksum") + require.Error(t, err) + assert.Nil(t, deployment) + assert.Contains(t, err.Error(), `env var "FORGED" on container "mcp" uses the reserved sentinel prefix`) + + var specErr *SpecValidationError + assert.ErrorAs(t, err, &specErr, "sentinel forgery must be a terminal typed error") +} + +// TestDeploymentForMCPServer_TrustedEgressPolicyInjectsNothing pins that sentinel +// injection is gated on spec.untrusted: a trusted workload declaring an +// egressPolicy gets no sentinel env. +func TestDeploymentForMCPServer_TrustedEgressPolicyInjectsNothing(t *testing.T) { + t.Parallel() + + mcpServer := v1beta1test.NewMCPServer("trusted-egress-policy", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + CredentialEnvName: "GITHUB_TOKEN", + }}} + }), + ) + + r, _ := setupUntrustedReconciler(t, mcpServer) + ctx := log.IntoContext(t.Context(), log.Log) + + deployment, err := r.deploymentForMCPServer(ctx, mcpServer, "test-checksum") + require.NoError(t, err) + require.NotNil(t, deployment) + + patch := podPatchFromDeployment(t, deployment) + for _, c := range patch.Spec.Containers { + for _, env := range c.Env { + assert.NotContains(t, env.Value, "thv-untrusted-sentinel:", + "trusted workloads must never receive sentinel env") + } + } +} + +// podPatchFromDeployment extracts and unmarshals the --k8s-pod-patch argument +// from the proxy container args of a built Deployment. +func podPatchFromDeployment(t *testing.T, deployment *appsv1.Deployment) *corev1.PodTemplateSpec { + t.Helper() + + var patchJSON string + for _, arg := range deployment.Spec.Template.Spec.Containers[0].Args { + if after, ok := strings.CutPrefix(arg, "--k8s-pod-patch="); ok { + patchJSON = after + break + } + } + require.NotEmpty(t, patchJSON, "Deployment should carry a --k8s-pod-patch argument") + + patch := &corev1.PodTemplateSpec{} + require.NoError(t, json.Unmarshal([]byte(patchJSON), patch)) + return patch +} + +// TestMCPServerReconciler_UntrustedGroupRefNotVMCPFronted pins the untrusted +// groupRef status check (ADR-0001 D4): an untrusted workload whose MCPGroup +// has no fronting VirtualMCPServer gets GroupRefValidated=False with the +// dedicated reason; the same workload reports valid once a vMCP fronts the +// group. Trusted workloads in the same un-fronted group are unaffected. +func TestMCPServerReconciler_UntrustedGroupRefNotVMCPFronted(t *testing.T) { + t.Parallel() + + readyGroup := func(name string) *mcpv1beta1.MCPGroup { + return &mcpv1beta1.MCPGroup{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Status: mcpv1beta1.MCPGroupStatus{Phase: mcpv1beta1.MCPGroupPhaseReady}, + } + } + untrustedInGroup := func(name, group string) *mcpv1beta1.MCPServer { + return v1beta1test.NewMCPServer(name, "default", + withUntrustedCompliantPolicy(), + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.GroupRef = &mcpv1beta1.MCPGroupRef{Name: group} + }), + ) + } + + t.Run("untrusted workload in an un-fronted group gets the dedicated condition", func(t *testing.T) { + t.Parallel() + group := readyGroup("lonely-group") + mcpServer := untrustedInGroup("untrusted-no-front", "lonely-group") + r, _ := setupUntrustedReconciler(t, group, mcpServer) + + r.validateGroupRef(t.Context(), mcpServer) + + updated := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), updated)) + condition := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionGroupRefValidated) + require.NotNil(t, condition) + assert.Equal(t, metav1.ConditionFalse, condition.Status) + assert.Equal(t, mcpv1beta1.ConditionReasonGroupRefNotVMCPFronted, condition.Reason) + assert.Equal(t, updated.Generation, condition.ObservedGeneration) + assert.Contains(t, condition.Message, "not fronted by a VirtualMCPServer") + }) + + t.Run("untrusted workload in a vMCP-fronted group validates", func(t *testing.T) { + t.Parallel() + group := readyGroup("fronted-group") + vmcp := v1beta1test.NewVirtualMCPServer("front", "default", v1beta1test.WithVMCPGroupRef("fronted-group")) + mcpServer := untrustedInGroup("untrusted-fronted", "fronted-group") + r, _ := setupUntrustedReconciler(t, group, vmcp, mcpServer) + + r.validateGroupRef(t.Context(), mcpServer) + + updated := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), updated)) + condition := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionGroupRefValidated) + require.NotNil(t, condition) + assert.Equal(t, metav1.ConditionTrue, condition.Status) + assert.Equal(t, mcpv1beta1.ConditionReasonGroupRefValidated, condition.Reason) + }) + + t.Run("trusted workload in an un-fronted group is unaffected", func(t *testing.T) { + t.Parallel() + group := readyGroup("trusted-group") + mcpServer := v1beta1test.NewMCPServer("trusted-no-front", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.GroupRef = &mcpv1beta1.MCPGroupRef{Name: "trusted-group"} + }), + ) + r, _ := setupUntrustedReconciler(t, group, mcpServer) + + r.validateGroupRef(t.Context(), mcpServer) + + updated := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), updated)) + condition := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionGroupRefValidated) + require.NotNil(t, condition) + assert.Equal(t, metav1.ConditionTrue, condition.Status, + "the vMCP-front requirement applies to untrusted workloads only") + }) +} diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go new file mode 100644 index 0000000000..8af5408f53 --- /dev/null +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go @@ -0,0 +1,778 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "context" + stderrors "errors" + "fmt" + "net" + "sort" + "strings" + "sync" + "time" + + "gopkg.in/yaml.v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +// Untrusted-mode wiring constants (ADR-0001 Wave 3). Resource naming is NOT +// re-declared here: it comes from egressbroker.ResourceNamesFor / +// BaseCASecretName / TrimGeneration, the shared naming contract both sides of +// the module boundary import (the vMCP side resolves the same names in +// pkg/vmcp/session/untrusted/egress.go). Data keys come from +// egressbroker.CAKeyCert / CAKeyKey. +const ( + // untrustedMCPServerUIDLabel is the label the operator stamps on the + // untrusted backend workload (via RunConfig ContainerLabels → the + // StatefulSet) so the vMCP pod lifecycle can resolve the clone template by + // rename-safe UID selector. It mirrors + // pkg/vmcp/session/untrusted.LabelMCPServerUID across the module boundary. + untrustedMCPServerUIDLabel = "toolhive.stacklok.dev/mcpserver-uid" + // untrustedNetworkPolicySuffix names the egress-lockdown NetworkPolicy. + untrustedNetworkPolicySuffix = "-egress" + + // untrustedCAGenerationAnnotation is stamped on the backend StatefulSet's + // pod template with the current bump-CA generation. The vMCP pod lifecycle + // reads it at clone time so a pod cloned mid-rotation mounts one + // consistent cert/key pair. It mirrors + // pkg/vmcp/session/untrusted.AnnotationCAGeneration across the module + // boundary; the contract is pinned by tests. + untrustedCAGenerationAnnotation = "toolhive.stacklok.dev/bump-ca-generation" +) + +// untrustedDNSLookup is the resolver for egress-policy destination hosts. +// Production leaves the stub nil (net.LookupIP). Tests install a stub via +// stubUntrustedDNSLookup, which SERIALIZES the whole calling test (parent or +// subtest): the installer holds untrustedDNSTestMu until the t.Cleanup +// restore runs. Tests that exercise an untrusted ensure must stub (via +// stubEgressDNS or reconcileOnce) — real round-robin DNS answers vary +// between calls and would churn the rendered allowlist if a parallel +// sibling's ensure hit them mid-test. All access is mutex-guarded: an +// unsynced package-level swap races between parallel tests. +var ( + untrustedDNSTestMu sync.Mutex // held for the lifetime of one stubbing test + untrustedDNSLookupMu sync.RWMutex + untrustedDNSLookupStub func(string) ([]net.IP, error) +) + +// untrustedDNSTestLock serializes a test (or subtest) that stubs the egress +// DNS lookup against every other such test and installs the stub for the +// caller's whole lifetime (released via the registered t.Cleanup). A +// same-chain re-entry (parent already holds the lock) installs the stub +// without re-locking, so serial subtests compose with a stubbing parent +// without deadlock. +func untrustedDNSTestLock(t interface { + Helper() + Cleanup(func()) +}, swap func(string) ([]net.IP, error)) { + t.Helper() + untrustedDNSLookupMu.RLock() + held := untrustedDNSLookupStub != nil + untrustedDNSLookupMu.RUnlock() + if !held { + untrustedDNSTestMu.Lock() + } + untrustedDNSLookupMu.Lock() + untrustedDNSLookupStub = swap + untrustedDNSLookupMu.Unlock() + if !held { + t.Cleanup(func() { + untrustedDNSLookupMu.Lock() + untrustedDNSLookupStub = nil + untrustedDNSLookupMu.Unlock() + untrustedDNSTestMu.Unlock() + }) + } +} + +// resolveEgressHost resolves one policy destination host through the +// installed stub (tests) or net.LookupIP (production). +func resolveEgressHost(host string) ([]net.IP, error) { + untrustedDNSLookupMu.RLock() + stub := untrustedDNSLookupStub + untrustedDNSLookupMu.RUnlock() + if stub != nil { + return stub(host) + } + return net.LookupIP(host) +} + +// ensureUntrustedResources reconciles the untrusted-mode data-plane +// resources for the MCPServer: the per-tenant bump CA (generation-named +// Secret + bundle ConfigMap), the egress-policy ConfigMap, and the +// egress-lockdown NetworkPolicy. All are ownerRef'd to the MCPServer (GC on +// deletion); the untrusted→trusted flip deletes them explicitly. +// +// Terminal vs transient: policy-shape problems (invalid hosts, a host that +// resolves to nothing, an empty resolved allowlist) are terminal +// *SpecValidationError*s — retrying cannot fix the spec. DNS lookup failures +// wrap egressbroker.ErrDNSResolution and are returned as ordinary errors so +// controller-runtime retries with backoff (a NXDOMAIN blip must not poison +// the workload). +func (r *MCPServerReconciler) ensureUntrustedResources(ctx context.Context, m *mcpv1beta1.MCPServer) error { + if !isUntrusted(m) { + return r.deleteUntrustedResources(ctx, m) + } + if m.Spec.EgressPolicy == nil { + return &SpecValidationError{ + Message: "spec.egressPolicy is required when spec.untrusted is true", + } + } + + // Compile the CRD policy once, resolve destinations, then render the + // ConfigMap document with the dial allowlist included so the sidecar's + // D7 guard and the NetworkPolicy share one source. + policyDoc, err := renderEgressPolicyYAML(m.Spec.EgressPolicy, nil) + if err != nil { + return &SpecValidationError{Message: err.Error()} + } + policy, err := egressbroker.ParsePolicy(policyDoc) + if err != nil { + return &SpecValidationError{Message: err.Error()} + } + dialAllowlist, err := egressbroker.ResolveDialAllowlist(policy, resolveEgressHost) + if err != nil { + if stderrors.Is(err, egressbroker.ErrDNSResolution) { + // Transient: DNS is an external dependency; retry with backoff. + return fmt.Errorf("failed to resolve untrusted egress destinations: %w", err) + } + return &SpecValidationError{Message: err.Error()} + } + policyDoc, err = renderEgressPolicyYAML(m.Spec.EgressPolicy, dialAllowlist) + if err != nil { + return &SpecValidationError{Message: err.Error()} + } + + generation, err := r.ensureUntrustedCA(ctx, m) + if err != nil { + return fmt.Errorf("failed to ensure bump CA: %w", err) + } + if err := r.ensureEgressPolicyConfigMap(ctx, m, policyDoc); err != nil { + return fmt.Errorf("failed to ensure egress policy ConfigMap: %w", err) + } + if err := r.ensureUntrustedNetworkPolicy(ctx, m, dialAllowlist); err != nil { + return fmt.Errorf("failed to ensure egress NetworkPolicy: %w", err) + } + if err := r.stampUntrustedCAGeneration(ctx, m, generation); err != nil { + return fmt.Errorf("failed to publish bump CA generation: %w", err) + } + return r.gcUntrustedCAGenerations(ctx, m, generation) +} + +// deleteUntrustedResources removes all untrusted-mode resources (used on the +// untrusted→trusted flip; MCPServer deletion itself is handled by GC through +// the owner references). CA generations are listed by label so every +// retained generation goes, not just the current one. +func (r *MCPServerReconciler) deleteUntrustedResources(ctx context.Context, m *mcpv1beta1.MCPServer) error { + secrets := &corev1.SecretList{} + if err := r.List(ctx, secrets, client.InNamespace(m.Namespace), + client.MatchingLabels(untrustedResourceLabels(m))); err != nil { + return fmt.Errorf("failed to list bump CA Secrets: %w", err) + } + for i := range secrets.Items { + name := secrets.Items[i].Name + if _, ok := egressbroker.TrimGeneration(name, egressbroker.BaseCASecretName(m.Name)); !ok { + continue + } + if err := r.deleteIfExists(ctx, &corev1.Secret{}, name, m.Namespace, "Secret"); err != nil { + return err + } + } + + bundles := &corev1.ConfigMapList{} + if err := r.List(ctx, bundles, client.InNamespace(m.Namespace), + client.MatchingLabels(untrustedResourceLabels(m))); err != nil { + return fmt.Errorf("failed to list bump CA bundles: %w", err) + } + for i := range bundles.Items { + name := bundles.Items[i].Name + if _, ok := egressbroker.TrimGeneration(name, egressbroker.BaseCABundleName(m.Name)); !ok { + continue + } + if err := r.deleteIfExists(ctx, &corev1.ConfigMap{}, name, m.Namespace, "ConfigMap"); err != nil { + return err + } + } + + if err := r.deleteIfExists(ctx, &corev1.ConfigMap{}, + egressbroker.ResourceNamesFor(m.Name, "").EgressPolicy, m.Namespace, "ConfigMap"); err != nil { + return err + } + return r.deleteIfExists(ctx, &networkingv1.NetworkPolicy{}, + m.Name+untrustedNetworkPolicySuffix, m.Namespace, "NetworkPolicy") +} + +// renderEgressPolicyYAML renders the CRD EgressPolicy into the ConfigMap +// document the sidecar compiles. The render is symmetric with +// egressbroker.ParsePolicy: exactly the CRD fields plus the operator-resolved +// dialAllowlist (the same CIDRs the NetworkPolicy ipBlocks carry, so D7 in +// the sidecar and the pod-level policy enforce the same destination set). +func renderEgressPolicyYAML(policy *mcpv1beta1.EgressPolicy, dialAllowlist []string) ([]byte, error) { + doc := struct { + Providers []egressbroker.ProviderPolicy `yaml:"providers"` + DialAllowlist []string `yaml:"dialAllowlist"` + }{ + Providers: make([]egressbroker.ProviderPolicy, 0, len(policy.Providers)), + DialAllowlist: dialAllowlist, + } + for _, p := range policy.Providers { + doc.Providers = append(doc.Providers, egressbroker.ProviderPolicy{ + Provider: p.Provider, + AllowedHosts: p.AllowedHosts, + AllowedMethods: p.AllowedMethods, + AllowedPathPrefixes: p.AllowedPathPrefixes, + }) + } + out, err := yaml.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("failed to render egress policy: %w", err) + } + return out, nil +} + +// ensureUntrustedCA ensures the per-tenant bump-CA Secret and public bundle +// ConfigMap for the CURRENT CA generation, rotating into a NEW generation +// when the CA enters the rotation window (50% of validity, D9). Returns the +// current generation (cert hash). +// +// Rotation is generation-named, never in-place: cert+key of one CA generation +// live in ONE Secret named -bump-ca-, and the bundle ConfigMap of +// the same generation carries only the public cert. The backend StatefulSet's +// pod template annotation (see stampUntrustedCAGeneration) names the current +// generation, so a session pod cloned mid-rotation mounts exactly one +// consistent cert/key pair — the old Update-in-place design could hand a pod +// new-key+old-cert during the Get/Update window. The private key exists only +// in the Secret, mounted by the trusted sidecar container. Key material is +// never logged. +func (r *MCPServerReconciler) ensureUntrustedCA(ctx context.Context, m *mcpv1beta1.MCPServer) (string, error) { + secrets := &corev1.SecretList{} + if err := r.List(ctx, secrets, client.InNamespace(m.Namespace), + client.MatchingLabels(untrustedResourceLabels(m))); err != nil { + return "", fmt.Errorf("failed to list bump CA Secrets: %w", err) + } + + certPEM, keyPEM, err := currentCAMaterial(m.Name, secrets.Items) + if err != nil { + return "", err + } + generation := egressbroker.CAGeneration(certPEM) + if keyPEM != nil { + names := egressbroker.ResourceNamesFor(m.Name, generation) + if err := r.createCASecretIfAbsent(ctx, m, names.CASecret, certPEM, keyPEM); err != nil { + return "", err + } + } + + // Bundle ConfigMaps carry the public cert of their own generation (never + // the key). Converge a bundle for EVERY parseable generation Secret — + // pods cloned before a rotation still mount the previous generation's + // pair while it is retained, so its bundle must exist and match its + // Secret's cert. The current Secret may have been created this pass and + // not be in the pre-pass list, so its cert is seeded explicitly. + bundleCerts := map[string][]byte{generation: certPEM} + for i := range secrets.Items { + s := &secrets.Items[i] + gen, ok := egressbroker.TrimGeneration(s.Name, egressbroker.BaseCASecretName(m.Name)) + if !ok || gen == generation { + continue + } + if _, err := egressbroker.ParseBumpCA(s.Data[egressbroker.CAKeyCert], s.Data[egressbroker.CAKeyKey]); err != nil { + continue + } + bundleCerts[gen] = s.Data[egressbroker.CAKeyCert] + } + for gen, cert := range bundleCerts { + bundle := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: egressbroker.ResourceNamesFor(m.Name, gen).CABundle, + Namespace: m.Namespace, + Labels: untrustedResourceLabels(m), + }, + Data: map[string]string{egressbroker.CAKeyCert: string(cert)}, + } + if err := controllerutil.SetControllerReference(m, bundle, r.Scheme); err != nil { + return "", fmt.Errorf("failed to set owner reference on bump CA bundle: %w", err) + } + if err := upsertConfigMapData(ctx, r.Client, bundle); err != nil { + return "", fmt.Errorf("failed to upsert bump CA bundle ConfigMap: %w", err) + } + } + return generation, nil +} + +// currentCAMaterial selects the current CA generation from the listed +// generation Secrets and returns its cert PEM (plus key PEM only when a fresh +// CA was minted this pass — no Secret, or rotation due). The current +// generation is the parseable Secret with the latest notAfter; unparseable +// Secrets are ignored here (the GC pass deletes them — they match no +// published generation). +func currentCAMaterial(mcpserverName string, secrets []corev1.Secret) (certPEM, keyPEM []byte, err error) { + var ( + current *corev1.Secret + currentCert *time.Time + ) + for i := range secrets { + s := &secrets[i] + if _, ok := egressbroker.TrimGeneration(s.Name, egressbroker.BaseCASecretName(mcpserverName)); !ok { + continue + } + loaded, err := egressbroker.ParseBumpCA(s.Data[egressbroker.CAKeyCert], s.Data[egressbroker.CAKeyKey]) + if err != nil { + continue + } + notAfter := loaded.NotAfter() + if currentCert == nil || notAfter.After(*currentCert) { + current, currentCert = s, ¬After + } + } + + if current == nil { + return mintBumpCA(mcpserverName) + } + loaded, err := egressbroker.ParseBumpCA( + current.Data[egressbroker.CAKeyCert], current.Data[egressbroker.CAKeyKey]) + if err != nil { + return nil, nil, fmt.Errorf("failed to re-parse bump CA Secret %q: %w", current.Name, err) + } + if !loaded.NeedsRotation(time.Now()) { + return current.Data[egressbroker.CAKeyCert], nil, nil + } + // Rotation due: mint a NEW generation alongside the old one. Existing + // pods keep trusting the CA their emptyDir was seeded with; new session + // pods clone the new template (no cross-pod TLS). + return mintBumpCA(mcpserverName) +} + +// createCASecretIfAbsent creates the generation-named bump-CA Secret when it +// does not exist yet. OwnerRef'd to the MCPServer for GC. Generation Secrets +// are immutable by construction: a name collides only when the same cert was +// re-minted (impossible — the mint includes a fresh key and serial), so an +// AlreadyExists on create means a concurrent reconcile won and its content +// is identical by definition of the generation hash; the call is then a +// no-op. +func (r *MCPServerReconciler) createCASecretIfAbsent( + ctx context.Context, m *mcpv1beta1.MCPServer, name string, certPEM, keyPEM []byte, +) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: m.Namespace, + Labels: untrustedResourceLabels(m), + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + egressbroker.CAKeyCert: certPEM, + egressbroker.CAKeyKey: keyPEM, + }, + } + if err := controllerutil.SetControllerReference(m, secret, r.Scheme); err != nil { + return fmt.Errorf("failed to set owner reference on bump CA Secret: %w", err) + } + if err := r.Create(ctx, secret); err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create bump CA Secret: %w", err) + } + return nil +} + +// gcUntrustedCAGenerations keeps the current CA generation and the previous +// one (N and N-1: pods cloned just before the rotation still mount N-1, and +// their idle TTL outlives the next rotation window), deleting every older +// generation's Secret and bundle. Deletions are best-effort-collected: a +// failure is returned so the reconcile retries (orphan generations would +// otherwise accumulate forever). +func (r *MCPServerReconciler) gcUntrustedCAGenerations(ctx context.Context, m *mcpv1beta1.MCPServer, currentGen string) error { + secrets := &corev1.SecretList{} + if err := r.List(ctx, secrets, client.InNamespace(m.Namespace), + client.MatchingLabels(untrustedResourceLabels(m))); err != nil { + return fmt.Errorf("failed to list bump CA Secrets for GC: %w", err) + } + previousGen := previousCAGenerationFromSecrets(secrets.Items, m.Name, currentGen) + keep := map[string]bool{currentGen: true} + if previousGen != "" { + keep[previousGen] = true + } + for i := range secrets.Items { + s := &secrets.Items[i] + gen, ok := egressbroker.TrimGeneration(s.Name, egressbroker.BaseCASecretName(m.Name)) + if !ok || keep[gen] { + continue + } + if err := r.deleteIfExists(ctx, &corev1.Secret{}, s.Name, m.Namespace, "Secret"); err != nil { + return err + } + } + + bundles := &corev1.ConfigMapList{} + if err := r.List(ctx, bundles, client.InNamespace(m.Namespace), + client.MatchingLabels(untrustedResourceLabels(m))); err != nil { + return fmt.Errorf("failed to list bump CA bundles for GC: %w", err) + } + for i := range bundles.Items { + b := &bundles.Items[i] + gen, ok := egressbroker.TrimGeneration(b.Name, egressbroker.BaseCABundleName(m.Name)) + if !ok || keep[gen] { + continue + } + if err := r.deleteIfExists(ctx, &corev1.ConfigMap{}, b.Name, m.Namespace, "ConfigMap"); err != nil { + return err + } + } + return nil +} + +// previousCAGenerationFromSecrets returns the parseable CA generation with +// the latest notAfter strictly older than currentGen's, or "" when there is +// none. notAfter (not creation order) orders generations: two rotations in +// quick succession still keep the immediately-previous CA. secrets is the +// caller's already-fetched list (the operator lists these Secrets once per +// reconcile pass). +func previousCAGenerationFromSecrets( + secrets []corev1.Secret, mcpserverName, currentGen string, +) string { + type candidate struct { + gen string + notAfter time.Time + } + var current *candidate + candidates := map[string]candidate{} + for i := range secrets { + s := &secrets[i] + gen, ok := egressbroker.TrimGeneration(s.Name, egressbroker.BaseCASecretName(mcpserverName)) + if !ok { + continue + } + loaded, err := egressbroker.ParseBumpCA(s.Data[egressbroker.CAKeyCert], s.Data[egressbroker.CAKeyKey]) + if err != nil { + continue + } + c := candidate{gen: gen, notAfter: loaded.NotAfter()} + candidates[gen] = c + if gen == currentGen { + cc := c + current = &cc + } + } + if current == nil { + return "" + } + previous := "" + for _, c := range candidates { + if c.gen == currentGen || !c.notAfter.Before(current.notAfter) { + continue + } + if previous == "" || c.notAfter.After(candidates[previous].notAfter) { + previous = c.gen + } + } + return previous +} + +// stampUntrustedCAGeneration publishes the current bump-CA generation on the +// backend StatefulSet's pod template annotation so the vMCP pod lifecycle +// clones pods mounting exactly that generation's Secret/bundle (mid-rotation +// consistency). A no-op when the annotation is already current (idempotent +// reconcile: no spurious STS write). +func (r *MCPServerReconciler) stampUntrustedCAGeneration( + ctx context.Context, m *mcpv1beta1.MCPServer, generation string, +) error { + sts, err := r.findUntrustedBackendSTS(ctx, m) + if err != nil || sts == nil { + // The STS may not exist yet (first reconcile precedes the workload + // deploy); the next reconcile stamps it. Session pods cannot be cloned + // before the STS exists either, so no pod ever runs unstamped. + return err + } + if sts.Spec.Template.Annotations[untrustedCAGenerationAnnotation] == generation { + return nil + } + patch := client.MergeFrom(sts.DeepCopy()) + annotations := map[string]string{} + for k, v := range sts.Spec.Template.Annotations { + annotations[k] = v + } + annotations[untrustedCAGenerationAnnotation] = generation + sts.Spec.Template.Annotations = annotations + if err := r.Patch(ctx, sts, patch); err != nil { + return fmt.Errorf("failed to stamp CA generation on backend StatefulSet: %w", err) + } + return nil +} + +// findUntrustedBackendSTS resolves the MCPServer's backend StatefulSet by the +// same selector the vMCP pod lifecycle lists with (mcpserver-uid label + +// toolhive=true). Returns (nil, nil) when no StatefulSet carries the label +// yet. More than one match is an error — the lifecycle would fail closed on +// the same ambiguity. +func (r *MCPServerReconciler) findUntrustedBackendSTS( + ctx context.Context, m *mcpv1beta1.MCPServer, +) (*appsv1.StatefulSet, error) { + list := &appsv1.StatefulSetList{} + if err := r.List(ctx, list, + client.InNamespace(m.Namespace), + client.MatchingLabels{untrustedMCPServerUIDLabel: string(m.UID), "toolhive": "true"}, + ); err != nil { + return nil, fmt.Errorf("failed to list backend StatefulSets: %w", err) + } + switch len(list.Items) { + case 0: + return nil, nil + case 1: + return &list.Items[0], nil + default: + return nil, fmt.Errorf("expected at most one backend StatefulSet for untrusted MCPServer %q, found %d", + m.Name, len(list.Items)) + } +} + +// ensureEgressPolicyConfigMap converges the sidecar-consumed policy document +// on the CRD's EgressPolicy. +func (r *MCPServerReconciler) ensureEgressPolicyConfigMap( + ctx context.Context, m *mcpv1beta1.MCPServer, policyDoc []byte, +) error { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: egressbroker.ResourceNamesFor(m.Name, "").EgressPolicy, + Namespace: m.Namespace, + Labels: untrustedResourceLabels(m), + }, + Data: map[string]string{"policy.yaml": string(policyDoc)}, + } + if err := controllerutil.SetControllerReference(m, cm, r.Scheme); err != nil { + return fmt.Errorf("failed to set owner reference on egress policy ConfigMap: %w", err) + } + return upsertConfigMapData(ctx, r.Client, cm) +} + +// ensureUntrustedNetworkPolicy converges the pod-level egress lockdown for +// untrusted session pods (D7): the only permitted egress is loopback (the +// sidecar), cluster DNS (kube-system/kube-dns pods only), and the +// policy-resolved destination CIDRs. +// +// NetworkPolicy is pod-level, not container-level — the backend container's +// confinement is enforced by routing (HTTP(S)_PROXY env) and by the fact +// that the credential never resides in the backend container at all; see +// spec-wave3 §4.1 for the documented container-separation limitation. The +// load-bearing controls are the sidecar's destination binding (D5) and +// per-dial IP validation (D7). +func (r *MCPServerReconciler) ensureUntrustedNetworkPolicy( + ctx context.Context, m *mcpv1beta1.MCPServer, dialAllowlist []string, +) error { + desired := desiredUntrustedNetworkPolicy(m, dialAllowlist) + if err := controllerutil.SetControllerReference(m, desired, r.Scheme); err != nil { + return fmt.Errorf("failed to set owner reference on egress NetworkPolicy: %w", err) + } + + existing := &networkingv1.NetworkPolicy{} + err := r.Get(ctx, types.NamespacedName{Name: desired.Name, Namespace: desired.Namespace}, existing) + if errors.IsNotFound(err) { + if err := r.Create(ctx, desired); err != nil { + return fmt.Errorf("failed to create egress NetworkPolicy: %w", err) + } + return nil + } + if err != nil { + return fmt.Errorf("failed to get egress NetworkPolicy: %w", err) + } + if networkPolicyNeedsUpdate(existing, desired) { + existing.Spec = desired.Spec + existing.Labels = desired.Labels + if err := r.Update(ctx, existing); err != nil { + return fmt.Errorf("failed to update egress NetworkPolicy: %w", err) + } + } + return nil +} + +const ( + // dnsNamespaceSelectorName is the namespace of the cluster DNS + // deployment on kubeadm/kind/managed clusters. Documented limitation: on + // clusters whose DNS lives elsewhere (e.g. a custom namespace), DNS + // egress is denied until the deployment is adjusted — an explicit, + // visible failure (backend DNS lookups fail loudly) rather than a silent + // port-53-to-anywhere hole. + dnsNamespaceSelectorName = "kube-system" + // dnsPodSelectorName matches CoreDNS/kube-dns pods by their standard + // k8s-app label (kubeadm, kind, GKE, EKS default addons all use it). + dnsPodSelectorName = "kube-dns" +) + +// desiredUntrustedNetworkPolicy renders the egress lockdown. +func desiredUntrustedNetworkPolicy( + m *mcpv1beta1.MCPServer, dialAllowlist []string, +) *networkingv1.NetworkPolicy { + udp := corev1.ProtocolUDP + tcp := corev1.ProtocolTCP + dnsPort := intstr.FromInt32(53) + + // Destination allowlist (operator-resolved AllowedHosts; best-effort + // defense-in-depth — the sidecar re-validates per dial, D7). The list is + // sorted by the resolver, so rule order is deterministic (no churn). + destinationRule := networkingv1.NetworkPolicyEgressRule{} + for _, cidr := range dialAllowlist { + destinationRule.To = append(destinationRule.To, + networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}}) + } + + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: m.Name + untrustedNetworkPolicySuffix, + Namespace: m.Namespace, + Labels: untrustedResourceLabels(m), + }, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{ + "toolhive.stacklok.dev/untrusted": "true", + "toolhive.stacklok.dev/mcpserver-uid": string(m.UID), + }}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, + Egress: []networkingv1.NetworkPolicyEgressRule{ + // Loopback: the in-pod sidecar (Envoy + broker). + {To: []networkingv1.NetworkPolicyPeer{ + {IPBlock: &networkingv1.IPBlock{CIDR: "127.0.0.1/32"}}, + {IPBlock: &networkingv1.IPBlock{CIDR: "::1/128"}}, + }}, + // DNS: restricted to the cluster DNS pods (kube-system + + // k8s-app=kube-dns) — never 0.0.0.0/0:53, which would let the + // backend run or reach an arbitrary DNS server to smuggle data + // or resolve attacker-controlled names off-policy. + { + To: []networkingv1.NetworkPolicyPeer{{ + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": dnsNamespaceSelectorName, + }}, + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "k8s-app": dnsPodSelectorName, + }}, + }}, + Ports: []networkingv1.NetworkPolicyPort{ + {Protocol: &udp, Port: &dnsPort}, + {Protocol: &tcp, Port: &dnsPort}, + }, + }, + destinationRule, + }, + }, + } +} + +// upsertConfigMapData creates the ConfigMap or updates only its Data (and +// labels) when they drift. A no-op reconcile performs no write (idempotency: +// unchanged ResourceVersion). The operator is the sole writer of these +// ConfigMaps, so a selective Update after Get is safe. +func upsertConfigMapData(ctx context.Context, c client.Client, desired *corev1.ConfigMap) error { + existing := &corev1.ConfigMap{} + err := c.Get(ctx, types.NamespacedName{Name: desired.Name, Namespace: desired.Namespace}, existing) + if errors.IsNotFound(err) { + if err := c.Create(ctx, desired); err != nil { + return fmt.Errorf("failed to create ConfigMap %s: %w", desired.Name, err) + } + return nil + } + if err != nil { + return fmt.Errorf("failed to get ConfigMap %s: %w", desired.Name, err) + } + if stringMapEqual(existing.Data, desired.Data) && stringMapEqual(existing.Labels, desired.Labels) { + return nil + } + existing.Data = desired.Data + existing.Labels = desired.Labels + if err := c.Update(ctx, existing); err != nil { + return fmt.Errorf("failed to update ConfigMap %s: %w", desired.Name, err) + } + return nil +} + +// mintBumpCA generates a fresh per-tenant bump CA and returns the PEM cert + +// key. The key bytes are handled only long enough to write the Secret; they +// are never logged. +func mintBumpCA(mcpserverName string) (certPEM, keyPEM []byte, err error) { + ca, err := egressbroker.GenerateBumpCA(mcpserverName+"-bump-ca", time.Now()) + if err != nil { + return nil, nil, err + } + key, err := ca.KeyPEM() + if err != nil { + return nil, nil, err + } + return ca.CertPEM(), key, nil +} + +// untrustedResourceLabels identifies operator-managed untrusted resources. +func untrustedResourceLabels(m *mcpv1beta1.MCPServer) map[string]string { + labels := labelsForMCPServer(m.Name) + labels["toolhive.stacklok.dev/untrusted-resource"] = "true" + return labels +} + +// stringMapEqual compares two string maps for exact equality. +func stringMapEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if bv, ok := b[k]; !ok || bv != v { + return false + } + } + return true +} + +// networkPolicyNeedsUpdate reports whether the live NetworkPolicy differs +// from desired in the fields the operator owns. +func networkPolicyNeedsUpdate(existing, desired *networkingv1.NetworkPolicy) bool { + if !stringMapEqual(existing.Labels, desired.Labels) { + return true + } + return renderedEgressRules(existing.Spec.Egress) != renderedEgressRules(desired.Spec.Egress) +} + +// renderedEgressRules canonicalizes egress rules for comparison: peers and +// ports are sorted per rule and rules are sorted, so semantically equal +// policies compare equal regardless of rule order. +func renderedEgressRules(rules []networkingv1.NetworkPolicyEgressRule) string { + rendered := make([]string, 0, len(rules)) + for _, rule := range rules { + var peers []string + for _, to := range rule.To { + if to.IPBlock != nil { + peers = append(peers, "ip:"+to.IPBlock.CIDR) + } + if to.PodSelector != nil { + peers = append(peers, fmt.Sprintf("pod:%v", to.PodSelector.MatchLabels)) + } + if to.NamespaceSelector != nil { + peers = append(peers, fmt.Sprintf("ns:%v", to.NamespaceSelector.MatchLabels)) + } + } + sort.Strings(peers) + var ports []string + for _, p := range rule.Ports { + proto := "" + if p.Protocol != nil { + proto = string(*p.Protocol) + } + port := "" + if p.Port != nil { + port = p.Port.String() + } + ports = append(ports, proto+"/"+port) + } + sort.Strings(ports) + rendered = append(rendered, strings.Join(peers, ",")+"|"+strings.Join(ports, ",")) + } + sort.Strings(rendered) + return strings.Join(rendered, ";") +} diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go new file mode 100644 index 0000000000..2a89c24004 --- /dev/null +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go @@ -0,0 +1,582 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" + "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1/v1beta1test" + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +func withEgressPolicy() v1beta1test.MCPServerOption { + return v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.Untrusted = true + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{ + Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + AllowedMethods: []string{"GET"}, + AllowedPathPrefixes: []string{"/repos/"}, + }}, + } + }) +} + +// Unit fixtures use hostname allowedHosts (the CRD grammar forbids IP +// literals) plus a stubbed DNS lookup — no real resolution in unit tests. The +// stub spans the test's whole lifetime (t.Cleanup restores it): a mid-test +// restore would let a parallel sibling's Reconcile hit real DNS or a stale +// stub. Tests calling this must NOT run in parallel (paralleltest nolint). +func stubEgressDNS(t *testing.T, ips map[string][]net.IP) { + t.Helper() + untrustedDNSTestLock(t, stubDNSFor(ips, false)) +} + +// stubEgressDNSStrict stubs DNS with exact map semantics (unknown hosts +// fail) for tests asserting resolution failure. +func stubEgressDNSStrict(t *testing.T, ips map[string][]net.IP) { + t.Helper() + untrustedDNSTestLock(t, stubDNSFor(ips, true)) +} + +// stubDNSFor builds a DNS stub over ips. Non-strict stubs resolve unknown +// hosts to the shared fixture IP so any test's stub is a valid stand-in for +// any other's (a parallel sibling may run under this stub after this test's +// cleanup already ran). Strict stubs fail unknown hosts (resolution-failure +// assertions). +func stubDNSFor(ips map[string][]net.IP, strict bool) func(string) ([]net.IP, error) { + return func(host string) ([]net.IP, error) { + if resolved, ok := ips[host]; ok { + return resolved, nil + } + if strict { + return nil, fmt.Errorf("no such host: %s", host) + } + return []net.IP{net.ParseIP("140.82.114.26")}, nil + } +} + +var githubDNS = map[string][]net.IP{"api.github.com": {net.ParseIP("140.82.114.26")}} + +func caSecretName(t *testing.T, r *MCPServerReconciler, m *mcpv1beta1.MCPServer) string { + t.Helper() + secrets := listCASecrets(t, r, m) + require.Len(t, secrets, 1, "expected exactly one current CA generation Secret") + return secrets[0].Name +} + +func listCASecrets(t *testing.T, r *MCPServerReconciler, m *mcpv1beta1.MCPServer) []corev1.Secret { + t.Helper() + secrets := &corev1.SecretList{} + require.NoError(t, r.List(t.Context(), secrets, + client.InNamespace(m.Namespace), client.MatchingLabels(untrustedResourceLabels(m)))) + return secrets.Items +} + +//nolint:tparallel // Subtests swap the package-level DNS lookup stub; they must run serially. +func TestEnsureUntrustedResources(t *testing.T) { + t.Parallel() + + //nolint:paralleltest // Swaps the package-level DNS lookup stub (restored after each ensure). + t.Run("untrusted MCPServer gets generation-named CA Secret, bundle, policy ConfigMap, NetworkPolicy", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m) + ctx := t.Context() + + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + + // Generation-named CA Secret with both keys, ownerRef'd. + secrets := listCASecrets(t, r, m) + require.Len(t, secrets, 1) + secret := secrets[0] + generation, ok := egressbroker.TrimGeneration(secret.Name, egressbroker.BaseCASecretName(m.Name)) + require.True(t, ok, "CA Secret must be generation-named, got %q", secret.Name) + assert.Equal(t, egressbroker.ResourceNamesFor(m.Name, generation).CASecret, secret.Name) + assert.NotEmpty(t, secret.Data["ca.crt"]) + assert.NotEmpty(t, secret.Data["ca.key"]) + assertOwnerRef(t, secret.OwnerReferences, m) + + // The generation is the hash of the cert; the CA parses and is fresh. + assert.Equal(t, egressbroker.CAGeneration(secret.Data["ca.crt"]), generation) + ca, err := egressbroker.ParseBumpCA(secret.Data["ca.crt"], secret.Data["ca.key"]) + require.NoError(t, err) + assert.False(t, ca.NeedsRotation(metav1.Now().Time)) + + // Bundle ConfigMap of the SAME generation carries the same public cert, no key. + bundle := &corev1.ConfigMap{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: egressbroker.ResourceNamesFor(m.Name, generation).CABundle, Namespace: "default"}, bundle)) + assert.Equal(t, string(secret.Data["ca.crt"]), bundle.Data["ca.crt"]) + assert.NotContains(t, bundle.Data, "ca.key") + assertOwnerRef(t, bundle.OwnerReferences, m) + + // Policy ConfigMap renders the CRD EgressPolicy and re-parses. + policyCM := &corev1.ConfigMap{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: "github-mcp-egress-policy", Namespace: "default"}, policyCM)) + assertOwnerRef(t, policyCM.OwnerReferences, m) + compiled, err := egressbroker.ParsePolicy([]byte(policyCM.Data["policy.yaml"])) + require.NoError(t, err, "rendered policy must round-trip through the sidecar's parser") + provider, ok := compiled.ProviderFor("api.github.com") + assert.True(t, ok) + assert.Equal(t, "github", provider) + assert.Equal(t, []string{"140.82.114.26/32"}, compiled.DialAllowlist()) + + // NetworkPolicy selects the untrusted session pods and locks egress. + np := &networkingv1.NetworkPolicy{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, np)) + assertOwnerRef(t, np.OwnerReferences, m) + assert.Equal(t, map[string]string{ + "toolhive.stacklok.dev/untrusted": "true", + "toolhive.stacklok.dev/mcpserver-uid": string(m.UID), + }, np.Spec.PodSelector.MatchLabels) + assert.Contains(t, np.Spec.PolicyTypes, networkingv1.PolicyTypeEgress) + + var cidrs []string + for _, rule := range np.Spec.Egress { + for _, peer := range rule.To { + if peer.IPBlock != nil { + cidrs = append(cidrs, peer.IPBlock.CIDR) + } + } + } + assert.Contains(t, cidrs, "127.0.0.1/32", "loopback (sidecar) must be permitted") + assert.Contains(t, cidrs, "140.82.114.26/32", "policy destinations must be permitted") + assertDNSRuleRestricted(t, np) + }) + + //nolint:paralleltest // Swaps the package-level DNS lookup stub (restored after each ensure). + t.Run("reconcile is idempotent: second ensure performs no spurious write", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m) + ctx := t.Context() + + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + secretName := caSecretName(t, r, m) + secretBefore := &corev1.Secret{} + require.NoError(t, r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: "default"}, secretBefore)) + npBefore := &networkingv1.NetworkPolicy{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, npBefore)) + + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + + secrets := listCASecrets(t, r, m) + require.Len(t, secrets, 1, "idempotent reconcile must not mint a new generation") + secretAfter := &corev1.Secret{} + require.NoError(t, r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: "default"}, secretAfter)) + assert.Equal(t, secretBefore.ResourceVersion, secretAfter.ResourceVersion, + "unchanged CA Secret must not be rewritten (a rewrite would churn pods)") + npAfter := &networkingv1.NetworkPolicy{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, npAfter)) + if npBefore.ResourceVersion != npAfter.ResourceVersion { + t.Fatalf("NetworkPolicy churned: before=%s after=%s\nbefore rules: %s\nafter rules: %s\nbefore labels: %v\nafter labels: %v", + npBefore.ResourceVersion, npAfter.ResourceVersion, + renderedEgressRules(npBefore.Spec.Egress), renderedEgressRules(npAfter.Spec.Egress), + npBefore.Labels, npAfter.Labels) + } + }) + + //nolint:paralleltest // Swaps the package-level DNS lookup stub (restored after each ensure). + t.Run("untrusted→trusted flip deletes all untrusted resources", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m) + ctx := t.Context() + + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + secretName := caSecretName(t, r, m) + gen, _ := egressbroker.TrimGeneration(secretName, egressbroker.BaseCASecretName(m.Name)) + + // Flip to trusted. + m.Spec.Untrusted = false + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + + secret := &corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: "default"}, secret) + assert.True(t, apierrors.IsNotFound(err), "CA Secret must be deleted on flip") + bundle := &corev1.ConfigMap{} + err = r.Get(ctx, types.NamespacedName{ + Name: egressbroker.ResourceNamesFor(m.Name, gen).CABundle, Namespace: "default"}, bundle) + assert.True(t, apierrors.IsNotFound(err), "CA bundle must be deleted on flip") + policyCM := &corev1.ConfigMap{} + err = r.Get(ctx, types.NamespacedName{Name: "github-mcp-egress-policy", Namespace: "default"}, policyCM) + assert.True(t, apierrors.IsNotFound(err), "policy ConfigMap must be deleted on flip") + np := &networkingv1.NetworkPolicy{} + err = r.Get(ctx, types.NamespacedName{Name: "github-mcp-egress", Namespace: "default"}, np) + assert.True(t, apierrors.IsNotFound(err), "NetworkPolicy must be deleted on flip") + }) + + //nolint:paralleltest // Serial with sibling subtests that swap the DNS lookup stub. + t.Run("missing EgressPolicy on untrusted server is a terminal spec error", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withUntrustedSpec()) + r, _ := setupUntrustedReconciler(t, m) + + err := r.ensureUntrustedResources(t.Context(), m) + require.Error(t, err) + var specErr *SpecValidationError + require.ErrorAs(t, err, &specErr) + }) + + //nolint:paralleltest // Serial with sibling subtests that swap the DNS lookup stub. + t.Run("DNS failure is transient, never a terminal spec error", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + stubEgressDNSStrict(t, map[string][]net.IP{}) // every lookup fails + r, _ := setupUntrustedReconciler(t, m) + + err := r.ensureUntrustedResources(t.Context(), m) + require.Error(t, err) + assert.ErrorIs(t, err, egressbroker.ErrDNSResolution) + var specErr *SpecValidationError + assert.False(t, errorAsSpecValidation(err, &specErr), + "a DNS blip must retry with backoff, not poison the workload: %v", err) + }) + + //nolint:paralleltest // Serial with sibling subtests that swap the DNS lookup stub. + t.Run("host resolving to no addresses is a terminal spec error", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + stubEgressDNSStrict(t, map[string][]net.IP{"api.github.com": {}}) // resolves to nothing + r, _ := setupUntrustedReconciler(t, m) + + err := r.ensureUntrustedResources(t.Context(), m) + require.Error(t, err) + var specErr *SpecValidationError + require.ErrorAs(t, err, &specErr) + }) + + //nolint:paralleltest // Swaps the package-level DNS lookup stub (restored after each ensure). + t.Run("unparseable CA Secret is garbage-collected and a fresh generation minted", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m) + ctx := t.Context() + + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + // Corrupt the current generation Secret. + secretName := caSecretName(t, r, m) + secret := &corev1.Secret{} + require.NoError(t, r.Get(ctx, types.NamespacedName{Name: secretName, Namespace: "default"}, secret)) + secret.Data = map[string][]byte{"ca.crt": []byte("garbage"), "ca.key": []byte("garbage")} + require.NoError(t, r.Update(ctx, secret)) + + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + secrets := listCASecrets(t, r, m) + require.Len(t, secrets, 1, "the corrupt generation must be GC'd, replaced by one fresh generation") + assert.NotEqual(t, secretName, secrets[0].Name) + _, err := egressbroker.ParseBumpCA(secrets[0].Data["ca.crt"], secrets[0].Data["ca.key"]) + require.NoError(t, err, "corrupt CA must be regenerated") + }) + + //nolint:paralleltest // Swaps the package-level DNS lookup stub (restored after each ensure). + t.Run("rotation mints a new generation, keeps N-1, GCs older, and re-stamps the STS template", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m) + ctx := t.Context() + + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + firstName := caSecretName(t, r, m) + first := &corev1.Secret{} + require.NoError(t, r.Get(ctx, types.NamespacedName{Name: firstName, Namespace: "default"}, first)) + + // Force rotation: replace the fresh generation with an almost-expired + // CA (test-only manipulation of the stored objects) so the next ensure + // rotates it. The fixture is backdated well past the second aged + // fixture below so generation ordering by notAfter is unambiguous. + oldCert, oldKey := mintAgedBumpCA(t, m.Name, -egressbroker.CAValidity/2-24*time.Hour) + staleGen := egressbroker.CAGeneration(oldCert) + staleName := egressbroker.ResourceNamesFor(m.Name, staleGen).CASecret + require.NoError(t, r.Delete(ctx, first)) + require.NoError(t, r.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: staleName, + Namespace: m.Namespace, + Labels: untrustedResourceLabels(m), + }, + Data: map[string][]byte{"ca.crt": oldCert, "ca.key": oldKey}, + })) + + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + + secrets := listCASecrets(t, r, m) + require.Len(t, secrets, 2, "rotation must keep N (new) and N-1 (aged) generations") + + var newSecret *corev1.Secret + for i := range secrets { + if secrets[i].Name != staleName { + newSecret = &secrets[i] + } + } + require.NotNil(t, newSecret, "a new generation Secret must exist") + newGen, _ := egressbroker.TrimGeneration(newSecret.Name, egressbroker.BaseCASecretName(m.Name)) + assert.NotEqual(t, staleGen, newGen) + + // The new bundle matches the new Secret's cert (same generation pair). + bundle := &corev1.ConfigMap{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: egressbroker.ResourceNamesFor(m.Name, newGen).CABundle, Namespace: "default"}, bundle)) + assert.Equal(t, string(newSecret.Data["ca.crt"]), bundle.Data["ca.crt"], + "bundle must always come from the same generation as the Secret") + // The stale generation's bundle is recreated from its own Secret's cert + // (N-1 pods keep a consistent pair), never from the new cert. + oldBundle := &corev1.ConfigMap{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: egressbroker.ResourceNamesFor(m.Name, staleGen).CABundle, Namespace: "default"}, oldBundle)) + assert.Equal(t, string(oldCert), oldBundle.Data["ca.crt"]) + + // The STS template annotation publishes the NEW generation. + sts := backendSTSFixture(m) + require.NoError(t, r.Create(ctx, sts)) + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + stamped := &appsv1.StatefulSet{} + require.NoError(t, r.Get(ctx, types.NamespacedName{Name: sts.Name, Namespace: m.Namespace}, stamped)) + assert.Equal(t, newGen, stamped.Spec.Template.Annotations[untrustedCAGenerationAnnotation]) + + // A third rotation GCs the oldest generation. + newest := &corev1.Secret{} + require.NoError(t, r.Get(ctx, types.NamespacedName{Name: newSecret.Name, Namespace: "default"}, newest)) + olderCert, olderKey := mintAgedBumpCA(t, m.Name, -egressbroker.CAValidity/2) + require.NoError(t, r.Delete(ctx, newest)) + // Replace "current" with another aged CA so the next ensure rotates again. + secondStaleGen := egressbroker.CAGeneration(olderCert) + require.NoError(t, r.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: egressbroker.ResourceNamesFor(m.Name, secondStaleGen).CASecret, + Namespace: m.Namespace, + Labels: untrustedResourceLabels(m), + }, + Data: map[string][]byte{"ca.crt": olderCert, "ca.key": olderKey}, + })) + require.NoError(t, r.ensureUntrustedResources(ctx, m)) + + secrets = listCASecrets(t, r, m) + require.Len(t, secrets, 2, "only N and N-1 generations survive") + names := []string{secrets[0].Name, secrets[1].Name} + assert.NotContains(t, names, staleName, "the oldest generation must be GC'd") + goneBundle := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{ + Name: egressbroker.ResourceNamesFor(m.Name, staleGen).CABundle, Namespace: "default"}, goneBundle) + assert.True(t, apierrors.IsNotFound(err), "the oldest generation's bundle must be GC'd") + }) +} + +// backendSTSFixture builds the labeled backend StatefulSet the generation +// stamp targets (same labels DeployWorkload derives from the RunConfig +// container labels). +func backendSTSFixture(m *mcpv1beta1.MCPServer) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: m.Name, + Namespace: m.Namespace, + Labels: map[string]string{ + untrustedMCPServerUIDLabel: string(m.UID), + "toolhive": "true", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": m.Name}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": m.Name}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "mcp", Image: "img"}}}, + }, + }, + } +} + +// mintAgedBumpCA mints a CA already inside the rotation window (past 50% of +// validity) so ensure triggers rotation on the next pass. age is the +// (negative) offset from now the CA's validity starts at; distinct ages make +// generation ordering by notAfter unambiguous. +func mintAgedBumpCA(t *testing.T, name string, age time.Duration) (certPEM, keyPEM []byte) { + t.Helper() + ca, err := egressbroker.GenerateBumpCA(name+"-bump-ca", metav1.Now().Add(age)) + require.NoError(t, err) + require.True(t, ca.NeedsRotation(metav1.Now().Time), "fixture CA must be inside the rotation window") + key, err := ca.KeyPEM() + require.NoError(t, err) + return ca.CertPEM(), key +} + +func errorAsSpecValidation(err error, target **SpecValidationError) bool { + for err != nil { + if se, ok := err.(*SpecValidationError); ok { + *target = se + return true + } + unwrapper, ok := err.(interface{ Unwrap() error }) + if !ok { + return false + } + err = unwrapper.Unwrap() + } + return false +} + +// TestCreateRunConfig_UntrustedUIDLabel pins that an untrusted MCPServer's +// RunConfig carries the rename-safe mcpserver-uid container label. The label +// flows to the backend StatefulSet (DeployWorkload applies ContainerLabels), +// which is how the vMCP pod lifecycle resolves the clone template by selector +// (LabelMCPServerUID + toolhive=true). A trusted MCPServer must NOT carry it. +func TestCreateRunConfig_UntrustedUIDLabel(t *testing.T) { + t.Parallel() + + newServer := func(opts ...v1beta1test.MCPServerOption) *mcpv1beta1.MCPServer { + return v1beta1test.NewMCPServer("github-mcp", "default", opts...) + } + + t.Run("untrusted server stamps the mcpserver-uid container label", func(t *testing.T) { + t.Parallel() + m := newServer(withUntrustedCompliantPolicy()) + r, _ := setupUntrustedReconciler(t, m) + rc, err := r.createRunConfigFromMCPServer(m) + require.NoError(t, err) + assert.Equal(t, string(m.UID), rc.ContainerLabels[untrustedMCPServerUIDLabel], + "untrusted RunConfig must carry the mcpserver-uid label for STS template resolution") + }) + + t.Run("trusted server does not stamp the label", func(t *testing.T) { + t.Parallel() + m := newServer() + r, _ := setupUntrustedReconciler(t, m) + rc, err := r.createRunConfigFromMCPServer(m) + require.NoError(t, err) + _, present := rc.ContainerLabels[untrustedMCPServerUIDLabel] + assert.False(t, present, "trusted RunConfig must not carry the untrusted UID label") + }) +} + +func assertOwnerRef(t *testing.T, refs []metav1.OwnerReference, m *mcpv1beta1.MCPServer) { + t.Helper() + require.Len(t, refs, 1) + assert.Equal(t, "MCPServer", refs[0].Kind) + assert.Equal(t, m.Name, refs[0].Name) + assert.Equal(t, m.UID, refs[0].UID) +} + +// assertDNSRuleRestricted pins the tightened DNS egress: DNS may flow only to +// the cluster DNS pods (kube-system + k8s-app=kube-dns) on port 53 — never to +// 0.0.0.0/0:53 (an arbitrary DNS server would smuggle data off-policy). +func assertDNSRuleRestricted(t *testing.T, np *networkingv1.NetworkPolicy) { + t.Helper() + for _, rule := range np.Spec.Egress { + has53 := false + for _, port := range rule.Ports { + if port.Port != nil && port.Port.IntVal == 53 { + has53 = true + } + } + if !has53 { + continue + } + require.Len(t, rule.To, 1, "the DNS rule must name exactly one peer (the cluster DNS pods)") + peer := rule.To[0] + assert.Nil(t, peer.IPBlock, "DNS must not be an IPBlock rule (that was the 0.0.0.0/0:53 hole)") + require.NotNil(t, peer.NamespaceSelector) + assert.Equal(t, map[string]string{"kubernetes.io/metadata.name": "kube-system"}, + peer.NamespaceSelector.MatchLabels) + require.NotNil(t, peer.PodSelector) + assert.Equal(t, map[string]string{"k8s-app": "kube-dns"}, peer.PodSelector.MatchLabels) + return + } + t.Fatal("NetworkPolicy must permit DNS (port 53) to the cluster DNS pods") +} + +// TestOperatorPolicyContractWithSidecar pins the cross-module contract the +// untrusted pod clone relies on (applyEgressBrokerSidecar in +// pkg/vmcp/session/untrusted/egress.go): the operator-rendered egress policy +// ConfigMap must live at the fixed, generation-free name the sidecar volume +// mounts (egressbroker.ResourceNamesFor(name, "").EgressPolicy), its document +// must round-trip through the sidecar's parser, and it must carry the +// operator-resolved dialAllowlist the broker's D7 guard enforces (the env +// override THV_EGRESSBROKER_DIAL_ALLOWLIST is unset in the clone wiring). +// +//nolint:paralleltest // Swaps the package-level DNS lookup stub (restored after the ensure call). +func TestOperatorPolicyContractWithSidecar(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m) + + require.NoError(t, r.ensureUntrustedResources(t.Context(), m)) + + // Name contract: fixed name, no generation suffix (unlike the CA objects, + // whose generation is read from the STS template annotation at clone time). + wantName := egressbroker.ResourceNamesFor(m.Name, "").EgressPolicy + assert.Equal(t, m.Name+"-egress-policy", wantName) + assert.NotContains(t, wantName, egressbroker.CAGeneration([]byte("x")), + "the policy ConfigMap name must not be generation-qualified") + + cm := &corev1.ConfigMap{} + require.NoError(t, r.Get(t.Context(), types.NamespacedName{Name: wantName, Namespace: m.Namespace}, cm)) + + // Content contract: the sidecar compiles exactly this document (policy + + // dialAllowlist); ParsePolicy is the sidecar's own parser. + policy, err := egressbroker.ParsePolicy([]byte(cm.Data["policy.yaml"])) + require.NoError(t, err) + provider, ok := policy.ProviderFor("api.github.com") + assert.True(t, ok) + assert.Equal(t, "github", provider) + assert.Equal(t, []string{"140.82.114.26/32"}, policy.DialAllowlist(), + "the broker's D7 dial allowlist comes from this document when the env override is unset") + + // The NetworkPolicy ipBlocks and the document's dialAllowlist are the same + // set (one resolver output feeds both). + np := &networkingv1.NetworkPolicy{} + require.NoError(t, r.Get(t.Context(), types.NamespacedName{ + Name: m.Name + untrustedNetworkPolicySuffix, Namespace: m.Namespace}, np)) + var cidrs []string + for _, rule := range np.Spec.Egress { + for _, peer := range rule.To { + if peer.IPBlock != nil && peer.IPBlock.CIDR != "127.0.0.1/32" && peer.IPBlock.CIDR != "::1/128" { + cidrs = append(cidrs, peer.IPBlock.CIDR) + } + } + } + assert.Equal(t, policy.DialAllowlist(), cidrs) +} + +func TestRenderEgressPolicyYAML(t *testing.T) { + t.Parallel() + + doc, err := renderEgressPolicyYAML(&mcpv1beta1.EgressPolicy{ + Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com", "*.githubusercontent.com"}, + AllowedMethods: []string{"GET", "POST"}, + AllowedPathPrefixes: []string{"/repos/"}, + }}, + }, []string{"140.82.112.0/20"}) + require.NoError(t, err) + + // The render must be symmetric with the sidecar parser: same policy in, + // same evaluation out — and the dial allowlist travels with the document + // (same source as the NetworkPolicy ipBlocks, for D7). + policy, err := egressbroker.ParsePolicy(doc) + require.NoError(t, err) + assert.Equal(t, []string{"140.82.112.0/20"}, policy.DialAllowlist()) + provider, ok := policy.ProviderFor("raw.githubusercontent.com") + assert.True(t, ok) + assert.Equal(t, "github", provider) + assert.True(t, policy.Allows("github", "POST", "/repos/x")) + assert.False(t, policy.Allows("github", "DELETE", "/repos/x")) + assert.False(t, policy.Allows("github", "GET", "/admin")) +} diff --git a/cmd/thv-operator/controllers/virtualmcpserver_deployment.go b/cmd/thv-operator/controllers/virtualmcpserver_deployment.go index 9b6df3c02f..67bcbafaa0 100644 --- a/cmd/thv-operator/controllers/virtualmcpserver_deployment.go +++ b/cmd/thv-operator/controllers/virtualmcpserver_deployment.go @@ -79,6 +79,11 @@ const ( vmcpDefaultMemoryRequest = "128Mi" vmcpDefaultCPULimit = "500m" vmcpDefaultMemoryLimit = "512Mi" + + // untrustedTokenStoreAddrEnvVar carries the auth-server Redis address to the + // vMCP, which clones it into untrusted egress-broker sidecars. The value is + // the non-secret host:port from spec.authServerConfig.storage.redis.addr. + untrustedTokenStoreAddrEnvVar = "THV_UNTRUSTED_TOKEN_STORE_REDIS_ADDR" // #nosec G101 -- env var name, not a credential ) // RBAC rules for VirtualMCPServer service account in inline mode @@ -109,6 +114,25 @@ var vmcpDiscoveredRBACRules = []rbacv1.PolicyRule{ Resources: []string{"configmaps", "secrets"}, Verbs: []string{"get", "list", "watch"}, }, + { + // Untrusted mode (ADR-0001 D4): vMCP is the session-lifecycle owner and + // creates/deletes one bare pod per (user, session, untrusted MCPServer), + // cloned from the operator-built backend StatefulSet template. Pods-only + // namespaced rule; the template-clone-only constraint and quota + // admission bound the blast radius of pod-write. + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch", "create", "delete"}, + }, + { + // Untrusted mode (ADR-0001 D4): the pod lifecycle resolves the + // operator-built backend StatefulSet by label selector + // (toolhive=true + mcpserver-uid) to clone its pod template. + // Read-only — vMCP never writes StatefulSets. + APIGroups: []string{"apps"}, + Resources: []string{"statefulsets"}, + Verbs: []string{"get", "list", "watch"}, + }, { APIGroups: []string{"toolhive.stacklok.dev"}, Resources: []string{ @@ -390,9 +414,41 @@ func (r *VirtualMCPServerReconciler) buildEnvVarsForVmcp( env = append(env, ctrlutil.GenerateAuthServerEnvVars(vmcp.Spec.AuthServerConfig)...) } + // Mount the auth-server token-store coordinates for untrusted-mode egress + // (Wave-3): the vMCP forwards these to the egress-broker sidecar at pod-clone + // time. The address is non-secret; the vMCP derives the per-tenant key prefix + // from its own identity (VMCP_NAME/VMCP_NAMESPACE) via DeriveKeyPrefix. + env = append(env, buildUntrustedTokenStoreEnvVars(vmcp)...) + return ctrlutil.EnsureRequiredEnvVars(ctx, env), nil } +// buildUntrustedTokenStoreEnvVars returns the auth-server Redis address env var +// the vMCP clones into untrusted egress-broker sidecars (THV_EGRESSBROKER_*). +// Empty when the embedded auth server does not use Redis storage — the broker +// then fails closed at startup, matching the untrusted-mode posture that +// upstream-token injection requires Redis-backed token storage. The KEK is +// deliberately NOT injected here: the CRD has no token-encryption field, and +// when encryption is enabled (enterprise RunConfig path) the sidecar's KEK is +// supplied via its own Secret env reference, never a ConfigMap. +func buildUntrustedTokenStoreEnvVars(vmcp *mcpv1beta1.VirtualMCPServer) []corev1.EnvVar { + as := vmcp.Spec.AuthServerConfig + if as == nil || as.Storage == nil || + as.Storage.Type != mcpv1beta1.AuthServerStorageTypeRedis || + as.Storage.Redis == nil { + return nil + } + // Sentinel-based auth-server Redis has no single address the sidecar can + // dial directly; untrusted egress requires a standalone/cluster addr. + if as.Storage.Redis.Addr == "" { + return nil + } + return []corev1.EnvVar{{ + Name: untrustedTokenStoreAddrEnvVar, + Value: as.Storage.Redis.Addr, + }} +} + // buildOIDCEnvVars builds environment variables for OIDC client secret mounting. func (r *VirtualMCPServerReconciler) buildOIDCEnvVars( ctx context.Context, vmcp *mcpv1beta1.VirtualMCPServer, diff --git a/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go b/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go new file mode 100644 index 0000000000..40ee9af45a --- /dev/null +++ b/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + rbacv1 "k8s.io/api/rbac/v1" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" + "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1/v1beta1test" +) + +// TestVmcpDiscoveredRBACRules_PodsRule pins the untrusted-mode pods grant: +// vMCP (discovered mode is the untrusted-mode front) needs get/list/watch/ +// create/delete on pods, and pods only, in its namespaced Role. +func TestVmcpDiscoveredRBACRules_PodsRule(t *testing.T) { + t.Parallel() + + var podsRules []rbacv1.PolicyRule + for _, rule := range vmcpDiscoveredRBACRules { + for _, res := range rule.Resources { + if res == "pods" { + podsRules = append(podsRules, rule) + } + } + } + + require.Len(t, podsRules, 1, "expected exactly one pods rule in vmcpDiscoveredRBACRules") + rule := podsRules[0] + assert.Equal(t, []string{""}, rule.APIGroups) + assert.Equal(t, []string{"pods"}, rule.Resources, "pods rule must be pods-only") + assert.ElementsMatch(t, []string{"get", "list", "watch", "create", "delete"}, rule.Verbs) + assert.NotContains(t, rule.Verbs, "update", "no update: pods are created/deleted, never mutated") + assert.NotContains(t, rule.Verbs, "patch", "no patch: pods are created/deleted, never mutated") +} + +// TestVmcpInlineRBACRules_NoPodsRule pins that inline mode (no discovery) +// does NOT gain pod-write: untrusted mode requires discovered backends. +func TestVmcpInlineRBACRules_NoPodsRule(t *testing.T) { + t.Parallel() + + for _, rule := range vmcpInlineRBACRules { + assert.NotContains(t, rule.Resources, "pods") + } +} + +// TestVmcpDiscoveredRBACRules_StatefulSetsRule pins the untrusted-mode +// statefulsets grant: the pod lifecycle resolves the backend StatefulSet by +// label selector (toolhive=true + mcpserver-uid) to clone its pod template. +// The grant is read-only — vMCP never writes StatefulSets. +func TestVmcpDiscoveredRBACRules_StatefulSetsRule(t *testing.T) { + t.Parallel() + + var stsRules []rbacv1.PolicyRule + for _, rule := range vmcpDiscoveredRBACRules { + for _, res := range rule.Resources { + if res == "statefulsets" { + stsRules = append(stsRules, rule) + } + } + } + + require.Len(t, stsRules, 1, "expected exactly one statefulsets rule in vmcpDiscoveredRBACRules") + rule := stsRules[0] + assert.Equal(t, []string{"apps"}, rule.APIGroups) + assert.Equal(t, []string{"statefulsets"}, rule.Resources, "statefulsets rule must be statefulsets-only") + assert.ElementsMatch(t, []string{"get", "list", "watch"}, rule.Verbs) + assert.NotContains(t, rule.Verbs, "create", "read-only: vMCP never writes StatefulSets") + assert.NotContains(t, rule.Verbs, "update", "read-only: vMCP never writes StatefulSets") + assert.NotContains(t, rule.Verbs, "patch", "read-only: vMCP never writes StatefulSets") + assert.NotContains(t, rule.Verbs, "delete", "read-only: vMCP never writes StatefulSets") +} + +// TestBuildUntrustedTokenStoreEnvVars pins the Wave-3 token-store coordinate +// injection: the vMCP Deployment gets the (non-secret) auth-server Redis +// address when the embedded auth server uses standalone/cluster Redis storage, +// and nothing otherwise. The KEK is never injected here — it is a sidecar-only +// Secret reference resolved at clone time. +func TestBuildUntrustedTokenStoreEnvVars(t *testing.T) { + t.Parallel() + + redisStorage := func(addr string) *mcpv1beta1.EmbeddedAuthServerConfig { + cfg := &mcpv1beta1.EmbeddedAuthServerConfig{ + Storage: &mcpv1beta1.AuthServerStorageConfig{ + Type: mcpv1beta1.AuthServerStorageTypeRedis, + Redis: &mcpv1beta1.RedisStorageConfig{ + Addr: addr, + ACLUserConfig: &mcpv1beta1.RedisACLUserConfig{ + PasswordSecretRef: &mcpv1beta1.SecretKeyRef{Name: "redis-creds", Key: "password"}, + }, + }, + }, + } + return cfg + } + + tests := []struct { + name string + authCfg *mcpv1beta1.EmbeddedAuthServerConfig + wantAddr string + wantEmpty bool + }{ + { + name: "nil auth server config produces no env var", + authCfg: nil, + wantEmpty: true, + }, + { + name: "memory storage produces no env var", + authCfg: &mcpv1beta1.EmbeddedAuthServerConfig{ + Storage: &mcpv1beta1.AuthServerStorageConfig{Type: mcpv1beta1.AuthServerStorageTypeMemory}, + }, + wantEmpty: true, + }, + { + name: "redis storage with addr produces the address env var", + authCfg: redisStorage("redis.auth:6379"), + wantAddr: "redis.auth:6379", + wantEmpty: false, + }, + { + name: "redis storage without addr (sentinel) produces no env var", + authCfg: &mcpv1beta1.EmbeddedAuthServerConfig{ + Storage: &mcpv1beta1.AuthServerStorageConfig{ + Type: mcpv1beta1.AuthServerStorageTypeRedis, + Redis: &mcpv1beta1.RedisStorageConfig{}, + }, + }, + wantEmpty: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", + v1beta1test.WithVMCPAuthServerConfig(tc.authCfg)) + env := buildUntrustedTokenStoreEnvVars(vmcp) + if tc.wantEmpty { + assert.Empty(t, env) + return + } + require.Len(t, env, 1) + assert.Equal(t, untrustedTokenStoreAddrEnvVar, env[0].Name) + assert.Equal(t, tc.wantAddr, env[0].Value) + assert.Nil(t, env[0].ValueFrom, "address is non-secret; must be a plain literal, never a Secret ref") + }) + } +} diff --git a/cmd/thv-operator/pkg/controllerutil/untrusted_sentinel.go b/cmd/thv-operator/pkg/controllerutil/untrusted_sentinel.go new file mode 100644 index 0000000000..ae97300488 --- /dev/null +++ b/cmd/thv-operator/pkg/controllerutil/untrusted_sentinel.go @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllerutil + +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" +) + +// UntrustedSentinelPrefix is the literal prefix injected as the value of each +// egressPolicy.providers[].credentialEnvName env var for untrusted workloads. The value +// is a pure compatibility shim (ADR-0001): servers that refuse to start tokenless still +// boot; the broker ignores the sentinel and injects the real credential. +const UntrustedSentinelPrefix = "thv-untrusted-sentinel:" + +// InjectUntrustedSentinels appends one literal env var per declared +// EgressPolicy provider CredentialEnvName to the container named containerName in +// the pod-template patch. Literal values deliberately use Value (never ValueFrom) +// so the injected result passes the Wave-0 gate ValidateNoSecretEnvForUntrusted. +// +// It is idempotent: an existing env var whose value is already the exact sentinel +// literal for the same provider is left untouched (re-reconcile is a no-op). +// It returns an error (a terminal spec error at the call site) when +// CredentialEnvName collides with an existing env var of a different value (the +// user already set that name in spec.env, or the builder did via WithSecrets), or +// when two providers declare the same non-empty CredentialEnvName. When the patch +// has no container named containerName yet (no user template and no spec.secrets — +// the expected untrusted shape) the container is created; the proxy runner +// strategic-merges the patch over its own base container. +// +// A nil podTemplate or nil policy is a no-op. The caller's podTemplate is mutated +// (it is the freshly-built patch owned by deploymentForMCPServer). +func InjectUntrustedSentinels( + podTemplate *corev1.PodTemplateSpec, containerName string, policy *mcpv1beta1.EgressPolicy, +) error { + if podTemplate == nil || policy == nil { + return nil + } + + providers := namedProviders(policy) + if err := checkDuplicateCredentialEnvNames(providers); err != nil { + return err + } + if len(providers) == 0 { + return nil + } + + for i := range podTemplate.Spec.Containers { + container := &podTemplate.Spec.Containers[i] + if container.Name != containerName { + continue + } + return injectSentinelsIntoContainer(container, containerName, providers) + } + + // The backend container is absent from the patch (no user template named it and + // no spec.secrets generated it — the expected untrusted shape). Create the + // patch container with exactly the sentinel env vars: the proxy runner + // strategic-merges the patch over its own base mcp container. + env := make([]corev1.EnvVar, 0, len(providers)) + for _, p := range providers { + env = append(env, corev1.EnvVar{Name: p.CredentialEnvName, Value: UntrustedSentinelPrefix + p.Provider}) + } + podTemplate.Spec.Containers = append(podTemplate.Spec.Containers, corev1.Container{ + Name: containerName, + Env: env, + }) + return nil +} + +// namedProviders returns the providers that declare a CredentialEnvName, in order. +func namedProviders(policy *mcpv1beta1.EgressPolicy) []mcpv1beta1.ProviderEgress { + var out []mcpv1beta1.ProviderEgress + for _, p := range policy.Providers { + if p.CredentialEnvName != "" { + out = append(out, p) + } + } + return out +} + +// checkDuplicateCredentialEnvNames rejects two providers fighting over the same env name. +func checkDuplicateCredentialEnvNames(providers []mcpv1beta1.ProviderEgress) error { + seen := map[string]string{} + for _, p := range providers { + if prev, dup := seen[p.CredentialEnvName]; dup { + return fmt.Errorf( + "untrusted workload: egressPolicy providers %q and %q both declare credentialEnvName %q; "+ + "each provider must use a distinct credentialEnvName", + prev, p.Provider, p.CredentialEnvName) + } + seen[p.CredentialEnvName] = p.Provider + } + return nil +} + +// injectSentinelsIntoContainer merges the sentinel literals into one existing container. +func injectSentinelsIntoContainer( + container *corev1.Container, containerName string, providers []mcpv1beta1.ProviderEgress, +) error { + existing := map[string]int{} + for j, env := range container.Env { + existing[env.Name] = j + } + for _, p := range providers { + sentinel := UntrustedSentinelPrefix + p.Provider + if j, found := existing[p.CredentialEnvName]; found { + if container.Env[j].Value != sentinel || container.Env[j].ValueFrom != nil { + return fmt.Errorf( + "untrusted workload: env var %q on container %q already exists with a different value; "+ + "egressPolicy provider %q cannot inject its sentinel over user-declared env", + p.CredentialEnvName, containerName, p.Provider) + } + // Already the exact sentinel literal: idempotent re-reconcile, no duplicate. + continue + } + container.Env = append(container.Env, corev1.EnvVar{Name: p.CredentialEnvName, Value: sentinel}) + existing[p.CredentialEnvName] = len(container.Env) - 1 + } + return nil +} + +// ValidateUntrustedSentinelForgery rejects literal env values on the container named +// containerName that start with the reserved UntrustedSentinelPrefix — a user forging +// a sentinel in spec.env (or a raw podTemplateSpec) would confuse operators reading pod +// specs into believing the broker manages a credential the broker has never heard of. +// The prefix is reserved for operator-injected values only. Sentinel injection runs +// before this check at the call site, so operator-injected literals are intentionally +// exempt (the check scopes to what the user declared, which is already validated by +// InjectUntrustedSentinels for collisions). +// +// A nil podTemplate is a no-op. +func ValidateUntrustedSentinelForgery( + podTemplate *corev1.PodTemplateSpec, containerName string, policy *mcpv1beta1.EgressPolicy, +) error { + if podTemplate == nil { + return nil + } + for _, container := range podTemplate.Spec.Containers { + if container.Name != containerName { + continue + } + for _, env := range container.Env { + if !strings.HasPrefix(env.Value, UntrustedSentinelPrefix) { + continue + } + if isOperatorInjectedSentinel(env, policy) { + continue + } + return fmt.Errorf( + "untrusted workload: env var %q on container %q uses the reserved sentinel prefix %q; "+ + "sentinel values are injected by the operator from egressPolicy and must not be declared by users", + env.Name, containerName, UntrustedSentinelPrefix) + } + } + return nil +} + +// isOperatorInjectedSentinel reports whether env is exactly the sentinel literal a +// declared provider would inject for credentialEnvName env.Name. +func isOperatorInjectedSentinel(env corev1.EnvVar, policy *mcpv1beta1.EgressPolicy) bool { + if policy == nil { + return false + } + for _, p := range policy.Providers { + if p.CredentialEnvName == env.Name && env.Value == UntrustedSentinelPrefix+p.Provider { + return true + } + } + return false +} diff --git a/cmd/thv-operator/pkg/controllerutil/untrusted_sentinel_test.go b/cmd/thv-operator/pkg/controllerutil/untrusted_sentinel_test.go new file mode 100644 index 0000000000..e1da4f5531 --- /dev/null +++ b/cmd/thv-operator/pkg/controllerutil/untrusted_sentinel_test.go @@ -0,0 +1,315 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllerutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" +) + +// mcpPodTemplate builds a pod template patch with a single backend (mcp) container +// carrying the given env vars. +func mcpPodTemplate(env ...corev1.EnvVar) *corev1.PodTemplateSpec { + return &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "mcp", Env: env}}, + }} +} + +// envByName returns the env var with the given name, or nil. +func envByName(envs []corev1.EnvVar, name string) *corev1.EnvVar { + for i := range envs { + if envs[i].Name == name { + return &envs[i] + } + } + return nil +} + +func TestInjectUntrustedSentinels(t *testing.T) { + t.Parallel() + + provider := func(name, envName string) mcpv1beta1.ProviderEgress { + return mcpv1beta1.ProviderEgress{ + Provider: name, + AllowedHosts: []string{"api.example.com"}, + CredentialEnvName: envName, + } + } + + tests := []struct { + name string + podTemplate *corev1.PodTemplateSpec + policy *mcpv1beta1.EgressPolicy + wantEnv map[string]string // expected literal env on the mcp container afterwards + wantAbsent []string // env names that must NOT be present afterwards + wantErr string + }{ + { + name: "single provider appends one literal sentinel env var", + podTemplate: mcpPodTemplate(), + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{provider("github", "GITHUB_TOKEN")}}, + wantEnv: map[string]string{"GITHUB_TOKEN": "thv-untrusted-sentinel:github"}, + }, + { + name: "provider with empty credentialEnvName injects nothing", + podTemplate: mcpPodTemplate(), + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{provider("github", "")}}, + wantAbsent: []string{"GITHUB_TOKEN"}, + }, + { + name: "collision with existing env name of a different value is an error", + podTemplate: mcpPodTemplate( + corev1.EnvVar{Name: "GITHUB_TOKEN", Value: "user-declared"}, + ), + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{provider("github", "GITHUB_TOKEN")}}, + wantErr: `env var "GITHUB_TOKEN" on container "mcp" already exists with a different value`, + }, + { + name: "collision with existing ValueFrom env is an error", + podTemplate: mcpPodTemplate( + corev1.EnvVar{Name: "GITHUB_TOKEN", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{Key: "token"}, + }}, + ), + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{provider("github", "GITHUB_TOKEN")}}, + wantErr: `env var "GITHUB_TOKEN" on container "mcp" already exists with a different value`, + }, + { + name: "existing exact sentinel literal is idempotent: no duplicate, no error", + podTemplate: mcpPodTemplate( + corev1.EnvVar{Name: "GITHUB_TOKEN", Value: "thv-untrusted-sentinel:github"}, + ), + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{provider("github", "GITHUB_TOKEN")}}, + wantEnv: map[string]string{"GITHUB_TOKEN": "thv-untrusted-sentinel:github"}, + }, + { + name: "two providers sharing a credentialEnvName is an error", + podTemplate: mcpPodTemplate(), + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{ + provider("github", "API_TOKEN"), + provider("gitlab", "API_TOKEN"), + }}, + wantErr: `both declare credentialEnvName "API_TOKEN"`, + }, + { + name: "two distinct providers each get their sentinel", + podTemplate: mcpPodTemplate(), + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{ + provider("github", "GITHUB_TOKEN"), + provider("google", "GOOGLE_TOKEN"), + }}, + wantEnv: map[string]string{ + "GITHUB_TOKEN": "thv-untrusted-sentinel:github", + "GOOGLE_TOKEN": "thv-untrusted-sentinel:google", + }, + }, + { + name: "nil policy is a no-op", + podTemplate: mcpPodTemplate(), + policy: nil, + }, + { + name: "nil pod template is a no-op", + podTemplate: nil, + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{provider("github", "GITHUB_TOKEN")}}, + }, + { + name: "absent backend container is created carrying the sentinels", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "not-mcp"}}, + }}, + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{provider("github", "GITHUB_TOKEN")}}, + wantEnv: map[string]string{"GITHUB_TOKEN": "thv-untrusted-sentinel:github"}, + }, + { + name: "empty pod template gets a new backend container with the sentinel", + podTemplate: &corev1.PodTemplateSpec{}, + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{provider("github", "GITHUB_TOKEN")}}, + wantEnv: map[string]string{"GITHUB_TOKEN": "thv-untrusted-sentinel:github"}, + }, + { + name: "env on other containers is not a collision", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "sidecar", Env: []corev1.EnvVar{{Name: "GITHUB_TOKEN", Value: "sidecar-owned"}}}, + {Name: "mcp"}, + }, + }}, + policy: &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{provider("github", "GITHUB_TOKEN")}}, + wantEnv: map[string]string{"GITHUB_TOKEN": "thv-untrusted-sentinel:github"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := InjectUntrustedSentinels(tt.podTemplate, "mcp", tt.policy) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + + if tt.podTemplate == nil { + return + } + var mcpEnv []corev1.EnvVar + for _, c := range tt.podTemplate.Spec.Containers { + if c.Name == "mcp" { + mcpEnv = c.Env + } + } + for name, wantValue := range tt.wantEnv { + env := envByName(mcpEnv, name) + require.NotNil(t, env, "expected env var %q on mcp container", name) + assert.Equal(t, wantValue, env.Value) + assert.Nil(t, env.ValueFrom, "sentinel env must be a literal Value, never ValueFrom") + // Idempotency pins exactly one entry per name. + count := 0 + for _, e := range mcpEnv { + if e.Name == name { + count++ + } + } + assert.Equal(t, 1, count, "env var %q must appear exactly once", name) + } + for _, name := range tt.wantAbsent { + assert.Nil(t, envByName(mcpEnv, name), "env var %q must not be injected", name) + } + }) + } +} + +// TestInjectUntrustedSentinels_ComposesWithGate proves the injected literals pass the +// Wave-0 gate: both functions compose at the deploymentForMCPServer seam. +func TestInjectUntrustedSentinels_ComposesWithGate(t *testing.T) { + t.Parallel() + + podTemplate := mcpPodTemplate() + policy := &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + CredentialEnvName: "GITHUB_TOKEN", + }}} + + require.NoError(t, InjectUntrustedSentinels(podTemplate, "mcp", policy)) + require.NoError(t, ValidateNoSecretEnvForUntrusted(podTemplate, "mcp", true), + "injected literal sentinels must pass the Wave-0 untrusted env gate") + + // Re-running the injection (re-reconcile) keeps the gate passing and adds nothing. + envBefore := podTemplate.Spec.Containers[0].Env + require.NoError(t, InjectUntrustedSentinels(podTemplate, "mcp", policy)) + assert.Equal(t, envBefore, podTemplate.Spec.Containers[0].Env, "re-injection must be a no-op") + require.NoError(t, ValidateNoSecretEnvForUntrusted(podTemplate, "mcp", true)) +} + +func TestValidateUntrustedSentinelForgery(t *testing.T) { + t.Parallel() + + policy := &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + CredentialEnvName: "GITHUB_TOKEN", + }}} + + tests := []struct { + name string + podTemplate *corev1.PodTemplateSpec + policy *mcpv1beta1.EgressPolicy + wantErr string + }{ + { + name: "user env value with the reserved sentinel prefix is rejected", + podTemplate: mcpPodTemplate( + corev1.EnvVar{Name: "MY_TOKEN", Value: "thv-untrusted-sentinel:fake"}, + ), + policy: policy, + wantErr: `env var "MY_TOKEN" on container "mcp" uses the reserved sentinel prefix`, + }, + { + name: "operator-injected sentinel for a declared provider is exempt", + podTemplate: mcpPodTemplate( + corev1.EnvVar{Name: "GITHUB_TOKEN", Value: "thv-untrusted-sentinel:github"}, + ), + policy: policy, + }, + { + name: "sentinel literal under a different env name is still forgery", + podTemplate: mcpPodTemplate( + corev1.EnvVar{Name: "SOME_OTHER_NAME", Value: "thv-untrusted-sentinel:github"}, + ), + policy: policy, + wantErr: `env var "SOME_OTHER_NAME" on container "mcp" uses the reserved sentinel prefix`, + }, + { + name: "sentinel literal for an undeclared provider is forgery", + podTemplate: mcpPodTemplate( + corev1.EnvVar{Name: "GITLAB_TOKEN", Value: "thv-untrusted-sentinel:gitlab"}, + ), + policy: policy, + wantErr: `env var "GITLAB_TOKEN" on container "mcp" uses the reserved sentinel prefix`, + }, + { + name: "ordinary literal env values pass", + podTemplate: mcpPodTemplate( + corev1.EnvVar{Name: "LOG_LEVEL", Value: "debug"}, + corev1.EnvVar{Name: "NOT_A_SENTINEL", Value: "thv-untrusted-but-not-the-prefix"}, + ), + policy: policy, + }, + { + name: "sentinel-prefixed env on a non-backend container is out of scope", + podTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "sidecar", Env: []corev1.EnvVar{{Name: "X", Value: "thv-untrusted-sentinel:fake"}}}, + {Name: "mcp"}, + }, + }}, + policy: policy, + }, + { + name: "nil pod template is a no-op", + podTemplate: nil, + policy: policy, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := ValidateUntrustedSentinelForgery(tt.podTemplate, "mcp", tt.policy) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + } else { + require.NoError(t, err) + } + }) + } +} + +// TestSentinelInjectionThenForgeryCheck exercises the exact call-site ordering: +// inject first, then the forgery check must accept the operator's own literals. +func TestSentinelInjectionThenForgeryCheck(t *testing.T) { + t.Parallel() + + podTemplate := mcpPodTemplate(corev1.EnvVar{Name: "LOG_LEVEL", Value: "info"}) + policy := &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + CredentialEnvName: "GITHUB_TOKEN", + }}} + + require.NoError(t, InjectUntrustedSentinels(podTemplate, "mcp", policy)) + require.NoError(t, ValidateUntrustedSentinelForgery(podTemplate, "mcp", policy), + "forgery check must not flag operator-injected sentinels") +} diff --git a/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_cel_integration_test.go b/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_cel_integration_test.go new file mode 100644 index 0000000000..10933ff0f9 --- /dev/null +++ b/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_cel_integration_test.go @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" +) + +// validUntrustedSpec returns a fully R1-R6-compliant untrusted MCPServer spec +// (untrusted + egressPolicy with one provider + groupRef). Cases mutate it to +// violate exactly one rule at a time. +func validUntrustedSpec() mcpv1beta1.MCPServerSpec { + return mcpv1beta1.MCPServerSpec{ + Image: "example/mcp-server:latest", + Untrusted: true, + GroupRef: &mcpv1beta1.MCPGroupRef{Name: "test-group"}, + EgressPolicy: &mcpv1beta1.EgressPolicy{ + Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + }}, + }, + } +} + +var _ = Describe("CEL Validation for MCPServer untrusted mode", Label("k8s", "cel", "validation"), func() { + type celCase struct { + mutate func(*mcpv1beta1.MCPServerSpec) // nil leaves the valid baseline untouched + wantErr string // empty means admission must accept + } + + DescribeTable("R1-R6 admission rules", + func(c celCase) { + spec := validUntrustedSpec() + if c.mutate != nil { + c.mutate(&spec) + } + server := &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "untrusted-cel-", + Namespace: "default", + }, + Spec: spec, + } + err := untrustedK8sClient.Create(untrustedCtx, server) + if c.wantErr == "" { + Expect(err).NotTo(HaveOccurred()) + Expect(untrustedK8sClient.Delete(untrustedCtx, server)).To(Succeed()) + return + } + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(c.wantErr)) + }, + Entry("R1a rejects untrusted without egressPolicy", celCase{ + mutate: func(s *mcpv1beta1.MCPServerSpec) { s.EgressPolicy = nil }, + wantErr: "egressPolicy with at least one provider is required when untrusted is true", + }), + Entry("R1b rejects untrusted with empty providers (schema MinItems fires first)", celCase{ + mutate: func(s *mcpv1beta1.MCPServerSpec) { s.EgressPolicy.Providers = nil }, + wantErr: "Required value", + }), + Entry("R1c accepts untrusted with one valid provider", celCase{}), + Entry("R2a rejects untrusted without groupRef", celCase{ + mutate: func(s *mcpv1beta1.MCPServerSpec) { s.GroupRef = nil }, + wantErr: "untrusted workloads must belong to an MCPGroup fronted by a VirtualMCPServer", + }), + Entry("R2b accepts untrusted with groupRef", celCase{}), + Entry("R3a rejects untrusted with spec.secrets", celCase{ + mutate: func(s *mcpv1beta1.MCPServerSpec) { + s.Secrets = []mcpv1beta1.SecretRef{{Name: "creds", Key: "token", TargetEnvName: "API_TOKEN"}} + }, + wantErr: "spec.secrets is forbidden when untrusted is true", + }), + Entry("R3b accepts untrusted without secrets", celCase{}), + Entry("R4a rejects untrusted with podTemplateSpec", celCase{ + mutate: func(s *mcpv1beta1.MCPServerSpec) { + s.PodTemplateSpec = &runtime.RawExtension{ + Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"A","value":"b"}]}]}}`), + } + }, + wantErr: "podTemplateSpec is forbidden when untrusted is true", + }), + Entry("R4b accepts untrusted without podTemplateSpec", celCase{}), + Entry("R5a rejects untrusted with sessionAffinity None", celCase{ + mutate: func(s *mcpv1beta1.MCPServerSpec) { s.SessionAffinity = "None" }, + wantErr: "untrusted workloads require sessionAffinity ClientIP", + }), + Entry("R5b accepts untrusted with sessionAffinity omitted (defaults ClientIP)", celCase{}), + Entry("R5c accepts untrusted with sessionAffinity ClientIP", celCase{ + mutate: func(s *mcpv1beta1.MCPServerSpec) { s.SessionAffinity = "ClientIP" }, + }), + Entry("R6a rejects untrusted with backendReplicas", celCase{ + mutate: func(s *mcpv1beta1.MCPServerSpec) { s.BackendReplicas = ptr.To(int32(2)) }, + wantErr: "backendReplicas is managed by the untrusted-mode session lifecycle and must not be set", + }), + Entry("R6b accepts untrusted without backendReplicas", celCase{}), + Entry("Trusted is inert for all rules at once", celCase{ + mutate: func(s *mcpv1beta1.MCPServerSpec) { + s.Untrusted = false + s.EgressPolicy = nil + s.GroupRef = nil + s.Secrets = []mcpv1beta1.SecretRef{{Name: "creds", Key: "token", TargetEnvName: "API_TOKEN"}} + s.PodTemplateSpec = &runtime.RawExtension{ + Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"A","value":"b"}]}]}}`), + } + s.SessionAffinity = "None" + s.BackendReplicas = ptr.To(int32(2)) + }, + }), + ) + + Context("EgressPolicy schema validation", func() { + It("should reject a provider without allowedHosts", func() { + spec := validUntrustedSpec() + spec.EgressPolicy.Providers[0].AllowedHosts = nil + server := &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{Name: "untrusted-no-hosts", Namespace: "default"}, + Spec: spec, + } + err := untrustedK8sClient.Create(untrustedCtx, server) + Expect(err).To(HaveOccurred()) + }) + + It("should reject an allowedHost with an invalid hostname", func() { + spec := validUntrustedSpec() + spec.EgressPolicy.Providers[0].AllowedHosts = []string{"https://api.github.com:443"} + server := &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{Name: "untrusted-bad-host", Namespace: "default"}, + Spec: spec, + } + err := untrustedK8sClient.Create(untrustedCtx, server) + Expect(err).To(HaveOccurred()) + }) + + It("should accept a one-label wildcard host", func() { + spec := validUntrustedSpec() + spec.EgressPolicy.Providers[0].AllowedHosts = []string{"*.githubusercontent.com"} + server := &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{Name: "untrusted-wildcard", Namespace: "default"}, + Spec: spec, + } + Expect(untrustedK8sClient.Create(untrustedCtx, server)).To(Succeed()) + Expect(untrustedK8sClient.Delete(untrustedCtx, server)).To(Succeed()) + }) + + It("should reject an invalid HTTP method", func() { + spec := validUntrustedSpec() + spec.EgressPolicy.Providers[0].AllowedMethods = []string{"YEET"} + server := &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{Name: "untrusted-bad-method", Namespace: "default"}, + Spec: spec, + } + err := untrustedK8sClient.Create(untrustedCtx, server) + Expect(err).To(HaveOccurred()) + }) + + It("should reject a duplicate provider name", func() { + spec := validUntrustedSpec() + spec.EgressPolicy.Providers = append(spec.EgressPolicy.Providers, mcpv1beta1.ProviderEgress{ + Provider: "github", + AllowedHosts: []string{"github.com"}, + }) + server := &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{Name: "untrusted-dup-provider", Namespace: "default"}, + Spec: spec, + } + err := untrustedK8sClient.Create(untrustedCtx, server) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_egress_integration_test.go b/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_egress_integration_test.go new file mode 100644 index 0000000000..0b8c832940 --- /dev/null +++ b/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_egress_integration_test.go @@ -0,0 +1,250 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +// getCurrentCASecret loads the single current-generation bump-CA Secret for +// the test server (fails while zero or several generations exist). +func getCurrentCASecret(into *corev1.Secret) error { + secrets := &corev1.SecretList{} + if err := untrustedK8sClient.List(untrustedCtx, secrets, + client.InNamespace("default"), + client.MatchingLabels{"toolhive.stacklok.dev/untrusted-resource": "true"}); err != nil { + return err + } + if len(secrets.Items) != 1 { + return fmt.Errorf("expected exactly one current CA Secret, found %d", len(secrets.Items)) + } + *into = secrets.Items[0] + return nil +} + +// mintAgedCAForRotation mints a CA already inside the rotation window (past +// 50% of validity) so the next reconcile rotates it. +func mintAgedCAForRotation() (certPEM, keyPEM []byte) { + ca, err := egressbroker.GenerateBumpCA("untrusted-egress-it-bump-ca", + time.Now().Add(-egressbroker.CAValidity/2)) + Expect(err).NotTo(HaveOccurred()) + Expect(ca.NeedsRotation(time.Now())).To(BeTrue(), "fixture CA must be inside the rotation window") + key, err := ca.KeyPEM() + Expect(err).NotTo(HaveOccurred()) + return ca.CertPEM(), key +} + +var _ = Describe("MCPServer untrusted egress resources", Label("k8s", "untrusted", "egress"), func() { + const ( + timeout = time.Second * 30 + interval = time.Millisecond * 250 + ) + + Context("When an untrusted MCPServer is reconciled", Ordered, func() { + const serverName = "untrusted-egress-it" + var mcpServer *mcpv1beta1.MCPServer + + BeforeAll(func() { + mcpServer = &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: serverName, + Namespace: "default", + }, + Spec: mcpv1beta1.MCPServerSpec{ + Image: "example/mcp-server:latest", + Transport: "stdio", + Untrusted: true, + GroupRef: &mcpv1beta1.MCPGroupRef{Name: "test-group"}, + EgressPolicy: &mcpv1beta1.EgressPolicy{ + Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + AllowedMethods: []string{"GET"}, + AllowedPathPrefixes: []string{"/repos/"}, + CredentialEnvName: "GITHUB_TOKEN", + }}, + }, + }, + } + Expect(untrustedK8sClient.Create(untrustedCtx, mcpServer)).To(Succeed()) + }) + + AfterAll(func() { + Expect(untrustedK8sClient.Delete(untrustedCtx, mcpServer)).To(Succeed()) + }) + + It("Should create the generation-named bump-CA Secret with an ownerRef", func() { + secret := &corev1.Secret{} + Eventually(func() error { + return getCurrentCASecret(secret) + }, timeout, interval).Should(Succeed()) + + generation, ok := egressbroker.TrimGeneration(secret.Name, egressbroker.BaseCASecretName(serverName)) + Expect(ok).To(BeTrue(), "CA Secret must be generation-named, got %q", secret.Name) + Expect(egressbroker.CAGeneration(secret.Data["ca.crt"])).To(Equal(generation), + "the generation must be the hash of the Secret's cert") + Expect(secret.Data).To(HaveKey("ca.crt")) + Expect(secret.Data).To(HaveKey("ca.key")) + Expect(secret.OwnerReferences).To(HaveLen(1)) + Expect(secret.OwnerReferences[0].Kind).To(Equal("MCPServer")) + Expect(secret.OwnerReferences[0].Name).To(Equal(serverName)) + }) + + It("Should create the CA-bundle ConfigMap with the same generation's public cert only", func() { + secret := &corev1.Secret{} + Eventually(func() error { + return getCurrentCASecret(secret) + }, timeout, interval).Should(Succeed()) + generation, _ := egressbroker.TrimGeneration(secret.Name, egressbroker.BaseCASecretName(serverName)) + + bundle := &corev1.ConfigMap{} + Eventually(func() error { + return untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: egressbroker.ResourceNamesFor(serverName, generation).CABundle, + Namespace: "default", + }, bundle) + }, timeout, interval).Should(Succeed()) + + Expect(bundle.Data).To(HaveKey("ca.crt")) + Expect(bundle.Data).ToNot(HaveKey("ca.key"), + "the CA private key must never appear outside the CA Secret") + + // The bundle cert must equal the SAME generation Secret's public cert. + Expect(bundle.Data["ca.crt"]).To(Equal(string(secret.Data["ca.crt"]))) + }) + + It("Should rotate into a new generation, keep N-1, and GC older generations", func() { + // Force rotation: age the current Secret's CA into the rotation + // window by replacing its material with an aged CA (name follows + // the cert hash, so create the aged generation and delete the + // fresh one). + current := &corev1.Secret{} + Eventually(func() error { + return getCurrentCASecret(current) + }, timeout, interval).Should(Succeed()) + + agedCert, agedKey := mintAgedCAForRotation() + agedGen := egressbroker.CAGeneration(agedCert) + aged := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: egressbroker.ResourceNamesFor(serverName, agedGen).CASecret, + Namespace: "default", + Labels: current.Labels, + }, + Data: map[string][]byte{"ca.crt": agedCert, "ca.key": agedKey}, + } + Expect(untrustedK8sClient.Create(untrustedCtx, aged)).To(Succeed()) + Expect(untrustedK8sClient.Delete(untrustedCtx, current)).To(Succeed()) + + // The reconcile must mint a NEW generation alongside the aged one. + Eventually(func() int { + secrets := &corev1.SecretList{} + if err := untrustedK8sClient.List(untrustedCtx, secrets, + client.InNamespace("default"), client.MatchingLabels(current.Labels)); err != nil { + return 0 + } + return len(secrets.Items) + }, timeout, interval).Should(Equal(2), "rotation keeps N and N-1 generations") + + // The aged generation's bundle converges on its own cert. + bundle := &corev1.ConfigMap{} + Eventually(func() error { + return untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: egressbroker.ResourceNamesFor(serverName, agedGen).CABundle, + Namespace: "default", + }, bundle) + }, timeout, interval).Should(Succeed()) + Expect(bundle.Data["ca.crt"]).To(Equal(string(agedCert)), + "the N-1 bundle must carry the N-1 cert (mid-rotation consistency)") + }) + + It("Should render the egress policy ConfigMap from the CRD EgressPolicy", func() { + cm := &corev1.ConfigMap{} + Eventually(func() error { + return untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: serverName + "-egress-policy", + Namespace: "default", + }, cm) + }, timeout, interval).Should(Succeed()) + + Expect(cm.Data).To(HaveKey("policy.yaml")) + Expect(cm.Data["policy.yaml"]).To(ContainSubstring("provider: github")) + Expect(cm.Data["policy.yaml"]).To(ContainSubstring("api.github.com")) + Expect(cm.Data["policy.yaml"]).To(ContainSubstring("/repos/")) + Expect(cm.OwnerReferences).To(HaveLen(1)) + Expect(cm.OwnerReferences[0].Kind).To(Equal("MCPServer")) + }) + + It("Should create the egress-lockdown NetworkPolicy selecting the untrusted session pods", func() { + np := &networkingv1.NetworkPolicy{} + Eventually(func() error { + return untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: serverName + "-egress", + Namespace: "default", + }, np) + }, timeout, interval).Should(Succeed()) + + Expect(np.Spec.PodSelector.MatchLabels).To(HaveKeyWithValue( + "toolhive.stacklok.dev/untrusted", "true")) + Expect(np.Spec.PodSelector.MatchLabels).To(HaveKey("toolhive.stacklok.dev/mcpserver-uid")) + Expect(np.Spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeEgress)) + + var cidrs []string + for _, rule := range np.Spec.Egress { + for _, peer := range rule.To { + if peer.IPBlock != nil { + cidrs = append(cidrs, peer.IPBlock.CIDR) + } + } + } + Expect(cidrs).To(ContainElement("127.0.0.1/32"), + "loopback (the in-pod sidecar) must be the only always-permitted egress besides DNS") + Expect(np.OwnerReferences).To(HaveLen(1)) + Expect(np.OwnerReferences[0].Kind).To(Equal("MCPServer")) + }) + + It("Should delete all untrusted resources on an untrusted→trusted flip", func() { + current := &mcpv1beta1.MCPServer{} + Expect(untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: serverName, + Namespace: "default", + }, current)).To(Succeed()) + current.Spec.Untrusted = false + Expect(untrustedK8sClient.Update(untrustedCtx, current)).To(Succeed()) + + gone := func(obj client.Object, name string) func() bool { + return func() bool { + err := untrustedK8sClient.Get(untrustedCtx, + types.NamespacedName{Name: name, Namespace: "default"}, obj) + return apierrors.IsNotFound(err) + } + } + Eventually(func() int { + secrets := &corev1.SecretList{} + if err := untrustedK8sClient.List(untrustedCtx, secrets, + client.InNamespace("default"), + client.MatchingLabels{"toolhive.stacklok.dev/untrusted-resource": "true"}); err != nil { + return -1 + } + return len(secrets.Items) + }, timeout, interval).Should(Equal(0), "every CA generation Secret must be deleted on flip") + Eventually(gone(&corev1.ConfigMap{}, serverName+"-egress-policy"), timeout, interval).Should(BeTrue()) + Eventually(gone(&networkingv1.NetworkPolicy{}, serverName+"-egress"), timeout, interval).Should(BeTrue(), + "NetworkPolicy must be deleted on flip") + }) + }) +}) diff --git a/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_sentinel_integration_test.go b/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_sentinel_integration_test.go new file mode 100644 index 0000000000..0dc31f6a03 --- /dev/null +++ b/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_sentinel_integration_test.go @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "encoding/json" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" +) + +// podPatchFromArgs extracts the --k8s-pod-patch JSON from the proxy container args. +func podPatchFromArgs(args []string) map[string]interface{} { + for _, arg := range args { + if patchJSON, ok := strings.CutPrefix(arg, "--k8s-pod-patch="); ok { + var patch map[string]interface{} + ExpectWithOffset(1, json.Unmarshal([]byte(patchJSON), &patch)).To(Succeed()) + return patch + } + } + return nil +} + +// sentinelEnvValues returns the credentialEnvName → value map of sentinel env vars +// on the mcp container of the pod patch. +func sentinelEnvValues(patch map[string]interface{}) map[string]string { + result := map[string]string{} + if patch == nil { + return result + } + spec, ok := patch["spec"].(map[string]interface{}) + if !ok { + return result + } + containers, ok := spec["containers"].([]interface{}) + if !ok { + return result + } + for _, c := range containers { + container, ok := c.(map[string]interface{}) + if !ok || container["name"] != "mcp" { + continue + } + envs, ok := container["env"].([]interface{}) + if !ok { + continue + } + for _, e := range envs { + env, ok := e.(map[string]interface{}) + if !ok { + continue + } + name, _ := env["name"].(string) + value, _ := env["value"].(string) + result[name] = value + } + } + return result +} + +var _ = Describe("MCPServer untrusted sentinel injection", Label("k8s", "untrusted", "sentinel"), func() { + const ( + timeout = time.Second * 30 + interval = time.Millisecond * 250 + ) + + Context("When an untrusted MCPServer declares an egressPolicy", Ordered, func() { + var ( + mcpServerName string + mcpServer *mcpv1beta1.MCPServer + ) + + BeforeAll(func() { + mcpServerName = "untrusted-sentinel-it" + mcpServer = &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: mcpServerName, + Namespace: "default", + }, + Spec: mcpv1beta1.MCPServerSpec{ + Image: "example/mcp-server:latest", + Transport: "stdio", + Untrusted: true, + GroupRef: &mcpv1beta1.MCPGroupRef{Name: "test-group"}, + EgressPolicy: &mcpv1beta1.EgressPolicy{ + Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + CredentialEnvName: "GITHUB_TOKEN", + }}, + }, + }, + } + Expect(untrustedK8sClient.Create(untrustedCtx, mcpServer)).To(Succeed()) + }) + + AfterAll(func() { + Expect(untrustedK8sClient.Delete(untrustedCtx, mcpServer)).To(Succeed()) + }) + + It("Should inject the sentinel env literal into the backend pod patch", func() { + deployment := &appsv1.Deployment{} + Eventually(func() error { + return untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: mcpServerName, + Namespace: "default", + }, deployment) + }, timeout, interval).Should(Succeed()) + + env := sentinelEnvValues(podPatchFromArgs(deployment.Spec.Template.Spec.Containers[0].Args)) + Expect(env).To(HaveKeyWithValue("GITHUB_TOKEN", "thv-untrusted-sentinel:github")) + }) + + It("Should stop injecting the sentinel after an untrusted→trusted flip", func() { + // Flip the workload to trusted. + current := &mcpv1beta1.MCPServer{} + Expect(untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: mcpServerName, + Namespace: "default", + }, current)).To(Succeed()) + current.Spec.Untrusted = false + Expect(untrustedK8sClient.Update(untrustedCtx, current)).To(Succeed()) + + deployment := &appsv1.Deployment{} + Eventually(func() map[string]string { + if err := untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: mcpServerName, + Namespace: "default", + }, deployment); err != nil { + return map[string]string{"__error__": err.Error()} + } + return sentinelEnvValues(podPatchFromArgs(deployment.Spec.Template.Spec.Containers[0].Args)) + }, timeout, interval).ShouldNot(HaveKey("GITHUB_TOKEN"), + "trusted reconcile must rebuild the pod patch without sentinel env") + }) + }) +}) + +var _ = Describe("MCPServer untrusted gate via spec field", Label("k8s", "untrusted", "gate"), func() { + It("Should be inert for a trusted MCPServer carrying the interim Wave-0 annotation", func() { + server := &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "untrusted-annotation-inert", + Namespace: "default", + Annotations: map[string]string{ + "toolhive.stacklok.dev/untrusted": "true", + }, + }, + Spec: mcpv1beta1.MCPServerSpec{ + Image: "example/mcp-server:latest", + Transport: "stdio", + // spec.secrets on a trusted workload is fine (annotation must not arm the gate). + Secrets: []mcpv1beta1.SecretRef{{Name: "creds", Key: "token", TargetEnvName: "API_TOKEN"}}, + }, + } + Expect(untrustedK8sClient.Create(untrustedCtx, server)).To(Succeed()) + defer func() { + Expect(untrustedK8sClient.Delete(untrustedCtx, server)).To(Succeed()) + }() + + // The gate would latch Valid=False; assert it never does. + Consistently(func() bool { + current := &mcpv1beta1.MCPServer{} + if err := untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: server.Name, + Namespace: "default", + }, current); err != nil { + return false + } + return !meta_FindFalseCondition(current.Status.Conditions, mcpv1beta1.ConditionTypeValid) + }, 5*time.Second, 250*time.Millisecond).Should(BeTrue(), + "the interim Wave-0 annotation must not arm the untrusted gate") + }) +}) + +func meta_FindFalseCondition(conditions []metav1.Condition, condType string) bool { + for _, c := range conditions { + if c.Type == condType && c.Status == metav1.ConditionFalse { + return true + } + } + return false +} diff --git a/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go b/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go new file mode 100644 index 0000000000..2af3ea43cd --- /dev/null +++ b/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package controllers contains integration tests for the MCPServer untrusted mode: +// CEL admission rules R1-R6 and sentinel env injection in the backend pod patch. +package controllers + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/stacklok/toolhive/cmd/thv-operator/controllers" + ctrlutil "github.com/stacklok/toolhive/cmd/thv-operator/pkg/controllerutil" + "github.com/stacklok/toolhive/cmd/thv-operator/test-integration/testutil" +) + +var ( + untrustedCfg *rest.Config + untrustedK8sClient client.Client + untrustedCtx context.Context + untrustedSuiteEnv *testutil.SuiteEnv +) + +func TestMCPServerUntrusted(t *testing.T) { + t.Parallel() + RegisterFailHandler(Fail) + + suiteConfig, reporterConfig := GinkgoConfiguration() + reporterConfig.Verbose = false + reporterConfig.VeryVerbose = false + reporterConfig.FullTrace = false + + RunSpecs(t, "MCPServer Untrusted Integration Test Suite", suiteConfig, reporterConfig) +} + +var _ = BeforeSuite(func() { + untrustedSuiteEnv = testutil.StartSuite(testutil.SuiteOptions{ + RegisterGroupRefIndexers: true, + }) + untrustedCfg = untrustedSuiteEnv.Cfg + untrustedK8sClient = untrustedSuiteEnv.Client + untrustedCtx = untrustedSuiteEnv.Ctx + + // Register the MCPGroup controller (needed for groupRef validation) + err := (&controllers.MCPGroupReconciler{ + Client: untrustedSuiteEnv.Manager.GetClient(), + }).SetupWithManager(untrustedSuiteEnv.Manager) + Expect(err).ToNot(HaveOccurred()) + + // Register the MCPServer controller + err = (&controllers.MCPServerReconciler{ + Client: untrustedSuiteEnv.Manager.GetClient(), + Scheme: untrustedSuiteEnv.Manager.GetScheme(), + PlatformDetector: ctrlutil.NewSharedPlatformDetector(), + }).SetupWithManager(untrustedSuiteEnv.Manager) + Expect(err).ToNot(HaveOccurred()) + + untrustedSuiteEnv.StartManager() +}) + +var _ = AfterSuite(func() { + untrustedSuiteEnv.Stop() +}) diff --git a/cmd/thv-operator/test-integration/testutil/envtest.go b/cmd/thv-operator/test-integration/testutil/envtest.go index bc2a6f5122..8eeaec8bf4 100644 --- a/cmd/thv-operator/test-integration/testutil/envtest.go +++ b/cmd/thv-operator/test-integration/testutil/envtest.go @@ -17,6 +17,7 @@ import ( "go.uber.org/zap/zapcore" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" @@ -196,6 +197,7 @@ func registerOperatorScheme() { mcpv1beta1.AddToScheme, appsv1.AddToScheme, corev1.AddToScheme, + networkingv1.AddToScheme, rbacv1.AddToScheme, } { Expect(add(scheme.Scheme)).To(Succeed()) diff --git a/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpservers.yaml b/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpservers.yaml index b2597e7f10..cd4c3ae980 100644 --- a/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpservers.yaml +++ b/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpservers.yaml @@ -238,6 +238,85 @@ spec: format: int32 minimum: 0 type: integer + egressPolicy: + description: |- + EgressPolicy declares which upstream providers this server may call and where the + broker may inject each provider's credential. Required when Untrusted is true. + When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend + pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar + Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + properties: + providers: + description: |- + Providers maps an upstream provider name (as configured in the embedded auth + server's upstream chain / upstream_inject strategy) to its egress constraints. + items: + description: |- + ProviderEgress is the per-provider egress constraint. Credential-to-destination + binding (ADR-0001 D5): the broker injects this provider's credential only to + AllowedHosts, only on AllowedMethods, only under AllowedPathPrefixes. + properties: + allowedHosts: + description: |- + AllowedHosts is the exact set of destination hosts the credential may be + injected for. Exact hostnames or one-label wildcards ("*.githubusercontent.com"); + no ports, no schemes. + items: + pattern: ^(\*\.)?[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$ + type: string + minItems: 1 + type: array + x-kubernetes-list-type: set + allowedMethods: + description: |- + AllowedMethods constrains HTTP methods the broker will inject on. + Empty means GET/HEAD/OPTIONS only (safe default — read-only). + items: + enum: + - GET + - HEAD + - OPTIONS + - POST + - PUT + - PATCH + - DELETE + type: string + type: array + x-kubernetes-list-type: set + allowedPathPrefixes: + description: |- + AllowedPathPrefixes constrains URL paths the broker will inject on + (prefix match, e.g. "/repos/"). Empty means "/" (all paths on the allowed hosts). + items: + type: string + type: array + x-kubernetes-list-type: set + credentialEnvName: + description: |- + CredentialEnvName names the backend env var that would normally carry this + provider's token (e.g. "GITHUB_TOKEN"). The operator injects a SENTINEL literal + here so servers that refuse to start tokenless still boot; the broker ignores + the sentinel value. Sentinel literal format: "thv-untrusted-sentinel:". + type: string + provider: + description: |- + Provider is the logical upstream provider name (e.g. "github", "google"). + Must match the provider name used by the auth server and the vMCP + upstream_inject strategy config for this workload. + minLength: 1 + type: string + required: + - allowedHosts + - provider + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - provider + x-kubernetes-list-type: map + required: + - providers + type: object endpointPrefix: description: |- EndpointPrefix is the path prefix to prepend to SSE endpoint URLs. @@ -786,6 +865,15 @@ spec: When enabled, the proxy will use X-Forwarded-Proto, X-Forwarded-Host, X-Forwarded-Port, and X-Forwarded-Prefix headers to construct endpoint URLs type: boolean + untrusted: + default: false + description: |- + Untrusted marks this MCP server as running untrusted code. When true, the operator + enforces the untrusted-mode invariants (ADR-0001): no Secret/ConfigMap-sourced env on + the backend container, single-tenant session-scoped backend pods, mandatory MCPGroup + membership behind a VirtualMCPServer, and egress only through the credential-broker + sidecar per EgressPolicy. K8s-only; ignored (and inert) in CLI/Docker mode. + type: boolean volumes: description: Volumes are volumes to mount in the MCP server container items: @@ -844,6 +932,24 @@ spec: or externalAuthConfigRef) rule: '!has(self.rateLimiting) || !has(self.rateLimiting.tools) || self.rateLimiting.tools.all(t, !has(t.perUser)) || has(self.oidcConfigRef) || has(self.externalAuthConfigRef)' + - message: egressPolicy with at least one provider is required when untrusted + is true + rule: '!self.untrusted || (has(self.egressPolicy) && size(self.egressPolicy.providers) + > 0)' + - message: untrusted workloads must belong to an MCPGroup fronted by a + VirtualMCPServer + rule: '!self.untrusted || has(self.groupRef)' + - message: spec.secrets is forbidden when untrusted is true; declare providers + in egressPolicy and use sentinel env + rule: '!self.untrusted || !has(self.secrets) || size(self.secrets) == + 0' + - message: podTemplateSpec is forbidden when untrusted is true + rule: '!self.untrusted || !has(self.podTemplateSpec)' + - message: untrusted workloads require sessionAffinity ClientIP + rule: '!self.untrusted || self.sessionAffinity == ''ClientIP''' + - message: backendReplicas is managed by the untrusted-mode session lifecycle + and must not be set + rule: '!self.untrusted || !has(self.backendReplicas)' status: description: MCPServerStatus defines the observed state of MCPServer properties: @@ -1182,6 +1288,85 @@ spec: format: int32 minimum: 0 type: integer + egressPolicy: + description: |- + EgressPolicy declares which upstream providers this server may call and where the + broker may inject each provider's credential. Required when Untrusted is true. + When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend + pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar + Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + properties: + providers: + description: |- + Providers maps an upstream provider name (as configured in the embedded auth + server's upstream chain / upstream_inject strategy) to its egress constraints. + items: + description: |- + ProviderEgress is the per-provider egress constraint. Credential-to-destination + binding (ADR-0001 D5): the broker injects this provider's credential only to + AllowedHosts, only on AllowedMethods, only under AllowedPathPrefixes. + properties: + allowedHosts: + description: |- + AllowedHosts is the exact set of destination hosts the credential may be + injected for. Exact hostnames or one-label wildcards ("*.githubusercontent.com"); + no ports, no schemes. + items: + pattern: ^(\*\.)?[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$ + type: string + minItems: 1 + type: array + x-kubernetes-list-type: set + allowedMethods: + description: |- + AllowedMethods constrains HTTP methods the broker will inject on. + Empty means GET/HEAD/OPTIONS only (safe default — read-only). + items: + enum: + - GET + - HEAD + - OPTIONS + - POST + - PUT + - PATCH + - DELETE + type: string + type: array + x-kubernetes-list-type: set + allowedPathPrefixes: + description: |- + AllowedPathPrefixes constrains URL paths the broker will inject on + (prefix match, e.g. "/repos/"). Empty means "/" (all paths on the allowed hosts). + items: + type: string + type: array + x-kubernetes-list-type: set + credentialEnvName: + description: |- + CredentialEnvName names the backend env var that would normally carry this + provider's token (e.g. "GITHUB_TOKEN"). The operator injects a SENTINEL literal + here so servers that refuse to start tokenless still boot; the broker ignores + the sentinel value. Sentinel literal format: "thv-untrusted-sentinel:". + type: string + provider: + description: |- + Provider is the logical upstream provider name (e.g. "github", "google"). + Must match the provider name used by the auth server and the vMCP + upstream_inject strategy config for this workload. + minLength: 1 + type: string + required: + - allowedHosts + - provider + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - provider + x-kubernetes-list-type: map + required: + - providers + type: object endpointPrefix: description: |- EndpointPrefix is the path prefix to prepend to SSE endpoint URLs. @@ -1730,6 +1915,15 @@ spec: When enabled, the proxy will use X-Forwarded-Proto, X-Forwarded-Host, X-Forwarded-Port, and X-Forwarded-Prefix headers to construct endpoint URLs type: boolean + untrusted: + default: false + description: |- + Untrusted marks this MCP server as running untrusted code. When true, the operator + enforces the untrusted-mode invariants (ADR-0001): no Secret/ConfigMap-sourced env on + the backend container, single-tenant session-scoped backend pods, mandatory MCPGroup + membership behind a VirtualMCPServer, and egress only through the credential-broker + sidecar per EgressPolicy. K8s-only; ignored (and inert) in CLI/Docker mode. + type: boolean volumes: description: Volumes are volumes to mount in the MCP server container items: @@ -1788,6 +1982,24 @@ spec: or externalAuthConfigRef) rule: '!has(self.rateLimiting) || !has(self.rateLimiting.tools) || self.rateLimiting.tools.all(t, !has(t.perUser)) || has(self.oidcConfigRef) || has(self.externalAuthConfigRef)' + - message: egressPolicy with at least one provider is required when untrusted + is true + rule: '!self.untrusted || (has(self.egressPolicy) && size(self.egressPolicy.providers) + > 0)' + - message: untrusted workloads must belong to an MCPGroup fronted by a + VirtualMCPServer + rule: '!self.untrusted || has(self.groupRef)' + - message: spec.secrets is forbidden when untrusted is true; declare providers + in egressPolicy and use sentinel env + rule: '!self.untrusted || !has(self.secrets) || size(self.secrets) == + 0' + - message: podTemplateSpec is forbidden when untrusted is true + rule: '!self.untrusted || !has(self.podTemplateSpec)' + - message: untrusted workloads require sessionAffinity ClientIP + rule: '!self.untrusted || self.sessionAffinity == ''ClientIP''' + - message: backendReplicas is managed by the untrusted-mode session lifecycle + and must not be set + rule: '!self.untrusted || !has(self.backendReplicas)' status: description: MCPServerStatus defines the observed state of MCPServer properties: diff --git a/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpservers.yaml b/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpservers.yaml index cc26407b74..2a7b8ca662 100644 --- a/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpservers.yaml +++ b/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpservers.yaml @@ -241,6 +241,85 @@ spec: format: int32 minimum: 0 type: integer + egressPolicy: + description: |- + EgressPolicy declares which upstream providers this server may call and where the + broker may inject each provider's credential. Required when Untrusted is true. + When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend + pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar + Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + properties: + providers: + description: |- + Providers maps an upstream provider name (as configured in the embedded auth + server's upstream chain / upstream_inject strategy) to its egress constraints. + items: + description: |- + ProviderEgress is the per-provider egress constraint. Credential-to-destination + binding (ADR-0001 D5): the broker injects this provider's credential only to + AllowedHosts, only on AllowedMethods, only under AllowedPathPrefixes. + properties: + allowedHosts: + description: |- + AllowedHosts is the exact set of destination hosts the credential may be + injected for. Exact hostnames or one-label wildcards ("*.githubusercontent.com"); + no ports, no schemes. + items: + pattern: ^(\*\.)?[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$ + type: string + minItems: 1 + type: array + x-kubernetes-list-type: set + allowedMethods: + description: |- + AllowedMethods constrains HTTP methods the broker will inject on. + Empty means GET/HEAD/OPTIONS only (safe default — read-only). + items: + enum: + - GET + - HEAD + - OPTIONS + - POST + - PUT + - PATCH + - DELETE + type: string + type: array + x-kubernetes-list-type: set + allowedPathPrefixes: + description: |- + AllowedPathPrefixes constrains URL paths the broker will inject on + (prefix match, e.g. "/repos/"). Empty means "/" (all paths on the allowed hosts). + items: + type: string + type: array + x-kubernetes-list-type: set + credentialEnvName: + description: |- + CredentialEnvName names the backend env var that would normally carry this + provider's token (e.g. "GITHUB_TOKEN"). The operator injects a SENTINEL literal + here so servers that refuse to start tokenless still boot; the broker ignores + the sentinel value. Sentinel literal format: "thv-untrusted-sentinel:". + type: string + provider: + description: |- + Provider is the logical upstream provider name (e.g. "github", "google"). + Must match the provider name used by the auth server and the vMCP + upstream_inject strategy config for this workload. + minLength: 1 + type: string + required: + - allowedHosts + - provider + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - provider + x-kubernetes-list-type: map + required: + - providers + type: object endpointPrefix: description: |- EndpointPrefix is the path prefix to prepend to SSE endpoint URLs. @@ -789,6 +868,15 @@ spec: When enabled, the proxy will use X-Forwarded-Proto, X-Forwarded-Host, X-Forwarded-Port, and X-Forwarded-Prefix headers to construct endpoint URLs type: boolean + untrusted: + default: false + description: |- + Untrusted marks this MCP server as running untrusted code. When true, the operator + enforces the untrusted-mode invariants (ADR-0001): no Secret/ConfigMap-sourced env on + the backend container, single-tenant session-scoped backend pods, mandatory MCPGroup + membership behind a VirtualMCPServer, and egress only through the credential-broker + sidecar per EgressPolicy. K8s-only; ignored (and inert) in CLI/Docker mode. + type: boolean volumes: description: Volumes are volumes to mount in the MCP server container items: @@ -847,6 +935,24 @@ spec: or externalAuthConfigRef) rule: '!has(self.rateLimiting) || !has(self.rateLimiting.tools) || self.rateLimiting.tools.all(t, !has(t.perUser)) || has(self.oidcConfigRef) || has(self.externalAuthConfigRef)' + - message: egressPolicy with at least one provider is required when untrusted + is true + rule: '!self.untrusted || (has(self.egressPolicy) && size(self.egressPolicy.providers) + > 0)' + - message: untrusted workloads must belong to an MCPGroup fronted by a + VirtualMCPServer + rule: '!self.untrusted || has(self.groupRef)' + - message: spec.secrets is forbidden when untrusted is true; declare providers + in egressPolicy and use sentinel env + rule: '!self.untrusted || !has(self.secrets) || size(self.secrets) == + 0' + - message: podTemplateSpec is forbidden when untrusted is true + rule: '!self.untrusted || !has(self.podTemplateSpec)' + - message: untrusted workloads require sessionAffinity ClientIP + rule: '!self.untrusted || self.sessionAffinity == ''ClientIP''' + - message: backendReplicas is managed by the untrusted-mode session lifecycle + and must not be set + rule: '!self.untrusted || !has(self.backendReplicas)' status: description: MCPServerStatus defines the observed state of MCPServer properties: @@ -1185,6 +1291,85 @@ spec: format: int32 minimum: 0 type: integer + egressPolicy: + description: |- + EgressPolicy declares which upstream providers this server may call and where the + broker may inject each provider's credential. Required when Untrusted is true. + When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend + pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar + Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + properties: + providers: + description: |- + Providers maps an upstream provider name (as configured in the embedded auth + server's upstream chain / upstream_inject strategy) to its egress constraints. + items: + description: |- + ProviderEgress is the per-provider egress constraint. Credential-to-destination + binding (ADR-0001 D5): the broker injects this provider's credential only to + AllowedHosts, only on AllowedMethods, only under AllowedPathPrefixes. + properties: + allowedHosts: + description: |- + AllowedHosts is the exact set of destination hosts the credential may be + injected for. Exact hostnames or one-label wildcards ("*.githubusercontent.com"); + no ports, no schemes. + items: + pattern: ^(\*\.)?[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$ + type: string + minItems: 1 + type: array + x-kubernetes-list-type: set + allowedMethods: + description: |- + AllowedMethods constrains HTTP methods the broker will inject on. + Empty means GET/HEAD/OPTIONS only (safe default — read-only). + items: + enum: + - GET + - HEAD + - OPTIONS + - POST + - PUT + - PATCH + - DELETE + type: string + type: array + x-kubernetes-list-type: set + allowedPathPrefixes: + description: |- + AllowedPathPrefixes constrains URL paths the broker will inject on + (prefix match, e.g. "/repos/"). Empty means "/" (all paths on the allowed hosts). + items: + type: string + type: array + x-kubernetes-list-type: set + credentialEnvName: + description: |- + CredentialEnvName names the backend env var that would normally carry this + provider's token (e.g. "GITHUB_TOKEN"). The operator injects a SENTINEL literal + here so servers that refuse to start tokenless still boot; the broker ignores + the sentinel value. Sentinel literal format: "thv-untrusted-sentinel:". + type: string + provider: + description: |- + Provider is the logical upstream provider name (e.g. "github", "google"). + Must match the provider name used by the auth server and the vMCP + upstream_inject strategy config for this workload. + minLength: 1 + type: string + required: + - allowedHosts + - provider + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - provider + x-kubernetes-list-type: map + required: + - providers + type: object endpointPrefix: description: |- EndpointPrefix is the path prefix to prepend to SSE endpoint URLs. @@ -1733,6 +1918,15 @@ spec: When enabled, the proxy will use X-Forwarded-Proto, X-Forwarded-Host, X-Forwarded-Port, and X-Forwarded-Prefix headers to construct endpoint URLs type: boolean + untrusted: + default: false + description: |- + Untrusted marks this MCP server as running untrusted code. When true, the operator + enforces the untrusted-mode invariants (ADR-0001): no Secret/ConfigMap-sourced env on + the backend container, single-tenant session-scoped backend pods, mandatory MCPGroup + membership behind a VirtualMCPServer, and egress only through the credential-broker + sidecar per EgressPolicy. K8s-only; ignored (and inert) in CLI/Docker mode. + type: boolean volumes: description: Volumes are volumes to mount in the MCP server container items: @@ -1791,6 +1985,24 @@ spec: or externalAuthConfigRef) rule: '!has(self.rateLimiting) || !has(self.rateLimiting.tools) || self.rateLimiting.tools.all(t, !has(t.perUser)) || has(self.oidcConfigRef) || has(self.externalAuthConfigRef)' + - message: egressPolicy with at least one provider is required when untrusted + is true + rule: '!self.untrusted || (has(self.egressPolicy) && size(self.egressPolicy.providers) + > 0)' + - message: untrusted workloads must belong to an MCPGroup fronted by a + VirtualMCPServer + rule: '!self.untrusted || has(self.groupRef)' + - message: spec.secrets is forbidden when untrusted is true; declare providers + in egressPolicy and use sentinel env + rule: '!self.untrusted || !has(self.secrets) || size(self.secrets) == + 0' + - message: podTemplateSpec is forbidden when untrusted is true + rule: '!self.untrusted || !has(self.podTemplateSpec)' + - message: untrusted workloads require sessionAffinity ClientIP + rule: '!self.untrusted || self.sessionAffinity == ''ClientIP''' + - message: backendReplicas is managed by the untrusted-mode session lifecycle + and must not be set + rule: '!self.untrusted || !has(self.backendReplicas)' status: description: MCPServerStatus defines the observed state of MCPServer properties: diff --git a/docs/operator/crd-api.md b/docs/operator/crd-api.md index dad8a42745..e4e67c465b 100644 --- a/docs/operator/crd-api.md +++ b/docs/operator/crd-api.md @@ -1874,6 +1874,24 @@ This provides a local name for use in the CRD status. +#### api.v1beta1.EgressPolicy + + + +EgressPolicy binds upstream providers to the exact destinations their credentials +may be injected for. The broker emits provider P's credential ONLY to hosts in +P's AllowedHosts, with method/path further constrained by the entry (ADR-0001 D5). + + + +_Appears in:_ +- [api.v1beta1.MCPServerSpec](#apiv1beta1mcpserverspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `providers` _[api.v1beta1.ProviderEgress](#apiv1beta1provideregress) array_ | Providers maps an upstream provider name (as configured in the embedded auth
server's upstream chain / upstream_inject strategy) to its egress constraints. | | MinItems: 1
| + + #### api.v1beta1.EmbeddedAuthServerCIMDConfig @@ -3208,6 +3226,8 @@ _Appears in:_ | `trustProxyHeaders` _boolean_ | TrustProxyHeaders indicates whether to trust X-Forwarded-* headers from reverse proxies
When enabled, the proxy will use X-Forwarded-Proto, X-Forwarded-Host, X-Forwarded-Port,
and X-Forwarded-Prefix headers to construct endpoint URLs | false | Optional: \{\}
| | `endpointPrefix` _string_ | EndpointPrefix is the path prefix to prepend to SSE endpoint URLs.
This is used to handle path-based ingress routing scenarios where the ingress
strips a path prefix before forwarding to the backend. | | Optional: \{\}
| | `groupRef` _[api.v1beta1.MCPGroupRef](#apiv1beta1mcpgroupref)_ | GroupRef references the MCPGroup this server belongs to.
The referenced MCPGroup must be in the same namespace. | | Optional: \{\}
| +| `untrusted` _boolean_ | Untrusted marks this MCP server as running untrusted code. When true, the operator
enforces the untrusted-mode invariants (ADR-0001): no Secret/ConfigMap-sourced env on
the backend container, single-tenant session-scoped backend pods, mandatory MCPGroup
membership behind a VirtualMCPServer, and egress only through the credential-broker
sidecar per EgressPolicy. K8s-only; ignored (and inert) in CLI/Docker mode. | false | Optional: \{\}
| +| `egressPolicy` _[api.v1beta1.EgressPolicy](#apiv1beta1egresspolicy)_ | EgressPolicy declares which upstream providers this server may call and where the
broker may inject each provider's credential. Required when Untrusted is true.
When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend
pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar
Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. | | Optional: \{\}
| | `sessionAffinity` _string_ | SessionAffinity controls whether the Service routes repeated client connections to the same pod.
MCP protocols (SSE, streamable-http) are stateful, so ClientIP is the default.
Set to "None" for stateless servers or when using an external load balancer with its own affinity. | ClientIP | Enum: [ClientIP None]
Optional: \{\}
| | `replicas` _integer_ | Replicas is the desired number of proxy runner (thv run) pod replicas.
MCPServer creates two separate Deployments: one for the proxy runner and one
for the MCP server backend. This field controls the proxy runner Deployment.
When nil, the operator does not set Deployment.Spec.Replicas, leaving replica
management to an HPA or other external controller. | | Minimum: 0
Optional: \{\}
| | `backendReplicas` _integer_ | BackendReplicas is the desired number of MCP server backend pod replicas.
This controls the backend Deployment (the MCP server container itself),
independent of the proxy runner controlled by Replicas.
When nil, the operator does not set Deployment.Spec.Replicas, leaving replica
management to an HPA or other external controller. | | Minimum: 0
Optional: \{\}
| @@ -3780,6 +3800,28 @@ _Appears in:_ | `enabled` _boolean_ | Enabled controls whether Prometheus metrics endpoint is exposed | false | Optional: \{\}
| +#### api.v1beta1.ProviderEgress + + + +ProviderEgress is the per-provider egress constraint. Credential-to-destination +binding (ADR-0001 D5): the broker injects this provider's credential only to +AllowedHosts, only on AllowedMethods, only under AllowedPathPrefixes. + + + +_Appears in:_ +- [api.v1beta1.EgressPolicy](#apiv1beta1egresspolicy) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `provider` _string_ | Provider is the logical upstream provider name (e.g. "github", "google").
Must match the provider name used by the auth server and the vMCP
upstream_inject strategy config for this workload. | | MinLength: 1
| +| `allowedHosts` _string array_ | AllowedHosts is the exact set of destination hosts the credential may be
injected for. Exact hostnames or one-label wildcards ("*.githubusercontent.com");
no ports, no schemes. | | MinItems: 1
items:Pattern: `^(\*\.)?[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$`
| +| `allowedMethods` _string array_ | AllowedMethods constrains HTTP methods the broker will inject on.
Empty means GET/HEAD/OPTIONS only (safe default — read-only). | | items:Enum: [GET HEAD OPTIONS POST PUT PATCH DELETE]
Optional: \{\}
| +| `allowedPathPrefixes` _string array_ | AllowedPathPrefixes constrains URL paths the broker will inject on
(prefix match, e.g. "/repos/"). Empty means "/" (all paths on the allowed hosts). | | Optional: \{\}
| +| `credentialEnvName` _string_ | CredentialEnvName names the backend env var that would normally carry this
provider's token (e.g. "GITHUB_TOKEN"). The operator injects a SENTINEL literal
here so servers that refuse to start tokenless still boot; the broker ignores
the sentinel value. Sentinel literal format: "thv-untrusted-sentinel:". | | Optional: \{\}
| + + #### api.v1beta1.ProxyDeploymentOverrides diff --git a/go.mod b/go.mod index d5d1cc60a2..e9adcf1165 100644 --- a/go.mod +++ b/go.mod @@ -84,14 +84,20 @@ require github.com/getsentry/sentry-go/otel v0.44.1 require github.com/hashicorp/golang-lru/v2 v2.0.7 -require go.starlark.net v0.0.0-20260630144053-529d8e869a14 +require ( + github.com/envoyproxy/go-control-plane/envoy v1.37.0 + go.starlark.net v0.0.0-20260630144053-529d8e869a14 +) require ( github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/go-openapi/runtime/server-middleware v0.30.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/modelcontextprotocol/go-sdk v1.6.1 // indirect github.com/oklog/ulid/v2 v2.1.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/segmentio/encoding v0.5.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect ) @@ -297,9 +303,9 @@ require ( golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68 // indirect - google.golang.org/grpc v1.82.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260523011958-0a33c5d7ca68 + google.golang.org/grpc v1.82.0 + google.golang.org/protobuf v1.36.11 gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 0bae8d9144..e6ba0bf6b7 100644 --- a/go.sum +++ b/go.sum @@ -134,6 +134,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= @@ -203,6 +205,10 @@ github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= @@ -680,6 +686,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/pkg/auth/upstreamtoken/strict_reader.go b/pkg/auth/upstreamtoken/strict_reader.go new file mode 100644 index 0000000000..10c88a0aad --- /dev/null +++ b/pkg/auth/upstreamtoken/strict_reader.go @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package upstreamtoken + +import ( + "context" + + "github.com/stacklok/toolhive/pkg/authserver/storage" +) + +// strictReader wraps a TokenReader and forces ExpectedBinding.Strict=true on every +// GetAllUpstreamCredentials call. Used for untrusted workloads (ADR-0001): a legacy row +// with no recorded owner must never be released to an untrusted backend's credential path. +// This is the untrusted-mode caller the Wave-0 Strict field was built for. +type strictReader struct{ inner TokenReader } + +// NewStrictTokenReader wraps inner so all reads fail closed on legacy/unowned rows: +// the wrapped reader asserts ExpectedBinding.Strict=true on every call, so a stored +// upstream-token row with an empty UserID yields storage.ErrInvalidBinding instead of +// being released under the permissive legacy rule. It does NOT relax any other binding +// dimension — caller-supplied UserID/ClientID/UpstreamSubject are preserved untouched. +// Callers (the untrusted-mode credential broker, Wave 3) are responsible for wrapping +// the reader at construction time; the per-JWT load path in TokenValidator must NOT be +// wrapped, since trusted workloads legitimately read legacy rows. +func NewStrictTokenReader(inner TokenReader) TokenReader { return &strictReader{inner: inner} } + +// GetAllUpstreamCredentials forces Strict=true on the expected binding, copying the +// caller's binding before mutating (copy-before-mutate: the caller's struct is never +// modified), and delegates to the inner reader. +func (s *strictReader) GetAllUpstreamCredentials( + ctx context.Context, sessionID string, expected *storage.ExpectedBinding, +) (map[string]UpstreamCredential, []string, error) { + e := &storage.ExpectedBinding{Strict: true} + if expected != nil { + *e = *expected + e.Strict = true + } + return s.inner.GetAllUpstreamCredentials(ctx, sessionID, e) +} diff --git a/pkg/auth/upstreamtoken/strict_reader_test.go b/pkg/auth/upstreamtoken/strict_reader_test.go new file mode 100644 index 0000000000..e5894a95f1 --- /dev/null +++ b/pkg/auth/upstreamtoken/strict_reader_test.go @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package upstreamtoken_test + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken" + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken/mocks" + "github.com/stacklok/toolhive/pkg/authserver/storage" +) + +func TestStrictTokenReader(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("forces Strict=true and preserves caller binding fields", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + inner := mocks.NewMockTokenReader(ctrl) + reader := upstreamtoken.NewStrictTokenReader(inner) + + expected := &storage.ExpectedBinding{ + UserID: "user-123", + ClientID: "client-abc", + UpstreamSubject: "upstream-sub", + } + inner.EXPECT(). + GetAllUpstreamCredentials(ctx, "session-1", gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, got *storage.ExpectedBinding, + ) (map[string]upstreamtoken.UpstreamCredential, []string, error) { + assert.True(t, got.Strict, "Strict must be forced true") + assert.Equal(t, "user-123", got.UserID, "caller UserID must be preserved") + assert.Equal(t, "client-abc", got.ClientID, "caller ClientID must be preserved") + assert.Equal(t, "upstream-sub", got.UpstreamSubject, "caller UpstreamSubject must be preserved") + return map[string]upstreamtoken.UpstreamCredential{"github": {AccessToken: "tok"}}, nil, nil + }) + + creds, failed, err := reader.GetAllUpstreamCredentials(ctx, "session-1", expected) + require.NoError(t, err) + assert.Empty(t, failed) + assert.Equal(t, "tok", creds["github"].AccessToken) + + assert.False(t, expected.Strict, "caller's expected binding must not be mutated (copy-before-mutate)") + }) + + t.Run("overrides a caller-supplied Strict=false", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + inner := mocks.NewMockTokenReader(ctrl) + reader := upstreamtoken.NewStrictTokenReader(inner) + + expected := &storage.ExpectedBinding{UserID: "user-123", Strict: false} + inner.EXPECT(). + GetAllUpstreamCredentials(ctx, "session-1", gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, got *storage.ExpectedBinding, + ) (map[string]upstreamtoken.UpstreamCredential, []string, error) { + assert.True(t, got.Strict, "Strict=false from the caller must be overridden") + return nil, nil, nil + }) + + _, _, err := reader.GetAllUpstreamCredentials(ctx, "session-1", expected) + require.NoError(t, err) + assert.False(t, expected.Strict, "caller's struct must not be mutated") + }) + + t.Run("nil expected is synthesized with only Strict=true", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + inner := mocks.NewMockTokenReader(ctrl) + reader := upstreamtoken.NewStrictTokenReader(inner) + + inner.EXPECT(). + GetAllUpstreamCredentials(ctx, "session-1", gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, got *storage.ExpectedBinding, + ) (map[string]upstreamtoken.UpstreamCredential, []string, error) { + require.NotNil(t, got) + assert.Equal(t, storage.ExpectedBinding{Strict: true}, *got) + return nil, nil, nil + }) + + _, _, err := reader.GetAllUpstreamCredentials(ctx, "session-1", nil) + require.NoError(t, err) + }) + + t.Run("inner ErrInvalidBinding propagates", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + inner := mocks.NewMockTokenReader(ctrl) + reader := upstreamtoken.NewStrictTokenReader(inner) + + inner.EXPECT(). + GetAllUpstreamCredentials(ctx, "session-1", gomock.Any()). + Return(nil, nil, fmt.Errorf("wrapped: %w", storage.ErrInvalidBinding)) + + _, _, err := reader.GetAllUpstreamCredentials(ctx, "session-1", &storage.ExpectedBinding{UserID: "u"}) + require.Error(t, err) + assert.ErrorIs(t, err, storage.ErrInvalidBinding) + }) + + t.Run("session ID and context are forwarded unchanged", func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + inner := mocks.NewMockTokenReader(ctrl) + reader := upstreamtoken.NewStrictTokenReader(inner) + + type ctxKey struct{} + keyedCtx := context.WithValue(ctx, ctxKey{}, "marker") + + inner.EXPECT(). + GetAllUpstreamCredentials(keyedCtx, "exact-session-id", gomock.Any()). + Return(nil, []string{"github"}, nil) + + _, failed, err := reader.GetAllUpstreamCredentials(keyedCtx, "exact-session-id", nil) + require.NoError(t, err) + assert.Equal(t, []string{"github"}, failed, "failed-provider list must pass through untouched") + }) +} diff --git a/pkg/authserver/tokenenc/tokenenc.go b/pkg/authserver/tokenenc/tokenenc.go index 1922787c2b..f8fb8d3d4c 100644 --- a/pkg/authserver/tokenenc/tokenenc.go +++ b/pkg/authserver/tokenenc/tokenenc.go @@ -247,17 +247,6 @@ func NeedsRotation(kr Keyring, value []byte) bool { return env.KID != activeID } -// EnvelopeKeyID extracts the key ID from an envelope value for observability -// (never for security decisions). ok is false for non-envelope values. -func EnvelopeKeyID(value []byte) (kid string, ok bool) { - var env envelope - if jsonErr := json.Unmarshal(value, &env); jsonErr != nil || - env.V != envelopeVersion || env.KID == "" || env.EDEK == "" || env.CT == "" { - return "", false - } - return env.KID, true -} - // gcmSeal encrypts plaintext with AES-256-GCM under key, returning // nonce|ciphertext|tag. aad is additional authenticated data (may be nil). func gcmSeal(key, plaintext, aad []byte) ([]byte, error) { diff --git a/pkg/authserver/tokenenc/tokenenc_test.go b/pkg/authserver/tokenenc/tokenenc_test.go index dcc463744f..c6713f0782 100644 --- a/pkg/authserver/tokenenc/tokenenc_test.go +++ b/pkg/authserver/tokenenc/tokenenc_test.go @@ -323,21 +323,6 @@ func TestIsLegacyValue(t *testing.T) { assert.True(t, IsLegacyValue([]byte("garbage"))) } -func TestEnvelopeKeyID(t *testing.T) { - t.Parallel() - - kr := newTestKeyring(t, "k1", map[string][]byte{"k1": testKey(1)}) - sealed, err := Seal(kr, "key1", []byte("payload")) - require.NoError(t, err) - - kid, ok := EnvelopeKeyID(sealed) - assert.True(t, ok) - assert.Equal(t, "k1", kid) - - _, ok = EnvelopeKeyID([]byte(`{"access_token":"tok"}`)) - assert.False(t, ok) -} - // TestSealOpen_RandomKeys guards against accidental dependence on the // deterministic testKey helper: real random KEKs must round-trip too. func TestSealOpen_RandomKeys(t *testing.T) { diff --git a/pkg/egressbroker/ca.go b/pkg/egressbroker/ca.go new file mode 100644 index 0000000000..8820ed165e --- /dev/null +++ b/pkg/egressbroker/ca.go @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "time" +) + +const ( + // CAValidity is the bump-CA lifetime (ADR D9). + CAValidity = 90 * 24 * time.Hour + // rotationLead is how long before expiry rotation becomes due (50% of + // validity, so the overlap window covers the backend rollout). + rotationLead = CAValidity / 2 + // leafValidity bounds each minted per-SNI cert. Short-lived: the cert is + // only ever validated by the co-located backend container against the + // per-tenant bump CA. + leafValidity = 24 * time.Hour +) + +// BumpCA is the per-tenant TLS-bump certificate authority (D9). It signs +// per-SNI leaf certificates so the backend container's HTTPS connection can +// be terminated at the sidecar, inspected, and re-originated. +// +// The private key never leaves the operator-owned CA Secret / the sidecar +// process; it is never logged. +type BumpCA struct { + cert *x509.Certificate + key *ecdsa.PrivateKey + // notAfter is carried explicitly so NeedsRotation is testable. + notAfter time.Time +} + +// GenerateBumpCA creates a self-signed ECDSA P-256 CA valid for CAValidity +// from now. commonName identifies the tenant (e.g. the MCPServer name). +func GenerateBumpCA(commonName string, now time.Time) (*BumpCA, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to generate bump CA key: %w", err) + } + serial, err := randomSerial() + if err != nil { + return nil, err + } + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: commonName}, + NotBefore: now.Add(-time.Minute), + NotAfter: now.Add(CAValidity), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to self-sign bump CA: %w", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to parse self-signed bump CA: %w", err) + } + return &BumpCA{cert: cert, key: key, notAfter: cert.NotAfter}, nil +} + +// ParseBumpCA loads a CA from PEM cert + PEM PKCS8/SEC1 private key bytes +// (the operator-owned Secret volume contents). Fails loudly on malformed or +// non-CA input — a broker that cannot mint certs must not start. +func ParseBumpCA(certPEM, keyPEM []byte) (*BumpCA, error) { + certBlock, _ := pem.Decode(certPEM) + if certBlock == nil || certBlock.Type != "CERTIFICATE" { + return nil, fmt.Errorf("egressbroker: bump CA cert PEM is missing or not a CERTIFICATE block") + } + cert, err := x509.ParseCertificate(certBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to parse bump CA cert: %w", err) + } + if !cert.IsCA { + return nil, fmt.Errorf("egressbroker: bump CA cert is not a CA certificate") + } + keyBlock, _ := pem.Decode(keyPEM) + if keyBlock == nil { + return nil, fmt.Errorf("egressbroker: bump CA key PEM is missing") + } + key, err := parseECPrivateKey(keyBlock.Bytes) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to parse bump CA key: %w", err) + } + return &BumpCA{cert: cert, key: key, notAfter: cert.NotAfter}, nil +} + +// CertPEM renders the public CA certificate (the bundle content distributed +// to backend containers). Key material is never rendered by any accessor. +func (c *BumpCA) CertPEM() []byte { + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.cert.Raw}) +} + +// KeyPEM renders the CA private key in PKCS8 PEM form. Used ONLY by the +// operator-side reconcile to populate the bump-CA Secret; never log or +// include the result in errors. +func (c *BumpCA) KeyPEM() ([]byte, error) { + der, err := x509.MarshalPKCS8PrivateKey(c.key) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to marshal bump CA key: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil +} + +// NeedsRotation reports whether the CA is within the rotation window (past +// 50% of validity) at now. +func (c *BumpCA) NeedsRotation(now time.Time) bool { + return !now.Before(c.notAfter.Add(-rotationLead)) +} + +// NotAfter returns the CA certificate's expiry. The operator orders CA +// generations by it when deciding which generations to retain across a +// rotation. +func (c *BumpCA) NotAfter() time.Time { + return c.notAfter +} + +// MintLeaf issues a short-lived leaf certificate for hostname (the SNI the +// backend dialed), signed by this CA. hostname must be a DNS name, not an IP: +// D5/D7 allowlists are name-based and IP-literal TLS bumps are refused +// (fail closed). +func (c *BumpCA) MintLeaf(hostname string, now time.Time) (certPEM, keyPEM []byte, err error) { + if hostname == "" { + return nil, nil, fmt.Errorf("egressbroker: cannot mint leaf for empty hostname") + } + if ip := net.ParseIP(hostname); ip != nil { + return nil, nil, fmt.Errorf("egressbroker: refusing to mint leaf for IP-literal SNI") + } + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("egressbroker: failed to generate leaf key: %w", err) + } + serial, err := randomSerial() + if err != nil { + return nil, nil, err + } + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: hostname}, + DNSNames: []string{hostname}, + NotBefore: now.Add(-time.Minute), + NotAfter: now.Add(leafValidity), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, c.cert, &key.PublicKey, c.key) + if err != nil { + return nil, nil, fmt.Errorf("egressbroker: failed to sign leaf for %q: %w", hostname, err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, nil, fmt.Errorf("egressbroker: failed to marshal leaf key: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), nil +} + +func randomSerial() (*big.Int, error) { + limit := new(big.Int).Lsh(big.NewInt(1), 128) + serial, err := rand.Int(rand.Reader, limit) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to generate cert serial: %w", err) + } + if serial.Sign() == 0 { + serial = big.NewInt(1) + } + return serial, nil +} + +func parseECPrivateKey(der []byte) (*ecdsa.PrivateKey, error) { + if key, err := x509.ParsePKCS8PrivateKey(der); err == nil { + if ecKey, ok := key.(*ecdsa.PrivateKey); ok { + return ecKey, nil + } + return nil, fmt.Errorf("key is not ECDSA") + } + return x509.ParseECPrivateKey(der) +} diff --git a/pkg/egressbroker/ca_test.go b/pkg/egressbroker/ca_test.go new file mode 100644 index 0000000000..e137cd948b --- /dev/null +++ b/pkg/egressbroker/ca_test.go @@ -0,0 +1,179 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "crypto/x509" + "encoding/pem" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +func TestBumpCA(t *testing.T) { + t.Parallel() + // CA-level dates use a fixed instant (rotation math is deterministic); + // leaf minting uses real time because leaf validity (24h) must cover the + // actual verification moment. + now := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + realNow := time.Now() + + t.Run("generation produces a valid self-signed CA", func(t *testing.T) { + t.Parallel() + ca, err := egressbroker.GenerateBumpCA("github-mcp-bump-ca", now) + require.NoError(t, err) + + block, _ := pem.Decode(ca.CertPEM()) + require.NotNil(t, block) + cert, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + assert.True(t, cert.IsCA) + assert.Equal(t, "github-mcp-bump-ca", cert.Subject.CommonName) + assert.WithinDuration(t, now.Add(egressbroker.CAValidity), cert.NotAfter, time.Minute) + + // Self-signed: verifies against its own pool. + roots := x509.NewCertPool() + roots.AddCert(cert) + _, err = cert.Verify(x509.VerifyOptions{Roots: roots, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}}) + require.NoError(t, err) + }) + + t.Run("PEM round-trip preserves signing ability", func(t *testing.T) { + t.Parallel() + ca, err := egressbroker.GenerateBumpCA("roundtrip", now) + require.NoError(t, err) + keyPEM, err := ca.KeyPEM() + require.NoError(t, err) + + loaded, err := egressbroker.ParseBumpCA(ca.CertPEM(), keyPEM) + require.NoError(t, err) + + // A leaf minted by the loaded CA must verify against the original's cert. + leafPEM, _, err := loaded.MintLeaf("api.github.com", realNow) + require.NoError(t, err) + assertLeafValidFor(t, ca.CertPEM(), leafPEM, "api.github.com") + }) + + t.Run("minted leaf verifies against the CA bundle for the SNI host", func(t *testing.T) { + t.Parallel() + ca, err := egressbroker.GenerateBumpCA("mint", now) + require.NoError(t, err) + leafPEM, _, err := ca.MintLeaf("api.github.com", realNow) + require.NoError(t, err) + assertLeafValidFor(t, ca.CertPEM(), leafPEM, "api.github.com") + }) + + t.Run("leaf for one host does not verify for another", func(t *testing.T) { + t.Parallel() + ca, err := egressbroker.GenerateBumpCA("mint2", now) + require.NoError(t, err) + leafPEM, _, err := ca.MintLeaf("api.github.com", realNow) + require.NoError(t, err) + + block, _ := pem.Decode(leafPEM) + require.NotNil(t, block) + leaf, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + caBlock, _ := pem.Decode(ca.CertPEM()) + caCert, err := x509.ParseCertificate(caBlock.Bytes) + require.NoError(t, err) + roots := x509.NewCertPool() + roots.AddCert(caCert) + _, err = leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "api.evil.com"}) + require.Error(t, err, "leaf must be bound to its SNI hostname") + }) + + t.Run("minting refuses empty hostname and IP literals (fail closed)", func(t *testing.T) { + t.Parallel() + ca, err := egressbroker.GenerateBumpCA("refuse", now) + require.NoError(t, err) + _, _, err = ca.MintLeaf("", now) + require.Error(t, err) + _, _, err = ca.MintLeaf("140.82.114.26", now) + require.Error(t, err, "IP-literal SNI must not get a bump cert") + _, _, err = ca.MintLeaf("::1", now) + require.Error(t, err) + }) + + t.Run("rotation window: due at 50% validity", func(t *testing.T) { + t.Parallel() + ca, err := egressbroker.GenerateBumpCA("rotate", now) + require.NoError(t, err) + + assert.False(t, ca.NeedsRotation(now), "fresh CA must not need rotation") + assert.False(t, ca.NeedsRotation(now.Add(egressbroker.CAValidity/2-time.Hour))) + assert.True(t, ca.NeedsRotation(now.Add(egressbroker.CAValidity/2)), + "CA at 50% validity must be due for rotation") + assert.True(t, ca.NeedsRotation(now.Add(egressbroker.CAValidity)), + "expired CA must need rotation") + }) + + t.Run("rotation overlap: old and new CA each verify their own leaves", func(t *testing.T) { + t.Parallel() + oldCA, err := egressbroker.GenerateBumpCA("overlap", realNow.Add(-egressbroker.CAValidity/2)) + require.NoError(t, err) + newCA, err := egressbroker.GenerateBumpCA("overlap", realNow) + require.NoError(t, err) + + oldLeaf, _, err := oldCA.MintLeaf("api.github.com", realNow) + require.NoError(t, err) + newLeaf, _, err := newCA.MintLeaf("api.github.com", realNow) + require.NoError(t, err) + + // Each pod trusts its own CA during the rollout overlap; there is no + // cross-pod TLS, so cross-verification must fail. + assertLeafValidFor(t, oldCA.CertPEM(), oldLeaf, "api.github.com") + assertLeafValidFor(t, newCA.CertPEM(), newLeaf, "api.github.com") + + block, _ := pem.Decode(newLeaf) + require.NotNil(t, block) + leaf, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + oldBlock, _ := pem.Decode(oldCA.CertPEM()) + oldCert, err := x509.ParseCertificate(oldBlock.Bytes) + require.NoError(t, err) + oldRoots := x509.NewCertPool() + oldRoots.AddCert(oldCert) + _, err = leaf.Verify(x509.VerifyOptions{Roots: oldRoots, DNSName: "api.github.com"}) + require.Error(t, err, "new-CA leaf must not verify against the old CA") + }) + + t.Run("parse rejects malformed and non-CA input (fail closed)", func(t *testing.T) { + t.Parallel() + ca, err := egressbroker.GenerateBumpCA("parse", now) + require.NoError(t, err) + keyPEM, err := ca.KeyPEM() + require.NoError(t, err) + + _, err = egressbroker.ParseBumpCA([]byte("garbage"), keyPEM) + require.Error(t, err) + _, err = egressbroker.ParseBumpCA(ca.CertPEM(), []byte("garbage")) + require.Error(t, err) + // A leaf (non-CA) cert must not load as a CA. + leafPEM, leafKey, err := ca.MintLeaf("api.github.com", realNow) + require.NoError(t, err) + _, err = egressbroker.ParseBumpCA(leafPEM, leafKey) + require.Error(t, err, "non-CA cert must be rejected") + }) +} + +func assertLeafValidFor(t *testing.T, caPEM, leafPEM []byte, dnsName string) { //nolint:unparam // varied call sites in prior revisions; kept parameterized for clarity + t.Helper() + caBlock, _ := pem.Decode(caPEM) + require.NotNil(t, caBlock) + caCert, err := x509.ParseCertificate(caBlock.Bytes) + require.NoError(t, err) + leafBlock, _ := pem.Decode(leafPEM) + require.NotNil(t, leafBlock) + leaf, err := x509.ParseCertificate(leafBlock.Bytes) + require.NoError(t, err) + roots := x509.NewCertPool() + roots.AddCert(caCert) + _, err = leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: dnsName}) + require.NoError(t, err, "leaf must verify against the CA bundle for %s", dnsName) +} diff --git a/pkg/egressbroker/config.go b/pkg/egressbroker/config.go new file mode 100644 index 0000000000..c61d86e49d --- /dev/null +++ b/pkg/egressbroker/config.go @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +// Environment-driven configuration (the sidecar is configured exclusively +// via env, injected at clone time — no flags, no config files the backend +// could influence). +const ( + // EnvPolicyFile is the mounted policy ConfigMap file path. + EnvPolicyFile = "THV_EGRESSBROKER_POLICY_FILE" + // EnvCAFile is the bump-CA public cert path (from the CA emptyDir). + EnvCAFile = "THV_EGRESSBROKER_CA_FILE" + // EnvCAKeyFile is the bump-CA private key path (sidecar-only Secret mount). + EnvCAKeyFile = "THV_EGRESSBROKER_CA_KEY_FILE" + // EnvListenAddress is the ext_authz gRPC listen address (default 127.0.0.1). + EnvListenAddress = "THV_EGRESSBROKER_LISTEN_ADDRESS" + // EnvListenPort is the ext_authz gRPC listen port (default 9001). + EnvListenPort = "THV_EGRESSBROKER_LISTEN_PORT" + // EnvDialAllowlist is a comma-separated CIDR/IP list for the D7 per-dial + // resolved-IP validation. + EnvDialAllowlist = "THV_EGRESSBROKER_DIAL_ALLOWLIST" +) + +// Defaults for the gRPC listener and the Envoy explicit-proxy port. +const ( + defaultListenAddress = "127.0.0.1" + defaultListenPort = 9001 + // DefaultProxyPort is the Envoy explicit-proxy port the backend's + // HTTP(S)_PROXY env points at (must match the clone-time wiring). + DefaultProxyPort = 15001 +) + +// Config is the validated runtime configuration of the broker process. +type Config struct { + // PolicyFile is the mounted policy ConfigMap file (required). + PolicyFile string + // CAFile is the bump-CA cert path (required). + CAFile string + // CAKeyFile is the bump-CA private key path (required). + CAKeyFile string + // ListenAddress for the ext_authz gRPC server. + ListenAddress string + // ListenPort for the ext_authz gRPC server. + ListenPort int + // DialAllowlist is the D7 per-dial IP allowlist (required, non-empty). + DialAllowlist []string +} + +// LoadConfig reads and validates the process environment. Fails loudly on +// any missing/invalid value — a misconfigured broker must not start (fail +// closed). +func LoadConfig(getenv func(string) string) (*Config, error) { + if getenv == nil { + return nil, fmt.Errorf("egressbroker: env lookup must not be nil") + } + cfg := &Config{ + PolicyFile: strings.TrimSpace(getenv(EnvPolicyFile)), + CAFile: strings.TrimSpace(getenv(EnvCAFile)), + CAKeyFile: strings.TrimSpace(getenv(EnvCAKeyFile)), + ListenAddress: strings.TrimSpace(getenv(EnvListenAddress)), + DialAllowlist: splitTrim(getenv(EnvDialAllowlist)), + } + if cfg.ListenAddress == "" { + cfg.ListenAddress = defaultListenAddress + } + cfg.ListenPort = defaultListenPort + if raw := strings.TrimSpace(getenv(EnvListenPort)); raw != "" { + port, err := strconv.Atoi(raw) + if err != nil || port <= 0 || port > 65535 { + return nil, fmt.Errorf("egressbroker: %s value %q is not a valid port", EnvListenPort, raw) + } + cfg.ListenPort = port + } + if cfg.PolicyFile == "" { + return nil, fmt.Errorf("egressbroker: %s must be set (policy ConfigMap mount)", EnvPolicyFile) + } + if cfg.CAFile == "" { + return nil, fmt.Errorf("egressbroker: %s must be set (bump CA cert)", EnvCAFile) + } + if cfg.CAKeyFile == "" { + return nil, fmt.Errorf("egressbroker: %s must be set (bump CA key, sidecar-only mount)", EnvCAKeyFile) + } + if len(cfg.DialAllowlist) > 0 { + // Operator-rendered policies carry the allowlist in the policy file + // (single source with the NetworkPolicy ipBlocks); the env var is an + // override for hand-rolled deployments. Validate eagerly either way. + if _, err := ParseIPAllowlist(cfg.DialAllowlist); err != nil { + return nil, err + } + } + if _, err := NewPodIdentityResolver(getenv); err != nil { + return nil, err + } + return cfg, nil +} + +// ResolveDialAllowlist returns the effective D7 dial allowlist: the env +// override when set, otherwise the operator-rendered policy document's list. +// An empty effective list is a startup error (fail closed). +func (c *Config) ResolveDialAllowlist(policy *EgressPolicy) ([]string, error) { + if len(c.DialAllowlist) > 0 { + return c.DialAllowlist, nil + } + if policy == nil || len(policy.DialAllowlist()) == 0 { + return nil, fmt.Errorf("egressbroker: no D7 dial allowlist: %s is unset and the policy document "+ + "carries no dialAllowlist (operator-rendered policy required)", EnvDialAllowlist) + } + return policy.DialAllowlist(), nil +} + +// ReadPolicyFile loads and compiles the mounted policy document. +func (c *Config) ReadPolicyFile() (*EgressPolicy, error) { + data, err := os.ReadFile(c.PolicyFile) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to read policy file: %w", err) + } + return ParsePolicy(data) +} + +// ReadBumpCA loads the bump CA from the mounted cert + key files. +func (c *Config) ReadBumpCA() (*BumpCA, error) { + certPEM, err := os.ReadFile(c.CAFile) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to read bump CA cert: %w", err) + } + keyPEM, err := os.ReadFile(c.CAKeyFile) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to read bump CA key: %w", err) + } + return ParseBumpCA(certPEM, keyPEM) +} + +func splitTrim(s string) []string { + if strings.TrimSpace(s) == "" { + return nil + } + parts := strings.Split(s, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} diff --git a/pkg/egressbroker/config_test.go b/pkg/egressbroker/config_test.go new file mode 100644 index 0000000000..d923d22439 --- /dev/null +++ b/pkg/egressbroker/config_test.go @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +func TestLoadConfig(t *testing.T) { + t.Parallel() + + base := map[string]string{ + egressbroker.EnvPolicyFile: "/etc/thv-policy/policy.yaml", + egressbroker.EnvCAFile: "/ca/ca.crt", + egressbroker.EnvCAKeyFile: "/ca/ca.key", + egressbroker.EnvIssuer: "https://issuer.example.com", + egressbroker.EnvSubjectRaw: "user-123", + egressbroker.EnvSessionID: "session-abc", + egressbroker.EnvMCPServer: "github-mcp", + egressbroker.EnvDialAllowlist: "140.82.112.0/20,185.199.108.133", + } + + t.Run("valid env loads with defaults", func(t *testing.T) { + t.Parallel() + cfg, err := egressbroker.LoadConfig(envFrom(base)) + require.NoError(t, err) + assert.Equal(t, "127.0.0.1", cfg.ListenAddress) + assert.Equal(t, 9001, cfg.ListenPort) + assert.Equal(t, []string{"140.82.112.0/20", "185.199.108.133"}, cfg.DialAllowlist) + }) + + t.Run("missing required values fail closed", func(t *testing.T) { + t.Parallel() + for _, key := range []string{ + egressbroker.EnvPolicyFile, egressbroker.EnvCAFile, egressbroker.EnvCAKeyFile, + egressbroker.EnvIssuer, egressbroker.EnvSubjectRaw, + egressbroker.EnvSessionID, egressbroker.EnvMCPServer, + } { + values := map[string]string{} + for k, v := range base { + values[k] = v + } + delete(values, key) + _, err := egressbroker.LoadConfig(envFrom(values)) + require.Error(t, err, "%s must be required", key) + } + }) + + t.Run("invalid port and invalid allowlist fail", func(t *testing.T) { + t.Parallel() + values := map[string]string{} + for k, v := range base { + values[k] = v + } + values[egressbroker.EnvListenPort] = "not-a-port" + _, err := egressbroker.LoadConfig(envFrom(values)) + require.Error(t, err) + + values[egressbroker.EnvListenPort] = "9001" + values[egressbroker.EnvDialAllowlist] = "not-an-ip" + _, err = egressbroker.LoadConfig(envFrom(values)) + require.Error(t, err) + }) +} + +func TestConfigResolveDialAllowlist(t *testing.T) { + t.Parallel() + + cfg := &egressbroker.Config{} + + t.Run("falls back to the policy document's dialAllowlist", func(t *testing.T) { + t.Parallel() + policy := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com"] +dialAllowlist: ["140.82.112.0/20"] +`) + out, err := cfg.ResolveDialAllowlist(policy) + require.NoError(t, err) + assert.Equal(t, []string{"140.82.112.0/20"}, out) + }) + + t.Run("env override wins", func(t *testing.T) { + t.Parallel() + cfgWithEnv := &egressbroker.Config{DialAllowlist: []string{"1.2.3.4/32"}} + policy := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com"] +dialAllowlist: ["140.82.112.0/20"] +`) + out, err := cfgWithEnv.ResolveDialAllowlist(policy) + require.NoError(t, err) + assert.Equal(t, []string{"1.2.3.4/32"}, out) + }) + + t.Run("no allowlist anywhere fails closed", func(t *testing.T) { + t.Parallel() + policy := mustParse(t, testPolicyYAML) + _, err := cfg.ResolveDialAllowlist(policy) + require.Error(t, err) + _, err = cfg.ResolveDialAllowlist(nil) + require.Error(t, err) + }) +} diff --git a/pkg/egressbroker/dialcontrol.go b/pkg/egressbroker/dialcontrol.go new file mode 100644 index 0000000000..308fff99ce --- /dev/null +++ b/pkg/egressbroker/dialcontrol.go @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "context" + "fmt" + "net" + "net/netip" + "strings" + "syscall" +) + +// IPAllowlist is the D7 resolved-IP guard: the set of destination IPs (as +// prefixes) the broker may dial. It mirrors the WithDialControl pattern from +// pkg/vmcp/client/client.go — validation happens after DNS resolution and +// before the TCP handshake, so a hostname that passes the name-based D5 +// allowlist but resolves to a rebinding-suspect IP is still refused. +// +// The allowlist is the union of the operator-rendered destination CIDRs for +// the policy's allowed hosts (same data the NetworkPolicy ipBlocks carry). +// Matching is exact: every IP outside the prefixes — loopback included — is +// refused; the broker dials upstream APIs and Redis, never pod-local services. +type IPAllowlist struct { + prefixes []netip.Prefix +} + +// ParseIPAllowlist compiles CIDR/IPv4/IPv6 strings into an ipAllowlist. +// Entries without a '/' are treated as single-host prefixes. Fails loudly on +// any malformed entry — a broker with a corrupt dial allowlist must not +// start. +func ParseIPAllowlist(entries []string) (*IPAllowlist, error) { + if len(entries) == 0 { + return nil, fmt.Errorf("egressbroker: dial IP allowlist must not be empty") + } + prefixes := make([]netip.Prefix, 0, len(entries)) + for _, entry := range entries { + entry = strings.TrimSpace(entry) + if entry == "" { + return nil, fmt.Errorf("egressbroker: dial IP allowlist contains an empty entry") + } + if !strings.Contains(entry, "/") { + addr, err := netip.ParseAddr(entry) + if err != nil { + return nil, fmt.Errorf("egressbroker: invalid IP %q in dial allowlist: %w", entry, err) + } + prefixes = append(prefixes, netip.PrefixFrom(addr, addr.BitLen())) + continue + } + prefix, err := netip.ParsePrefix(entry) + if err != nil { + return nil, fmt.Errorf("egressbroker: invalid CIDR %q in dial allowlist: %w", entry, err) + } + prefixes = append(prefixes, prefix.Masked()) + } + return &IPAllowlist{prefixes: prefixes}, nil +} + +// Allows reports whether addr (a host:port or bare IP dial target) resolves +// inside the allowlist. Fails closed: unparseable targets, and IPs outside +// every prefix, are refused. +func (a *IPAllowlist) Allows(addr string) bool { + host, _, err := net.SplitHostPort(addr) + if err != nil { + host = addr + } + ip, err := netip.ParseAddr(strings.Trim(host, "[]")) + if err != nil { + return false + } + ip = ip.Unmap() + for _, prefix := range a.prefixes { + if prefix.Contains(ip) { + return true + } + } + return false +} + +// DialControl returns a net.Dialer.Control hook enforcing the allowlist on +// every TCP dial (D7): the resolved peer IP must be in the allowlist or the +// dial is refused. The signature matches net.Dialer.Control exactly. +func (a *IPAllowlist) DialControl(_, address string, _ syscall.RawConn) error { + if !a.Allows(address) { + return fmt.Errorf("egressbroker: refused dial to %s: resolved IP outside destination allowlist (D7)", address) + } + return nil +} + +// DialContext returns a DialContext function enforcing the allowlist on every +// dial (D7), for components that dial by hostname rather than raw IP (the +// go-redis client takes DialContext, not net.Dialer.Control). +func (a *IPAllowlist) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + if !a.Allows(address) { + return nil, fmt.Errorf("egressbroker: refused dial to %s: resolved IP outside destination allowlist (D7)", address) + } + d := &net.Dialer{Control: a.DialControl} + return d.DialContext(ctx, network, address) +} diff --git a/pkg/egressbroker/dialcontrol_test.go b/pkg/egressbroker/dialcontrol_test.go new file mode 100644 index 0000000000..73f209c7cc --- /dev/null +++ b/pkg/egressbroker/dialcontrol_test.go @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +func TestIPAllowlist(t *testing.T) { + t.Parallel() + + // Not exported: behavior is verified through Config validation + the + // exported ParseIPAllowlist/DialControl surface. + allow, err := egressbroker.ParseIPAllowlist([]string{"140.82.112.0/20", "185.199.108.133"}) + require.NoError(t, err) + + t.Run("constructor validation", func(t *testing.T) { + t.Parallel() + _, err := egressbroker.ParseIPAllowlist(nil) + require.Error(t, err, "empty allowlist must fail closed") + _, err = egressbroker.ParseIPAllowlist([]string{"not-an-ip"}) + require.Error(t, err) + _, err = egressbroker.ParseIPAllowlist([]string{"10.0.0.0/33"}) + require.Error(t, err) + _, err = egressbroker.ParseIPAllowlist([]string{""}) + require.Error(t, err) + }) + + t.Run("resolved IP not in allowlist → dial refused", func(t *testing.T) { + t.Parallel() + err := allow.DialControl("tcp", "8.8.8.8:443", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "allowlist") + }) + + t.Run("resolved IP in allowlist CIDR → dial allowed", func(t *testing.T) { + t.Parallel() + require.NoError(t, allow.DialControl("tcp", "140.82.114.26:443", nil)) + }) + + t.Run("single-host entry matches exactly", func(t *testing.T) { + t.Parallel() + require.NoError(t, allow.DialControl("tcp", "185.199.108.133:443", nil)) + require.Error(t, allow.DialControl("tcp", "185.199.108.134:443", nil)) + }) + + t.Run("rebinding: allowlisted name resolving to internal IP → refused", func(t *testing.T) { + t.Parallel() + for _, internal := range []string{ + "10.0.0.5:443", "192.168.1.1:443", "172.16.0.1:443", + "169.254.169.254:80", // cloud metadata + "100.64.0.1:443", // CGNAT + "[fd00::1]:443", // IPv6 ULA + } { + require.Error(t, allow.DialControl("tcp", internal, nil), + "dial to %s must be refused", internal) + } + }) + + t.Run("IPv4-mapped IPv6 is unmapped before matching", func(t *testing.T) { + t.Parallel() + // ::ffff:140.82.114.26 must match the IPv4 CIDR (not slip past it, not + // be wrongly refused either). + require.NoError(t, allow.DialControl("tcp", "[::ffff:140.82.114.26]:443", nil)) + // And a mapped internal address must still be refused. + require.Error(t, allow.DialControl("tcp", "[::ffff:10.0.0.5]:443", nil)) + }) + + t.Run("unparseable target fails closed", func(t *testing.T) { + t.Parallel() + require.Error(t, allow.DialControl("tcp", "not-an-ip-literal:443", nil)) + }) + + t.Run("loopback refused unless explicitly allowlisted", func(t *testing.T) { + t.Parallel() + require.Error(t, allow.DialControl("tcp", "127.0.0.1:9001", nil)) + require.Error(t, allow.DialControl("tcp", "[::1]:9001", nil)) + + withLoopback, err := egressbroker.ParseIPAllowlist([]string{"127.0.0.1/32"}) + require.NoError(t, err) + require.NoError(t, withLoopback.DialControl("tcp", "127.0.0.1:9001", nil)) + require.Error(t, withLoopback.DialControl("tcp", "127.0.0.2:9001", nil)) + }) +} diff --git a/pkg/egressbroker/envoyconfig.go b/pkg/egressbroker/envoyconfig.go new file mode 100644 index 0000000000..a613f8d249 --- /dev/null +++ b/pkg/egressbroker/envoyconfig.go @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "fmt" + "os" + "sort" + "strings" +) + +// EnvoyConfigParams carries the rendered Envoy bootstrap's variable parts. +type EnvoyConfigParams struct { + // ExtAuthzAddress is the broker's gRPC listen address (loopback-only); + // ext_authz AND on-demand SDS are served on the same socket. + ExtAuthzAddress string + // ExtAuthzPort is the broker's gRPC listen port. + ExtAuthzPort int + // ProxyPort is the explicit-proxy listener the backend reaches via + // HTTP_PROXY/HTTPS_PROXY (loopback-only). + ProxyPort int + // AllowedHosts is the policy's host allowlist; requests to any other host + // get no route → connection refused (D6b). Also the set of SNI names the + // SDS server will mint bump certs for — anything else fails the + // downstream handshake. + AllowedHosts []string +} + +// Validate fails loudly on an unrenderable bootstrap (constructor +// validation: misconfiguration must fail at startup). +func (p EnvoyConfigParams) Validate() error { + if p.ExtAuthzAddress == "" { + return fmt.Errorf("egressbroker: ext_authz address must not be empty") + } + if p.ExtAuthzPort <= 0 || p.ExtAuthzPort > 65535 { + return fmt.Errorf("egressbroker: ext_authz port %d out of range", p.ExtAuthzPort) + } + if p.ProxyPort <= 0 || p.ProxyPort > 65535 { + return fmt.Errorf("egressbroker: proxy port %d out of range", p.ProxyPort) + } + if len(p.AllowedHosts) == 0 { + return fmt.Errorf("egressbroker: allowed hosts must not be empty") + } + return nil +} + +// envoyBootstrapTemplate is the explicit-forward-proxy TLS-bump bootstrap +// (D9/D10). The backend reaches the proxy via HTTP_PROXY/HTTPS_PROXY; CONNECT +// tunnels are TLS-bumped with per-SNI certs fetched on demand over SDS from +// the broker process, then re-encrypted upstream to the real destination. +// +// Bump mechanics (the request path, top to bottom): +// 1. tls_inspector reads the CONNECT authority / tunneled ClientHello SNI. +// 2. sni_dynamic_forward_proxy sets the upstream host from SNI. +// 3. The CONNECT route upgrade (connect_config) terminates the tunnel: the +// downstream_tls_context has NO static cert, so Envoy on-demand-fetches +// the SDS resource "host:" from the broker's sds_grpc cluster. The +// broker mints the cert only for policy-allowlisted SNI; anything else +// gets no cert → handshake failure (fail closed). +// 4. Tunneled requests pass ext_authz (credential injection, D5) and route +// to dynamic_forward_proxy_cluster, whose upstream TLS context +// re-encrypts to the destination, SNI from the CONNECT authority, +// validated against the system CA bundle (real upstream certs). +// +// Security invariants baked into the template (do not relax): +// - ext_authz failure_mode_allow: false — a dead injector denies, never +// passes unauthenticated traffic. +// - The route table matches only allowlisted hosts; anything else has no +// route → connection refused before ext_authz is even consulted. +// - No redirect-following filter exists: Envoy as a forward proxy passes +// 3xx responses through untouched (D6a). +// - Listeners and the broker cluster bind loopback only; the SDS secret +// stream never leaves the pod. +// - upstream_validation_context pins trust to the system CA bundle — the +// bump CA is only ever presented to the backend container, never used to +// validate real upstreams. +const envoyBootstrapTemplate = `static_resources: + listeners: + - name: https_proxy + address: + socket_address: + address: 127.0.0.1 + port_value: {{PROXY_PORT}} + listener_filters: + - name: envoy.filters.listener.tls_inspector + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + filter_chains: + - filters: + - name: envoy.filters.network.sni_dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.sni_dynamic_forward_proxy.v3.FilterConfig + port_value: 443 + dns_cache_config: + name: dynamic_forward_proxy_cache + dns_lookup_family: V4_ONLY + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: egress_broker + http2_protocol_options: + allow_connect: true + upgrade_configs: + - upgrade_type: CONNECT + enabled: true + route_config: + name: egress_routes + virtual_hosts: +{{VHOSTS}} + http_filters: + - name: envoy.filters.http.ext_authz + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz + failure_mode_allow: false + with_request_body: + max_request_bytes: 0 + grpc_service: + envoy_grpc: + cluster_name: egress_broker + timeout: 5s + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + tls_certificate_sds_secret_configs: + - name: envoy.transport_sockets.tls.v3.SdsSecretConfig + sds_config: + api_config_source: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: egress_broker + clusters: + - name: egress_broker + type: STATIC + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: egress_broker + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{EXT_AUTHZ_ADDRESS}} + port_value: {{EXT_AUTHZ_PORT}} + - name: dynamic_forward_proxy_cluster + lb_policy: CLUSTER_PROVIDED + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: + name: dynamic_forward_proxy_cache + dns_lookup_family: V4_ONLY + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + validation_context: + trusted_ca: + filename: /etc/ssl/certs/ca-certificates.crt +` + +// vhostTemplate is one allowlisted-host route entry. The CONNECT upgrade +// terminates the tunnel downstream via the filter chain's on-demand SDS cert +// (resource "host:") and re-originates upstream through the dynamic +// forward proxy; every request — the CONNECT itself and the decrypted tunneled +// requests — passes ext_authz (no per-route disable: the credential injection +// on tunneled traffic IS the feature). Non-CONNECT routes (plain absolute-form +// HTTP through the proxy) share the same upstream. +const vhostTemplate = ` - name: {{HOST_NAME}} + domains: ["{{HOST}}"] + routes: + - match: + connect_matcher: {} + route: + cluster: dynamic_forward_proxy_cluster + upgrade_configs: + - upgrade_type: CONNECT + connect_config: + terminate_connect: true + - match: + prefix: / + route: + cluster: dynamic_forward_proxy_cluster +` + +// RenderEnvoyBootstrap renders the Envoy bootstrap YAML for params. The +// result contains no secrets — the bump-CA material is delivered to Envoy +// via SDS at runtime, not through the bootstrap file. AllowedHosts is sorted +// so the rendered document is deterministic. +func RenderEnvoyBootstrap(params EnvoyConfigParams) ([]byte, error) { + if err := params.Validate(); err != nil { + return nil, err + } + hosts := make([]string, len(params.AllowedHosts)) + copy(hosts, params.AllowedHosts) + sort.Strings(hosts) + + var vhosts strings.Builder + for i, host := range hosts { + entry := strings.ReplaceAll(vhostTemplate, "{{HOST_NAME}}", sanitizeRouteName(host, i)) + entry = strings.ReplaceAll(entry, "{{HOST}}", host) + vhosts.WriteString(entry) + } + out := strings.ReplaceAll(envoyBootstrapTemplate, "{{PROXY_PORT}}", fmt.Sprintf("%d", params.ProxyPort)) + out = strings.ReplaceAll(out, "{{EXT_AUTHZ_ADDRESS}}", params.ExtAuthzAddress) + out = strings.ReplaceAll(out, "{{EXT_AUTHZ_PORT}}", fmt.Sprintf("%d", params.ExtAuthzPort)) + out = strings.ReplaceAll(out, "{{VHOSTS}}", vhosts.String()) + return []byte(out), nil +} + +// WriteEnvoyBootstrap renders and writes the bootstrap to path with +// owner-only permissions. +func WriteEnvoyBootstrap(path string, params EnvoyConfigParams) error { + data, err := RenderEnvoyBootstrap(params) + if err != nil { + return err + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("egressbroker: failed to write envoy bootstrap: %w", err) + } + return nil +} + +// sanitizeRouteName makes a valid Envoy route/virtual-host name from a host +// pattern. Names carry no security semantics (domains do the matching), so +// collisions are broken with the index. +func sanitizeRouteName(host string, i int) string { + name := strings.NewReplacer("*", "wildcard", ":", "_").Replace(host) + return fmt.Sprintf("host-%d-%s", i, name) +} diff --git a/pkg/egressbroker/envoyconfig_test.go b/pkg/egressbroker/envoyconfig_test.go new file mode 100644 index 0000000000..114d54caf3 --- /dev/null +++ b/pkg/egressbroker/envoyconfig_test.go @@ -0,0 +1,301 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "context" + "net" + "testing" + "time" + + envoytls "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + envoydiscovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +func testParams() egressbroker.EnvoyConfigParams { + return egressbroker.EnvoyConfigParams{ + ExtAuthzAddress: "127.0.0.1", + ExtAuthzPort: 9001, + ProxyPort: 15001, + AllowedHosts: []string{"api.github.com", "*.example.com"}, + } +} + +func render(t *testing.T, params egressbroker.EnvoyConfigParams) map[string]any { + t.Helper() + data, err := egressbroker.RenderEnvoyBootstrap(params) + require.NoError(t, err) + var doc map[string]any + require.NoError(t, yaml.Unmarshal(data, &doc), "rendered bootstrap must be valid YAML") + return doc +} + +// dig walks nested maps/slices: dig(doc, "static_resources", "listeners"). +func dig(t *testing.T, node any, path ...string) any { + t.Helper() + cur := node + for _, key := range path { + m, ok := cur.(map[string]any) + require.True(t, ok, "path %q: expected map at %q, got %T", path, key, cur) + cur, ok = m[key] + require.True(t, ok, "path %q: missing key %q", path, key) + } + return cur +} + +func TestRenderEnvoyBootstrap(t *testing.T) { + t.Parallel() + + t.Run("validation fails loudly", func(t *testing.T) { + t.Parallel() + for _, params := range []egressbroker.EnvoyConfigParams{ + {ExtAuthzPort: 9001, ProxyPort: 15001, AllowedHosts: []string{"a.example.com"}}, + {ExtAuthzAddress: "127.0.0.1", ProxyPort: 15001, AllowedHosts: []string{"a.example.com"}}, + {ExtAuthzAddress: "127.0.0.1", ExtAuthzPort: 9001, AllowedHosts: []string{"a.example.com"}}, + {ExtAuthzAddress: "127.0.0.1", ExtAuthzPort: 9001, ProxyPort: 15001}, + } { + _, err := egressbroker.RenderEnvoyBootstrap(params) + require.Error(t, err) + } + }) + + t.Run("bootstrap is deterministic regardless of host order", func(t *testing.T) { + t.Parallel() + a, err := egressbroker.RenderEnvoyBootstrap(testParams()) + require.NoError(t, err) + p2 := testParams() + p2.AllowedHosts = []string{"*.example.com", "api.github.com"} + b, err := egressbroker.RenderEnvoyBootstrap(p2) + require.NoError(t, err) + assert.Equal(t, string(a), string(b)) + }) + + t.Run("listener has tls_inspector and a loopback explicit-proxy address", func(t *testing.T) { + t.Parallel() + doc := render(t, testParams()) + + listeners := dig(t, doc, "static_resources", "listeners").([]any) + require.Len(t, listeners, 1) + listener := listeners[0].(map[string]any) + + addr := dig(t, listener, "address", "socket_address").(map[string]any) + assert.Equal(t, "127.0.0.1", addr["address"]) + assert.Equal(t, 15001, addr["port_value"]) + + lfs := listener["listener_filters"].([]any) + require.Len(t, lfs, 1) + assert.Equal(t, "envoy.filters.listener.tls_inspector", lfs[0].(map[string]any)["name"]) + }) + + t.Run("filter chain TLS-bumps via on-demand SDS from the broker cluster", func(t *testing.T) { + t.Parallel() + doc := render(t, testParams()) + + chains := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any) + require.Len(t, chains, 1) + chain := chains[0].(map[string]any) + + // Network filters: sni_dynamic_forward_proxy then the HCM. + filters := chain["filters"].([]any) + require.Len(t, filters, 2) + assert.Equal(t, "envoy.filters.network.sni_dynamic_forward_proxy", filters[0].(map[string]any)["name"]) + assert.Equal(t, "envoy.filters.network.http_connection_manager", filters[1].(map[string]any)["name"]) + + // Downstream transport socket: no static certs — on-demand SDS only, + // sourced from the broker's gRPC cluster. This is the wiring that lets + // Envoy fetch a per-SNI bump cert for allowlisted SNI. + ts := dig(t, chain, "transport_socket", "typed_config").(map[string]any) + assert.Equal(t, + "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", ts["@type"]) + sdsCfgs := dig(t, ts, "common_tls_context", "tls_certificate_sds_secret_configs").([]any) + require.Len(t, sdsCfgs, 1) + sds := dig(t, sdsCfgs[0], "sds_config", "api_config_source").(map[string]any) + assert.Equal(t, "GRPC", sds["api_type"]) + services := sds["grpc_services"].([]any) + require.Len(t, services, 1) + cluster := dig(t, services[0], "envoy_grpc").(map[string]any)["cluster_name"] + assert.Equal(t, "egress_broker", cluster) + // Crucially: no inline tls_certificates — certs come from SDS only. + assert.NotContains(t, dig(t, ts, "common_tls_context").(map[string]any), "tls_certificates") + }) + + t.Run("ext_authz fails closed and points at the broker cluster", func(t *testing.T) { + t.Parallel() + doc := render(t, testParams()) + chain := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any)[0] + hcm := dig(t, chain, "filters").([]any)[1].(map[string]any) + httpFilters := dig(t, hcm, "typed_config", "http_filters").([]any) + require.Len(t, httpFilters, 2) + + extAuthz := dig(t, httpFilters[0], "typed_config").(map[string]any) + assert.Equal(t, false, extAuthz["failure_mode_allow"], "ext_authz must deny when the broker is down") + cluster := dig(t, extAuthz, "grpc_service", "envoy_grpc").(map[string]any)["cluster_name"] + assert.Equal(t, "egress_broker", cluster) + assert.Equal(t, "envoy.filters.http.router", httpFilters[1].(map[string]any)["name"]) + }) + + t.Run("routes cover exactly the allowlisted hosts; CONNECT terminates with upgrade config", func(t *testing.T) { + t.Parallel() + doc := render(t, testParams()) + chain := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any)[0] + vhostList := dig(t, chain, "filters").([]any)[1].(map[string]any) + vl := dig(t, vhostList, "typed_config", "route_config", "virtual_hosts").([]any) + require.Len(t, vl, 2, "exactly one virtual host per allowlisted host pattern") + domains := map[string]bool{} + for _, v := range vl { + vh := v.(map[string]any) + for _, d := range vh["domains"].([]any) { + domains[d.(string)] = true + } + // CONNECT route terminates the tunnel (TLS-bump data path) and + // re-originates through the dynamic forward proxy. + routes := vh["routes"].([]any) + require.NotEmpty(t, routes) + connect := routes[0].(map[string]any) + upgrade := dig(t, connect, "route", "upgrade_configs").([]any)[0].(map[string]any) + assert.Equal(t, "CONNECT", upgrade["upgrade_type"]) + assert.Equal(t, true, dig(t, upgrade, "connect_config").(map[string]any)["terminate_connect"]) + assert.Equal(t, "dynamic_forward_proxy_cluster", + dig(t, connect, "route").(map[string]any)["cluster"]) + } + assert.True(t, domains["api.github.com"]) + assert.True(t, domains["*.example.com"]) + assert.False(t, domains["evil.com"], "non-allowlisted hosts must have no route") + }) + + t.Run("upstream cluster re-encrypts with system-CA validation", func(t *testing.T) { + t.Parallel() + doc := render(t, testParams()) + clusters := dig(t, doc, "static_resources", "clusters").([]any) + var dfp map[string]any + for _, c := range clusters { + if c.(map[string]any)["name"] == "dynamic_forward_proxy_cluster" { + dfp = c.(map[string]any) + } + } + require.NotNil(t, dfp) + ts := dig(t, dfp, "transport_socket", "typed_config").(map[string]any) + assert.Equal(t, + "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", ts["@type"]) + ca := dig(t, ts, "common_tls_context", "validation_context", "trusted_ca").(map[string]any) + assert.Equal(t, "/etc/ssl/certs/ca-certificates.crt", ca["filename"], + "upstream certs must validate against the system bundle, never the bump CA") + }) + + t.Run("no redirect-following filter exists (D6a)", func(t *testing.T) { + t.Parallel() + data, err := egressbroker.RenderEnvoyBootstrap(testParams()) + require.NoError(t, err) + assert.NotContains(t, string(data), "redirect") + assert.NotContains(t, string(data), "internal_redirect") + }) + + t.Run("bootstrap carries no secret material", func(t *testing.T) { + t.Parallel() + data, err := egressbroker.RenderEnvoyBootstrap(testParams()) + require.NoError(t, err) + assert.NotContains(t, string(data), "private_key") + assert.NotContains(t, string(data), "BEGIN ") + }) +} + +// TestSDSServesCertsForAllowlistedSNIOnly exercises the SDS server the +// bootstrap points at: Envoy's on-demand fetch ("host:") yields a minted +// cert for allowlisted SNI and is refused for anything else (handshake then +// fails closed). +func TestSDSServesCertsForAllowlistedSNIOnly(t *testing.T) { + t.Parallel() + + policy, err := egressbroker.ParsePolicy([]byte(` +providers: +- provider: github + allowedHosts: ["api.github.com"] + allowedMethods: ["GET"] +dialAllowlist: ["203.0.113.10/32"] +`)) + require.NoError(t, err) + ca, err := egressbroker.GenerateBumpCA("test-bump-ca", time.Now()) + require.NoError(t, err) + sds, err := egressbroker.NewSecretDiscoveryServer(ca, policy) + require.NoError(t, err) + + ctx := context.Background() + + t.Run("allowlisted SNI yields a TLS certificate secret", func(t *testing.T) { + t.Parallel() + resp, err := sds.FetchSecrets(ctx, &envoydiscovery.DiscoveryRequest{ + ResourceNames: []string{"host:api.github.com"}, + }) + require.NoError(t, err) + require.Len(t, resp.GetResources(), 1) + var secret envoytls.Secret + // The resource is an envoy Secret; verify it carries cert+key. + require.NoError(t, resp.GetResources()[0].UnmarshalTo(&secret)) + assert.Equal(t, "host:api.github.com", secret.GetName()) + assert.NotEmpty(t, secret.GetTlsCertificate().GetCertificateChain().GetInlineBytes()) + assert.NotEmpty(t, secret.GetTlsCertificate().GetPrivateKey().GetInlineBytes()) + }) + + t.Run("non-allowlisted SNI is refused (no cert, handshake fails closed)", func(t *testing.T) { + t.Parallel() + _, err := sds.FetchSecrets(ctx, &envoydiscovery.DiscoveryRequest{ + ResourceNames: []string{"host:evil.com"}, + }) + require.Error(t, err) + }) + + t.Run("malformed resource names are refused", func(t *testing.T) { + t.Parallel() + _, err := sds.FetchSecrets(ctx, &envoydiscovery.DiscoveryRequest{ + ResourceNames: []string{"ca-key"}, + }) + require.Error(t, err) + _, err = sds.FetchSecrets(ctx, &envoydiscovery.DiscoveryRequest{}) + require.Error(t, err) + }) +} + +// TestBrokerClusterIsLoopback asserts the rendered broker cluster (ext_authz + +// SDS) never leaves the pod. +func TestBrokerClusterIsLoopback(t *testing.T) { + t.Parallel() + doc := render(t, testParams()) + clusters := dig(t, doc, "static_resources", "clusters").([]any) + for _, c := range clusters { + cm := c.(map[string]any) + if cm["name"] != "egress_broker" { + continue + } + endpoints := dig(t, cm, "load_assignment", "endpoints").([]any) + for _, e := range endpoints { + for _, lb := range e.(map[string]any)["lb_endpoints"].([]any) { + addr := dig(t, lb, "endpoint", "address", "socket_address").(map[string]any)["address"].(string) + require.True(t, net.ParseIP(addr).IsLoopback(), "broker cluster endpoint %s must be loopback", addr) + } + } + } +} + +// Example output guard: the rendered bootstrap mentions the SDS resource +// prefix the SDS server expects ("host:") nowhere — the resource NAME is +// derived from SNI at runtime — but the fetch wiring must be present. +func TestBootstrapMentionsSDSFetchWiring(t *testing.T) { + t.Parallel() + data, err := egressbroker.RenderEnvoyBootstrap(testParams()) + require.NoError(t, err) + for _, want := range []string{ + "tls_certificate_sds_secret_configs", + "tls_inspector", + "sni_dynamic_forward_proxy", + "terminate_connect: true", + "failure_mode_allow: false", + } { + assert.Contains(t, string(data), want) + } +} diff --git a/pkg/egressbroker/identity.go b/pkg/egressbroker/identity.go new file mode 100644 index 0000000000..48e1631d7d --- /dev/null +++ b/pkg/egressbroker/identity.go @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" +) + +// Pod-annotation ↔ broker-env contract between the vMCP untrusted pod +// lifecycle (writer, pkg/vmcp/session/untrusted) and the egress broker +// sidecar (reader, this package's PodIdentityResolver). The pod is the +// registry: the vMCP stamps these annotations at clone time, and the broker +// receives them via downward-API env (THV_UNTRUSTED_*), injected by the clone +// wiring from IdentityEnvMappings. Both sides import this package — the +// annotation keys and env names exist nowhere else. The operator side keeps +// its own mirrors because cmd/thv-operator is a separate module boundary. +const ( + // AnnotationIssuer carries the raw OIDC issuer (public, non-sensitive). + AnnotationIssuer = "toolhive.stacklok.dev/iss" + // AnnotationSubjectRaw carries the raw OIDC subject. The broker needs the + // raw sub for its ExpectedBinding{UserID: sub, Strict: true} check when + // loading the user's upstream credential. An annotation (not a label) + // carries it: annotations have no charset limit and are reachable only + // via the downward API from inside the pod itself; K8s RBAC on pods + // (list/watch/get) is an operator/vMCP-only grant. + AnnotationSubjectRaw = "toolhive.stacklok.dev/sub-raw" + // AnnotationSessionID carries the vMCP session ID the pod serves; it is + // the token-store session key (tsid) and is checked on pod reuse (a + // mismatch means a hash collision or a cross-session pod — fail closed, + // never attach). + AnnotationSessionID = "toolhive.stacklok.dev/session-id" + // AnnotationMCPServerName carries the MCPServer CR name the pod backends. + AnnotationMCPServerName = "toolhive.stacklok.dev/mcpserver-name" +) + +// Downward-API env var names the clone-time wiring sets on the broker +// container; each mirrors one pod annotation (see AnnotationForEnv). +const ( + // EnvIssuer mirrors the pod's AnnotationIssuer annotation. + EnvIssuer = "THV_UNTRUSTED_ISS" + // EnvSubjectRaw mirrors the pod's AnnotationSubjectRaw annotation (raw + // OIDC sub; a hashed value is unusable for binding). + EnvSubjectRaw = "THV_UNTRUSTED_SUB_RAW" + // EnvSessionID mirrors the pod's AnnotationSessionID annotation. + EnvSessionID = "THV_UNTRUSTED_SESSION" + // EnvMCPServer mirrors the pod's AnnotationMCPServerName annotation. + EnvMCPServer = "THV_UNTRUSTED_MCPSERVER" +) + +// EnvToAnnotation is the complete downward-API contract: each env var name to +// the pod annotation it mirrors. Iteration order is irrelevant — consumers +// index by env name. +var EnvToAnnotation = map[string]string{ + EnvIssuer: AnnotationIssuer, + EnvSubjectRaw: AnnotationSubjectRaw, + EnvSessionID: AnnotationSessionID, + EnvMCPServer: AnnotationMCPServerName, +} + +// AnnotationForEnv returns the pod annotation the given broker identity env +// var mirrors, and whether the env var is part of the contract. +func AnnotationForEnv(envName string) (annotation string, ok bool) { + annotation, ok = EnvToAnnotation[envName] + return annotation, ok +} + +// AnnotationFieldRef is the downward-API fieldPath form of an annotation key, +// e.g. metadata.annotations['toolhive.stacklok.dev/iss']. +func AnnotationFieldRef(annotation string) string { + return fmt.Sprintf("metadata.annotations['%s']", annotation) +} + +// ResourceNaming derives the operator-managed data-plane resource names for +// one MCPServer from its CR name. The bump-CA objects are generation-named: +// the Secret carries cert+key of ONE CA generation together, so a pod cloned +// mid-rotation always mounts a consistent cert/key pair (no Update-in-place +// rotation race). The bundle ConfigMap carries only the public cert of the +// same generation. +type ResourceNaming struct { + // CASecret is the bump-CA Secret name (generation-qualified). + CASecret string + // CABundle is the bump-CA public bundle ConfigMap name + // (generation-qualified, same generation as CASecret). + CABundle string + // EgressPolicy is the egress-policy ConfigMap name. + EgressPolicy string + // Generation is the CA generation suffix (sha256 of the CA cert, hex, + // first 16 chars); empty for the un-generated base names. + Generation string +} + +const ( + // caSecretSuffix is the base suffix of the per-tenant bump-CA Secret. + caSecretSuffix = "-bump-ca" + // caBundleSuffix is the base suffix of the bump-CA public bundle ConfigMap. + caBundleSuffix = "-bump-ca-bundle" + // egressPolicySuffix is the suffix of the egress-policy ConfigMap. + egressPolicySuffix = "-egress-policy" + + // caGenerationLen bounds the cert-hash generation suffix. + caGenerationLen = 16 +) + +// CAKeyCert is the public-cert data key of the bump-CA Secret and bundle +// ConfigMap. +const CAKeyCert = "ca.crt" + +// CAKeyKey is the private-key data key of the bump-CA Secret. The key lives +// in the same Secret object as the cert precisely so a mounted pair is always +// one consistent generation. +const CAKeyKey = "ca.key" //nolint:gosec // G101: a Secret data key name, not a credential + +// CAGeneration returns the generation suffix for a CA certificate: the first +// 16 hex chars of its sha256. Two distinct certs never share a generation; +// the same cert always maps to the same generation (idempotent reconcile). +func CAGeneration(certPEM []byte) string { + sum := sha256.Sum256(certPEM) + return hex.EncodeToString(sum[:])[:caGenerationLen] +} + +// ResourceNamesFor renders the naming for one CA generation of the named +// MCPServer. generation must come from CAGeneration of the Secret's cert. +// Names stay under the 63-char label limit by construction (MCPServer names +// are themselves DNS-1123). +func ResourceNamesFor(mcpserverName, generation string) ResourceNaming { + genSuffix := "" + if generation != "" { + genSuffix = "-" + generation + } + return ResourceNaming{ + CASecret: mcpserverName + caSecretSuffix + genSuffix, + CABundle: mcpserverName + caBundleSuffix + genSuffix, + EgressPolicy: mcpserverName + egressPolicySuffix, + Generation: generation, + } +} + +// BaseCASecretName is the generation-less prefix every CA generation Secret +// shares; used for label-free generation sweeps (GC lists by prefix). +func BaseCASecretName(mcpserverName string) string { + return mcpserverName + caSecretSuffix +} + +// BaseCABundleName is the generation-less prefix every bundle generation +// shares. +func BaseCABundleName(mcpserverName string) string { + return mcpserverName + caBundleSuffix +} + +// TrimGeneration splits a generation-qualified CA resource name into base +// name and generation. ok is false when name does not carry the base prefix +// plus a generation suffix. +func TrimGeneration(name, base string) (generation string, ok bool) { + rest, found := strings.CutPrefix(name, base+"-") + if !found || rest == "" { + return "", false + } + return rest, true +} + +// PodIdentity is the (user, session, server) tuple the broker serves, +// resolved from the broker's own pod metadata. The token-selection key and +// the Strict binding user both come from here — never from anything the +// request carries (D5: credential choice is driven by pod identity, not by a +// server-supplied value). +type PodIdentity struct { + // Issuer is the OIDC issuer of the user this pod serves. + Issuer string + // Subject is the raw OIDC sub; asserted as ExpectedBinding.UserID on every + // credential load. + Subject string + // SessionID is the vMCP session ID; used as the tsid token-store key. + SessionID string + // MCPServer is the MCPServer CR name this pod backends. + MCPServer string +} + +// PodIdentityResolver resolves the broker's identity from its environment +// (downward API). Resolution happens once at construction; the hot path +// performs no K8s or Redis reads (security: the pod object is the registry). +type PodIdentityResolver struct { + identity PodIdentity +} + +// NewPodIdentityResolver validates that all four downward-API identity env +// vars are present and non-empty. Missing identity is a startup error — a +// broker that cannot prove which user/session it serves must never run +// (fail closed on unknown pod identity). +func NewPodIdentityResolver(getenv func(string) string) (*PodIdentityResolver, error) { + if getenv == nil { + return nil, fmt.Errorf("egressbroker: env lookup must not be nil") + } + identity := PodIdentity{ + Issuer: strings.TrimSpace(getenv(EnvIssuer)), + Subject: strings.TrimSpace(getenv(EnvSubjectRaw)), + SessionID: strings.TrimSpace(getenv(EnvSessionID)), + MCPServer: strings.TrimSpace(getenv(EnvMCPServer)), + } + for name, value := range map[string]string{ + EnvIssuer: identity.Issuer, + EnvSubjectRaw: identity.Subject, + EnvSessionID: identity.SessionID, + EnvMCPServer: identity.MCPServer, + } { + if value == "" { + return nil, fmt.Errorf("egressbroker: required identity env var %s is empty or unset "+ + "(downward-API annotation missing); refusing to start", name) + } + } + return &PodIdentityResolver{identity: identity}, nil +} + +// PodIdentity returns the validated identity. +func (r *PodIdentityResolver) PodIdentity() PodIdentity { + return r.identity +} diff --git a/pkg/egressbroker/identity_test.go b/pkg/egressbroker/identity_test.go new file mode 100644 index 0000000000..7a4f78e345 --- /dev/null +++ b/pkg/egressbroker/identity_test.go @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +func envFrom(values map[string]string) func(string) string { + return func(key string) string { return values[key] } +} + +func TestPodIdentityResolver(t *testing.T) { + t.Parallel() + + full := map[string]string{ + egressbroker.EnvIssuer: "https://issuer.example.com", + egressbroker.EnvSubjectRaw: "user-123", + egressbroker.EnvSessionID: "session-abc", + egressbroker.EnvMCPServer: "github-mcp", + } + + t.Run("valid env resolves identity", func(t *testing.T) { + t.Parallel() + r, err := egressbroker.NewPodIdentityResolver(envFrom(full)) + require.NoError(t, err) + id := r.PodIdentity() + assert.Equal(t, "https://issuer.example.com", id.Issuer) + assert.Equal(t, "user-123", id.Subject) + assert.Equal(t, "session-abc", id.SessionID) + assert.Equal(t, "github-mcp", id.MCPServer) + }) + + t.Run("nil env lookup → error", func(t *testing.T) { + t.Parallel() + _, err := egressbroker.NewPodIdentityResolver(nil) + require.Error(t, err) + }) + + t.Run("missing/empty downward-API env fails closed at startup", func(t *testing.T) { + t.Parallel() + for _, missing := range []string{ + egressbroker.EnvIssuer, + egressbroker.EnvSubjectRaw, + egressbroker.EnvSessionID, + egressbroker.EnvMCPServer, + } { + for _, variant := range map[string]string{"missing": "", "whitespace-only": " "} { + values := map[string]string{} + for k, v := range full { + values[k] = v + } + values[missing] = variant + _, err := egressbroker.NewPodIdentityResolver(envFrom(values)) + require.Error(t, err, "%s %s must fail construction", missing, variant) + assert.Contains(t, err.Error(), missing) + } + } + }) +} + +func TestResourceNaming(t *testing.T) { + t.Parallel() + + t.Run("generation-qualified names share the base prefix and are deterministic", func(t *testing.T) { + t.Parallel() + gen := egressbroker.CAGeneration([]byte("cert-pem-bytes")) + require.Len(t, gen, 16) + + n := egressbroker.ResourceNamesFor("github-mcp", gen) + assert.Equal(t, "github-mcp-bump-ca-"+gen, n.CASecret) + assert.Equal(t, "github-mcp-bump-ca-bundle-"+gen, n.CABundle) + assert.Equal(t, "github-mcp-egress-policy", n.EgressPolicy) + assert.Equal(t, gen, n.Generation) + + // Same cert → same generation → same names (idempotent reconcile). + assert.Equal(t, n, egressbroker.ResourceNamesFor("github-mcp", egressbroker.CAGeneration([]byte("cert-pem-bytes")))) + // Different cert → different generation. + assert.NotEqual(t, gen, egressbroker.CAGeneration([]byte("other-cert"))) + }) + + t.Run("TrimGeneration round-trips generation-qualified names", func(t *testing.T) { + t.Parallel() + base := egressbroker.BaseCASecretName("github-mcp") + gen, ok := egressbroker.TrimGeneration("github-mcp-bump-ca-0123456789abcdef", base) + require.True(t, ok) + assert.Equal(t, "0123456789abcdef", gen) + + _, ok = egressbroker.TrimGeneration("github-mcp-bump-ca", base) + assert.False(t, ok, "base name without generation must not parse") + _, ok = egressbroker.TrimGeneration("github-mcp-bump-ca-", base) + assert.False(t, ok, "empty generation must not parse") + _, ok = egressbroker.TrimGeneration("other-bump-ca-abc", base) + assert.False(t, ok, "foreign name must not parse") + }) + + t.Run("identity contract: every env maps to a distinct annotation", func(t *testing.T) { + t.Parallel() + seen := map[string]string{} + for env, annotation := range egressbroker.EnvToAnnotation { + got, ok := egressbroker.AnnotationForEnv(env) + require.True(t, ok) + assert.Equal(t, annotation, got) + _, dup := seen[annotation] + assert.False(t, dup, "annotation %s mapped twice", annotation) + seen[annotation] = env + assert.Contains(t, egressbroker.AnnotationFieldRef(annotation), annotation) + } + _, ok := egressbroker.AnnotationForEnv("THV_UNTRUSTED_NOPE") + assert.False(t, ok) + }) +} diff --git a/pkg/egressbroker/injector.go b/pkg/egressbroker/injector.go new file mode 100644 index 0000000000..3000e6e2f2 --- /dev/null +++ b/pkg/egressbroker/injector.go @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "context" + "fmt" + "log/slog" + "slices" + + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken" + "github.com/stacklok/toolhive/pkg/authserver/storage" +) + +// AuthorizationHeader is the header the injector mutates on allow. +const AuthorizationHeader = "authorization" + +// Destination is the request's destination, extracted from the bumped +// request. Host must be port-stripped by the caller. +type Destination struct { + Host string + Method string + Path string +} + +// Decision is the injector's verdict. On Deny, HeaderName/HeaderValue are +// guaranteed empty: no deny path may ever carry header material. +type Decision struct { + Allow bool + DenyReason string + HeaderName string + HeaderValue string //nolint:gosec // G117: field legitimately holds sensitive data +} + +// CredentialInjector enforces destination binding (ADR-0001 D5) and injects +// the pod-owner's upstream credential. +// +// Ordering is the security invariant and MUST NOT be reordered: +// 1. Provider lookup (which provider's allowlist contains the destination?) +// 2. Method/path policy check +// 3. ONLY THEN the credential load (Strict binding, ExpectedBinding from the +// pod's own identity — never from request data) +// 4. Header mutation +// +// Every failure mode denies; no deny path touches the token reader or writes +// a header. The token reader must be wrapped in upstreamtoken.NewStrictTokenReader +// by the caller (the constructor does it here so the wrap cannot be forgotten). +type CredentialInjector struct { + identity PodIdentity + policy *EgressPolicy + tokenReader upstreamtoken.TokenReader +} + +// NewCredentialInjector wires the injector. tokenReader is wrapped with +// NewStrictTokenReader internally so every read fails closed on legacy +// (unowned) token rows. Fails loudly on nil dependencies. +func NewCredentialInjector( + identity PodIdentity, + policy *EgressPolicy, + tokenReader upstreamtoken.TokenReader, +) (*CredentialInjector, error) { + if policy == nil { + return nil, fmt.Errorf("egressbroker: policy must not be nil") + } + if tokenReader == nil { + return nil, fmt.Errorf("egressbroker: token reader must not be nil") + } + if identity.Issuer == "" || identity.Subject == "" || identity.SessionID == "" || identity.MCPServer == "" { + return nil, fmt.Errorf("egressbroker: pod identity is incomplete; refusing to build injector") + } + return &CredentialInjector{ + identity: identity, + policy: policy, + tokenReader: upstreamtoken.NewStrictTokenReader(tokenReader), + }, nil +} + +// Evaluate decides whether dest may carry the pod-owner's credential and, if +// so, returns the Authorization header mutation. It never logs credential +// material; deny reasons carry no token data. +func (i *CredentialInjector) Evaluate(ctx context.Context, dest Destination) Decision { + // D5 step 1: provider lookup — before any credential load. + provider, ok := i.policy.ProviderFor(dest.Host) + if !ok { + return deny("no provider allowlists this destination") + } + // D5 step 2: method + path check — still before any credential load. + if !i.policy.Allows(provider, dest.Method, dest.Path) { + return deny("method/path outside provider policy") + } + + // Step 3: credential load, bound to the pod's own identity. The expected + // binding's UserID is the pod-owner's raw sub from the downward-API + // registry; Strict (forced by NewStrictTokenReader) fails closed on rows + // that cannot prove their owner. + creds, failed, err := i.tokenReader.GetAllUpstreamCredentials(ctx, i.identity.SessionID, + &storage.ExpectedBinding{UserID: i.identity.Subject, Strict: true}) + if err != nil { + // Fail closed on storage errors (Redis down, binding violations): + // never passthrough. + slog.DebugContext(ctx, "egressbroker: credential load failed; denying", + "provider", provider, "error", err) + return deny("upstream credential load failed") + } + cred, ok := creds[provider] + if !ok || slices.Contains(failed, provider) { + return deny("upstream credential unavailable; re-consent required") + } + if cred.AccessToken == "" { + // Defensive: a successful load with an empty token must never produce + // an "Authorization: Bearer " header. + return deny("upstream credential unavailable; re-consent required") + } + + return Decision{ + Allow: true, + HeaderName: AuthorizationHeader, + HeaderValue: "Bearer " + cred.AccessToken, + } +} + +func deny(reason string) Decision { + return Decision{Allow: false, DenyReason: reason} +} diff --git a/pkg/egressbroker/injector_test.go b/pkg/egressbroker/injector_test.go new file mode 100644 index 0000000000..1c88d2c1b6 --- /dev/null +++ b/pkg/egressbroker/injector_test.go @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken" + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken/mocks" + "github.com/stacklok/toolhive/pkg/authserver/storage" + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +var testIdentity = egressbroker.PodIdentity{ + Issuer: "https://issuer.example.com", + Subject: "user-123", + SessionID: "session-abc", + MCPServer: "github-mcp", +} + +func mustInjector(t *testing.T, reader upstreamtoken.TokenReader) *egressbroker.CredentialInjector { + t.Helper() + policy := mustParse(t, testPolicyYAML) + inj, err := egressbroker.NewCredentialInjector(testIdentity, policy, reader) + require.NoError(t, err) + return inj +} + +func TestCredentialInjector(t *testing.T) { + t.Parallel() + ctx := context.Background() + githubDest := egressbroker.Destination{Host: "api.github.com", Method: "GET", Path: "/repos/foo"} + + t.Run("happy path injects Authorization header", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, got *storage.ExpectedBinding, + ) (map[string]upstreamtoken.UpstreamCredential, []string, error) { + // The binding must carry the pod-owner's raw sub and be forced Strict. + assert.Equal(t, "user-123", got.UserID) + assert.True(t, got.Strict, "injector must force Strict binding via NewStrictTokenReader") + return map[string]upstreamtoken.UpstreamCredential{ + "github": {AccessToken: "gho_secret"}, + }, nil, nil + }) + + decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + assert.True(t, decision.Allow) + assert.Equal(t, egressbroker.AuthorizationHeader, decision.HeaderName) + assert.Equal(t, "Bearer gho_secret", decision.HeaderValue) + assert.Empty(t, decision.DenyReason) + }) + + t.Run("D5 ordering: credential load never happens on policy deny", func(t *testing.T) { + t.Parallel() + + denyDests := map[string]egressbroker.Destination{ + "non-allowlisted host": {Host: "evil.example.com", Method: "GET", Path: "/"}, + "superdomain of allowed": {Host: "github.com", Method: "GET", Path: "/"}, + "disallowed method": {Host: "api.github.com", Method: "DELETE", Path: "/repos/foo"}, + "disallowed path": {Host: "api.github.com", Method: "GET", Path: "/admin"}, + "method default is read-only": {Host: "slack.com", Method: "POST", Path: "/api/chat"}, + } + for name, dest := range denyDests { + t.Run(name, func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + // No EXPECT: any call to the token reader fails the test. + // This proves D5 is evaluated before any credential load. + decision := mustInjector(t, reader).Evaluate(ctx, dest) + assert.False(t, decision.Allow) + assert.NotEmpty(t, decision.DenyReason) + assert.Empty(t, decision.HeaderName, "deny path must never carry header material") + assert.Empty(t, decision.HeaderValue, "deny path must never carry header material") + }) + } + }) + + t.Run("non-allowlisted host gets no credential even with a valid session", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + // The reader would return a valid credential for ANY provider — but the + // destination binding must deny before it is ever consulted. + decision := mustInjector(t, reader).Evaluate(ctx, + egressbroker.Destination{Host: "attacker-controlled.example.com", Method: "GET", Path: "/"}) + assert.False(t, decision.Allow) + assert.Empty(t, decision.HeaderValue) + }) + + t.Run("credential missing → deny", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + Return(map[string]upstreamtoken.UpstreamCredential{}, nil, nil) + + decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + assert.False(t, decision.Allow) + assert.Contains(t, decision.DenyReason, "re-consent") + assert.Empty(t, decision.HeaderValue) + }) + + t.Run("credential in failed list → deny", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + Return(map[string]upstreamtoken.UpstreamCredential{}, []string{"github"}, nil) + + decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + assert.False(t, decision.Allow) + assert.Empty(t, decision.HeaderValue) + }) + + t.Run("empty access token → deny (no Bearer-with-nothing header)", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + Return(map[string]upstreamtoken.UpstreamCredential{ + "github": {AccessToken: ""}, + }, nil, nil) + + decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + assert.False(t, decision.Allow) + assert.Empty(t, decision.HeaderValue) + }) + + t.Run("Strict binding error → deny (fail closed)", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + Return(nil, nil, fmt.Errorf("row has no owner: %w", storage.ErrInvalidBinding)) + + decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + assert.False(t, decision.Allow) + assert.Empty(t, decision.HeaderValue) + // The deny reason must not leak storage internals. + assert.NotContains(t, decision.DenyReason, "row") + }) + + t.Run("store down (Redis error) → deny, never passthrough", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + Return(nil, nil, errors.New("connection refused")) + + decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + assert.False(t, decision.Allow) + assert.Empty(t, decision.HeaderValue) + }) + + t.Run("wildcard destination carries credential for its own provider only", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + Return(map[string]upstreamtoken.UpstreamCredential{ + "github": {AccessToken: "gho_secret"}, + "slack": {AccessToken: "xoxb_secret"}, + }, nil, nil) + + // Wildcard github host: github credential, never the slack one. + decision := mustInjector(t, reader).Evaluate(ctx, + egressbroker.Destination{Host: "raw.githubusercontent.com", Method: "GET", Path: "/repos/x"}) + assert.True(t, decision.Allow) + assert.Equal(t, "Bearer gho_secret", decision.HeaderValue) + assert.NotContains(t, decision.HeaderValue, "xoxb") + }) +} + +func TestNewCredentialInjectorValidation(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + policy := mustParse(t, testPolicyYAML) + + t.Run("nil policy", func(t *testing.T) { + t.Parallel() + _, err := egressbroker.NewCredentialInjector(testIdentity, nil, reader) + require.Error(t, err) + }) + t.Run("nil token reader", func(t *testing.T) { + t.Parallel() + _, err := egressbroker.NewCredentialInjector(testIdentity, policy, nil) + require.Error(t, err) + }) + t.Run("incomplete identity", func(t *testing.T) { + t.Parallel() + for name, id := range map[string]egressbroker.PodIdentity{ + "empty issuer": {Issuer: "", Subject: "s", SessionID: "x", MCPServer: "m"}, + "empty subject": {Issuer: "i", Subject: "", SessionID: "x", MCPServer: "m"}, + "empty session": {Issuer: "i", Subject: "s", SessionID: "", MCPServer: "m"}, + "empty mcpserver": {Issuer: "i", Subject: "s", SessionID: "x", MCPServer: ""}, + } { + _, err := egressbroker.NewCredentialInjector(id, policy, reader) + require.Error(t, err, name) + } + }) +} diff --git a/pkg/egressbroker/policy.go b/pkg/egressbroker/policy.go new file mode 100644 index 0000000000..5e968279d7 --- /dev/null +++ b/pkg/egressbroker/policy.go @@ -0,0 +1,273 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package egressbroker implements the credential-broker data plane for +// untrusted MCP servers (ADR-0001 D3/D5/D6/D7): a per-pod ext_authz service +// that resolves its own pod's identity from downward-API env, enforces +// destination binding (D5) before loading any credential, and returns the +// Authorization header mutation for destinations the EgressPolicy allows. +// +// The broker is the only component in the untrusted pod that ever possesses +// an upstream credential; the backend container never sees one. +package egressbroker + +import ( + "fmt" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// ProviderPolicy is one provider's egress constraint (credential-to-destination +// binding, ADR-0001 D5). This is the runtime mirror of the MCPServer CRD's +// ProviderEgress, rendered into the operator-generated policy ConfigMap. +type ProviderPolicy struct { + // Provider is the logical upstream provider name (e.g. "github"). + Provider string `yaml:"provider"` + // AllowedHosts is the exact set of destination hosts the credential may be + // injected for: exact hostnames or one-label wildcards ("*.example.com"). + AllowedHosts []string `yaml:"allowedHosts"` + // AllowedMethods constrains the HTTP methods injected on. Empty means + // GET/HEAD/OPTIONS only (read-only safe default). + AllowedMethods []string `yaml:"allowedMethods,omitempty"` + // AllowedPathPrefixes constrains URL paths injected on (prefix match). + // Empty means all paths on the allowed hosts. + AllowedPathPrefixes []string `yaml:"allowedPathPrefixes,omitempty"` +} + +// policyDocument is the YAML wire format of the policy ConfigMap. +type policyDocument struct { + Providers []ProviderPolicy `yaml:"providers"` + // DialAllowlist carries the operator-resolved destination CIDRs for the + // D7 per-dial IP validation. Rendered by the operator alongside the + // policy so the sidecar's dial guard and the NetworkPolicy ipBlocks are + // the same data. Optional in the wire format only so a hand-written + // policy document can still compile; LoadConfig requires it non-empty + // (fail closed). + DialAllowlist []string `yaml:"dialAllowlist,omitempty"` +} + +// EgressPolicy is the immutable, startup-compiled lookup the injector +// consults. All matching is case-normalized; the policy is rebuilt only by +// process restart (v1: no hot reload). +type EgressPolicy struct { + providers []ProviderPolicy + dialAllowlist []string +} + +// DialAllowlist returns the operator-rendered D7 dial allowlist (may be +// empty when the document predates the field; callers must fail closed). +func (e *EgressPolicy) DialAllowlist() []string { + return e.dialAllowlist +} + +// ParsePolicy compiles the policy ConfigMap document into an immutable +// EgressPolicy. It fails loudly on any malformed or unsafe entry (empty +// provider, empty/invalid host, unknown method, empty path prefix): a +// sidecar that cannot trust its policy must not start (fail closed). +func ParsePolicy(data []byte) (*EgressPolicy, error) { + var doc policyDocument + if err := yaml.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("egressbroker: failed to parse egress policy: %w", err) + } + if len(doc.Providers) == 0 { + return nil, fmt.Errorf("egressbroker: egress policy has no providers") + } + providers := make([]ProviderPolicy, 0, len(doc.Providers)) + seen := make(map[string]struct{}, len(doc.Providers)) + for _, p := range doc.Providers { + norm, err := normalizeProvider(p) + if err != nil { + return nil, err + } + if _, dup := seen[norm.Provider]; dup { + return nil, fmt.Errorf("egressbroker: duplicate provider %q in egress policy", norm.Provider) + } + seen[norm.Provider] = struct{}{} + providers = append(providers, norm) + } + return &EgressPolicy{providers: providers, dialAllowlist: doc.DialAllowlist}, nil +} + +// ProviderNames returns the sorted provider names (used to render the Envoy +// route allowlist deterministically). +func (e *EgressPolicy) ProviderNames() []string { + names := make([]string, 0, len(e.providers)) + for _, p := range e.providers { + names = append(names, p.Provider) + } + sort.Strings(names) + return names +} + +// ProviderFor returns the provider whose AllowedHosts match host. Host +// matching is exact or one-label wildcard; if multiple providers could match, +// the one with the longest (most specific) matched host pattern wins. This is +// the first half of D5: it must be consulted before any credential load. +func (e *EgressPolicy) ProviderFor(host string) (string, bool) { + host = normalizeHost(host) + best := -1 + provider := "" + for i := range e.providers { + for _, pattern := range e.providers[i].AllowedHosts { + if hostMatches(pattern, host) && len(pattern) > best { + best = len(pattern) + provider = e.providers[i].Provider + } + } + } + return provider, best >= 0 +} + +// Allows reports whether the provider's policy permits method+path (the +// second half of D5, evaluated before any credential load or header write). +// provider must be a name previously returned by ProviderFor. +func (e *EgressPolicy) Allows(provider, method, path string) bool { + for i := range e.providers { + if e.providers[i].Provider != provider { + continue + } + p := e.providers[i] + if !methodAllowed(p, method) { + return false + } + return pathAllowed(p, path) + } + return false +} + +// HostAllowlist returns the union of all providers' AllowedHosts (used to +// render Envoy route matches; requests to other hosts get no route → denied +// before any credential logic runs). +func (e *EgressPolicy) HostAllowlist() []string { + seen := map[string]struct{}{} + var out []string + for _, p := range e.providers { + for _, h := range p.AllowedHosts { + if _, ok := seen[h]; !ok { + seen[h] = struct{}{} + out = append(out, h) + } + } + } + sort.Strings(out) + return out +} + +// normalizeProvider lowercases and validates one policy entry. +func normalizeProvider(p ProviderPolicy) (ProviderPolicy, error) { + if strings.TrimSpace(p.Provider) == "" { + return ProviderPolicy{}, fmt.Errorf("egressbroker: provider with empty name in egress policy") + } + if len(p.AllowedHosts) == 0 { + return ProviderPolicy{}, fmt.Errorf("egressbroker: provider %q has no allowedHosts", p.Provider) + } + out := ProviderPolicy{ + Provider: p.Provider, + AllowedHosts: make([]string, 0, len(p.AllowedHosts)), + AllowedMethods: make([]string, 0, len(p.AllowedMethods)), + AllowedPathPrefixes: make([]string, 0, len(p.AllowedPathPrefixes)), + } + for _, h := range p.AllowedHosts { + h = normalizeHost(h) + if err := validateHostPattern(h); err != nil { + return ProviderPolicy{}, fmt.Errorf("egressbroker: provider %q: %w", p.Provider, err) + } + out.AllowedHosts = append(out.AllowedHosts, h) + } + for _, m := range p.AllowedMethods { + m = strings.ToUpper(strings.TrimSpace(m)) + if !knownMethod(m) { + return ProviderPolicy{}, fmt.Errorf("egressbroker: provider %q has unknown method %q", p.Provider, m) + } + out.AllowedMethods = append(out.AllowedMethods, m) + } + for _, prefix := range p.AllowedPathPrefixes { + if !strings.HasPrefix(prefix, "/") { + return ProviderPolicy{}, fmt.Errorf( + "egressbroker: provider %q has path prefix %q not starting with '/'", p.Provider, prefix) + } + out.AllowedPathPrefixes = append(out.AllowedPathPrefixes, prefix) + } + return out, nil +} + +// normalizeHost strips a trailing dot and lowercases; host patterns and +// request hosts are always compared in this form. +func normalizeHost(host string) string { + return strings.TrimSuffix(strings.ToLower(strings.TrimSpace(host)), ".") +} + +// validateHostPattern enforces the CRD's host grammar at the data plane too +// (defense-in-depth): hostnames only — no IP literals or CIDRs (the CRD +// grammar forbids them and dial-time IP validation is D7's job, not D5's), no +// ports, no schemes, no path, at most one leading one-label wildcard. +func validateHostPattern(host string) error { + if host == "" { + return fmt.Errorf("allowedHosts entry is empty") + } + if strings.ContainsAny(host, "/:@ \t") { + return fmt.Errorf("allowedHosts entry %q must be a bare hostname (no scheme, port, path, or userinfo)", host) + } + body := strings.TrimPrefix(host, "*.") + if body == "" || !strings.Contains(body, ".") { + return fmt.Errorf("allowedHosts entry %q is not a valid hostname", host) + } + if strings.Contains(body, "*") { + return fmt.Errorf("allowedHosts entry %q: wildcards only allowed as a leading one-label '*.'", host) + } + return nil +} + +// hostMatches reports whether pattern (exact or one-label "*.suffix") matches +// host. Both must already be normalized. "*.example.com" matches +// "api.example.com" but NOT "example.com" itself (one-label wildcard). +func hostMatches(pattern, host string) bool { + if suffix, wildcard := strings.CutPrefix(pattern, "*."); wildcard { + return strings.HasSuffix(host, "."+suffix) && len(host) > len(suffix)+1 + } + return pattern == host +} + +// methodAllowed: empty AllowedMethods defaults to safe read-only methods. +func methodAllowed(p ProviderPolicy, method string) bool { + method = strings.ToUpper(method) + if len(p.AllowedMethods) == 0 { + switch method { + case "GET", "HEAD", "OPTIONS": + return true + } + return false + } + for _, m := range p.AllowedMethods { + if m == method { + return true + } + } + return false +} + +// pathAllowed: empty AllowedPathPrefixes means all paths ("/" prefix). +func pathAllowed(p ProviderPolicy, path string) bool { + if path == "" { + path = "/" + } + if len(p.AllowedPathPrefixes) == 0 { + return true + } + for _, prefix := range p.AllowedPathPrefixes { + if strings.HasPrefix(path, prefix) { + return true + } + } + return false +} + +func knownMethod(m string) bool { + switch m { + case "GET", "HEAD", "OPTIONS", "POST", "PUT", "PATCH", "DELETE": + return true + } + return false +} diff --git a/pkg/egressbroker/policy_test.go b/pkg/egressbroker/policy_test.go new file mode 100644 index 0000000000..c419b30fcd --- /dev/null +++ b/pkg/egressbroker/policy_test.go @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +const testPolicyYAML = ` +providers: +- provider: github + allowedHosts: ["api.github.com", "*.githubusercontent.com"] + allowedMethods: ["GET", "POST"] + allowedPathPrefixes: ["/repos/", "/user"] +- provider: slack + allowedHosts: ["slack.com"] + allowedPathPrefixes: ["/api/"] +` + +func mustParse(t *testing.T, doc string) *egressbroker.EgressPolicy { + t.Helper() + p, err := egressbroker.ParsePolicy([]byte(doc)) + require.NoError(t, err) + return p +} + +func TestParsePolicyValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + doc string + wantErr string + }{ + {"empty document", ``, "no providers"}, + {"no providers", `providers: []`, "no providers"}, + {"invalid yaml", `{{{`, "failed to parse"}, + {"empty provider name", `providers: [{provider: "", allowedHosts: ["a.example.com"]}]`, "empty name"}, + {"no allowed hosts", `providers: [{provider: "github"}]`, "no allowedHosts"}, + {"host with scheme", `providers: [{provider: "g", allowedHosts: ["https://api.example.com"]}]`, "bare hostname"}, + {"host with port", `providers: [{provider: "g", allowedHosts: ["api.example.com:443"]}]`, "bare hostname"}, + {"host with path", `providers: [{provider: "g", allowedHosts: ["api.example.com/v1"]}]`, "bare hostname"}, + {"wildcard mid-label", `providers: [{provider: "g", allowedHosts: ["api.*.example.com"]}]`, "wildcards only allowed"}, + {"bare wildcard suffix", `providers: [{provider: "g", allowedHosts: ["*.com"]}]`, "not a valid hostname"}, + {"single-label host", `providers: [{provider: "g", allowedHosts: ["localhost"]}]`, "not a valid hostname"}, + {"unknown method", `providers: [{provider: "g", allowedHosts: ["a.example.com"], allowedMethods: ["YEET"]}]`, "unknown method"}, + {"path prefix not absolute", + `providers: [{provider: "g", allowedHosts: ["a.example.com"], allowedPathPrefixes: ["repos"]}]`, "not starting with '/'"}, + {"duplicate provider", + "providers:\n- {provider: g, allowedHosts: [a.example.com]}\n- {provider: g, allowedHosts: [b.example.com]}", + "duplicate provider"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, err := egressbroker.ParsePolicy([]byte(tt.doc)) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestProviderFor(t *testing.T) { + t.Parallel() + p := mustParse(t, testPolicyYAML) + + tests := []struct { + name string + host string + provider string + ok bool + }{ + {"exact host", "api.github.com", "github", true}, + {"exact host case-insensitive", "API.GitHub.COM", "github", true}, + {"exact host trailing dot", "api.github.com.", "github", true}, + {"wildcard one label", "raw.githubusercontent.com", "github", true}, + {"wildcard does not match bare suffix", "githubusercontent.com", "", false}, + {"wildcard matches deep subdomain", "a.b.githubusercontent.com", "github", true}, + {"unrelated host", "evil.example.com", "", false}, + {"superdomain of exact host", "github.com", "", false}, + {"exact host in other provider", "slack.com", "slack", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + provider, ok := p.ProviderFor(tt.host) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.provider, provider) + }) + } +} + +func TestAllows(t *testing.T) { + t.Parallel() + p := mustParse(t, testPolicyYAML) + + tests := []struct { + name string + provider string + method string + path string + want bool + }{ + {"allowed method + prefix", "github", "GET", "/repos/foo/bar", true}, + {"allowed method case-insensitive", "github", "get", "/repos/foo", true}, + {"POST declared", "github", "POST", "/repos/foo", true}, + {"DELETE not declared", "github", "DELETE", "/repos/foo", false}, + {"path outside prefixes", "github", "GET", "/admin/users", false}, + {"prefix boundary: /user matches /user", "github", "GET", "/user", true}, + {"prefix match is a string prefix", "github", "GET", "/user/repos", true}, + {"empty path treated as /", "github", "GET", "", false}, + {"unknown provider", "nonexistent", "GET", "/anything", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, p.Allows(tt.provider, tt.method, tt.path)) + }) + } +} + +func TestAllowsMethodDefault(t *testing.T) { + t.Parallel() + // slack entry declares no methods → safe read-only default. + p := mustParse(t, testPolicyYAML) + + assert.True(t, p.Allows("slack", "GET", "/api/chat")) + assert.True(t, p.Allows("slack", "HEAD", "/api/chat")) + assert.True(t, p.Allows("slack", "OPTIONS", "/api/chat")) + assert.False(t, p.Allows("slack", "POST", "/api/chat"), + "empty allowedMethods must default to read-only") +} + +func TestAllowsPathDefault(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: google + allowedHosts: ["www.googleapis.com"] +`) + assert.True(t, p.Allows("google", "GET", "/"), + "empty allowedPathPrefixes must allow all paths on allowed hosts") + assert.True(t, p.Allows("google", "GET", "/anything/deep")) + assert.False(t, p.Allows("google", "POST", "/"), + "method default still applies") +} + +func TestHostAllowlist(t *testing.T) { + t.Parallel() + p := mustParse(t, testPolicyYAML) + assert.Equal(t, + []string{"*.githubusercontent.com", "api.github.com", "slack.com"}, + p.HostAllowlist()) +} diff --git a/pkg/egressbroker/resolve.go b/pkg/egressbroker/resolve.go new file mode 100644 index 0000000000..c3efc223b7 --- /dev/null +++ b/pkg/egressbroker/resolve.go @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "errors" + "fmt" + "net" + "net/netip" + "slices" + "strings" +) + +// ErrDNSResolution marks DNS lookup failures during dial-allowlist resolution. +// Callers (the operator reconciler) treat it as transient — retry with +// backoff — unlike policy-shape errors, which are terminal spec violations. +var ErrDNSResolution = errors.New("egressbroker: DNS resolution failed") + +// ResolveDialAllowlist resolves the policy's allowed hosts to the concrete +// IP/CIDR set used for both the operator-rendered NetworkPolicy ipBlocks and +// the sidecar's D7 per-dial allowlist. allowedHosts are hostnames by CRD +// grammar, so every entry goes through lookupHost (DNS at operator reconcile +// time; v1: best-effort defense-in-depth — the load-bearing control is D5 + +// D7 in the sidecar). +// +// A lookup error wraps ErrDNSResolution (transient: the caller retries with +// backoff). A host that resolves to nothing (or only to non-public rebinding +// suspects) is a plain error — the policy destinations are wrong, so the +// caller surfaces it as a terminal spec violation rather than rendering an +// empty-allowlist sidecar that silently refuses all egress. +// +// The result is sorted: DNS answer order is unstable, and the allowlist is +// rendered into NetworkPolicy ipBlocks and the policy ConfigMap — an +// order-only difference would churn both every reconcile. +func ResolveDialAllowlist(policy *EgressPolicy, lookupHost func(string) ([]net.IP, error)) ([]string, error) { + if policy == nil { + return nil, fmt.Errorf("egressbroker: policy must not be nil") + } + if lookupHost == nil { + return nil, fmt.Errorf("egressbroker: DNS lookup must not be nil") + } + seen := map[string]struct{}{} + var out []string + for _, host := range policy.HostAllowlist() { + // Wildcard patterns cannot resolve; strip the marker and resolve the + // base domain (best-effort — CDN fronted APIs may move; D7 in the + // sidecar re-validates at dial time). + name := strings.TrimPrefix(host, "*.") + ips, err := lookupHost(name) + if err != nil { + return nil, fmt.Errorf("%w: policy host %q: %w", ErrDNSResolution, name, err) + } + if len(ips) == 0 { + return nil, fmt.Errorf("egressbroker: policy host %q resolved to no addresses", name) + } + for _, ip := range ips { + addr, ok := netip.AddrFromSlice(ip) + if !ok { + continue + } + addr = addr.Unmap() + if isNonPublic(addr) { + continue + } + prefix := netip.PrefixFrom(addr, addr.BitLen()).String() + if _, dup := seen[prefix]; !dup { + seen[prefix] = struct{}{} + out = append(out, prefix) + } + } + } + if len(out) == 0 { + return nil, fmt.Errorf("egressbroker: policy resolved to an empty dial allowlist") + } + slices.Sort(out) + return out, nil +} + +// nonPublicPrefixes are never allowlisted from DNS answers (OWASP SSRF set: +// loopback, RFC 1918, link-local, CGNAT, IPv6 ULA/link-local). A name that +// resolves into these is a rebinding suspect and must not widen the dial +// allowlist. +var nonPublicPrefixes = []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/8"), + netip.MustParsePrefix("10.0.0.0/8"), + netip.MustParsePrefix("100.64.0.0/10"), + netip.MustParsePrefix("127.0.0.0/8"), + netip.MustParsePrefix("169.254.0.0/16"), + netip.MustParsePrefix("172.16.0.0/12"), + netip.MustParsePrefix("192.0.0.0/24"), + netip.MustParsePrefix("192.0.2.0/24"), + netip.MustParsePrefix("192.168.0.0/16"), + netip.MustParsePrefix("198.18.0.0/15"), + netip.MustParsePrefix("198.51.100.0/24"), + netip.MustParsePrefix("203.0.113.0/24"), + netip.MustParsePrefix("224.0.0.0/4"), + netip.MustParsePrefix("240.0.0.0/4"), + netip.MustParsePrefix("::1/128"), + netip.MustParsePrefix("fc00::/7"), + netip.MustParsePrefix("fe80::/10"), + netip.MustParsePrefix("ff00::/8"), +} + +func isNonPublic(addr netip.Addr) bool { + for _, prefix := range nonPublicPrefixes { + if prefix.Contains(addr) { + return true + } + } + return false +} diff --git a/pkg/egressbroker/resolve_test.go b/pkg/egressbroker/resolve_test.go new file mode 100644 index 0000000000..9d7c8d45c3 --- /dev/null +++ b/pkg/egressbroker/resolve_test.go @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "fmt" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +func TestResolveDialAllowlist(t *testing.T) { + t.Parallel() + + lookup := func(ips map[string][]net.IP) func(string) ([]net.IP, error) { + return func(host string) ([]net.IP, error) { + if ips, ok := ips[host]; ok { + return ips, nil + } + return nil, fmt.Errorf("no such host: %s", host) + } + } + + t.Run("resolves policy hosts to deduplicated IP prefixes", func(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com"] +`) + out, err := egressbroker.ResolveDialAllowlist(p, lookup(map[string][]net.IP{ + "api.github.com": {net.ParseIP("140.82.114.26"), net.ParseIP("140.82.114.26")}, + })) + require.NoError(t, err) + assert.Equal(t, []string{"140.82.114.26/32"}, out) + }) + + t.Run("output is sorted regardless of DNS answer order (anti-churn)", func(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com", "*.githubusercontent.com"] +`) + resolve := func(ips map[string][]net.IP) []string { + out, err := egressbroker.ResolveDialAllowlist(p, lookup(ips)) + require.NoError(t, err) + return out + } + shuffled := map[string][]net.IP{ + "api.github.com": {net.ParseIP("140.82.116.6"), net.ParseIP("140.82.112.5"), net.ParseIP("140.82.114.26")}, + "githubusercontent.com": {net.ParseIP("185.199.110.133"), net.ParseIP("185.199.108.133")}, + } + reversed := map[string][]net.IP{ + "api.github.com": {net.ParseIP("140.82.114.26"), net.ParseIP("140.82.112.5"), net.ParseIP("140.82.116.6")}, + "githubusercontent.com": {net.ParseIP("185.199.108.133"), net.ParseIP("185.199.110.133")}, + } + first := resolve(shuffled) + assert.Equal(t, first, resolve(reversed), "DNS answer order must not change the rendered allowlist") + assert.IsIncreasingf(t, first, "allowlist must be sorted (NetworkPolicy/ConfigMap anti-churn)") + }) + + t.Run("wildcard resolves the base domain", func(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["*.githubusercontent.com"] +`) + out, err := egressbroker.ResolveDialAllowlist(p, lookup(map[string][]net.IP{ + "githubusercontent.com": {net.ParseIP("185.199.108.133")}, + })) + require.NoError(t, err) + assert.Equal(t, []string{"185.199.108.133/32"}, out) + }) + + t.Run("DNS failure wraps ErrDNSResolution (transient: the operator retries)", func(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com"] +`) + _, err := egressbroker.ResolveDialAllowlist(p, lookup(map[string][]net.IP{})) + require.Error(t, err) + assert.ErrorIs(t, err, egressbroker.ErrDNSResolution) + }) + + t.Run("rebinding-suspect DNS answers never widen the allowlist", func(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com"] +`) + _, err := egressbroker.ResolveDialAllowlist(p, lookup(map[string][]net.IP{ + "api.github.com": {net.ParseIP("10.0.0.5"), net.ParseIP("127.0.0.1"), net.ParseIP("169.254.169.254")}, + })) + require.Error(t, err, "a host resolving only to non-global IPs must fail, not allowlist them") + assert.Contains(t, err.Error(), "empty dial allowlist") + }) + + t.Run("mixed global + private answers keep only the global ones", func(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com"] +`) + out, err := egressbroker.ResolveDialAllowlist(p, lookup(map[string][]net.IP{ + "api.github.com": {net.ParseIP("10.0.0.5"), net.ParseIP("140.82.114.26")}, + })) + require.NoError(t, err) + assert.Equal(t, []string{"140.82.114.26/32"}, out) + }) + + t.Run("nil inputs fail loudly", func(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com"] +`) + _, err := egressbroker.ResolveDialAllowlist(nil, net.LookupIP) + require.Error(t, err) + _, err = egressbroker.ResolveDialAllowlist(p, nil) + require.Error(t, err) + }) +} diff --git a/pkg/egressbroker/server.go b/pkg/egressbroker/server.go new file mode 100644 index 0000000000..dd045280cb --- /dev/null +++ b/pkg/egressbroker/server.go @@ -0,0 +1,266 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "context" + "fmt" + "log/slog" + "net" + "strings" + "sync" + "time" + + envoycore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoytls "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + envoyauth "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" + envoydiscovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + envoysecret "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" + "google.golang.org/genproto/googleapis/rpc/code" + "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + grpcstatus "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" +) + +// AuthorizationServer is the Envoy ext_authz gRPC endpoint. It translates +// Check requests into CredentialInjector decisions. Every failure mode — +// malformed request, missing destination, injector deny, internal error — +// returns a denial: ext_authz must never pass traffic on doubt. +type AuthorizationServer struct { + envoyauth.UnimplementedAuthorizationServer + injector *CredentialInjector +} + +// NewAuthorizationServer builds the ext_authz endpoint on an injector. +func NewAuthorizationServer(injector *CredentialInjector) (*AuthorizationServer, error) { + if injector == nil { + return nil, fmt.Errorf("egressbroker: injector must not be nil") + } + return &AuthorizationServer{injector: injector}, nil +} + +// Check implements envoy.service.auth.v3.Authorization. +func (s *AuthorizationServer) Check(ctx context.Context, req *envoyauth.CheckRequest) (*envoyauth.CheckResponse, error) { + dest, err := destinationFromCheckRequest(req) + if err != nil { + slog.DebugContext(ctx, "egressbroker: denying request with unparseable destination", "error", err) + return deniedResponse(codes.InvalidArgument, "unparseable destination"), nil + } + decision := s.injector.Evaluate(ctx, dest) + if !decision.Allow { + slog.DebugContext(ctx, "egressbroker: denied egress", + "host", dest.Host, "method", dest.Method, "reason", decision.DenyReason) + return deniedResponse(codes.PermissionDenied, decision.DenyReason), nil + } + return okResponse(decision.HeaderName, decision.HeaderValue), nil +} + +// destinationFromCheckRequest extracts host/method/path from the attribute +// context. Host is port-stripped and lowercased; an empty host or method +// fails closed. +func destinationFromCheckRequest(req *envoyauth.CheckRequest) (Destination, error) { + httpReq := req.GetAttributes().GetRequest().GetHttp() + if httpReq == nil { + return Destination{}, fmt.Errorf("check request carries no HTTP attributes") + } + host := httpReq.GetHeaders()[":authority"] + if host == "" { + host = httpReq.GetHost() + } + host, _, err := net.SplitHostPort(host) + if err != nil { + // SplitHostPort fails when there is no port — use the raw value. + host = httpReq.GetHeaders()[":authority"] + if host == "" { + host = httpReq.GetHost() + } + } + host = normalizeHost(host) + if host == "" { + return Destination{}, fmt.Errorf("check request carries no destination host") + } + method := httpReq.GetMethod() + if method == "" { + return Destination{}, fmt.Errorf("check request carries no HTTP method") + } + path := httpReq.GetPath() + if path == "" { + path = "/" + } + // CONNECT requests carry no scheme; others do — the path for CONNECT is + // authority-form, which the policy's path prefixes will correctly refuse + // unless "/" is allowed. Envoy routes CONNECT through upgrade config, and + // the tunneled requests are re-checked individually. + return Destination{Host: host, Method: method, Path: path}, nil +} + +// okResponse allows the request and mutates exactly one header. The header +// value is the only credential material that ever crosses this boundary. +func okResponse(headerName, headerValue string) *envoyauth.CheckResponse { + return &envoyauth.CheckResponse{ + Status: &status.Status{Code: int32(code.Code_OK)}, + HttpResponse: &envoyauth.CheckResponse_OkResponse{ + OkResponse: &envoyauth.OkHttpResponse{ + Headers: []*envoycore.HeaderValueOption{{ + Header: &envoycore.HeaderValue{Key: headerName, Value: headerValue}, + }}, + // Never echo the credential anywhere but the upstream request + // header: response headers and dynamic metadata stay empty. + }, + }, + } +} + +// deniedResponse denies with a static body carrying the reason (never +// credential material). Envoy maps a non-OK status to 403 by default; the +// denied response makes that explicit. +func deniedResponse(rpcCode codes.Code, reason string) *envoyauth.CheckResponse { + return &envoyauth.CheckResponse{ + Status: &status.Status{Code: int32(rpcCode)}, //nolint:gosec // G115: gRPC codes are small non-negative ints + HttpResponse: &envoyauth.CheckResponse_DeniedResponse{ + DeniedResponse: &envoyauth.DeniedHttpResponse{ + Body: "egress denied: " + reason, + }, + }, + } +} + +// SecretDiscoveryServer mints per-SNI TLS-bump certificates on demand (D9) +// and serves them to Envoy over SDS. Only the TLS listener's per-host +// certificate resources are served; requests for any other resource (or for +// the CA key) are refused. The CA private key is never served. +// +// Certs are minted once per hostname and cached for the process lifetime +// (short-lived leaves; a process restart re-mints). +type SecretDiscoveryServer struct { + envoysecret.UnimplementedSecretDiscoveryServiceServer + ca *BumpCA + policy *EgressPolicy + + mu sync.Mutex + cache map[string]*envoytls.Secret +} + +// NewSecretDiscoveryServer builds the SDS endpoint. policy constrains which +// hostnames certs are minted for — a cert is only ever minted for a host the +// egress policy allowlists (fail closed: no policy match, no cert). +func NewSecretDiscoveryServer(ca *BumpCA, policy *EgressPolicy) (*SecretDiscoveryServer, error) { + if ca == nil { + return nil, fmt.Errorf("egressbroker: bump CA must not be nil") + } + if policy == nil { + return nil, fmt.Errorf("egressbroker: policy must not be nil") + } + return &SecretDiscoveryServer{ca: ca, policy: policy, cache: map[string]*envoytls.Secret{}}, nil +} + +// FetchSecrets implements SecretDiscoveryService. Used by Envoy's +// on-demand SDS for downstream TLS contexts. +func (s *SecretDiscoveryServer) FetchSecrets( + ctx context.Context, req *envoydiscovery.DiscoveryRequest, +) (*envoydiscovery.DiscoveryResponse, error) { + if len(req.GetResourceNames()) == 0 { + return nil, grpcstatus.Error(codes.InvalidArgument, "SDS request must name resources") + } + resources := make([]*anypb.Any, 0, len(req.GetResourceNames())) + for _, name := range req.GetResourceNames() { + secret, err := s.secretForHost(name) + if err != nil { + // Fail closed: no cert for a non-allowlisted host. The listener + // handshake fails instead of falling back to any other cert. + slog.DebugContext(ctx, "egressbroker: refusing to mint cert", "resource", name, "error", err) + return nil, grpcstatus.Error(codes.PermissionDenied, "no certificate for resource") + } + packed, err := anypb.New(secret) + if err != nil { + return nil, grpcstatus.Error(codes.Internal, "failed to pack secret") + } + resources = append(resources, packed) + } + return &envoydiscovery.DiscoveryResponse{ + VersionInfo: fmt.Sprintf("%d", time.Now().UnixNano()), + Resources: resources, + TypeUrl: "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret", + }, nil +} + +// secretForHost returns (minting+caching) the bump cert for a resource name +// of the form "host:". The hostname must be policy-allowlisted. +func (s *SecretDiscoveryServer) secretForHost(resourceName string) (*envoytls.Secret, error) { + host, found := strings.CutPrefix(resourceName, "host:") + if !found || host == "" { + return nil, fmt.Errorf("unknown SDS resource name") + } + host = normalizeHost(host) + if _, ok := s.policy.ProviderFor(host); !ok { + return nil, fmt.Errorf("host not in egress policy") + } + + s.mu.Lock() + defer s.mu.Unlock() + if cached, ok := s.cache[host]; ok { + return cached, nil + } + certPEM, keyPEM, err := s.ca.MintLeaf(host, time.Now()) + if err != nil { + return nil, err + } + secret := &envoytls.Secret{ + Name: resourceName, + Type: &envoytls.Secret_TlsCertificate{ + TlsCertificate: &envoytls.TlsCertificate{ + CertificateChain: &envoycore.DataSource{ + Specifier: &envoycore.DataSource_InlineBytes{InlineBytes: certPEM}, + }, + PrivateKey: &envoycore.DataSource{ + Specifier: &envoycore.DataSource_InlineBytes{InlineBytes: keyPEM}, + }, + }, + }, + } + s.cache[host] = secret + return secret, nil +} + +// Server bundles the gRPC listener serving ext_authz + SDS on one socket +// (loopback-only). +type Server struct { + grpcServer *grpc.Server + listener net.Listener +} + +// NewServer creates the gRPC server on listenAddress:port, registering the +// ext_authz and SDS services. +func NewServer( + listenAddress string, + port int, + authz *AuthorizationServer, + sds *SecretDiscoveryServer, +) (*Server, error) { + if authz == nil || sds == nil { + return nil, fmt.Errorf("egressbroker: authz and sds servers must not be nil") + } + listener, err := net.Listen("tcp", net.JoinHostPort(listenAddress, fmt.Sprintf("%d", port))) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to listen on %s:%d: %w", listenAddress, port, err) + } + grpcServer := grpc.NewServer() + envoyauth.RegisterAuthorizationServer(grpcServer, authz) + envoysecret.RegisterSecretDiscoveryServiceServer(grpcServer, sds) + return &Server{grpcServer: grpcServer, listener: listener}, nil +} + +// Run serves until ctx is canceled, then gracefully stops. +func (s *Server) Run(ctx context.Context) error { + go func() { + <-ctx.Done() + s.grpcServer.GracefulStop() + }() + if err := s.grpcServer.Serve(s.listener); err != nil { + return fmt.Errorf("egressbroker: gRPC server failed: %w", err) + } + return nil +} diff --git a/pkg/egressbroker/server_test.go b/pkg/egressbroker/server_test.go new file mode 100644 index 0000000000..da1868ef8d --- /dev/null +++ b/pkg/egressbroker/server_test.go @@ -0,0 +1,231 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "context" + "crypto/x509" + "encoding/pem" + "testing" + "time" + + envoycore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoytls "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + envoyauth "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" + envoydiscovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "google.golang.org/genproto/googleapis/rpc/code" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken" + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken/mocks" + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +func checkRequest(host, method, path string) *envoyauth.CheckRequest { + return &envoyauth.CheckRequest{ + Attributes: &envoyauth.AttributeContext{ + Request: &envoyauth.AttributeContext_Request{ + Http: &envoyauth.AttributeContext_HttpRequest{ + Method: method, + Path: path, + Host: host, + }, + }, + }, + } +} + +func mustAuthzServer(t *testing.T, reader upstreamtoken.TokenReader) *egressbroker.AuthorizationServer { + t.Helper() + inj := mustInjector(t, reader) + srv, err := egressbroker.NewAuthorizationServer(inj) + require.NoError(t, err) + return srv +} + +func TestAuthorizationServerCheck(t *testing.T) { + t.Parallel() + ctx := context.Background() + + t.Run("allowlisted request → OK with Authorization header set", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + Return(map[string]upstreamtoken.UpstreamCredential{ + "github": {AccessToken: "gho_secret"}, + }, nil, nil) + + resp, err := mustAuthzServer(t, reader).Check(ctx, + checkRequest("api.github.com:443", "GET", "/repos/foo")) + require.NoError(t, err) + assert.Equal(t, int32(code.Code_OK), resp.GetStatus().GetCode()) + + ok := resp.GetOkResponse() + require.NotNil(t, ok) + require.Len(t, ok.GetHeaders(), 1) + assert.Equal(t, egressbroker.AuthorizationHeader, ok.GetHeaders()[0].GetHeader().GetKey()) + assert.Equal(t, "Bearer gho_secret", ok.GetHeaders()[0].GetHeader().GetValue()) + // The response must not leak the token anywhere else. + assert.Empty(t, ok.GetResponseHeadersToAdd()) + assert.Nil(t, resp.GetDeniedResponse()) + }) + + t.Run("non-allowlisted host → deny, no header mutation", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + // Token reader never consulted (D5 ordering proven at injector level). + + resp, err := mustAuthzServer(t, reader).Check(ctx, + checkRequest("evil.example.com:443", "GET", "/")) + require.NoError(t, err) + assert.Equal(t, int32(code.Code_PERMISSION_DENIED), resp.GetStatus().GetCode()) + assert.Nil(t, resp.GetOkResponse(), "deny must never carry header material") + require.NotNil(t, resp.GetDeniedResponse()) + }) + + t.Run("credential unavailable → deny", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + Return(map[string]upstreamtoken.UpstreamCredential{}, nil, nil) + + resp, err := mustAuthzServer(t, reader).Check(ctx, + checkRequest("api.github.com", "GET", "/repos/foo")) + require.NoError(t, err) + assert.Equal(t, int32(code.Code_PERMISSION_DENIED), resp.GetStatus().GetCode()) + assert.Nil(t, resp.GetOkResponse()) + }) + + t.Run("missing destination → deny (fail closed)", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + + for name, req := range map[string]*envoyauth.CheckRequest{ + "no http attributes": {}, + "empty host": checkRequest("", "GET", "/"), + "empty method": checkRequest("api.github.com", "", "/"), + } { + resp, err := mustAuthzServer(t, reader).Check(ctx, req) + require.NoError(t, err, name) + assert.Equal(t, int32(code.Code_INVALID_ARGUMENT), resp.GetStatus().GetCode(), name) + assert.Nil(t, resp.GetOkResponse(), name) + } + }) + + t.Run("deny body carries the reason, never credential material", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + reader := mocks.NewMockTokenReader(ctrl) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). + Return(map[string]upstreamtoken.UpstreamCredential{}, []string{"github"}, nil) + + resp, err := mustAuthzServer(t, reader).Check(ctx, + checkRequest("api.github.com", "GET", "/repos/foo")) + require.NoError(t, err) + denied := resp.GetDeniedResponse() + require.NotNil(t, denied) + assert.Contains(t, denied.GetBody(), "re-consent") + assert.NotContains(t, denied.GetBody(), "gho_") + }) +} + +func mustSDSServer(t *testing.T) (*egressbroker.SecretDiscoveryServer, *egressbroker.BumpCA) { + t.Helper() + ca, err := egressbroker.GenerateBumpCA("sds-test", time.Now()) + require.NoError(t, err) + srv, err := egressbroker.NewSecretDiscoveryServer(ca, mustParse(t, testPolicyYAML)) + require.NoError(t, err) + return srv, ca +} + +func TestSecretDiscoveryServer(t *testing.T) { + t.Parallel() + ctx := context.Background() + + t.Run("mints a verifiable bump cert for an allowlisted host", func(t *testing.T) { + t.Parallel() + srv, ca := mustSDSServer(t) + resp, err := srv.FetchSecrets(ctx, &envoydiscovery.DiscoveryRequest{ + ResourceNames: []string{"host:api.github.com"}, + }) + require.NoError(t, err) + require.Len(t, resp.GetResources(), 1) + + var secret envoytls.Secret + require.NoError(t, resp.GetResources()[0].UnmarshalTo(&secret)) + certPEM := secret.GetTlsCertificate().GetCertificateChain().GetInlineBytes() + keyPEM := secret.GetTlsCertificate().GetPrivateKey().GetInlineBytes() + require.NotEmpty(t, certPEM) + require.NotEmpty(t, keyPEM) + + block, _ := pem.Decode(certPEM) + require.NotNil(t, block) + leaf, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + caBlock, _ := pem.Decode(ca.CertPEM()) + require.NotNil(t, caBlock) + caCert, err := x509.ParseCertificate(caBlock.Bytes) + require.NoError(t, err) + roots := x509.NewCertPool() + roots.AddCert(caCert) + _, err = leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: "api.github.com"}) + require.NoError(t, err, "SDS-served cert must verify against the tenant bump CA") + }) + + t.Run("refuses non-allowlisted host (fail closed, no cert)", func(t *testing.T) { + t.Parallel() + srv, _ := mustSDSServer(t) + _, err := srv.FetchSecrets(ctx, &envoydiscovery.DiscoveryRequest{ + ResourceNames: []string{"host:evil.example.com"}, + }) + require.Error(t, err) + assert.Equal(t, codes.PermissionDenied, status.Code(err)) + }) + + t.Run("refuses unknown resource names and empty requests", func(t *testing.T) { + t.Parallel() + srv, _ := mustSDSServer(t) + _, err := srv.FetchSecrets(ctx, &envoydiscovery.DiscoveryRequest{ + ResourceNames: []string{"ca-key"}, + }) + require.Error(t, err, "only host: resources are servable; the CA key is never served") + _, err = srv.FetchSecrets(ctx, &envoydiscovery.DiscoveryRequest{}) + require.Error(t, err) + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + }) + + t.Run("wildcard host pattern matches subdomain cert requests", func(t *testing.T) { + t.Parallel() + srv, _ := mustSDSServer(t) + resp, err := srv.FetchSecrets(ctx, &envoydiscovery.DiscoveryRequest{ + ResourceNames: []string{"host:raw.githubusercontent.com"}, + }) + require.NoError(t, err) + require.Len(t, resp.GetResources(), 1) + }) + + t.Run("constructor validation", func(t *testing.T) { + t.Parallel() + policy := mustParse(t, testPolicyYAML) + ca, err := egressbroker.GenerateBumpCA("x", time.Now()) + require.NoError(t, err) + _, err = egressbroker.NewSecretDiscoveryServer(nil, policy) + require.Error(t, err) + _, err = egressbroker.NewSecretDiscoveryServer(ca, nil) + require.Error(t, err) + }) + + var _ = envoycore.HeaderValue{} // keep import used if assertions change +} diff --git a/pkg/transport/session/session_data_storage_redis.go b/pkg/transport/session/session_data_storage_redis.go index c8b7f47c70..d1edbd821b 100644 --- a/pkg/transport/session/session_data_storage_redis.go +++ b/pkg/transport/session/session_data_storage_redis.go @@ -62,6 +62,23 @@ func (s *RedisSessionDataStorage) key(id string) string { return s.keyPrefix + id } +// RedisAccess exposes the underlying Redis client and per-tenant key prefix so +// the untrusted-mode wiring (pkg/vmcp/session/untrusted) can back its admission +// counters and pod leases from the same connection and ':'-terminated prefix as +// the session store, rather than a second connection that could drift from the +// session store's coordinates. The returned client is the same instance owned +// by this storage — callers must NOT close it (Close remains the storage's +// responsibility). +// +// A composition root that constructs the RedisSessionDataStorage itself (e.g. +// an enterprise build wiring the untrusted stack before the server exists) uses +// this to hand NewStack the shared client. The OSS vMCP CLI builds its own +// connection from cfg.SessionStorage instead because the storage is created +// inside Serve, after the session factory (which carries the resolver) exists. +func (s *RedisSessionDataStorage) RedisAccess() (redis.UniversalClient, string) { + return s.client, s.keyPrefix +} + // Load retrieves metadata from Redis and refreshes the key's TTL via GETEX. // Returns ErrSessionNotFound if the key does not exist. func (s *RedisSessionDataStorage) Load(ctx context.Context, id string) (map[string]string, error) { diff --git a/pkg/transport/session/session_data_storage_test.go b/pkg/transport/session/session_data_storage_test.go index bdc35fc64d..f3b6b4f4fe 100644 --- a/pkg/transport/session/session_data_storage_test.go +++ b/pkg/transport/session/session_data_storage_test.go @@ -335,6 +335,27 @@ func newTestRedisDataStorage(t *testing.T) (*RedisSessionDataStorage, *miniredis return s, mr } +// TestRedisSessionDataStorage_RedisAccess pins the accessor the untrusted-mode +// wiring uses to share the session store's connection and prefix: it returns +// the live client (usable for the admission/pod-lease keys) and the exact +// ':'-terminated key prefix the store writes under. +func TestRedisSessionDataStorage_RedisAccess(t *testing.T) { + t.Parallel() + s, _ := newTestRedisDataStorage(t) + + client, prefix := s.RedisAccess() + assert.Equal(t, "test:data:", prefix) + require.NotNil(t, client) + + // The returned client is the live connection: a write through it lands + // under the same Redis the store reads. + ctx := context.Background() + require.NoError(t, client.Set(ctx, prefix+"probe", "1", 0).Err()) + got, err := client.Get(ctx, prefix+"probe").Result() + require.NoError(t, err) + assert.Equal(t, "1", got) +} + func TestRedisSessionDataStorage(t *testing.T) { t.Parallel() diff --git a/pkg/vmcp/cli/serve.go b/pkg/vmcp/cli/serve.go index e25bdc0a64..4b1053f50e 100644 --- a/pkg/vmcp/cli/serve.go +++ b/pkg/vmcp/cli/serve.go @@ -17,6 +17,7 @@ import ( "path/filepath" "time" + "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" "gopkg.in/yaml.v3" "k8s.io/client-go/rest" @@ -46,6 +47,7 @@ import ( ratelimitfactory "github.com/stacklok/toolhive/pkg/vmcp/ratelimit/factory" vmcprouter "github.com/stacklok/toolhive/pkg/vmcp/router" vmcpserver "github.com/stacklok/toolhive/pkg/vmcp/server" + "github.com/stacklok/toolhive/pkg/vmcp/server/sessionmanager" vmcpsession "github.com/stacklok/toolhive/pkg/vmcp/session" "github.com/stacklok/toolhive/pkg/vmcp/session/optimizerdec" vmcpstatus "github.com/stacklok/toolhive/pkg/vmcp/status" @@ -348,9 +350,27 @@ func Serve(ctx context.Context, cfg ServeConfig) error { slog.Debug("VMCP_SESSION_HMAC_SECRET is set but no longer used after #5306; ignoring", "env_var", "VMCP_SESSION_HMAC_SECRET") } + + // Untrusted backend mode (ADR-0001): when the group contains an untrusted + // MCPServer, wire the per-session pod lifecycle (resolver) and GC reaper. + // Gated on actual untrusted backends, so trusted-only deployments are + // unaffected. The reaper is started after the server is built (below). + var untrustedMeterProvider metric.MeterProvider + if telemetryProvider != nil { + untrustedMeterProvider = telemetryProvider.MeterProvider() + } + untrustedStk, err := buildUntrustedStack(ctx, vmcpCfg, backends, vmcpNamespace(), vmcpCfg.Name, untrustedMeterProvider) + if err != nil { + return fmt.Errorf("failed to initialize untrusted backend mode: %w", err) + } + // The factory never aggregates — the core is the single source of capability // aggregation (agg feeds it via Config.Aggregator below). - sessionFactory := vmcpsession.NewSessionFactory(outgoingRegistry) + var sessionFactoryOpts []vmcpsession.MultiSessionFactoryOption + if untrustedStk != nil { + sessionFactoryOpts = append(sessionFactoryOpts, vmcpsession.WithUntrustedResolver(untrustedStk.resolver)) + } + sessionFactory := vmcpsession.NewSessionFactory(outgoingRegistry, sessionFactoryOpts...) // When the optimizer is enabled, its meta-tools are pass-through tools. // Authz uses this for optimizer-aware authorization/filtering. @@ -443,6 +463,16 @@ func Serve(ctx context.Context, cfg ServeConfig) error { Authz: authzConfig, }) + // Untrusted mode: forward the resolver + lifecycle to the session manager so + // Terminate deletes per-session pods and restores use the cold-start budget. + if untrustedStk != nil { + serverCfg.Untrusted = &sessionmanager.UntrustedConfig{ + Resolver: untrustedStk.resolver, + Lifecycle: untrustedStk.lifecycle, + CacheCapacity: untrustedCacheCapacity(), + } + } + // Assign Watcher only when backendWatcher is non-nil. A typed nil // *k8s.BackendWatcher assigned to the Watcher interface produces a // non-nil interface value, which panics on the first /readyz probe. @@ -465,6 +495,16 @@ func Serve(ctx context.Context, cfg ServeConfig) error { return fmt.Errorf("failed to create Virtual MCP Server: %w", err) } + // Start the untrusted-mode reaper with the server's lifecycle: it runs until + // ctx is cancelled (srv.Start returns), then the shared Redis client is + // closed. Started only when the group has untrusted backends. The reaper's + // session-liveness probe comes from the session manager (the owner of + // session storage) — never a re-derived storage key. + if untrustedStk != nil { + shutdown := runUntrustedReaper(ctx, untrustedStk, srv.VMCPSessionManager().SessionExists) + defer shutdown() + } + slog.Info(fmt.Sprintf("Starting Virtual MCP Server at %s", srv.Address())) return srv.Start(ctx) } diff --git a/pkg/vmcp/cli/untrusted.go b/pkg/vmcp/cli/untrusted.go new file mode 100644 index 0000000000..19308afdf2 --- /dev/null +++ b/pkg/vmcp/cli/untrusted.go @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "context" + "fmt" + "log/slog" + "os" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/metric" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + + tcredis "github.com/stacklok/toolhive-core/redis" + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" + authstorage "github.com/stacklok/toolhive/pkg/authserver/storage" + thvk8s "github.com/stacklok/toolhive/pkg/k8s" + "github.com/stacklok/toolhive/pkg/vmcp" + "github.com/stacklok/toolhive/pkg/vmcp/config" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" +) + +// untrustedTokenStoreAddrEnvVar mirrors the operator-side env var injected on +// the vMCP Deployment carrying the auth-server Redis address (see +// cmd/thv-operator/controllers/virtualmcpserver_deployment.go). The two +// packages must not import each other; the contract is pinned by tests. +const untrustedTokenStoreAddrEnvVar = "THV_UNTRUSTED_TOKEN_STORE_REDIS_ADDR" // #nosec G101 -- env var name, not a credential + +// defaultSessionKeyPrefix mirrors the fallback in server.buildSessionDataStorage. +const defaultSessionKeyPrefix = "thv:vmcp:session:" + +// untrustedBundle carries the wired untrusted-mode components for the vMCP +// serve path: the resolver installed on the session factory, the pod lifecycle +// (DeletePod on session Terminate), and the reaper owning pod GC, plus the +// Redis client they share (closed on shutdown). +type untrustedBundle struct { + resolver untrusted.BackendAddressResolver + lifecycle untrusted.PodLifecycle + reaper *untrusted.Reaper + redisClient redis.UniversalClient +} + +// groupHasUntrustedBackend reports whether any backend in the discovered group +// is marked untrusted by the K8s workload discoverer +// (pkg/vmcp/workloads/k8s.go). This is the production feature gate: the +// untrusted stack (pod lifecycle + reaper) starts only when the group actually +// contains an untrusted MCPServer, so trusted-only deployments pay nothing. +func groupHasUntrustedBackend(backends []vmcp.Backend) bool { + for i := range backends { + if backends[i].Metadata[untrusted.MetadataKeyUntrusted] == "true" { + return true + } + } + return false +} + +// buildUntrustedStack wires the untrusted-mode stack for the vMCP serve path. +// +// It returns (nil, nil) — untrusted mode off — unless the group contains at +// least one untrusted backend (the feature gate). When the gate is on it +// requires Redis-backed session storage (multi-pod admission counters and pod +// leases are Redis state) and a resolvable vMCP namespace (untrusted mode is +// Kubernetes-only); both are hard startup errors rather than silent +// degradation, so a misconfigured untrusted deployment fails loudly. +// +// The Redis client uses the same coordinates as the session store +// (cfg.SessionStorage address/db/prefix + THV_SESSION_REDIS_PASSWORD) so +// admission and the reaper operate on the same keys as session metadata. +// +// The token-store coordinates for the egress-broker sidecar are resolved from +// the vMCP's own identity: the key prefix is derived via +// storage.DeriveKeyPrefix(namespace, vmcpName) (the auth server the vMCP +// embeds owns the token rows) and the Redis address from the operator-injected +// THV_UNTRUSTED_TOKEN_STORE_REDIS_ADDR env var. When the address is absent the +// TokenStore is left nil and the broker fails closed at startup. +// +//nolint:gocyclo // startup wiring with explicit fail-loud gates is clearest linear. +func buildUntrustedStack( + ctx context.Context, + cfg *config.Config, + backends []vmcp.Backend, + namespace string, + vmcpName string, + meterProvider metric.MeterProvider, +) (*untrustedBundle, error) { + if !groupHasUntrustedBackend(backends) { + return nil, nil + } + + if cfg.SessionStorage == nil || cfg.SessionStorage.Provider != "redis" { + return nil, fmt.Errorf( + "untrusted backend present in group %q but session storage is not Redis-backed; "+ + "untrusted mode requires sessionStorage.provider=redis for cross-replica admission", cfg.Group) + } + if namespace == "" || namespace == "local" { + return nil, fmt.Errorf( + "untrusted backend present in group %q but the vMCP namespace could not be resolved; "+ + "untrusted mode is Kubernetes-only", cfg.Group) + } + + keyPrefix := cfg.SessionStorage.KeyPrefix + if keyPrefix == "" { + keyPrefix = defaultSessionKeyPrefix + } + redisClient, err := tcredis.NewClient(ctx, &tcredis.Config{ + Addr: cfg.SessionStorage.Address, + Password: os.Getenv(config.RedisPasswordEnvVar), + DB: int(cfg.SessionStorage.DB), + }) + if err != nil { + return nil, fmt.Errorf("failed to connect to Redis for untrusted mode: %w", err) + } + + k8sClient, err := untrustedK8sClient() + if err != nil { + _ = redisClient.Close() + return nil, fmt.Errorf("failed to build K8s client for untrusted mode: %w", err) + } + + stack, err := untrusted.NewStack(untrusted.WiringConfig{ + K8sClient: k8sClient, + RedisClient: redisClient, + KeyPrefix: keyPrefix, + Namespace: namespace, + VMCPUId: untrustedVMCPUId(), + Admission: untrusted.AdmissionConfig{CacheCapacity: untrustedCacheCapacity()}, + TokenStore: resolveTokenStoreConfig(namespace, vmcpName), + MeterProvider: meterProvider, + }) + if err != nil { + _ = redisClient.Close() + return nil, fmt.Errorf("failed to wire untrusted stack: %w", err) + } + + slog.Info("untrusted backend mode enabled", "group", cfg.Group, "namespace", namespace) + return &untrustedBundle{ + resolver: stack.Resolver, lifecycle: stack.Lifecycle, reaper: stack.Reaper, redisClient: redisClient, + }, nil +} + +// resolveTokenStoreConfig builds the egress-broker token-store coordinates from +// the vMCP's identity and the operator-injected Redis address. Returns nil +// (broker fails closed) when the address is absent. +func resolveTokenStoreConfig(namespace, vmcpName string) *untrusted.TokenStoreConfig { + addr := os.Getenv(untrustedTokenStoreAddrEnvVar) + if addr == "" { + slog.Warn("untrusted mode active but the auth-server Redis address is not configured; "+ + "egress-broker sidecars will fail closed (no upstream credential injection)", + "env_var", untrustedTokenStoreAddrEnvVar) + return nil + } + return &untrusted.TokenStoreConfig{ + RedisAddr: addr, + KeyPrefix: authstorage.DeriveKeyPrefix(namespace, vmcpName), + } +} + +// untrustedK8sClient builds the in-cluster controller-runtime client the pod +// lifecycle uses. The scheme needs core pods + apps statefulsets (via +// clientgoscheme) + the MCPServer CRD (for owner references). +func untrustedK8sClient() (client.Client, error) { + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to add client-go scheme: %w", err) + } + if err := mcpv1beta1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to add MCP v1beta1 scheme: %w", err) + } + return thvk8s.NewControllerRuntimeClient(scheme) +} + +// untrustedVMCPUId returns a stable identity for this vMCP instance used for +// reaper heartbeats and zombie detection. The pod UID (downward API via +// VMCP_POD_UID) is preferred; a random UUID is the local-dev fallback. +func untrustedVMCPUId() string { + if podUID := os.Getenv("VMCP_POD_UID"); podUID != "" { + return podUID + } + return uuid.NewString() +} + +// untrustedCacheCapacity resolves the session cache capacity the admission +// global-cap factor applies to. The vMCP serve path uses the session manager +// default (sessionmanager.defaultCacheCapacity = 1000); there is no config +// knob today, so this mirrors that default. +func untrustedCacheCapacity() int { + return 1000 +} + +// runUntrustedReaper starts the reaper goroutine bound to ctx and returns a +// shutdown function that closes the shared Redis client after the reaper's Run +// loop returns (ctx cancellation). sessionExists is the session manager's +// storage-backed liveness probe (the storage seam — this package never +// re-derives the session key shape). The reaper stops itself when ctx is done; +// the returned func only releases the connection. +func runUntrustedReaper( + ctx context.Context, + b *untrustedBundle, + sessionExists func(ctx context.Context, sessionID string) bool, +) func() { + done := make(chan struct{}) + go func() { + defer close(done) + b.reaper.Run(ctx, sessionExists) + }() + return func() { + <-done + if err := b.redisClient.Close(); err != nil { + slog.Warn("failed to close untrusted-mode Redis client", "error", err) + } + } +} diff --git a/pkg/vmcp/cli/untrusted_test.go b/pkg/vmcp/cli/untrusted_test.go new file mode 100644 index 0000000000..164d6bb193 --- /dev/null +++ b/pkg/vmcp/cli/untrusted_test.go @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/vmcp" + "github.com/stacklok/toolhive/pkg/vmcp/config" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" +) + +func untrustedTestBackend(id string) vmcp.Backend { + return vmcp.Backend{ + ID: id, + Name: id, + Metadata: map[string]string{ + untrusted.MetadataKeyUntrusted: "true", + untrusted.MetadataKeyMCPServerUID: "uid-1", + }, + } +} + +// trustedTestBackend returns a backend with no untrusted metadata. +func trustedTestBackend() vmcp.Backend { + return vmcp.Backend{ID: "a", Name: "a", Metadata: map[string]string{}} +} + +func TestGroupHasUntrustedBackend(t *testing.T) { + t.Parallel() + + assert.False(t, groupHasUntrustedBackend(nil), "no backends = off") + assert.False(t, groupHasUntrustedBackend([]vmcp.Backend{trustedTestBackend()}), "trusted-only = off") + assert.True(t, groupHasUntrustedBackend( + []vmcp.Backend{trustedTestBackend(), untrustedTestBackend("b")}), "any untrusted = on") +} + +//nolint:paralleltest // t.Setenv modifies the process environment; subtests cannot run in parallel. +func TestResolveTokenStoreConfig(t *testing.T) { + t.Run("absent addr returns nil (broker fails closed)", func(t *testing.T) { + // No THV_UNTRUSTED_TOKEN_STORE_REDIS_ADDR set. + assert.Nil(t, resolveTokenStoreConfig("toolhive", "my-vmcp")) + }) + + t.Run("addr present derives the per-tenant prefix from vMCP identity", func(t *testing.T) { + t.Setenv(untrustedTokenStoreAddrEnvVar, "redis.auth:6379") + ts := resolveTokenStoreConfig("toolhive", "my-vmcp") + require.NotNil(t, ts) + assert.Equal(t, "redis.auth:6379", ts.RedisAddr) + // DeriveKeyPrefix(ns, name) — the same prefix the embedded auth server uses. + assert.Equal(t, "thv:auth:{toolhive:my-vmcp}:", ts.KeyPrefix) + assert.Nil(t, ts.KEKSecretRef, "no KEK unless encryption is configured") + }) +} + +// TestBuildUntrustedStack_Gating pins the startup feature gate: the stack is +// only wired when the group actually contains an untrusted backend, and the +// hard prerequisites (Redis session storage, resolvable namespace) fail loudly +// rather than silently degrading. +func TestBuildUntrustedStack_Gating(t *testing.T) { + t.Parallel() + ctx := t.Context() + + redisCfg := &config.Config{ + Group: "g", + SessionStorage: &config.SessionStorageConfig{ + Provider: "redis", + Address: "127.0.0.1:0", // unroutable; only reached when the gate opens + }, + } + + t.Run("off when no untrusted backend (nil, no error)", func(t *testing.T) { + t.Parallel() + bundle, err := buildUntrustedStack(ctx, redisCfg, []vmcp.Backend{trustedTestBackend()}, "toolhive", "vmcp", nil) + require.NoError(t, err) + assert.Nil(t, bundle) + }) + + t.Run("off ignores missing session storage when gate is closed", func(t *testing.T) { + t.Parallel() + noStorage := &config.Config{Group: "g"} + bundle, err := buildUntrustedStack(ctx, noStorage, []vmcp.Backend{trustedTestBackend()}, "toolhive", "vmcp", nil) + require.NoError(t, err) + assert.Nil(t, bundle) + }) + + t.Run("untrusted backend + non-Redis storage is a hard error", func(t *testing.T) { + t.Parallel() + noStorage := &config.Config{Group: "g"} + _, err := buildUntrustedStack(ctx, noStorage, []vmcp.Backend{untrustedTestBackend("b")}, "toolhive", "vmcp", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "sessionStorage.provider=redis") + }) + + t.Run("untrusted backend + unresolvable namespace is a hard error", func(t *testing.T) { + t.Parallel() + _, err := buildUntrustedStack(ctx, redisCfg, []vmcp.Backend{untrustedTestBackend("b")}, "local", "vmcp", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "namespace") + }) +} diff --git a/pkg/vmcp/server/serve.go b/pkg/vmcp/server/serve.go index 43ec94dee3..4a8b519030 100644 --- a/pkg/vmcp/server/serve.go +++ b/pkg/vmcp/server/serve.go @@ -349,6 +349,8 @@ func Serve(ctx context.Context, v core.VMCP, cfg *ServerConfig) (*Server, error) // built directly in Serve from ServerConfig.SessionManagerConfig (a pre-built // *sessionmanager.FactoryConfig that carries the session factory and optimizer // wiring), not via Config→New, so these Config fields are unused on the Serve path. +// - Untrusted: consumed by New into FactoryConfig.Untrusted (session-manager +// untrusted-mode wiring), not a transport field — never mapped into Config. func buildServeConfig(cfg *ServerConfig) *Config { return &Config{ Name: cfg.Name, diff --git a/pkg/vmcp/server/serve_test.go b/pkg/vmcp/server/serve_test.go index fb6b0f45d8..0ac5c0241a 100644 --- a/pkg/vmcp/server/serve_test.go +++ b/pkg/vmcp/server/serve_test.go @@ -344,6 +344,7 @@ func TestBuildServeConfigMapsSharedFields(t *testing.T) { "RateLimiter": {}, // consumed by New to wrap the core (rate-limit decorator) before Serve; not a transport field "Aggregator": {}, // core collaborator: fed to core.New via deriveCoreConfig, not the transport "Authz": {}, // core collaborator: fed to the core admission seam via deriveCoreConfig + "Untrusted": {}, // session-manager wiring: consumed by New into FactoryConfig.Untrusted, not a transport field } // Every field set to a non-zero value so a dropped mapping surfaces as a zero diff --git a/pkg/vmcp/server/server.go b/pkg/vmcp/server/server.go index 991c317aee..8406304799 100644 --- a/pkg/vmcp/server/server.go +++ b/pkg/vmcp/server/server.go @@ -262,6 +262,14 @@ type Config struct { // from the HTTP middleware. When non-nil, Name must be non-empty (the Cedar resource // entity name). Authz *authz.Config + + // Untrusted, when non-nil, enables untrusted-mode behavior in the vMCP session + // manager: untrusted-aware restore-budget selection and DeletePod on session + // Terminate. It is forwarded to sessionmanager.FactoryConfig.Untrusted. Nil + // (the default) leaves untrusted-mode session-manager behavior off; the + // per-session address resolver is installed on Config.SessionFactory separately + // (session.WithUntrustedResolver) by the composition root. + Untrusted *sessionmanager.UntrustedConfig } // Server is the Virtual MCP Server that aggregates multiple backends. @@ -498,6 +506,7 @@ func New( OptimizerFactory: cfg.OptimizerFactory, TelemetryProvider: cfg.TelemetryProvider, AdvertiseFromCore: true, + Untrusted: cfg.Untrusted, } srv, err := Serve(ctx, coreVMCP, deriveServerConfig(resolved, backendRegistry, sessMgrCfg)) @@ -963,12 +972,20 @@ func (s *Server) handleReadiness(w http.ResponseWriter, r *http.Request) { } } -// SessionManager returns the session manager instance. -// This is useful for testing and monitoring. +// SessionManager returns the session manager instance.// This is useful for testing and monitoring. func (s *Server) SessionManager() *transportsession.Manager { return s.sessionManager } +// VMCPSessionManager returns the vMCP session manager (live MultiSessions, +// backend connections, session-storage ownership). Composition roots use it +// to wire storage-derived probes — e.g. the untrusted reaper's SessionExists +// liveness check — through the storage's own API rather than re-deriving its +// key shape. +func (s *Server) VMCPSessionManager() SessionManager { + return s.vmcpSessionMgr +} + // MCPServer returns the underlying mcpcompat *server.MCPServer instance // servicing this vMCP server's /mcp endpoint. // diff --git a/pkg/vmcp/server/session_manager_interface.go b/pkg/vmcp/server/session_manager_interface.go index 547b2e0873..1268e3d818 100644 --- a/pkg/vmcp/server/session_manager_interface.go +++ b/pkg/vmcp/server/session_manager_interface.go @@ -43,6 +43,12 @@ type SessionManager interface { // Terminate terminates the session with the given ID, closing all backend connections. Terminate(sessionID string) (bool, error) + // SessionExists reports whether the session metadata key for sessionID is + // alive in shared storage, through the storage's own API. It is the + // liveness probe for the untrusted-mode reaper's refresh rule and fails + // false on any storage error. + SessionExists(ctx context.Context, sessionID string) bool + // NotifyBackendExpired updates session metadata in storage to reflect that the // backend identified by workloadID is no longer connected. The caller must // supply the session metadata it already holds (e.g. from MultiSession.GetMetadata); diff --git a/pkg/vmcp/server/sessionmanager/factory.go b/pkg/vmcp/server/sessionmanager/factory.go index 38f1747e87..9411ec79ce 100644 --- a/pkg/vmcp/server/sessionmanager/factory.go +++ b/pkg/vmcp/server/sessionmanager/factory.go @@ -26,6 +26,7 @@ import ( vmcpsession "github.com/stacklok/toolhive/pkg/vmcp/session" "github.com/stacklok/toolhive/pkg/vmcp/session/optimizerdec" sessiontypes "github.com/stacklok/toolhive/pkg/vmcp/session/types" + vmcpsessionuntrusted "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) const instrumentationName = "github.com/stacklok/toolhive/pkg/vmcp" @@ -81,6 +82,27 @@ type FactoryConfig struct { // effect when the optimizer is disabled. The legacy server.New path leaves this // false, so its optimizer decorator is unchanged. AdvertiseFromCore bool + + // Untrusted, when non-nil, enables untrusted-mode behavior in the session + // manager: untrusted-aware restore budget selection and DeletePod on + // Terminate. The Resolver itself is installed on cfg.Base by the caller + // (WithUntrustedResolver); the reaper returned alongside the stack is + // started by the composition root that owns the process context. + Untrusted *UntrustedConfig +} + +// UntrustedConfig carries the untrusted-mode wiring for the session manager. +type UntrustedConfig struct { + // Resolver, when non-nil, marks untrusted backends present in restored + // backend sets for restore-budget selection. It is the same instance the + // base session factory was built with (informational here; the factory + // performs the actual rewrite). + Resolver vmcpsessionuntrusted.BackendAddressResolver + // Lifecycle deletes session pods on Terminate. + Lifecycle vmcpsessionuntrusted.PodLifecycle + // CacheCapacity mirrors FactoryConfig.CacheCapacity (0 = default) so the + // untrusted-aware budget selection has the resolved value. + CacheCapacity int } // resolveOptimizer wires the optimizer factory from cfg, applying telemetry diff --git a/pkg/vmcp/server/sessionmanager/session_manager.go b/pkg/vmcp/server/sessionmanager/session_manager.go index 2d1161fd32..93b596d8ee 100644 --- a/pkg/vmcp/server/sessionmanager/session_manager.go +++ b/pkg/vmcp/server/sessionmanager/session_manager.go @@ -29,7 +29,9 @@ import ( "github.com/stacklok/toolhive/pkg/vmcp" "github.com/stacklok/toolhive/pkg/vmcp/optimizer" vmcpsession "github.com/stacklok/toolhive/pkg/vmcp/session" + "github.com/stacklok/toolhive/pkg/vmcp/session/binding" sessiontypes "github.com/stacklok/toolhive/pkg/vmcp/session/types" + vmcpsessionuntrusted "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) const ( @@ -89,6 +91,10 @@ type Manager struct { // path can build a per-session optimizer over the core's tools. The store and // cleanup remain owned by this Manager (cleanup returned from New). optimizerFactory func(context.Context, []mcpserver.ServerTool) (optimizer.Optimizer, error) + + // untrusted, when non-nil, enables untrusted-mode behavior (restore budget + // selection, DeletePod on Terminate). Nil = untrusted mode off. + untrusted *UntrustedConfig } // New creates a Manager backed by the given SessionDataStorage and backend @@ -124,6 +130,7 @@ func New( sm := &Manager{ storage: storage, backendReg: backendRegistry, + untrusted: cfg.Untrusted, } // Surface the resolved optimizer factory to the Serve path ONLY when @@ -147,6 +154,10 @@ func New( slog.Warn("session cache: error closing evicted session", "session_id", id, "error", closeErr) } + // Untrusted sessions own per-session backend pods: eviction must + // release them like Terminate does, not leave them for the reaper's + // idle TTL (the reaper is the backstop, not the primary teardown). + sm.deleteUntrustedSessionPods(id, sess.GetMetadata()) slog.Warn("session cache: session evicted from node-local cache", "session_id", id) }, @@ -210,6 +221,13 @@ const restoreMetadataWriteTimeout = 5 * time.Second // (15 s) since both involve backend HTTP round-trips. const restoreSessionTimeout = 15 * time.Second +// restoreSessionTimeoutUntrusted bounds factory.RestoreSession when the +// restored backend set includes at least one untrusted backend: the restore +// may cold-start per-session pods (scheduling + image pull + container start), +// which the 15 s trusted budget cannot cover. 60 s is the cold-start SLO +// (ADR-0001 §7.3); the per-pod hard cap stays 120 s inside WaitReady. +const restoreSessionTimeoutUntrusted = 60 * time.Second + // terminateTimeout is the context deadline applied to storage operations inside // Terminate(). Terminate() is called on client DELETE requests and on auth // failures, each of which performs at most one Delete + one Load + one Store @@ -465,6 +483,28 @@ func (sm *Manager) Validate(sessionID string) (isTerminated bool, err error) { return false, nil } +// SessionExists reports whether the session metadata key for sessionID is +// alive in shared storage, through the storage's own API (the session manager +// owns session storage — no caller may re-derive its key shape). It is the +// liveness probe for the untrusted-mode reaper's refresh rule: a session that +// no longer exists stops getting pod-lease renewals, so its pods are reaped +// at idle-TTL lapse. +// +// Fail-closed on errors: any storage error (including Redis unavailability) +// reports false, so a pod is never kept alive on unreadable evidence. A +// placeholder terminated by another replica (MetadataKeyTerminated) also +// reports false. +func (sm *Manager) SessionExists(ctx context.Context, sessionID string) bool { + if sessionID == "" { + return false + } + metadata, err := sm.storage.Load(ctx, sessionID) + if err != nil { + return false + } + return metadata[MetadataKeyTerminated] != MetadataValTrue +} + // Terminate implements the SDK's SessionIdManager.Terminate(). // // The two session types are handled asymmetrically to prevent a race condition @@ -511,6 +551,7 @@ func (sm *Manager) Terminate(sessionID string) (isNotAllowed bool, err error) { if deleteErr := sm.storage.Delete(ctx, sessionID); deleteErr != nil { return false, fmt.Errorf("Manager.Terminate: failed to delete session from storage: %w", deleteErr) } + sm.deleteUntrustedSessionPods(sessionID, metadata) slog.Info("Manager.Terminate: session terminated", "session_id", sessionID) return false, nil } @@ -733,9 +774,11 @@ func (sm *Manager) loadSession(ctx context.Context, sessionID string) (vmcpsessi return nil, transportsession.ErrSessionNotFound } - restoreCtx, restoreCancel := context.WithTimeout(context.WithoutCancel(ctx), restoreSessionTimeout) + allBackends := sm.listAllBackends(ctx) + budget := sm.restoreBudgetFor(metadata, allBackends) + restoreCtx, restoreCancel := context.WithTimeout(context.WithoutCancel(ctx), budget) defer restoreCancel() - restored, restoreErr := sm.factory.RestoreSession(restoreCtx, sessionID, metadata, sm.listAllBackends(restoreCtx)) + restored, restoreErr := sm.factory.RestoreSession(restoreCtx, sessionID, metadata, allBackends) if restoreErr != nil { slog.Warn("Manager.loadSession: failed to restore session from storage", "session_id", sessionID, "error", restoreErr) @@ -829,3 +872,78 @@ func (sm *Manager) listAllBackends(ctx context.Context) []*vmcp.Backend { } return backends } + +// restoreBudgetFor selects the RestoreSession budget: the untrusted budget +// when the restored backend set includes at least one untrusted backend (pod +// cold start may be required), the trusted default otherwise. +func (sm *Manager) restoreBudgetFor(metadata map[string]string, allBackends []*vmcp.Backend) time.Duration { + if sm.untrusted == nil { + return restoreSessionTimeout + } + storedIDs := metadata[vmcpsession.MetadataKeyBackendIDs] + if storedIDs == "" { + return restoreSessionTimeout + } + idSet := make(map[string]struct{}) + for _, p := range strings.Split(storedIDs, ",") { + if t := strings.TrimSpace(p); t != "" { + idSet[t] = struct{}{} + } + } + for _, b := range allBackends { + if b == nil { + continue + } + if _, ok := idSet[b.ID]; ok && b.Metadata[vmcpsessionuntrusted.MetadataKeyUntrusted] == "true" { + return restoreSessionTimeoutUntrusted + } + } + return restoreSessionTimeout +} + +// deleteUntrustedSessionPods best-effort deletes the per-session untrusted +// backend pods of a terminated session. Pod names are deterministic from +// (mcpserverUID, userKey, sessionID), so no lookup is needed. Failures are +// logged and swallowed — the reaper's idle-TTL rule is the backstop. +func (sm *Manager) deleteUntrustedSessionPods(sessionID string, metadata map[string]string) { + if sm.untrusted == nil || sm.untrusted.Lifecycle == nil { + return + } + storedBinding := metadata[sessiontypes.MetadataKeyIdentityBinding] + iss, sub, ok := binding.Parse(storedBinding) + if !ok { + // Anonymous sessions never own untrusted pods (the resolver soft-fails + // them); malformed bindings have no derivable user key either. + return + } + userKey, err := binding.Format(iss, sub) + if err != nil { + return + } + + storedIDs := metadata[vmcpsession.MetadataKeyBackendIDs] + if storedIDs == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), terminateTimeout) + defer cancel() + for _, p := range strings.Split(storedIDs, ",") { + backendID := strings.TrimSpace(p) + if backendID == "" { + continue + } + b := sm.backendReg.Get(ctx, backendID) + if b == nil || b.Metadata[vmcpsessionuntrusted.MetadataKeyUntrusted] != "true" { + continue + } + uid := b.Metadata[vmcpsessionuntrusted.MetadataKeyMCPServerUID] + if uid == "" { + continue + } + podName := vmcpsessionuntrusted.PodNameFor(uid, userKey, sessionID) + if delErr := sm.untrusted.Lifecycle.DeletePod(ctx, podName); delErr != nil { + slog.Warn("Manager.Terminate: failed to delete untrusted session pod; reaper will collect it", + "session_id", sessionID, "backend", backendID, "error", delErr) + } + } +} diff --git a/pkg/vmcp/server/sessionmanager/session_manager_untrusted_test.go b/pkg/vmcp/server/sessionmanager/session_manager_untrusted_test.go new file mode 100644 index 0000000000..16201080ac --- /dev/null +++ b/pkg/vmcp/server/sessionmanager/session_manager_untrusted_test.go @@ -0,0 +1,306 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package sessionmanager + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + + transportsession "github.com/stacklok/toolhive/pkg/transport/session" + "github.com/stacklok/toolhive/pkg/vmcp" + "github.com/stacklok/toolhive/pkg/vmcp/session/binding" + sessionfactorymocks "github.com/stacklok/toolhive/pkg/vmcp/session/mocks" + sessiontypes "github.com/stacklok/toolhive/pkg/vmcp/session/types" + sessionmocks "github.com/stacklok/toolhive/pkg/vmcp/session/types/mocks" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" +) + +// fakeLifecycle records DeletePod calls. +type fakeLifecycle struct { + deleted []string +} + +func (*fakeLifecycle) EnsurePod(_ context.Context, _ untrusted.EnsurePodRequest) (*corev1.Pod, error) { + return nil, nil +} +func (*fakeLifecycle) WaitReady(_ context.Context, _ *corev1.Pod, _ time.Duration) error { + return nil +} +func (f *fakeLifecycle) DeletePod(_ context.Context, name string) error { + f.deleted = append(f.deleted, name) + return nil +} + +//nolint:unparam // uid kept as a parameter for readability; callers consistently use "uid-1" +func untrustedBackendMeta(uid string) map[string]string { + return map[string]string{ + untrusted.MetadataKeyUntrusted: "true", + untrusted.MetadataKeyMCPServerUID: uid, + } +} + +func TestRestoreBudgetFor(t *testing.T) { + t.Parallel() + + newManager := func(t *testing.T, untrustedCfg *UntrustedConfig, backends ...vmcp.Backend) *Manager { + t.Helper() + ctrl := gomock.NewController(t) + sess := newMockSession(t, ctrl, "s", nil) + factory := sessionfactorymocks.NewMockMultiSessionFactory(ctrl) + factory.EXPECT().MakeSessionWithID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(sess, nil).AnyTimes() + registry := &fakeBackendRegistry{backends: backends} + storage := newTestSessionDataStorage(t) + sm, cleanup, err := New(storage, &FactoryConfig{Base: factory, CacheCapacity: 100, Untrusted: untrustedCfg}, registry) + require.NoError(t, err) + t.Cleanup(func() { _ = cleanup(context.Background()) }) + return sm + } + + metadata := func(ids string) map[string]string { + return map[string]string{"vmcp.backend.ids": ids} + } + + t.Run("untrusted backend in restored set selects 60s budget", func(t *testing.T) { + t.Parallel() + sm := newManager(t, &UntrustedConfig{}, vmcp.Backend{ID: "b1", Metadata: untrustedBackendMeta("uid-1")}) + assert.Equal(t, restoreSessionTimeoutUntrusted, sm.restoreBudgetFor(metadata("b1"), sm.listAllBackends(context.Background()))) + }) + + t.Run("trusted-only set keeps 15s budget", func(t *testing.T) { + t.Parallel() + sm := newManager(t, &UntrustedConfig{}, vmcp.Backend{ID: "b1", Metadata: map[string]string{}}) + assert.Equal(t, restoreSessionTimeout, sm.restoreBudgetFor(metadata("b1"), sm.listAllBackends(context.Background()))) + }) + + t.Run("untrusted config nil keeps 15s even with untrusted metadata", func(t *testing.T) { + t.Parallel() + sm := newManager(t, nil, vmcp.Backend{ID: "b1", Metadata: untrustedBackendMeta("uid-1")}) + assert.Equal(t, restoreSessionTimeout, sm.restoreBudgetFor(metadata("b1"), sm.listAllBackends(context.Background()))) + }) + + t.Run("untrusted backend not in restored set keeps 15s", func(t *testing.T) { + t.Parallel() + sm := newManager(t, &UntrustedConfig{}, + vmcp.Backend{ID: "b1", Metadata: map[string]string{}}, + vmcp.Backend{ID: "b2", Metadata: untrustedBackendMeta("uid-1")}) + assert.Equal(t, restoreSessionTimeout, sm.restoreBudgetFor(metadata("b1"), sm.listAllBackends(context.Background()))) + }) +} + +func TestTerminateDeletesUntrustedSessionPods(t *testing.T) { + t.Parallel() + + bound, err := binding.Format("https://iss", "sub-1") + require.NoError(t, err) + + setup := func(t *testing.T, lifecycle *fakeLifecycle, backends ...vmcp.Backend) (*Manager, transportsession.DataStorage) { + t.Helper() + ctrl := gomock.NewController(t) + sess := newMockSession(t, ctrl, "sess", nil) + factory := newMockFactory(t, ctrl, sess) + registry := &fakeBackendRegistry{backends: backends} + storage := newTestSessionDataStorage(t) + sm, cleanup, err := New(storage, &FactoryConfig{ + Base: factory, + CacheCapacity: 100, + Untrusted: &UntrustedConfig{Lifecycle: lifecycle}, + }, registry) + require.NoError(t, err) + t.Cleanup(func() { _ = cleanup(context.Background()) }) + return sm, storage + } + + t.Run("full session terminate deletes deterministic pod names", func(t *testing.T) { + t.Parallel() + lifecycle := &fakeLifecycle{} + backend := vmcp.Backend{ID: "b1", Metadata: untrustedBackendMeta("uid-1")} + sm, storage := setup(t, lifecycle, backend) + + sessionID := sm.Generate() + // Upgrade to full session metadata: identity binding + backend IDs. + _, err := storage.Update(context.Background(), sessionID, map[string]string{ + sessiontypes.MetadataKeyIdentityBinding: bound, + "vmcp.backend.ids": "b1", + }) + require.NoError(t, err) + + isNotAllowed, err := sm.Terminate(sessionID) + require.NoError(t, err) + assert.False(t, isNotAllowed) + + userKey, err := binding.Format("https://iss", "sub-1") + require.NoError(t, err) + want := untrusted.PodNameFor("uid-1", userKey, sessionID) + assert.Equal(t, []string{want}, lifecycle.deleted) + }) + + t.Run("trusted backends produce no pod deletes", func(t *testing.T) { + t.Parallel() + lifecycle := &fakeLifecycle{} + backend := vmcp.Backend{ID: "b1", Metadata: map[string]string{}} + sm, storage := setup(t, lifecycle, backend) + + sessionID := sm.Generate() + _, err := storage.Update(context.Background(), sessionID, map[string]string{ + sessiontypes.MetadataKeyIdentityBinding: bound, + "vmcp.backend.ids": "b1", + }) + require.NoError(t, err) + + _, err = sm.Terminate(sessionID) + require.NoError(t, err) + assert.Empty(t, lifecycle.deleted) + }) + + t.Run("anonymous session produces no pod deletes", func(t *testing.T) { + t.Parallel() + lifecycle := &fakeLifecycle{} + backend := vmcp.Backend{ID: "b1", Metadata: untrustedBackendMeta("uid-1")} + sm, storage := setup(t, lifecycle, backend) + + sessionID := sm.Generate() + _, err := storage.Update(context.Background(), sessionID, map[string]string{ + sessiontypes.MetadataKeyIdentityBinding: binding.UnauthenticatedSentinel, + "vmcp.backend.ids": "b1", + }) + require.NoError(t, err) + + _, err = sm.Terminate(sessionID) + require.NoError(t, err) + assert.Empty(t, lifecycle.deleted) + }) +} + +// TestEvictionDeletesUntrustedSessionPods pins the onEvict behavior: an +// LRU-evicted untrusted session must release its per-session pods through the +// lifecycle, same as Terminate (the reaper's idle TTL is the backstop, not +// the primary teardown). +func TestEvictionDeletesUntrustedSessionPods(t *testing.T) { + t.Parallel() + + bound, err := binding.Format("https://iss", "sub-1") + require.NoError(t, err) + metadata := map[string]string{ + sessiontypes.MetadataKeyIdentityBinding: bound, + "vmcp.backend.ids": "b1", + } + + ctrl := gomock.NewController(t) + lifecycle := &fakeLifecycle{} + registry := &fakeBackendRegistry{backends: []vmcp.Backend{{ID: "b1", Metadata: untrustedBackendMeta("uid-1")}}} + storage := newTestSessionDataStorage(t) + + // Sessions are inserted directly: only the LRU eviction path is under + // test, not placeholder/storage ceremony. checkSession is bypassed (no + // Get) so the sessions are never validated against storage. + sessFor := func(id string) *sessionmocks.MockMultiSession { + sess := sessionmocks.NewMockMultiSession(ctrl) + sess.EXPECT().ID().Return(id).AnyTimes() + // GetMetadata must return the session's full metadata: eviction reads + // the identity binding + backend IDs from it. Copy per call — the map + // is shared across two sessions here. + sess.EXPECT().GetMetadata().DoAndReturn(func() map[string]string { + m := make(map[string]string, len(metadata)) + for k, v := range metadata { + m[k] = v + } + return m + }).AnyTimes() + sess.EXPECT().Close().Return(nil).AnyTimes() + return sess + } + factory := sessionfactorymocks.NewMockMultiSessionFactory(ctrl) + factory.EXPECT().MakeSessionWithID(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(sessFor("unused"), nil).AnyTimes() + + // Capacity 1: inserting the second session evicts the first (LRU). + sm, cleanup, err := New(storage, &FactoryConfig{ + Base: factory, + CacheCapacity: 1, + Untrusted: &UntrustedConfig{Lifecycle: lifecycle}, + }, registry) + require.NoError(t, err) + t.Cleanup(func() { _ = cleanup(context.Background()) }) + + first := sm.Generate() + second := sm.Generate() + + sm.sessions.Set(first, sessFor(first)) + require.Empty(t, lifecycle.deleted, "no eviction before the cache is full") + + sm.sessions.Set(second, sessFor(second)) + + userKey, err := binding.Format("https://iss", "sub-1") + require.NoError(t, err) + want := untrusted.PodNameFor("uid-1", userKey, first) + require.Equal(t, []string{want}, lifecycle.deleted, + "evicting the LRU session must delete its untrusted pods") +} + +// TestSessionExists pins the storage-seam liveness probe the untrusted reaper +// consumes: live session → true; terminated placeholder, unknown session, or +// storage error → false (fail closed). +func TestSessionExists(t *testing.T) { + t.Parallel() + ctx := context.Background() + + setup := func(t *testing.T) (*Manager, transportsession.DataStorage) { + t.Helper() + ctrl := gomock.NewController(t) + sess := newMockSession(t, ctrl, "sess", nil) + sm, cleanup, err := New(newTestSessionDataStorage(t), + &FactoryConfig{Base: newMockFactory(t, ctrl, sess), CacheCapacity: 10}, + &fakeBackendRegistry{}) + require.NoError(t, err) + t.Cleanup(func() { _ = cleanup(context.Background()) }) + return sm, sm.storage + } + + t.Run("live session exists", func(t *testing.T) { + t.Parallel() + sm, _ := setup(t) + sessionID := sm.Generate() + assert.True(t, sm.SessionExists(ctx, sessionID)) + }) + + t.Run("terminated placeholder does not exist", func(t *testing.T) { + t.Parallel() + sm, storage := setup(t) + sessionID := sm.Generate() + _, err := storage.Update(ctx, sessionID, map[string]string{MetadataKeyTerminated: MetadataValTrue}) + require.NoError(t, err) + assert.False(t, sm.SessionExists(ctx, sessionID)) + }) + + t.Run("unknown session does not exist", func(t *testing.T) { + t.Parallel() + sm, _ := setup(t) + assert.False(t, sm.SessionExists(ctx, "never-created")) + }) + + t.Run("empty session ID does not exist", func(t *testing.T) { + t.Parallel() + sm, _ := setup(t) + assert.False(t, sm.SessionExists(ctx, "")) + }) + + t.Run("storage error fails closed", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + sess := newMockSession(t, ctrl, "sess", nil) + sm, cleanup, err := New(alwaysFailDataStorage{}, + &FactoryConfig{Base: newMockFactory(t, ctrl, sess), CacheCapacity: 10}, + &fakeBackendRegistry{}) + require.NoError(t, err) + t.Cleanup(func() { _ = cleanup(context.Background()) }) + assert.False(t, sm.SessionExists(ctx, "any")) + }) +} diff --git a/pkg/vmcp/session/factory.go b/pkg/vmcp/session/factory.go index 14e3115a6d..0818e95382 100644 --- a/pkg/vmcp/session/factory.go +++ b/pkg/vmcp/session/factory.go @@ -15,6 +15,7 @@ import ( "time" "github.com/stacklok/toolhive/pkg/auth" + "github.com/stacklok/toolhive/pkg/k8s" transportsession "github.com/stacklok/toolhive/pkg/transport/session" "github.com/stacklok/toolhive/pkg/vmcp" vmcpauth "github.com/stacklok/toolhive/pkg/vmcp/auth" @@ -22,6 +23,7 @@ import ( "github.com/stacklok/toolhive/pkg/vmcp/session/internal/backend" "github.com/stacklok/toolhive/pkg/vmcp/session/internal/security" sessiontypes "github.com/stacklok/toolhive/pkg/vmcp/session/types" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) const ( @@ -112,6 +114,7 @@ type backendConnector func( // defaultMultiSessionFactory is the production MultiSessionFactory implementation. type defaultMultiSessionFactory struct { connector backendConnector + resolver untrusted.BackendAddressResolver // nil = untrusted mode off maxConcurrency int backendInitTimeout time.Duration } @@ -139,6 +142,15 @@ func WithBackendInitTimeout(d time.Duration) MultiSessionFactoryOption { } } +// WithUntrustedResolver installs the per-session backend address resolver for +// untrusted backends. A nil resolver (the default) leaves the trusted-mode +// session path bit-for-bit unchanged. +func WithUntrustedResolver(r untrusted.BackendAddressResolver) MultiSessionFactoryOption { + return func(f *defaultMultiSessionFactory) { + f.resolver = r + } +} + // NewSessionFactory creates a MultiSessionFactory that connects to backends // over HTTP using the given outgoing auth registry. func NewSessionFactory(registry vmcpauth.OutgoingAuthRegistry, opts ...MultiSessionFactoryOption) MultiSessionFactory { @@ -296,10 +308,23 @@ func populateBackendMetadata(transportSess transportsession.Session, results []i // WITHOUT applying the session-binding security wrapper. // Callers are responsible for wrapping the result with the appropriate decorator // (BindSession for new sessions, RestoreSessionBinding for restored ones). +// +// identityIssuer/identitySubject carry the (iss, sub) tuple the untrusted +// resolver binds pods to: from the live identity on the create path, from the +// stored binding on the restore path. Both empty = anonymous (untrusted +// backends soft-fail). +// +// When untrusted mode provisions a FRESH pod for a backend (EnsurePod create, +// not adopt), that backend's stored Mcp-Session-Id hint is dropped before the +// connector runs: the hint names a backend-side session on the pod that was +// just deleted — a backend that trusts it would resume state from the pod's +// previous incarnation. A stale hint is never sent to a fresh pod. func (f *defaultMultiSessionFactory) makeBaseSession( ctx context.Context, sessID string, identity *auth.Identity, + identityIssuer string, + identitySubject string, backends []*vmcp.Backend, sessionHints map[string]string, ) *defaultMultiSession { @@ -313,6 +338,28 @@ func (f *defaultMultiSessionFactory) makeBaseSession( } backends = filtered + // Untrusted seam: rewrite per-session backend targets to their + // single-tenant pods before any connection is opened. The resolver + // soft-fails individual backends; trusted backends pass through + // unmodified. Nil resolver = untrusted mode off (behavior unchanged). + if f.resolver != nil { + sessRef := untrusted.SessionRef{ + SessionID: sessID, + Issuer: identityIssuer, + Subject: identitySubject, + Namespace: currentNamespace(), + } + // Hint-drop seam: when the lifecycle CREATES a pod (rather than + // adopting the session's existing one), the backend's stored hint is + // stale — it names a backend-side session on the pod that was just + // deleted. The callback is wired through the resolver and fires only + // on fresh creates; makeBaseSession runs single-threaded here, so the + // plain-map write needs no synchronization. + backends = f.resolver.ResolveTargets(ctx, sessRef, backends, func(backendID string) { + delete(sessionHints, backendID) + }) + } + rawResults := make([]*initResult, len(backends)) sem := make(chan struct{}, f.maxConcurrency) var wg sync.WaitGroup @@ -378,7 +425,11 @@ func (f *defaultMultiSessionFactory) makeSession( identity *auth.Identity, backends []*vmcp.Backend, ) (MultiSession, error) { - baseSession := f.makeBaseSession(ctx, sessID, identity, backends, nil) + // Extract (iss, sub) for the untrusted resolver — the same tuple + // BindSession extracts (security.go extractBindingID). Anonymous sessions + // carry empty halves; the resolver soft-fails their untrusted backends. + iss, sub := identityClaims(identity) + baseSession := f.makeBaseSession(ctx, sessID, identity, iss, sub, backends, nil) // Apply session binding: extracts the (iss, sub) identity tuple, stores it in // session metadata under MetadataKeyIdentityBinding, and wraps the session with @@ -418,6 +469,10 @@ func (f *defaultMultiSessionFactory) RestoreSession( } // Filter allBackends to the subset originally connected in this session. + // sessionHints is non-nil even when empty so the hint-drop callback in + // makeBaseSession (untrusted mode) can always delete from it. + sessionHints := make(map[string]string, len(allBackends)) + filteredBackends := filterBackendsByStoredIDs(allBackends, storedBackendIDs) // Validate and read the stored identity binding. This key is written by @@ -444,15 +499,19 @@ func (f *defaultMultiSessionFactory) RestoreSession( // a non-nil *auth.Identity is always fully populated (see pkg/auth/identity.go). // Backend connectors receive nil identity; live tool calls already carry a // complete identity on req.Context() from TokenValidator.Middleware. See #5336. + var restoredIss, restoredSub string if !binding.IsUnauthenticated(storedBinding) { - if _, _, ok := binding.Parse(storedBinding); !ok { + iss, sub, ok := binding.Parse(storedBinding) + if !ok { return nil, fmt.Errorf("RestoreSession: stored identity binding is malformed: %q", storedBinding) } + restoredIss, restoredSub = iss, sub } // Extract stored per-backend session IDs as hints so each backend can // resume its session (via Mcp-Session-Id) rather than starting a new one. - sessionHints := make(map[string]string, len(filteredBackends)) + // A hint for an untrusted backend whose pod must be recreated is dropped + // by the resolver's fresh-pod callback before the connector runs. for _, b := range filteredBackends { if hint := storedMetadata[MetadataKeyBackendSessionPrefix+b.ID]; hint != "" { sessionHints[b.ID] = hint @@ -460,8 +519,10 @@ func (f *defaultMultiSessionFactory) RestoreSession( } // Build the base session (backend connections + routing table) without the - // security wrapper. Pass nil identity — see comment above. - baseSession := f.makeBaseSession(ctx, id, nil, filteredBackends, sessionHints) + // security wrapper. Pass nil identity — see comment above. The restored + // (iss, sub) from the parsed binding feeds the untrusted resolver, which + // finds the session's existing pods by deterministic name. + baseSession := f.makeBaseSession(ctx, id, nil, restoredIss, restoredSub, filteredBackends, sessionHints) // Restore only the identity-binding key from stored metadata. The other // keys (MetadataKeyBackendIDs, MetadataKeyBackendSessionPrefix.*) are @@ -478,6 +539,25 @@ func (f *defaultMultiSessionFactory) RestoreSession( return restored, nil } +// identityClaims extracts (iss, sub) from an identity the same way +// security.extractBindingID does. Missing/non-string claims yield empty +// strings — the untrusted resolver treats that as anonymous and soft-fails +// untrusted backends, matching BindSession's own fail-closed posture (a +// malformed identity never binds a pod). +func identityClaims(identity *auth.Identity) (iss, sub string) { + if identity == nil { + return "", "" + } + iss, _ = identity.Claims["iss"].(string) + sub, _ = identity.Claims["sub"].(string) + return iss, sub +} + +// currentNamespace resolves the vMCP's own namespace for untrusted pod +// provisioning. Var indirection keeps the session package testable without an +// in-cluster environment. +var currentNamespace = k8s.GetCurrentNamespace + // filterBackendsByStoredIDs returns the subset of allBackends whose ID appears in // the comma-separated storedIDs string. If storedIDs is empty, nil is returned (no backends). // diff --git a/pkg/vmcp/session/factory_untrusted_test.go b/pkg/vmcp/session/factory_untrusted_test.go new file mode 100644 index 0000000000..b47eb13ef1 --- /dev/null +++ b/pkg/vmcp/session/factory_untrusted_test.go @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/auth" + "github.com/stacklok/toolhive/pkg/vmcp" + "github.com/stacklok/toolhive/pkg/vmcp/session/binding" + internalbk "github.com/stacklok/toolhive/pkg/vmcp/session/internal/backend" + sessiontypes "github.com/stacklok/toolhive/pkg/vmcp/session/types" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" +) + +// captureResolver records the SessionRef it was invoked with and rewrites +// nothing (passes backends through). newPodFor, when non-empty, fires the +// onNewPod callback for that backend ID (simulating a fresh pod create). +type captureResolver struct { + called int + sessRefs []untrusted.SessionRef + newPodFor string +} + +func (r *captureResolver) ResolveTargets( + _ context.Context, sess untrusted.SessionRef, backends []*vmcp.Backend, onNewPod func(string), +) []*vmcp.Backend { + r.called++ + r.sessRefs = append(r.sessRefs, sess) + if r.newPodFor != "" && onNewPod != nil { + onNewPod(r.newPodFor) + } + return backends +} + +//nolint:unparam // connector signature fixed by backendConnector +func passThroughConnector( + _ context.Context, _ *vmcp.BackendTarget, _ *auth.Identity, _ string, +) (internalbk.Session, *vmcp.CapabilityList, error) { + return &mockConnectedBackend{sessID: "bs-1"}, &vmcp.CapabilityList{}, nil +} + +// identityWithIssSub builds an *auth.Identity carrying the (iss, sub) claims +// the untrusted resolver reads. (identityWithClaims already exists in this +// package's test files with a different shape.) +func identityWithIssSub(iss, sub string) *auth.Identity { + return &auth.Identity{ + PrincipalInfo: auth.PrincipalInfo{Claims: map[string]any{"iss": iss, "sub": sub}}, + Token: "bearer-token", + } +} + +//nolint:unparam // id kept as a parameter for readability; callers consistently use "b1" +func testBackend(id string) *vmcp.Backend { + return &vmcp.Backend{ + ID: id, + Name: id, + BaseURL: "http://svc:8080/mcp", + TransportType: "streamable-http", + Metadata: map[string]string{}, + } +} + +func TestFactoryResolverSeam(t *testing.T) { + t.Parallel() + + t.Run("nil resolver: trusted path unchanged", func(t *testing.T) { + t.Parallel() + factory := newSessionFactoryWithConnector(passThroughConnector) + sess, err := factory.MakeSessionWithID(context.Background(), "sess-1", + identityWithIssSub("iss", "sub"), []*vmcp.Backend{testBackend("b1")}) + require.NoError(t, err) + require.NoError(t, sess.Close()) + }) + + t.Run("create path: resolver invoked with identity (iss, sub)", func(t *testing.T) { + t.Parallel() + resolver := &captureResolver{} + factory := newSessionFactoryWithConnector(passThroughConnector, WithUntrustedResolver(resolver)) + + sess, err := factory.MakeSessionWithID(context.Background(), "sess-1", + identityWithIssSub("https://issuer", "user-1"), []*vmcp.Backend{testBackend("b1")}) + require.NoError(t, err) + require.NoError(t, sess.Close()) + require.Equal(t, 1, resolver.called) + ref := resolver.sessRefs[0] + assert.Equal(t, "sess-1", ref.SessionID) + assert.Equal(t, "https://issuer", ref.Issuer) + assert.Equal(t, "user-1", ref.Subject) + }) + + t.Run("restore path: resolver invoked with parsed binding (iss, sub), not nil identity", func(t *testing.T) { + t.Parallel() + resolver := &captureResolver{} + factory := newSessionFactoryWithConnector(passThroughConnector, WithUntrustedResolver(resolver)) + + storedBinding, err := binding.Format("https://issuer", "user-1") + require.NoError(t, err) + metadata := map[string]string{ + MetadataKeyBackendIDs: "b1", + sessiontypes.MetadataKeyIdentityBinding: storedBinding, + } + sess, err := factory.RestoreSession(context.Background(), "sess-restored", metadata, []*vmcp.Backend{testBackend("b1")}) + require.NoError(t, err) + require.NoError(t, sess.Close()) + require.Equal(t, 1, resolver.called) + ref := resolver.sessRefs[0] + assert.Equal(t, "sess-restored", ref.SessionID) + assert.Equal(t, "https://issuer", ref.Issuer) + assert.Equal(t, "user-1", ref.Subject) + }) + + t.Run("restore with stale hint + fresh pod: hint is not sent to the connector", func(t *testing.T) { + t.Parallel() + resolver := &captureResolver{newPodFor: "b1"} + + hints := make(chan string, 1) + connector := func( + _ context.Context, _ *vmcp.BackendTarget, _ *auth.Identity, hint string, + ) (internalbk.Session, *vmcp.CapabilityList, error) { + hints <- hint + return &mockConnectedBackend{sessID: "bs-new"}, &vmcp.CapabilityList{}, nil + } + factory := newSessionFactoryWithConnector(connector, WithUntrustedResolver(resolver)) + + storedBinding, err := binding.Format("https://issuer", "user-1") + require.NoError(t, err) + metadata := map[string]string{ + MetadataKeyBackendIDs: "b1", + MetadataKeyBackendSessionPrefix + "b1": "stale-hint-from-deleted-pod", + sessiontypes.MetadataKeyIdentityBinding: storedBinding, + } + sess, err := factory.RestoreSession(context.Background(), "sess-restored", metadata, []*vmcp.Backend{testBackend("b1")}) + require.NoError(t, err) + require.NoError(t, sess.Close()) + + select { + case hint := <-hints: + assert.Empty(t, hint, "a stale hint must never be sent to a freshly created pod") + default: + t.Fatal("connector was not invoked") + } + }) + + t.Run("restore with hint + adopted pod: hint is preserved and sent", func(t *testing.T) { + t.Parallel() + resolver := &captureResolver{} // no newPodFor: the pod was adopted + + hints := make(chan string, 1) + connector := func( + _ context.Context, _ *vmcp.BackendTarget, _ *auth.Identity, hint string, + ) (internalbk.Session, *vmcp.CapabilityList, error) { + hints <- hint + return &mockConnectedBackend{sessID: "bs-1"}, &vmcp.CapabilityList{}, nil + } + factory := newSessionFactoryWithConnector(connector, WithUntrustedResolver(resolver)) + + storedBinding, err := binding.Format("https://issuer", "user-1") + require.NoError(t, err) + metadata := map[string]string{ + MetadataKeyBackendIDs: "b1", + MetadataKeyBackendSessionPrefix + "b1": "live-hint", + sessiontypes.MetadataKeyIdentityBinding: storedBinding, + } + sess, err := factory.RestoreSession(context.Background(), "sess-restored", metadata, []*vmcp.Backend{testBackend("b1")}) + require.NoError(t, err) + require.NoError(t, sess.Close()) + + select { + case hint := <-hints: + assert.Equal(t, "live-hint", hint, "an adopted pod's hint must be sent so the backend resumes") + default: + t.Fatal("connector was not invoked") + } + }) + + t.Run("restore path: anonymous session yields empty (iss, sub)", func(t *testing.T) { + t.Parallel() + resolver := &captureResolver{} + factory := newSessionFactoryWithConnector(passThroughConnector, WithUntrustedResolver(resolver)) + + metadata := map[string]string{ + MetadataKeyBackendIDs: "b1", + sessiontypes.MetadataKeyIdentityBinding: binding.UnauthenticatedSentinel, + } + sess, err := factory.RestoreSession(context.Background(), "sess-anon", metadata, []*vmcp.Backend{testBackend("b1")}) + require.NoError(t, err) + require.NoError(t, sess.Close()) + require.Equal(t, 1, resolver.called) + ref := resolver.sessRefs[0] + assert.Empty(t, ref.Issuer) + assert.Empty(t, ref.Subject) + }) +} diff --git a/pkg/vmcp/session/untrusted/addressing.go b/pkg/vmcp/session/untrusted/addressing.go new file mode 100644 index 0000000000..b08890852f --- /dev/null +++ b/pkg/vmcp/session/untrusted/addressing.go @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "net/url" + "strconv" + "strings" + "time" + + "k8s.io/apimachinery/pkg/types" + + "github.com/stacklok/toolhive/pkg/vmcp" +) + +// MetadataKeyUntrusted marks a discovered backend as untrusted (set by the +// k8s workload discoverer from MCPServer.spec.untrusted). +const MetadataKeyUntrusted = "toolhive.stacklok.dev/untrusted" + +// MetadataKeyMCPServerUID carries the MCPServer CR UID on a discovered +// backend (rename-safe identity; set by the k8s workload discoverer). +const MetadataKeyMCPServerUID = "toolhive.stacklok.dev/mcpserver-uid" + +// MetadataKeyUntrustedPodName records which pod serves this session's +// backend. Observability only; routing uses BaseURL. +const MetadataKeyUntrustedPodName = "toolhive.stacklok.dev/untrusted-pod" + +// MetadataKeyUntrustedPodUID records the pod UID for observability. +const MetadataKeyUntrustedPodUID = "toolhive.stacklok.dev/untrusted-pod-uid" + +// BackendAddressResolver rewrites per-session backend targets for untrusted +// backends so they point at the session's single-tenant pod. +type BackendAddressResolver interface { + // ResolveTargets returns backends for this session with untrusted backends' + // BaseURL rewritten to their per-session pod. May create pods. Soft-fails + // individual backends (absent from the result), matching the partial-init + // semantics of the session factory. Trusted backends pass through + // unmodified. Registry backends are never mutated. + // + // onNewPod, when non-nil, is invoked with the backend ID each time + // provisioning CREATES a fresh pod (not when an existing pod is adopted): + // the caller drops the backend's stale Mcp-Session-Id hint so a session ID + // from the deleted pod's previous incarnation is never sent to the new pod. + ResolveTargets( + ctx context.Context, sess SessionRef, backends []*vmcp.Backend, onNewPod func(backendID string), + ) []*vmcp.Backend +} + +// podAddressResolver is the production BackendAddressResolver. +type podAddressResolver struct { + lifecycle PodLifecycle + admission Admission + readyBudget time.Duration + tokenStore *TokenStoreConfig + metrics *untrustedMetrics +} + +// NewPodAddressResolver creates the resolver. readyBudget bounds the per-pod +// cold-start wait (DefaultReadyBudget when zero). +func NewPodAddressResolver(lifecycle PodLifecycle, adm Admission, readyBudget time.Duration) BackendAddressResolver { + if readyBudget <= 0 { + readyBudget = DefaultReadyBudget + } + return &podAddressResolver{lifecycle: lifecycle, admission: adm, readyBudget: readyBudget} +} + +// podAddressResolverWithTokenStore is the internal constructor that also wires +// the egress-broker token-store coordinates and metrics into every provisioned +// pod. Kept unexported: production wiring goes through NewStack, which resolves +// the TokenStoreConfig and metrics once; NewPodAddressResolver stays the public +// seam for tests that exercise the resolver without a token store or metrics. +func podAddressResolverWithTokenStore( + lifecycle PodLifecycle, adm Admission, readyBudget time.Duration, ts *TokenStoreConfig, m *untrustedMetrics, +) *podAddressResolver { + r := NewPodAddressResolver(lifecycle, adm, readyBudget).(*podAddressResolver) + r.tokenStore = ts + r.metrics = m + return r +} + +// ResolveTargets implements BackendAddressResolver. Anonymous/unauthenticated +// sessions (empty issuer or subject) soft-fail every untrusted backend: there +// is no user to bind a pod to. +func (r *podAddressResolver) ResolveTargets( + ctx context.Context, sess SessionRef, backends []*vmcp.Backend, onNewPod func(backendID string), +) []*vmcp.Backend { + out := make([]*vmcp.Backend, 0, len(backends)) + authenticated := sess.Issuer != "" && sess.Subject != "" + + for _, b := range backends { + if b == nil { + // Defensive: the factory filters nils before invoking the resolver, + // but a nil must never be passed through to the init fan-out. + continue + } + if b.Metadata[MetadataKeyUntrusted] != "true" { + out = append(out, b) + continue + } + if !authenticated { + slog.Warn("untrusted backend excluded: session is anonymous; there is no user to bind a pod to", + "backend", b.Name) + continue + } + resolved, err := r.resolveOne(ctx, sess, b, onNewPod) + if err != nil { + logSoftFailure(b, err) + continue + } + out = append(out, resolved) + } + return out +} + +// resolveOne provisions the pod for one untrusted backend and returns a copy +// of the backend with the per-session BaseURL. The input is never mutated. +// onNewPod (may be nil) fires with the backend ID only when EnsurePod created +// the pod — the caller drops the stale session hint for that backend. +func (r *podAddressResolver) resolveOne( + ctx context.Context, sess SessionRef, b *vmcp.Backend, onNewPod func(backendID string), +) (*vmcp.Backend, error) { + userKey := sess.UserKey() + mcpserverUID := b.Metadata[MetadataKeyMCPServerUID] + if mcpserverUID == "" { + return nil, fmt.Errorf("untrusted backend %q has no MCPServer UID metadata", b.Name) + } + + if r.admission != nil { + if err := r.admission.Check(ctx, userKey, mcpserverUID); err != nil { + r.metrics.recordAdmission(ctx, admissionResult(err)) + return nil, err + } + r.metrics.recordAdmission(ctx, "admitted") + } + + port, suffix, err := BackendPortAndSuffix(b.BaseURL) + if err != nil { + return nil, fmt.Errorf("untrusted backend %q has unparseable BaseURL: %w", b.Name, err) + } + + pod, err := r.lifecycle.EnsurePod(ctx, EnsurePodRequest{ + Session: sess, + MCPServerUID: mcpserverUID, + MCPServerName: b.Name, + Port: port, + OwnerRefUID: types.UID(mcpserverUID), + TokenStore: r.tokenStore, + OnNewPod: func(string) { + if onNewPod != nil { + onNewPod(b.ID) + } + }, + }) + if err != nil { + return nil, err + } + + if err := r.lifecycle.WaitReady(ctx, pod, r.readyBudget); err != nil { + return nil, err + } + + // Copy before mutating: the registry backend is shared across sessions. + b2 := *b + b2.Metadata = maps.Clone(b.Metadata) + if b2.Metadata == nil { + b2.Metadata = make(map[string]string) + } + b2.BaseURL = fmt.Sprintf("http://%s:%d%s", pod.Status.PodIP, port, suffix) + b2.Metadata[MetadataKeyUntrustedPodName] = pod.Name + b2.Metadata[MetadataKeyUntrustedPodUID] = string(pod.UID) + return &b2, nil +} + +// admissionResult maps an admission error to the low-cardinality result label +// for untrusted_pod_admissions_total. Deliberately a fixed vocabulary — never +// a user, pod, or server identifier (ADR D11). +func admissionResult(err error) string { + if errors.Is(err, ErrQuotaExceeded) { + return "quota_exceeded" + } + return "error" +} + +// logSoftFailure logs a per-backend soft failure at WARN. The reason string is +// deliberately coarse (no user identifiers beyond the backend name). +func logSoftFailure(b *vmcp.Backend, err error) { + reason := "ensure_pod_failed" + if errors.Is(err, ErrQuotaExceeded) { + reason = "quota_exceeded" + } + slog.Warn("untrusted backend excluded from session", + "backend", b.Name, + "reason", reason, + "error", err, + ) +} + +// BackendPortAndSuffix extracts the MCP port and URL path/fragment suffix from +// the discovered shared BaseURL (built by transport.GenerateMCPServerURL — +// SSE/stdio targets carry a "#" fragment) so the pod-IP rewrite +// preserves transport semantics, e.g. "http://svc:8080#name" → +// "http://:8080#name". +func BackendPortAndSuffix(baseURL string) (port int32, suffix string, err error) { + u, err := url.Parse(baseURL) + if err != nil { + return 0, "", fmt.Errorf("failed to parse base URL: %w", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return 0, "", fmt.Errorf("unsupported scheme %q in base URL", u.Scheme) + } + portStr := u.Port() + if portStr == "" { + return 0, "", fmt.Errorf("base URL %q has no explicit port", baseURL) + } + p, err := strconv.ParseInt(portStr, 10, 32) + if err != nil || p <= 0 { + return 0, "", fmt.Errorf("invalid port %q in base URL", portStr) + } + + suffix = u.EscapedPath() + if u.RawQuery != "" { + suffix += "?" + u.RawQuery + } + if u.Fragment != "" { + suffix += "#" + u.Fragment + } + // Reject URLs whose host:port would leak credentials or user-info into the + // rewritten address. + if u.User != nil || strings.Contains(u.Host, "@") { + return 0, "", fmt.Errorf("base URL %q must not carry user info", baseURL) + } + return int32(p), suffix, nil +} diff --git a/pkg/vmcp/session/untrusted/addressing_external_test.go b/pkg/vmcp/session/untrusted/addressing_external_test.go new file mode 100644 index 0000000000..c6afb826ec --- /dev/null +++ b/pkg/vmcp/session/untrusted/addressing_external_test.go @@ -0,0 +1,226 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stacklok/toolhive/pkg/vmcp" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted/mocks" +) + +// userQuotaKeyForTest builds the quota key the admission layer reads +// (untrusted:userquota:). +func userQuotaKeyForTest(prefix, userKey string) string { + sum := sha256.Sum256([]byte(userKey)) + return prefix + "untrusted:userquota:" + hex.EncodeToString(sum[:])[:40] +} + +func untrustedBackend(id, uid, baseURL string) *vmcp.Backend { + return &vmcp.Backend{ + ID: id, + Name: id, + BaseURL: baseURL, + TransportType: "streamable-http", + Metadata: map[string]string{ + untrusted.MetadataKeyUntrusted: "true", + untrusted.MetadataKeyMCPServerUID: uid, + }, + } +} + +func trustedBackend(id, baseURL string) *vmcp.Backend { + return &vmcp.Backend{ + ID: id, + Name: id, + BaseURL: baseURL, + TransportType: "streamable-http", + Metadata: map[string]string{}, + } +} + +func authSession() untrusted.SessionRef { + return untrusted.SessionRef{SessionID: "sess-1", Issuer: "https://iss", Subject: "sub", Namespace: "toolhive"} +} + +func readyPod(name, ip string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "toolhive", UID: "pod-uid-1"}, + Status: corev1.PodStatus{PodIP: ip}, + } +} + +func TestBackendPortAndSuffix(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + baseURL string + wantPort int32 + wantSuffix string + wantErr string + }{ + {name: "streamable-http path", baseURL: "http://svc:8080/mcp", wantPort: 8080, wantSuffix: "/mcp"}, + {name: "sse path+fragment", baseURL: "http://svc:8080/sse#github-mcp", wantPort: 8080, wantSuffix: "/sse#github-mcp"}, + {name: "root path", baseURL: "http://svc:9090", wantPort: 9090, wantSuffix: ""}, + {name: "https scheme", baseURL: "https://svc:8443/mcp", wantPort: 8443, wantSuffix: "/mcp"}, + {name: "missing port", baseURL: "http://svc/mcp", wantErr: "no explicit port"}, + {name: "bad port", baseURL: "http://svc:notaport/mcp", wantErr: "invalid port"}, + {name: "no scheme", baseURL: "svc:8080/mcp", wantErr: "unsupported scheme"}, + {name: "user info rejected", baseURL: "http://user@svc:8080/mcp", wantErr: "user info"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + port, suffix, err := untrusted.BackendPortAndSuffix(tc.baseURL) + if tc.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantPort, port) + assert.Equal(t, tc.wantSuffix, suffix) + }) + } +} + +func TestResolveTargets(t *testing.T) { + t.Parallel() + ctx := context.Background() + + t.Run("trusted backends pass through unmodified (same pointer)", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + lc := mocks.NewMockPodLifecycle(ctrl) + resolver := untrusted.NewPodAddressResolver(lc, nil, time.Second) + + trusted := trustedBackend("trusted-1", "http://svc:8080/mcp") + out := resolver.ResolveTargets(ctx, authSession(), []*vmcp.Backend{trusted}, nil) + require.Len(t, out, 1) + assert.Same(t, trusted, out[0], "trusted backend must not be copied or rewritten") + }) + + t.Run("untrusted backend BaseURL rewritten to pod IP with port+suffix preserved", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + lc := mocks.NewMockPodLifecycle(ctrl) + pod := readyPod("untrusted-uid-abc", "10.1.2.3") + lc.EXPECT().EnsurePod(gomock.Any(), gomock.Any()).Return(pod, nil) + lc.EXPECT().WaitReady(gomock.Any(), pod, gomock.Any()).Return(nil) + resolver := untrusted.NewPodAddressResolver(lc, nil, time.Second) + + b := untrustedBackend("github-mcp", "uid-abc", "http://svc:8080/sse#github-mcp") + out := resolver.ResolveTargets(ctx, authSession(), []*vmcp.Backend{b}, nil) + require.Len(t, out, 1) + assert.Equal(t, "http://10.1.2.3:8080/sse#github-mcp", out[0].BaseURL) + assert.Equal(t, pod.Name, out[0].Metadata[untrusted.MetadataKeyUntrustedPodName]) + assert.Equal(t, "pod-uid-1", out[0].Metadata[untrusted.MetadataKeyUntrustedPodUID]) + + // Registry backend untouched. + assert.Equal(t, "http://svc:8080/sse#github-mcp", b.BaseURL) + _, hasPodKey := b.Metadata[untrusted.MetadataKeyUntrustedPodName] + assert.False(t, hasPodKey, "registry backend metadata must not be mutated") + }) + + t.Run("EnsurePod failure soft-fails the backend only", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + lc := mocks.NewMockPodLifecycle(ctrl) + lc.EXPECT().EnsurePod(gomock.Any(), gomock.Any()).Return(nil, errors.New("boom")) + resolver := untrusted.NewPodAddressResolver(lc, nil, time.Second) + + trusted := trustedBackend("trusted-1", "http://svc:8080/mcp") + untrustedB := untrustedBackend("github-mcp", "uid-abc", "http://svc:8080/mcp") + out := resolver.ResolveTargets(ctx, authSession(), []*vmcp.Backend{trusted, untrustedB}, nil) + require.Len(t, out, 1) + assert.Equal(t, "trusted-1", out[0].ID) + }) + + t.Run("WaitReady failure soft-fails the backend", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + lc := mocks.NewMockPodLifecycle(ctrl) + pod := readyPod("p", "10.0.0.1") + lc.EXPECT().EnsurePod(gomock.Any(), gomock.Any()).Return(pod, nil) + lc.EXPECT().WaitReady(gomock.Any(), pod, gomock.Any()).Return(errors.New("timeout")) + resolver := untrusted.NewPodAddressResolver(lc, nil, time.Second) + + out := resolver.ResolveTargets(ctx, authSession(), []*vmcp.Backend{untrustedBackend("b", "uid", "http://svc:8080/mcp")}, nil) + assert.Empty(t, out) + }) + + t.Run("anonymous session soft-fails untrusted backends without touching lifecycle", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + lc := mocks.NewMockPodLifecycle(ctrl) // no EXPECT: any call fails the test + resolver := untrusted.NewPodAddressResolver(lc, nil, time.Second) + + anon := untrusted.SessionRef{SessionID: "sess-1", Namespace: "toolhive"} + out := resolver.ResolveTargets(ctx, anon, []*vmcp.Backend{ + untrustedBackend("github-mcp", "uid-abc", "http://svc:8080/mcp"), + trustedBackend("trusted-1", "http://svc:8080/mcp"), + }, nil) + require.Len(t, out, 1) + assert.Equal(t, "trusted-1", out[0].ID) + }) + + t.Run("missing MCPServer UID soft-fails", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + lc := mocks.NewMockPodLifecycle(ctrl) + resolver := untrusted.NewPodAddressResolver(lc, nil, time.Second) + + b := untrustedBackend("github-mcp", "uid-abc", "http://svc:8080/mcp") + delete(b.Metadata, untrusted.MetadataKeyMCPServerUID) + out := resolver.ResolveTargets(ctx, authSession(), []*vmcp.Backend{b}, nil) + assert.Empty(t, out) + }) + + t.Run("admission denial soft-fails with quota reason", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + lc := mocks.NewMockPodLifecycle(ctrl) // no pod ops expected + + mr := miniredis.RunT(t) + rc := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + adm, err := untrusted.NewAdmission(rc, "thv:vmcp:session:", "toolhive", untrusted.AdmissionConfig{ + PerUserPodQuota: 1, + PerUserCreateRate: 100, + PerMCPServerCap: 100, + GlobalCapFactor: 0.8, + CacheCapacity: 100, + }) + require.NoError(t, err) + // Exhaust the user's quota (key shape: untrusted:userquota:). + require.NoError(t, rc.Set(ctx, userQuotaKeyForTest("thv:vmcp:session:", authSession().UserKey()), 1, time.Minute).Err()) + + resolver := untrusted.NewPodAddressResolver(lc, adm, time.Second) + out := resolver.ResolveTargets(ctx, authSession(), []*vmcp.Backend{untrustedBackend("b", "uid", "http://svc:8080/mcp")}, nil) + assert.Empty(t, out) + }) + + t.Run("nil backends are skipped", func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + lc := mocks.NewMockPodLifecycle(ctrl) + resolver := untrusted.NewPodAddressResolver(lc, nil, time.Second) + out := resolver.ResolveTargets(ctx, authSession(), []*vmcp.Backend{nil}, nil) + assert.Empty(t, out) + }) +} diff --git a/pkg/vmcp/session/untrusted/admission.go b/pkg/vmcp/session/untrusted/admission.go new file mode 100644 index 0000000000..c039cf9d4e --- /dev/null +++ b/pkg/vmcp/session/untrusted/admission.go @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +// ErrQuotaExceeded is returned when an admission control denies provisioning +// of an untrusted backend pod (per-user quota, create rate, per-server cap, or +// global capacity). Callers treat it as a soft failure: the backend is excluded +// from the session, matching partial-init semantics. +var ErrQuotaExceeded = errors.New("untrusted backend capacity exceeded") + +// AdmissionConfig tunes the DoS admission controls enforced before any pod +// write. Zero-valued fields fall back to the documented defaults; the +// constructor rejects out-of-range values (fail loudly, per go-style). +type AdmissionConfig struct { + // PerUserPodQuota caps concurrent untrusted pods per user across + // MCPServers. Default 10. + PerUserPodQuota int + // PerUserCreateRate caps pod creations per user per minute. Default 5. + PerUserCreateRate int + // PerMCPServerCap caps concurrent untrusted pods per MCPServer UID. + // Default 200. + PerMCPServerCap int + // GlobalCapFactor is the fraction of the session cache capacity that + // bounds total untrusted pods (SCARD of the global pod set). Default 0.8. + GlobalCapFactor float64 + // CacheCapacity is the session-manager cache capacity the global factor + // applies to. Required when GlobalCapFactor is used. + CacheCapacity int +} + +const ( + defaultPerUserPodQuota = 10 + defaultPerUserCreateRate = 5 + defaultPerMCPServerCap = 200 + defaultGlobalCapFactor = 0.8 + + // rateLimitWindowTTL keeps per-minute rate keys for two windows. + rateLimitWindowTTL = 2 * time.Minute +) + +// resolved fills defaults and validates. Returns an error on out-of-range +// config (fail loudly rather than silently admitting). +func (c AdmissionConfig) resolved() (AdmissionConfig, error) { + if c.PerUserPodQuota == 0 { + c.PerUserPodQuota = defaultPerUserPodQuota + } + if c.PerUserCreateRate == 0 { + c.PerUserCreateRate = defaultPerUserCreateRate + } + if c.PerMCPServerCap == 0 { + c.PerMCPServerCap = defaultPerMCPServerCap + } + if c.GlobalCapFactor == 0 { + c.GlobalCapFactor = defaultGlobalCapFactor + } + if c.PerUserPodQuota < 0 || c.PerUserCreateRate < 0 || c.PerMCPServerCap < 0 { + return c, fmt.Errorf("untrusted admission: quota values must be non-negative") + } + if c.GlobalCapFactor <= 0 || c.GlobalCapFactor > 1 { + return c, fmt.Errorf("untrusted admission: GlobalCapFactor must be in (0, 1], got %v", c.GlobalCapFactor) + } + if c.CacheCapacity <= 0 { + return c, fmt.Errorf("untrusted admission: CacheCapacity must be positive, got %d", c.CacheCapacity) + } + return c, nil +} + +// Admission is the DoS admission gate checked before any untrusted pod write. +type Admission interface { + // Check admits or denies provisioning one pod for the (user, server) pair. + // Returns nil to admit, an error wrapping ErrQuotaExceeded to deny. + Check(ctx context.Context, userKey, mcpserverUID string) error +} + +// admission enforces all DoS controls before any pod write. Every control +// fails closed when Redis is unavailable: an error reaching Redis denies +// admission, matching the existing Redis-required posture for multi-pod +// session metadata. +type admission struct { + store *redisStore + cfg AdmissionConfig + now func() time.Time +} + +// NewAdmission builds the admission gate from raw parts. Exported for tests +// and composition roots that wire components individually; production wiring +// goes through NewStack. +func NewAdmission(client redis.UniversalClient, keyPrefix, namespace string, cfg AdmissionConfig) (Admission, error) { + store, err := newRedisStore(client, keyPrefix, namespace) + if err != nil { + return nil, err + } + return newAdmission(store, cfg, time.Now) +} + +func newAdmission(store *redisStore, cfg AdmissionConfig, now func() time.Time) (*admission, error) { + resolved, err := cfg.resolved() + if err != nil { + return nil, err + } + return &admission{store: store, cfg: resolved, now: now}, nil +} + +// Check runs every admission control for the (user, server) pair. Returns nil +// to admit, ErrQuotaExceeded (wrapped with the control that fired) to deny. +// Checks run cheapest-first and before any pod write, so rejection costs only +// Redis reads. +// +// Every check here is a read-only pre-flight. The per-user quota counter is +// authoritatively enforced by the pod lifecycle's atomic INCR-with-cap inside +// recordPodCreate (an admitted race can never push the counter past the cap, +// and the over-quota pod is deleted by the caller); the reaper recomputes the +// counters from the authoritative pod LIST each tick. +func (a *admission) Check(ctx context.Context, userKey, mcpserverUID string) error { + // Per-user concurrent quota (read-only pre-flight; see doc comment). + quota, err := a.store.client.Get(ctx, a.store.userQuotaKey(userKey)).Int64() + if err != nil && !errors.Is(err, redis.Nil) { + return fmt.Errorf("untrusted admission: quota check failed (failing closed): %w", err) + } + if quota >= int64(a.cfg.PerUserPodQuota) { + return fmt.Errorf("%w: per-user pod quota (%d) reached", ErrQuotaExceeded, a.cfg.PerUserPodQuota) + } + + // Creation rate limit: INCR + EXPIRE on the current minute bucket. + rateKey := a.store.rateLimitKey(userKey, a.now()) + n, err := a.store.client.Incr(ctx, rateKey).Result() + if err != nil { + return fmt.Errorf("untrusted admission: rate-limit check failed (failing closed): %w", err) + } + if n == 1 { + if err := a.store.client.Expire(ctx, rateKey, rateLimitWindowTTL).Err(); err != nil { + return fmt.Errorf("untrusted admission: rate-limit expiry failed (failing closed): %w", err) + } + } + if n > int64(a.cfg.PerUserCreateRate) { + return fmt.Errorf("%w: per-user create rate (%d/min) exceeded", ErrQuotaExceeded, a.cfg.PerUserCreateRate) + } + + // Per-MCPServer fan-out cap. + serverCount, err := a.store.client.SCard(ctx, a.store.serverPodsSetKey(mcpserverUID)).Result() + if err != nil { + return fmt.Errorf("untrusted admission: per-server cap check failed (failing closed): %w", err) + } + if serverCount >= int64(a.cfg.PerMCPServerCap) { + return fmt.Errorf("%w: per-MCPServer cap (%d) reached", ErrQuotaExceeded, a.cfg.PerMCPServerCap) + } + + // vMCP-wide capacity. + global, err := a.store.client.SCard(ctx, a.store.podsSetKey()).Result() + if err != nil { + return fmt.Errorf("untrusted admission: global capacity check failed (failing closed): %w", err) + } + globalCap := int64(float64(a.cfg.CacheCapacity) * a.cfg.GlobalCapFactor) + if global >= globalCap { + return fmt.Errorf("%w: global untrusted pod capacity (%d) reached", ErrQuotaExceeded, globalCap) + } + + return nil +} diff --git a/pkg/vmcp/session/untrusted/admission_test.go b/pkg/vmcp/session/untrusted/admission_test.go new file mode 100644 index 0000000000..267be0826b --- /dev/null +++ b/pkg/vmcp/session/untrusted/admission_test.go @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestStore(t *testing.T) (*redisStore, *miniredis.Miniredis) { + t.Helper() + mr := miniredis.RunT(t) + store, err := newRedisStore(redis.NewClient(&redis.Options{Addr: mr.Addr()}), "thv:vmcp:session:", "toolhive") + require.NoError(t, err) + return store, mr +} + +func testAdmissionConfig() AdmissionConfig { + return AdmissionConfig{ + PerUserPodQuota: 3, + PerUserCreateRate: 2, + PerMCPServerCap: 4, + GlobalCapFactor: 0.8, + CacheCapacity: 10, + } +} + +func TestNewRedisStoreValidation(t *testing.T) { + t.Parallel() + + mr := miniredis.RunT(t) + c := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + + _, err := newRedisStore(nil, "thv:vmcp:session:", "ns") + require.Error(t, err, "nil client must be rejected") + + _, err = newRedisStore(c, "no-trailing-colon", "ns") + require.Error(t, err, "prefix without ':' must be rejected") + + _, err = newRedisStore(c, "", "ns") + require.Error(t, err, "empty prefix must be rejected") + + _, err = newRedisStore(c, "thv:vmcp:session:", "") + require.Error(t, err, "empty namespace must be rejected") +} + +func TestAdmissionConfigValidation(t *testing.T) { + t.Parallel() + + store, _ := newTestStore(t) + + _, err := newAdmission(store, AdmissionConfig{GlobalCapFactor: 1.5, CacheCapacity: 10}, time.Now) + require.Error(t, err, "GlobalCapFactor > 1 must be rejected") + + _, err = newAdmission(store, AdmissionConfig{CacheCapacity: 0}, time.Now) + require.Error(t, err, "CacheCapacity 0 must be rejected") + + _, err = newAdmission(store, testAdmissionConfig(), time.Now) + require.NoError(t, err) +} + +func TestAdmissionCheck(t *testing.T) { + t.Parallel() + ctx := context.Background() + userKey := "iss\x00sub" + serverUID := "uid-1" + + setup := func(t *testing.T) (*admission, *redisStore, *miniredis.Miniredis) { + t.Helper() + store, mr := newTestStore(t) + adm, err := newAdmission(store, testAdmissionConfig(), time.Now) + require.NoError(t, err) + return adm, store, mr + } + + t.Run("admits under all limits", func(t *testing.T) { + t.Parallel() + adm, _, _ := setup(t) + require.NoError(t, adm.Check(ctx, userKey, serverUID)) + }) + + t.Run("denies at per-user quota", func(t *testing.T) { + t.Parallel() + adm, store, _ := setup(t) + require.NoError(t, store.client.Set(ctx, store.userQuotaKey(userKey), 3, time.Minute).Err()) + err := adm.Check(ctx, userKey, serverUID) + require.ErrorIs(t, err, ErrQuotaExceeded) + assert.Contains(t, err.Error(), "per-user pod quota") + }) + + t.Run("denies above per-user create rate", func(t *testing.T) { + t.Parallel() + adm, _, _ := setup(t) + require.NoError(t, adm.Check(ctx, userKey, serverUID)) + require.NoError(t, adm.Check(ctx, userKey, serverUID)) + err := adm.Check(ctx, userKey, serverUID) + require.ErrorIs(t, err, ErrQuotaExceeded) + assert.Contains(t, err.Error(), "create rate") + }) + + t.Run("denies at per-server cap", func(t *testing.T) { + t.Parallel() + adm, store, _ := setup(t) + require.NoError(t, store.client.SAdd(ctx, store.serverPodsSetKey(serverUID), "p1", "p2", "p3", "p4").Err()) + err := adm.Check(ctx, userKey, serverUID) + require.ErrorIs(t, err, ErrQuotaExceeded) + assert.Contains(t, err.Error(), "per-MCPServer cap") + }) + + t.Run("denies at global capacity factor", func(t *testing.T) { + t.Parallel() + adm, store, _ := setup(t) + // Cap = 10 * 0.8 = 8. + members := []any{"p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8"} + require.NoError(t, store.client.SAdd(ctx, store.podsSetKey(), members...).Err()) + err := adm.Check(ctx, userKey, serverUID) + require.ErrorIs(t, err, ErrQuotaExceeded) + assert.Contains(t, err.Error(), "global untrusted pod capacity") + }) + + t.Run("rate bucket expires", func(t *testing.T) { + t.Parallel() + adm, store, mr := setup(t) + require.NoError(t, adm.Check(ctx, userKey, serverUID)) + require.NoError(t, adm.Check(ctx, userKey, serverUID)) + require.Error(t, adm.Check(ctx, userKey, serverUID)) + // Advance past the 2-minute bucket TTL. + mr.FastForward(3 * time.Minute) + rateKey := store.rateLimitKey(userKey, time.Now().Add(-3*time.Minute)) + assert.Equal(t, int64(0), store.client.Exists(ctx, rateKey).Val()) + }) + + t.Run("fails closed when Redis is down", func(t *testing.T) { + t.Parallel() + adm, _, mr := setup(t) + mr.Close() + err := adm.Check(ctx, userKey, serverUID) + require.Error(t, err) + assert.False(t, errors.Is(err, ErrQuotaExceeded), "infra failure must not masquerade as quota denial") + assert.Contains(t, err.Error(), "failing closed") + }) +} diff --git a/pkg/vmcp/session/untrusted/egress.go b/pkg/vmcp/session/untrusted/egress.go new file mode 100644 index 0000000000..f70f7c8d14 --- /dev/null +++ b/pkg/vmcp/session/untrusted/egress.go @@ -0,0 +1,281 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "fmt" + "sort" + + corev1 "k8s.io/api/core/v1" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +// Wave-3 egress-broker wiring constants. Data-plane resource names come from +// egressbroker.ResourceNamesFor (the shared naming contract — generation- +// named CA objects so a mid-rotation clone always mounts a consistent +// cert/key pair); the operator side mirrors the same functions because +// cmd/thv-operator is a separate module boundary. +const ( + // caKeyCert is the public-cert data key of the CA Secret/bundle. + caKeyCert = egressbroker.CAKeyCert + // caKeyKey is the private-key data key of the CA Secret. + caKeyKey = egressbroker.CAKeyKey + + // EgressBrokerImage is the broker sidecar image (ext_authz + SDS). + EgressBrokerImage = "ghcr.io/stacklok/toolhive/egressbroker:latest" + // EnvoyProxyImage is the Envoy data-plane image. + EnvoyProxyImage = "envoyproxy/envoy:v1.36-latest" + + // brokerListenPort is the ext_authz/SDS gRPC port (loopback). + brokerListenPort = 9001 + // proxyPort is the Envoy explicit-proxy port the backend's + // HTTP(S)_PROXY env points at (loopback). + proxyPort = 15001 + + // caSharedVolumeName is the emptyDir shared between the CA-seeding + // init-container and the backend container (read-only there). + caSharedVolumeName = "thv-bump-ca" + // caBundleVolumeName is the init-container's CA-bundle ConfigMap volume. + caBundleVolumeName = "thv-bump-ca-bundle" + // policyVolumeName is the sidecar's policy ConfigMap volume. + policyVolumeName = "thv-egress-policy" + // caSecretVolumeName is the sidecar's bump-CA Secret volume. + caSecretVolumeName = "thv-bump-ca-secret" //nolint:gosec // G101: a volume name, not a credential + // envoyConfigVolumeName is the Envoy bootstrap volume the broker renders. + envoyConfigVolumeName = "thv-envoy-config" + + // caMountPath is where the backend and init containers see the CA file. + caMountPath = "/etc/thv-ca" + // caFileName is the CA file inside the shared emptyDir. + caFileName = "ca.crt" + // envoyConfigPath is the shared emptyDir the broker writes the Envoy + // bootstrap into. + envoyConfigPath = "/etc/thv-envoy" + + // caSeedInitContainerName is the trusted init-container that seeds the + // CA emptyDir. + caSeedInitContainerName = "thv-ca-seed" + // envoyContainerName is the Envoy data-plane container. + envoyContainerName = "thv-envoy" + // brokerContainerName is the credential-broker container. + brokerContainerName = "thv-egressbroker" +) + +// sidecarEnv is the deterministic (sorted) view of the shared annotation→env +// contract (egressbroker.EnvToAnnotation): the broker's THV_UNTRUSTED_* env +// vars are downward-API mirrors of the pod annotations this package stamps. +var sidecarEnv = func() []struct { + name string + annotation string +} { + names := make([]string, 0, len(egressbroker.EnvToAnnotation)) + for name := range egressbroker.EnvToAnnotation { + names = append(names, name) + } + sort.Strings(names) + out := make([]struct { + name string + annotation string + }, 0, len(names)) + for _, name := range names { + annotation, _ := egressbroker.AnnotationForEnv(name) + out = append(out, struct { + name string + annotation string + }{name: name, annotation: annotation}) + } + return out +}() + +// applyEgressBrokerSidecar extends a freshly cloned untrusted pod with the +// Wave-3 egress data plane (ADR-0001 D3/D9/D10): +// +// 1. a trusted init-container that writes the bump-CA public cert into a +// shared emptyDir mounted read-only into the backend container (the CA +// cannot be a ConfigMap volume: the Wave-0 gate forbids ALL ConfigMap +// volumes in untrusted pod templates); +// 2. the Envoy data-plane container (explicit forward proxy, loopback); +// 3. the credential-broker container (ext_authz + SDS), with downward-API +// identity env, the bump-CA Secret mount (sidecar-only), and the policy +// ConfigMap mount; +// 4. HTTP(S)_PROXY + the runtime CA-env matrix on the backend container. +// +// The volumes this function adds are why reverifyNoSecretMaterial runs +// BEFORE it in clonePodFromTemplate: the gate is the authority on what the +// operator-built template may carry, and the sidecar volumes (Secret, +// ConfigMap) are operator-managed additions appended after verification. +// +// The broker's token-store env (THV_EGRESSBROKER_REDIS_ADDR / +// _REDIS_KEY_PREFIX, and the tokenenc KEK) is injected here from +// req.TokenStore. The address and key prefix are non-secret and are rendered +// as env literals; the KEK, when encryption is enabled, is referenced from a +// Secret via SecretKeyRef (never a ConfigMap or env literal). A nil TokenStore +// renders no token-store env at all — the broker then refuses to start +// (fail closed, per cmd/thv-egressbroker). +func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { + if pod == nil { + return fmt.Errorf("untrusted egress wiring: pod must not be nil") + } + if req.MCPServerName == "" { + return fmt.Errorf("untrusted egress wiring: MCPServer name must not be empty") + } + if req.TokenStore != nil { + if err := req.TokenStore.validate(); err != nil { + return err + } + } + + // Generation-qualified CA objects: the pod mounts the exact CA generation + // the operator published in the STS template annotation (cert+key in ONE + // Secret), so a pod cloned mid-rotation can never get new-key+old-cert. + caGen := req.Session.CAGeneration + if caGen == "" { + return fmt.Errorf("untrusted egress wiring: backend StatefulSet template carries no %s annotation "+ + "(operator must publish the current CA generation)", AnnotationCAGeneration) + } + names := egressbroker.ResourceNamesFor(req.MCPServerName, caGen) + readOnly := true + + pod.Spec.Volumes = append(pod.Spec.Volumes, + corev1.Volume{Name: caSharedVolumeName, VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }}, + corev1.Volume{Name: policyVolumeName, VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: names.EgressPolicy}, + }, + }}, + corev1.Volume{Name: caSecretVolumeName, VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: names.CASecret}, + }}, + corev1.Volume{Name: envoyConfigVolumeName, VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }}, + ) + + // Trusted init-container: seeds the shared CA emptyDir from the + // operator-owned bundle ConfigMap. Runs before every app container; the + // backend's TLS trust matrix points at the seeded file. ConfigMap volumes + // are forbidden by reverifyNoSecretMaterial only on the OPERATOR template; + // this pod is assembled by the vMCP after verification, and only the + // trusted init-container mounts the bundle — the backend container never + // gets a ConfigMap-backed path. + pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{ + Name: caSeedInitContainerName, + Image: EgressBrokerImage, + Command: []string{"sh", "-c"}, + Args: []string{ + fmt.Sprintf("cp /bundle/%s /ca/%s && chmod 0444 /ca/%s", caKeyCert, caFileName, caFileName), + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: caSharedVolumeName, MountPath: "/ca"}, + {Name: caBundleVolumeName, MountPath: "/bundle", ReadOnly: true}, + }, + }) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: caBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: names.CABundle}, + }, + }, + }) + + // Envoy data plane: consumes the broker-rendered bootstrap. + pod.Spec.Containers = append(pod.Spec.Containers, + corev1.Container{ + Name: envoyContainerName, + Image: EnvoyProxyImage, + Command: []string{"envoy", "-c", envoyConfigPath + "/envoy.yaml"}, + Ports: []corev1.ContainerPort{ + {Name: "egress-proxy", ContainerPort: proxyPort}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: envoyConfigVolumeName, MountPath: envoyConfigPath, ReadOnly: true}, + }, + }, + ) + + // Credential broker: ext_authz + SDS + cert minting + Envoy bootstrap + // rendering. The only container that ever holds credential material. + brokerEnv := []corev1.EnvVar{ + {Name: "THV_EGRESSBROKER_POLICY_FILE", Value: "/etc/thv-policy/policy.yaml"}, + {Name: "THV_EGRESSBROKER_CA_FILE", Value: "/ca-secret/" + caKeyCert}, + {Name: "THV_EGRESSBROKER_CA_KEY_FILE", Value: "/ca-secret/" + caKeyKey}, + {Name: "THV_EGRESSBROKER_LISTEN_ADDRESS", Value: "127.0.0.1"}, + {Name: "THV_EGRESSBROKER_LISTEN_PORT", Value: fmt.Sprintf("%d", brokerListenPort)}, + {Name: "THV_EGRESSBROKER_ENVOY_BOOTSTRAP_OUT", Value: envoyConfigPath + "/envoy.yaml"}, + } + for _, mapping := range sidecarEnv { + brokerEnv = append(brokerEnv, corev1.EnvVar{ + Name: mapping.name, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: egressbroker.AnnotationFieldRef(mapping.annotation), + }, + }, + }) + } + // Token-store coordinates (Wave-3 deviation 1). Address/prefix are non-secret + // literals; the KEK comes from a Secret env reference so the key value never + // appears in the pod spec or a ConfigMap. A nil TokenStore renders none of + // these — the broker then fails closed at startup. + if req.TokenStore != nil { + brokerEnv = append(brokerEnv, + corev1.EnvVar{Name: "THV_EGRESSBROKER_REDIS_ADDR", Value: req.TokenStore.RedisAddr}, + corev1.EnvVar{Name: "THV_EGRESSBROKER_REDIS_KEY_PREFIX", Value: req.TokenStore.KeyPrefix}, + ) + if ref := req.TokenStore.KEKSecretRef; ref != nil { + brokerEnv = append(brokerEnv, corev1.EnvVar{ + Name: "THV_EGRESSBROKER_KEK", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: ref.Name}, + Key: ref.Key, + }, + }, + }) + } + } + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ + Name: brokerContainerName, + Image: EgressBrokerImage, + Env: brokerEnv, + Ports: []corev1.ContainerPort{ + {Name: "ext-authz", ContainerPort: brokerListenPort}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: caSecretVolumeName, MountPath: "/ca-secret", ReadOnly: true}, + {Name: policyVolumeName, MountPath: "/etc/thv-policy", ReadOnly: true}, + {Name: caSharedVolumeName, MountPath: caMountPath, ReadOnly: true}, + {Name: envoyConfigVolumeName, MountPath: envoyConfigPath}, + }, + }) + + // Backend container: proxy env (D10) + runtime CA trust matrix (D9). + // Appended to the mcp container only; env literals are inert config, not + // credentials. + for i := range pod.Spec.Containers { + c := &pod.Spec.Containers[i] + if c.Name != mcpContainerName { + continue + } + proxyURL := fmt.Sprintf("http://127.0.0.1:%d", proxyPort) + c.Env = append(c.Env, + corev1.EnvVar{Name: "HTTP_PROXY", Value: proxyURL}, + corev1.EnvVar{Name: "HTTPS_PROXY", Value: proxyURL}, + corev1.EnvVar{Name: "NO_PROXY", Value: "127.0.0.1,localhost,::1"}, + corev1.EnvVar{Name: "SSL_CERT_FILE", Value: caMountPath + "/" + caFileName}, + corev1.EnvVar{Name: "NODE_EXTRA_CA_CERTS", Value: caMountPath + "/" + caFileName}, + corev1.EnvVar{Name: "REQUESTS_CA_BUNDLE", Value: caMountPath + "/" + caFileName}, + corev1.EnvVar{Name: "CURL_CA_BUNDLE", Value: caMountPath + "/" + caFileName}, + ) + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: caSharedVolumeName, MountPath: caMountPath, ReadOnly: readOnly, + }) + return nil + } + return fmt.Errorf("untrusted egress wiring: pod has no %q container", mcpContainerName) +} diff --git a/pkg/vmcp/session/untrusted/egress_test.go b/pkg/vmcp/session/untrusted/egress_test.go new file mode 100644 index 0000000000..d9549769d2 --- /dev/null +++ b/pkg/vmcp/session/untrusted/egress_test.go @@ -0,0 +1,271 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +func TestApplyEgressBrokerSidecar(t *testing.T) { + t.Parallel() + + newPod := func() *corev1.Pod { + pod, err := clonePodFromTemplate(validTemplate(), "backend-app", testRequest(), "vmcp-1") + require.NoError(t, err) + return pod + } + + t.Run("sidecar, envoy, and init containers are appended", func(t *testing.T) { + t.Parallel() + pod := newPod() + + names := []string{} + for _, c := range pod.Spec.Containers { + names = append(names, c.Name) + } + assert.Contains(t, names, "mcp") + assert.Contains(t, names, envoyContainerName) + assert.Contains(t, names, brokerContainerName) + + initNames := []string{} + for _, c := range pod.Spec.InitContainers { + initNames = append(initNames, c.Name) + } + assert.Contains(t, initNames, caSeedInitContainerName) + }) + + t.Run("backend gets proxy env and the runtime CA matrix, never a credential", func(t *testing.T) { + t.Parallel() + pod := newPod() + backend := findContainer(t, pod, "mcp") + + env := map[string]corev1.EnvVar{} + for _, e := range backend.Env { + env[e.Name] = e + } + for _, name := range []string{"HTTP_PROXY", "HTTPS_PROXY"} { + require.Contains(t, env, name) + assert.Equal(t, fmt.Sprintf("http://127.0.0.1:%d", proxyPort), env[name].Value) + } + assert.Equal(t, "127.0.0.1,localhost,::1", env["NO_PROXY"].Value) + caFile := caMountPath + "/" + caFileName + for _, name := range []string{"SSL_CERT_FILE", "NODE_EXTRA_CA_CERTS", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"} { + require.Contains(t, env, name) + assert.Equal(t, caFile, env[name].Value) + } + for _, e := range backend.Env { + if e.ValueFrom != nil { + assert.NotNil(t, e.ValueFrom.FieldRef, + "backend env %s must not source from Secret/ConfigMap", e.Name) + } + } + }) + + t.Run("backend CA mount is the shared emptyDir, read-only, not a ConfigMap", func(t *testing.T) { + t.Parallel() + pod := newPod() + backend := findContainer(t, pod, "mcp") + + var mount *corev1.VolumeMount + for i := range backend.VolumeMounts { + if backend.VolumeMounts[i].Name == caSharedVolumeName { + mount = &backend.VolumeMounts[i] + } + } + require.NotNil(t, mount, "backend must mount the shared CA emptyDir") + assert.Equal(t, caMountPath, mount.MountPath) + assert.True(t, mount.ReadOnly, "backend CA mount must be read-only") + + // The backend must not mount any ConfigMap/Secret volume directly. + volumesByName := map[string]corev1.Volume{} + for _, v := range pod.Spec.Volumes { + volumesByName[v.Name] = v + } + for _, m := range backend.VolumeMounts { + v := volumesByName[m.Name] + assert.Nil(t, v.ConfigMap, "backend mounts ConfigMap volume %s", m.Name) + assert.Nil(t, v.Secret, "backend mounts Secret volume %s", m.Name) + } + }) + + t.Run("broker container gets downward-API identity env mirroring the pod annotations", func(t *testing.T) { + t.Parallel() + pod := newPod() + broker := findContainer(t, pod, brokerContainerName) + + env := map[string]corev1.EnvVar{} + for _, e := range broker.Env { + env[e.Name] = e + } + for _, mapping := range sidecarEnv { + require.Contains(t, env, mapping.name) + e := env[mapping.name] + require.NotNil(t, e.ValueFrom, "%s must come from the downward API", mapping.name) + require.NotNil(t, e.ValueFrom.FieldRef) + expected := fmt.Sprintf("metadata.annotations['%s']", mapping.annotation) + assert.Equal(t, expected, e.ValueFrom.FieldRef.FieldPath) + // The annotation the env mirrors must actually be on the pod. + assert.Contains(t, pod.Annotations, mapping.annotation) + } + // Static config env. + assert.Equal(t, "/etc/thv-policy/policy.yaml", env["THV_EGRESSBROKER_POLICY_FILE"].Value) + }) + + t.Run("bump CA private key mount exists only on the broker container", func(t *testing.T) { + t.Parallel() + pod := newPod() + + volumesByName := map[string]corev1.Volume{} + for _, v := range pod.Spec.Volumes { + volumesByName[v.Name] = v + } + require.NotNil(t, volumesByName[caSecretVolumeName].Secret, + "CA volume must be the operator-owned Secret") + wantNames := egressbroker.ResourceNamesFor("github-mcp", "gen-aaa111") + assert.Equal(t, wantNames.CASecret, volumesByName[caSecretVolumeName].Secret.SecretName) + + for i := range pod.Spec.Containers { + c := &pod.Spec.Containers[i] + mountsCA := false + for _, m := range c.VolumeMounts { + if m.Name == caSecretVolumeName { + mountsCA = true + } + } + if c.Name == brokerContainerName { + assert.True(t, mountsCA, "broker must mount the CA Secret") + } else { + assert.False(t, mountsCA, "container %s must not mount the CA Secret", c.Name) + } + } + }) + + t.Run("policy and CA resource names match the operator contract", func(t *testing.T) { + t.Parallel() + pod := newPod() + volumesByName := map[string]corev1.Volume{} + for _, v := range pod.Spec.Volumes { + volumesByName[v.Name] = v + } + wantNames := egressbroker.ResourceNamesFor("github-mcp", "gen-aaa111") + assert.Equal(t, wantNames.EgressPolicy, + volumesByName[policyVolumeName].ConfigMap.Name) + assert.Equal(t, wantNames.CABundle, + volumesByName[caBundleVolumeName].ConfigMap.Name) + }) + + t.Run("init-container seeds the CA file into the shared emptyDir", func(t *testing.T) { + t.Parallel() + pod := newPod() + var init *corev1.Container + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == caSeedInitContainerName { + init = &pod.Spec.InitContainers[i] + } + } + require.NotNil(t, init) + mounts := map[string]string{} + for _, m := range init.VolumeMounts { + mounts[m.Name] = m.MountPath + } + assert.Contains(t, mounts, caSharedVolumeName) + assert.Contains(t, mounts, caBundleVolumeName) + }) + + t.Run("missing CA generation fails closed (no mid-rotation split pair possible)", func(t *testing.T) { + t.Parallel() + req := testRequest() + req.Session.CAGeneration = "" + _, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.Error(t, err) + assert.Contains(t, err.Error(), AnnotationCAGeneration) + }) + + t.Run("fails loudly on nil pod or missing mcp container", func(t *testing.T) { + t.Parallel() + require.Error(t, applyEgressBrokerSidecar(nil, testRequest())) + require.Error(t, applyEgressBrokerSidecar(&corev1.Pod{}, testRequest())) + require.Error(t, applyEgressBrokerSidecar(&corev1.Pod{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "other"}}}, + }, testRequest())) + noName := testRequest() + noName.MCPServerName = "" + require.Error(t, applyEgressBrokerSidecar(&corev1.Pod{}, noName)) + }) + + t.Run("token store env rendered on the broker from TokenStoreConfig", func(t *testing.T) { + t.Parallel() + req := testRequest() + req.TokenStore = &TokenStoreConfig{ + RedisAddr: "redis.auth:6379", + KeyPrefix: "thv:auth:{toolhive:my-vmcp}:", + KEKSecretRef: &SecretKeyRef{ + Name: "my-vmcp-token-kek", + Key: "kek", + }, + } + pod, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.NoError(t, err) + broker := findContainer(t, pod, brokerContainerName) + + env := map[string]corev1.EnvVar{} + for _, e := range broker.Env { + env[e.Name] = e + } + assert.Equal(t, "redis.auth:6379", env["THV_EGRESSBROKER_REDIS_ADDR"].Value) + assert.Equal(t, "thv:auth:{toolhive:my-vmcp}:", env["THV_EGRESSBROKER_REDIS_KEY_PREFIX"].Value) + + // The KEK is a Secret env reference, never a literal or ConfigMap. + kek := env["THV_EGRESSBROKER_KEK"] + require.NotNil(t, kek.ValueFrom, "KEK must come from a Secret env reference") + require.NotNil(t, kek.ValueFrom.SecretKeyRef) + assert.Equal(t, "my-vmcp-token-kek", kek.ValueFrom.SecretKeyRef.Name) + assert.Equal(t, "kek", kek.ValueFrom.SecretKeyRef.Key) + assert.Empty(t, kek.Value, "KEK value must never be a pod-spec literal") + }) + + t.Run("nil TokenStore renders no token-store env (broker fails closed)", func(t *testing.T) { + t.Parallel() + pod, err := clonePodFromTemplate(validTemplate(), "backend-app", testRequest(), "vmcp-1") + require.NoError(t, err) + broker := findContainer(t, pod, brokerContainerName) + for _, e := range broker.Env { + assert.NotContains(t, []string{ + "THV_EGRESSBROKER_REDIS_ADDR", + "THV_EGRESSBROKER_REDIS_KEY_PREFIX", + "THV_EGRESSBROKER_KEK", + }, e.Name, "no token-store env without a TokenStore config") + } + }) + + t.Run("invalid TokenStore is a clone-time error (fail closed)", func(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + ts *TokenStoreConfig + }{ + {"empty addr", &TokenStoreConfig{KeyPrefix: "thv:auth:{a:b}:"}}, + {"prefix missing colon", &TokenStoreConfig{RedisAddr: "r:6379", KeyPrefix: "thv:auth"}}, + {"kek ref missing key", &TokenStoreConfig{ + RedisAddr: "r:6379", KeyPrefix: "thv:auth:{a:b}:", + KEKSecretRef: &SecretKeyRef{Name: "s"}, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + req := testRequest() + req.TokenStore = tc.ts + _, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.Error(t, err) + }) + } + }) +} diff --git a/pkg/vmcp/session/untrusted/lifecycle.go b/pkg/vmcp/session/untrusted/lifecycle.go new file mode 100644 index 0000000000..837991768b --- /dev/null +++ b/pkg/vmcp/session/untrusted/lifecycle.go @@ -0,0 +1,593 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:generate mockgen -destination=mocks/mock_lifecycle.go -package=mocks github.com/stacklok/toolhive/pkg/vmcp/session/untrusted PodLifecycle,BackendAddressResolver + +package untrusted + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/redis/go-redis/v9" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// PodLifecycle owns creation, readiness, and deletion of per-session untrusted +// backend pods. Implementations must be idempotent: EnsurePod returns the +// existing pod when called twice with the same request, and DeletePod tolerates +// NotFound. +type PodLifecycle interface { + // EnsurePod returns the live pod for the request, creating it from the + // backend StatefulSet's pod template if absent. On a fresh create the + // admission bookkeeping (sets, quota counter, TTL lease) is recorded and + // req.OnNewPod is invoked with the deterministic pod name. + EnsurePod(ctx context.Context, req EnsurePodRequest) (*corev1.Pod, error) + // WaitReady blocks until the pod has an IP and a Ready condition, or the + // budget expires. The pod's UID is re-checked before its IP is trusted + // (guards against IP reuse by an unrelated replacement pod). + WaitReady(ctx context.Context, pod *corev1.Pod, budget time.Duration) error + // DeletePod deletes the named pod and its admission bookkeeping. NotFound + // is not an error. + DeletePod(ctx context.Context, name string) error +} + +// EnsurePodRequest is the identity tuple a pod is provisioned for. +type EnsurePodRequest struct { + Session SessionRef + MCPServerUID string + MCPServerName string + Port int32 + // OwnerRefUID is the MCPServer CR UID, set as a non-controller + // ownerReference so session pods cascade-delete with the MCPServer. + OwnerRefUID types.UID + // TokenStore, when non-nil, wires the egress-broker sidecar's auth-server + // token-store coordinates into the cloned pod (Wave-3). Nil leaves the + // broker without a token store (it fails closed at startup — only valid + // where the broker is not expected to inject credentials). + TokenStore *TokenStoreConfig + // OnNewPod, when non-nil, is invoked exactly once per fresh pod create + // with the deterministic pod name (used by the resolver to drop stale + // session hints). Not invoked when an existing pod is reused. + OnNewPod func(podName string) +} + +const ( + // DefaultReadyBudget is the per-pod cold-start hard cap. + DefaultReadyBudget = 120 * time.Second + + // waitReadyInitialPoll is the first poll interval for WaitReady; it + // doubles each poll up to waitReadyMaxPoll. + waitReadyInitialPoll = 500 * time.Millisecond + waitReadyMaxPoll = 5 * time.Second + + // readinessTimeout is the reaper's failed-cold-start threshold. + readinessTimeout = DefaultReadyBudget + + // zombieHeartbeatGrace is how long a missing vMCP heartbeat is tolerated + // before the zombie rule deletes a pod (2x the 5-minute heartbeat TTL). + zombieHeartbeatGrace = 10 * time.Minute +) + +// k8sPodLifecycle is the production PodLifecycle: bare pods cloned from the +// operator-built backend StatefulSet's pod template, in the vMCP's namespace. +// +// Quota ownership: the lifecycle (not the admission gate) owns the per-user +// quota counters. recordPodCreate INCRs the user's counter atomically with the +// pod registration (Lua), rolls it back if the cap is already reached, and the +// caller compensates by deleting the pod it just created — so the counter can +// never exceed the cap, and no over-quota pod survives. Admission.Check +// remains a read-only pre-flight (avoids a wasted pod create in the common +// case); the lifecycle write is the enforcement point. The reaper recomputes +// the counters from the authoritative pod LIST each tick (self-healing on +// crashed creates/deletes). +type k8sPodLifecycle struct { + k8s client.Client + store *redisStore + vmcpUID string + idleTTL time.Duration + adoptTTL time.Duration + quota int // per-user concurrent pod cap enforced inside recordPodCreate + + mu sync.Mutex + stsUIDs map[string]types.UID // mcpserver UID -> StatefulSet UID (process-lifetime cache) +} + +// NewK8sPodLifecycle creates a PodLifecycle backed by the given +// controller-runtime client and Redis admission store. idleTTL is the pod +// liveness lease the reaper enforces; adoptTTL is the short lease written when +// a pre-existing pod is adopted, bounded so an abandoned adoption lapses +// quickly (the reaper refreshes the full idle TTL while the owning session +// lives). perUserQuota is the same cap admission checks read-only; it must be +// positive (fail loudly on misconfiguration). +func NewK8sPodLifecycle( + k8sClient client.Client, + store *redisStore, + vmcpUID string, + idleTTL, adoptTTL time.Duration, + perUserQuota int, +) (PodLifecycle, error) { + if perUserQuota <= 0 { + return nil, fmt.Errorf("untrusted pod lifecycle: per-user quota must be positive, got %d", perUserQuota) + } + return &k8sPodLifecycle{ + k8s: k8sClient, + store: store, + vmcpUID: vmcpUID, + idleTTL: idleTTL, + adoptTTL: adoptTTL, + quota: perUserQuota, + stsUIDs: make(map[string]types.UID), + }, nil +} + +// EnsurePod implements PodLifecycle. +func (l *k8sPodLifecycle) EnsurePod(ctx context.Context, req EnsurePodRequest) (*corev1.Pod, error) { + name := PodNameFor(req.MCPServerUID, req.Session.UserKey(), req.Session.SessionID) + + existing, err := l.getVerifiedPod(ctx, name, req) + if err != nil { + return nil, err + } + if existing != nil { + // Adopting a pre-existing pod: write a bounded lease so the pod is not + // reaped mid-session even if the session metadata never lands (e.g. + // EnsurePod succeeds but session creation then fails). The reaper + // refreshes the full idle TTL while the owning session exists. + if err := l.store.writePodLease(ctx, existing, l.adoptTTL); err != nil { + return nil, fmt.Errorf("failed to write pod lease for %q: %w", name, err) + } + return existing, nil + } + + template, appLabel, err := l.resolveTemplate(ctx, req.MCPServerUID, req.Session.Namespace) + if err != nil { + return nil, err + } + + // The operator publishes the current bump-CA generation on the template; + // the clone mounts exactly that generation's Secret/bundle (consistent + // cert/key pair mid-rotation). + req.Session.CAGeneration = template.Annotations[AnnotationCAGeneration] + + pod, err := clonePodFromTemplate(template, appLabel, req, l.vmcpUID) + if err != nil { + return nil, err + } + + if err := l.k8s.Create(ctx, pod); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, fmt.Errorf("failed to create untrusted pod %q: %w", name, err) + } + // Lost a create race with another vMCP replica: adopt the winner's pod. + existing, err = l.getVerifiedPod(ctx, name, req) + if err != nil { + return nil, err + } + if existing == nil { + return nil, fmt.Errorf("untrusted pod %q reported created but not retrievable", name) + } + if err := l.store.writePodLease(ctx, existing, l.adoptTTL); err != nil { + return nil, fmt.Errorf("failed to write pod lease for %q: %w", name, err) + } + return existing, nil + } + + if err := l.store.recordPodCreate(ctx, pod, l.idleTTL, l.quota); err != nil { + // Compensate: do not leave an untracked pod behind on Redis failure. + deleteCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if delErr := l.deletePodObject(deleteCtx, name); delErr != nil { + slog.Warn("untrusted pod bookkeeping failed and compensating delete failed; reaper will collect the pod", + "pod", name, "error", delErr) + } + if errors.Is(err, ErrQuotaExceeded) { + cancel() + return nil, err + } + cancel() + return nil, fmt.Errorf("failed to record untrusted pod %q: %w", name, err) + } + + if req.OnNewPod != nil { + req.OnNewPod(name) + } + return pod, nil +} + +// WaitReady implements PodLifecycle. Polls (request-scoped one-shot, not an +// informer) with exponential backoff until podIP is assigned and the pod is +// Ready, hard-capped by budget. +func (l *k8sPodLifecycle) WaitReady(ctx context.Context, pod *corev1.Pod, budget time.Duration) error { + if pod == nil { + return fmt.Errorf("WaitReady: pod must not be nil") + } + waitCtx, cancel := context.WithTimeout(ctx, budget) + defer cancel() + + interval := waitReadyInitialPoll + for { + latest := &corev1.Pod{} + err := l.k8s.Get(waitCtx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, latest) + switch { + case apierrors.IsNotFound(err): + return fmt.Errorf("untrusted pod %q disappeared while waiting for readiness", pod.Name) + case err != nil: + if waitCtx.Err() != nil { + return fmt.Errorf("untrusted pod %q not ready within %s", pod.Name, budget) + } + // Transient API error: keep polling until the budget expires. + default: + if latest.UID == pod.UID && latest.Status.PodIP != "" && isPodReady(latest) { + return nil + } + } + + timer := time.NewTimer(interval) + select { + case <-waitCtx.Done(): + timer.Stop() + return fmt.Errorf("untrusted pod %q not ready within %s", pod.Name, budget) + case <-timer.C: + } + if interval < waitReadyMaxPoll { + interval *= 2 + if interval > waitReadyMaxPoll { + interval = waitReadyMaxPoll + } + } + } +} + +// DeletePod implements PodLifecycle. +func (l *k8sPodLifecycle) DeletePod(ctx context.Context, name string) error { + pod := &corev1.Pod{} + key := types.NamespacedName{Name: name, Namespace: l.namespace()} + var labels map[string]string + if err := l.k8s.Get(ctx, key, pod); err == nil { + labels = pod.Labels + } else if !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get untrusted pod %q for deletion: %w", name, err) + } + + if err := l.deletePodObject(ctx, name); err != nil { + return err + } + + // Bookkeeping is best-effort after the authoritative delete succeeded; + // the reaper's reconciliation step heals any residual drift. + if err := l.store.recordPodDelete(ctx, name, labels); err != nil { + slog.Warn("failed to clean untrusted pod bookkeeping; reaper will reconcile", + "pod", name, "error", err) + } + return nil +} + +// namespace returns the namespace this lifecycle operates in. Untrusted mode +// is single-namespace by construction; reaper and DeletePod paths derive it +// from the store's configured namespace. +func (l *k8sPodLifecycle) namespace() string { + return l.store.namespace +} + +// getVerifiedPod fetches the named pod and validates it is a live member of +// this (session, server) tuple: untrusted label set, same mcpserver UID, same +// session ID annotation, and not terminating. Returns (nil, nil) when no +// usable pod exists. Any mismatch is a hard failure — attaching to a pod +// whose annotations do not name this session could cross user bindings. +func (l *k8sPodLifecycle) getVerifiedPod(ctx context.Context, name string, req EnsurePodRequest) (*corev1.Pod, error) { + pod := &corev1.Pod{} + err := l.k8s.Get(ctx, types.NamespacedName{Name: name, Namespace: req.Session.Namespace}, pod) + if apierrors.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to get untrusted pod %q: %w", name, err) + } + if pod.Labels[LabelUntrusted] != "true" || pod.Labels[LabelMCPServerUID] != req.MCPServerUID { + return nil, fmt.Errorf("untrusted pod %q exists with mismatched identity labels; refusing to attach", name) + } + if pod.Annotations[AnnotationSessionID] != req.Session.SessionID { + return nil, fmt.Errorf("untrusted pod %q is bound to a different session; refusing to attach", name) + } + if pod.DeletionTimestamp != nil { + // Terminating pods are not reused; a concurrent create gets + // AlreadyExists until the apiserver finishes the delete, at which + // point the next session attempt recreates cleanly. + return nil, fmt.Errorf("untrusted pod %q is terminating; cannot provision session backend", name) + } + return pod, nil +} + +// resolveTemplate finds the backend StatefulSet for the MCPServer. The +// StatefulSet UID is cached per MCPServer UID for the process lifetime (the +// cache key changes when the MCPServer is recreated, invalidating the entry); +// the template itself is always re-read so image bumps roll forward naturally. +// The resolved StatefulSet must carry the app label (cloned onto session pods +// for NetworkPolicy selector parity) — a missing label is a hard error, never +// a silent empty string. +func (l *k8sPodLifecycle) resolveTemplate( + ctx context.Context, + mcpserverUID string, + namespace string, +) (*corev1.PodTemplateSpec, string, error) { + l.mu.Lock() + cachedUID, cached := l.stsUIDs[mcpserverUID] + l.mu.Unlock() + + sts := &appsv1.StatefulSet{} + if cached { + uidKey := types.NamespacedName{Name: string(cachedUID), Namespace: namespace} + if err := l.k8s.Get(ctx, uidKey, sts); err == nil { + return validateTemplateSTS(sts, mcpserverUID) + } + // Deleted/renamed underneath us: fall through to the list path. + } + + list := &appsv1.StatefulSetList{} + if err := l.k8s.List(ctx, list, + client.InNamespace(namespace), + client.MatchingLabels{LabelMCPServerUID: mcpserverUID, "toolhive": "true"}, + ); err != nil { + return nil, "", fmt.Errorf("failed to list backend StatefulSets for untrusted MCPServer: %w", err) + } + if len(list.Items) != 1 { + return nil, "", fmt.Errorf( + "expected exactly one backend StatefulSet for untrusted MCPServer (uid %s), found %d", + mcpserverUID, len(list.Items)) + } + sts = &list.Items[0] + + template, appLabel, err := validateTemplateSTS(sts, mcpserverUID) + if err != nil { + return nil, "", err + } + l.mu.Lock() + l.stsUIDs[mcpserverUID] = types.UID(sts.Name) + l.mu.Unlock() + return template, appLabel, nil +} + +// validateTemplateSTS enforces the resolved-STS invariants before any clone: +// the app label must be present (it is cloned onto session pods so they match +// the shared NetworkPolicy selectors; an empty label would silently strand +// them outside every selector). +func validateTemplateSTS(sts *appsv1.StatefulSet, mcpserverUID string) (*corev1.PodTemplateSpec, string, error) { + appLabel := sts.Labels[LabelApp] + if appLabel == "" { + return nil, "", fmt.Errorf( + "backend StatefulSet %q for untrusted MCPServer (uid %s) has no %q label; refusing to clone", + sts.Name, mcpserverUID, LabelApp) + } + return &sts.Spec.Template, appLabel, nil +} + +// deletePodObject issues the K8s delete only; NotFound is not an error. +func (l *k8sPodLifecycle) deletePodObject(ctx context.Context, name string) error { + pod := &corev1.Pod{} + key := types.NamespacedName{Name: name, Namespace: l.namespace()} + if err := l.k8s.Get(ctx, key, pod); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to get untrusted pod %q: %w", name, err) + } + if err := l.k8s.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete untrusted pod %q: %w", name, err) + } + return nil +} + +// isPodReady reports whether the pod has a Ready condition with status True. +func isPodReady(pod *corev1.Pod) bool { + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +// Redis Lua scripts. All single-key-per-statement, consistent with the +// session-storage posture. +var ( + // recordCreateScript registers a fresh pod atomically: INCR the per-user + // quota counter (rolled back → return 0 when the cap ARGV[4] is already + // reached), set its TTL (refreshed each create so the counter outlives a + // burst of activity), register the quota key in the reaper's GC registry, + // add global/server set membership, and write the idle TTL lease. + // ARGV: [1]=idleTTL ms, [2]=pod name, [3]=user hash, [4]=per-user cap. + recordCreateScript = redis.NewScript(` + local n = redis.call('INCR', KEYS[1]) + if n > tonumber(ARGV[4]) then + redis.call('DECR', KEYS[1]) + return 0 + end + redis.call('PEXPIRE', KEYS[1], ARGV[1]) + redis.call('SADD', KEYS[5], KEYS[1]) + redis.call('SADD', KEYS[2], ARGV[2]) + redis.call('SADD', KEYS[3], ARGV[2]) + redis.call('SET', KEYS[4], ARGV[3], 'PX', ARGV[1]) + return 1 + `) + + // recordDeleteScript releases a pod: DECR the user's quota counter (floor + // 0; the key disappears at 0 so a missing counter and a zero counter are + // the same state), removes set membership, and drops the TTL lease. + recordDeleteScript = redis.NewScript(` + if redis.call('SISMEMBER', KEYS[2], ARGV[1]) == 1 then + redis.call('SREM', KEYS[2], ARGV[1]) + redis.call('SREM', KEYS[3], ARGV[1]) + local n = redis.call('DECR', KEYS[1]) + if n <= 0 then + redis.call('DEL', KEYS[1]) + end + end + redis.call('DEL', KEYS[4]) + return 1 + `) + + // writeLeaseScript refreshes (or writes) a pod's TTL lease without + // touching quota or set membership (used when adopting existing pods). + writeLeaseScript = redis.NewScript(` + redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2]) + redis.call('SADD', KEYS[2], ARGV[3]) + redis.call('SADD', KEYS[3], ARGV[3]) + return 1 + `) + + // reconcileScript rebuilds the global pod set from the authoritative pod + // list and corrects every registered per-user quota counter to the live + // count (keys whose user owns no live pod are deleted). ARGV[1] is a + // '|'-joined list of per-user live counts as "=" pairs; + // ARGV[2..] are live pod names. The script parses pairs positionally + // (find from the end) so hashes can never collide with the separator. + reconcileScript = redis.NewScript(` + redis.call('DEL', KEYS[1]) + if #ARGV > 1 then + local pods = {} + for i = 2, #ARGV do pods[#pods + 1] = ARGV[i] end + redis.call('SADD', KEYS[1], unpack(pods)) + end + local counts = {} + for pair in string.gmatch(ARGV[1], '([^|]+)') do + local pos = string.find(pair, '=[^=]*$') + if pos then + counts[string.sub(pair, 1, pos - 1)] = tonumber(string.sub(pair, pos + 1)) + end + end + for _, qk in ipairs(redis.call('SMEMBERS', KEYS[2])) do + local user = string.match(qk, 'userquota:(.+)$') + if user then + local live = counts[user] + if live and live > 0 then + redis.call('SET', qk, live, 'KEEPTTL') + else + redis.call('DEL', qk) + end + end + end + return 1 + `) + + // rebuildServerSetScript swaps one per-server pod set with the + // authoritative members from the pod list. + rebuildServerSetScript = redis.NewScript(` + redis.call('DEL', KEYS[1]) + if #ARGV > 0 then + redis.call('SADD', KEYS[1], unpack(ARGV)) + end + return 1 + `) +) + +// redisStore owns all untrusted-mode Redis state: pod TTL leases, admission +// sets, quota counters, and vMCP heartbeats. Redis holds only counters and +// TTLs — the pod→user mapping lives in pod labels/annotations. +type redisStore struct { + client redis.UniversalClient + prefix string + namespace string +} + +// newRedisStore builds the store. prefix must be the tenant's ':'-terminated +// session-storage prefix (same validation as RedisStorage). +func newRedisStore(redisClient redis.UniversalClient, prefix, namespace string) (*redisStore, error) { + if redisClient == nil { + return nil, fmt.Errorf("untrusted redis store: client must not be nil") + } + if prefix == "" || !strings.HasSuffix(prefix, ":") { + return nil, fmt.Errorf("untrusted redis store: key prefix must be non-empty and end with ':'") + } + if namespace == "" { + return nil, fmt.Errorf("untrusted redis store: namespace must not be empty") + } + return &redisStore{client: redisClient, prefix: prefix, namespace: namespace}, nil +} + +func (s *redisStore) quotaKeyForUserHash(hash string) string { + return s.prefix + "untrusted:userquota:" + hash +} + +// quotaKeysRegistryKey is a Redis set of all per-user quota key names, +// maintained so the reaper can find and garbage-collect orphaned counters +// without a SCAN. +func (s *redisStore) quotaKeysRegistryKey() string { + return s.prefix + "untrusted:quotakeys" +} + +// recordPodCreate atomically registers a freshly created pod and takes one of +// the user's quota slots. When the user's counter is already at perUserQuota +// the script rolls its own INCR back and the call returns ErrQuotaExceeded — +// the caller must compensate by deleting the pod it just created. +func (s *redisStore) recordPodCreate(ctx context.Context, pod *corev1.Pod, idleTTL time.Duration, perUserQuota int) error { + userKeyHash := pod.Labels[LabelUntrustedUser] + admitted, err := recordCreateScript.Run(ctx, s.client, []string{ + s.quotaKeyForUserHash(userKeyHash), + s.podsSetKey(), + s.serverPodsSetKey(pod.Labels[LabelMCPServerUID]), + s.podTTLKey(pod.Name), + s.quotaKeysRegistryKey(), + }, idleTTL.Milliseconds(), pod.Name, userKeyHash, perUserQuota).Int() + if err != nil { + return err + } + if admitted == 0 { + return fmt.Errorf("%w: per-user pod quota (%d) reached", ErrQuotaExceeded, perUserQuota) + } + return nil +} + +func (s *redisStore) recordPodDelete(ctx context.Context, name string, labels map[string]string) error { + return recordDeleteScript.Run(ctx, s.client, []string{ + s.quotaKeyForUserHash(labels[LabelUntrustedUser]), + s.podsSetKey(), + s.serverPodsSetKey(labels[LabelMCPServerUID]), + s.podTTLKey(name), + }, name).Err() +} + +func (s *redisStore) writePodLease(ctx context.Context, pod *corev1.Pod, ttl time.Duration) error { + return writeLeaseScript.Run(ctx, s.client, []string{ + s.podTTLKey(pod.Name), + s.podsSetKey(), + s.serverPodsSetKey(pod.Labels[LabelMCPServerUID]), + }, pod.Labels[LabelUntrustedUser], ttl.Milliseconds(), pod.Name).Err() +} + +// touchPodLeaseScript refreshes the idle TTL on an existing lease (sliding +// window). No-op when the lease is absent — the reaper decides creation vs +// deletion. +var touchPodLeaseScript = redis.NewScript(` + if redis.call('EXISTS', KEYS[1]) == 1 then + redis.call('PEXPIRE', KEYS[1], ARGV[1]) + return 1 + end + return 0 +`) + +func (s *redisStore) touchPodLease(ctx context.Context, podName string, idleTTL time.Duration) error { + return touchPodLeaseScript.Run(ctx, s.client, []string{s.podTTLKey(podName)}, idleTTL.Milliseconds()).Err() +} + +func (s *redisStore) podLeaseExists(ctx context.Context, podName string) (bool, error) { + n, err := s.client.Exists(ctx, s.podTTLKey(podName)).Result() + return n > 0, err +} + +func (s *redisStore) heartbeatExists(ctx context.Context, vmcpUID string) (bool, error) { + n, err := s.client.Exists(ctx, s.heartbeatKey(vmcpUID)).Result() + return n > 0, err +} + +func (s *redisStore) writeHeartbeat(ctx context.Context, vmcpUID string, ttl time.Duration) error { + return s.client.Set(ctx, s.heartbeatKey(vmcpUID), "1", ttl).Err() +} diff --git a/pkg/vmcp/session/untrusted/lifecycle_integration_test.go b/pkg/vmcp/session/untrusted/lifecycle_integration_test.go new file mode 100644 index 0000000000..750a4fac0c --- /dev/null +++ b/pkg/vmcp/session/untrusted/lifecycle_integration_test.go @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build integration + +package untrusted_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "path/filepath" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/redis/go-redis/v9" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + "github.com/stacklok/toolhive/pkg/vmcp/session/binding" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" +) + +// Integration tests for the untrusted PodLifecycle against a real apiserver +// (envtest). These cover the paths the fake client cannot: ownerRef cascade +// delete, real AlreadyExists semantics, and status-patching for WaitReady. + +var ( + intCfg *rest.Config + intClient client.Client + intEnv *envtest.Environment + intMR *miniredis.Miniredis +) + +// newIntLifecycle builds a PodLifecycle against the shared envtest apiserver +// and a per-suite miniredis. +func newIntLifecycle() untrusted.PodLifecycle { + return newIntLifecycleWithQuota(10) +} + +func newIntLifecycleWithQuota(quota int) untrusted.PodLifecycle { + rc := redis.NewClient(&redis.Options{Addr: intMR.Addr()}) + lc, err := untrusted.NewK8sPodLifecycleFromParts( + intClient, rc, "thv:vmcp:session:", intNamespace, "vmcp-int", 30*time.Minute, 2*time.Minute, quota) + Expect(err).NotTo(HaveOccurred()) + return lc +} + +func TestUntrustedLifecycleIntegration(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Untrusted PodLifecycle Integration Suite") +} + +var _ = BeforeSuite(func() { + intEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "deploy", "charts", "operator-crds", "files", "crds")}, + ErrorIfCRDPathMissing: true, + } + var err error + intCfg, err = intEnv.Start() + Expect(err).NotTo(HaveOccurred()) + + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + Expect(appsv1.AddToScheme(scheme)).To(Succeed()) + + intClient, err = client.New(intCfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "untrusted-int"}} + Expect(intClient.Create(context.Background(), ns)).To(Succeed()) + + intMR = miniredis.RunT(GinkgoT()) +}) + +var _ = AfterSuite(func() { + Expect(intEnv.Stop()).To(Succeed()) +}) + +const ( + intNamespace = "untrusted-int" + intMCPServerUID = "uid-int-1234567890" +) + +func intRequest(sessionID string) untrusted.EnsurePodRequest { + return untrusted.EnsurePodRequest{ + Session: untrusted.SessionRef{ + SessionID: sessionID, + Issuer: "https://issuer.example.com", + Subject: "user-int", + Namespace: intNamespace, + }, + MCPServerUID: intMCPServerUID, + MCPServerName: "int-mcp", + Port: 8080, + OwnerRefUID: types.UID(intMCPServerUID), + } +} + +func intSTS() *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "int-backend-sts", + Namespace: intNamespace, + Labels: map[string]string{ + untrusted.LabelMCPServerUID: intMCPServerUID, + "toolhive": "true", + untrusted.LabelApp: "int-backend", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"x": "y"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"x": "y"}}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "mcp", + Image: "ghcr.io/example/backend:int", + Env: []corev1.EnvVar{{Name: "SENTINEL", Value: "literal"}}, + }}, + }, + }, + }, + } +} + +var _ = Describe("PodLifecycle (envtest)", func() { + ctx := context.Background() + + It("creates a pod from the STS template, idempotently", func() { + Expect(intClient.Create(ctx, intSTS())).To(Succeed()) + + lifecycle := newIntLifecycle() + req := intRequest("sess-int-1") + + pod, err := lifecycle.EnsurePod(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(len(pod.Name)).To(BeNumerically("<=", 63)) + Expect(pod.Labels[untrusted.LabelUntrusted]).To(Equal("true")) + Expect(pod.Spec.Containers[0].Image).To(Equal("ghcr.io/example/backend:int")) + + userKey, err := binding.Format(req.Session.Issuer, req.Session.Subject) + Expect(err).NotTo(HaveOccurred()) + Expect(pod.Name).To(Equal(untrusted.PodNameFor(intMCPServerUID, userKey, "sess-int-1"))) + + // OwnerRef: non-controller, pointing at the MCPServer identity. + Expect(pod.OwnerReferences).To(HaveLen(1)) + Expect(pod.OwnerReferences[0].Kind).To(Equal("MCPServer")) + Expect(pod.OwnerReferences[0].Controller).To(BeNil()) + + // Idempotency: second EnsurePod returns the same live pod (AlreadyExists + // treated as success at the apiserver level). + again, err := lifecycle.EnsurePod(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(again.UID).To(Equal(pod.UID)) + }) + + It("delete is idempotent (NotFound swallowed)", func() { + lifecycle := newIntLifecycle() + req := intRequest("sess-int-2") + pod, err := lifecycle.EnsurePod(ctx, req) + Expect(err).NotTo(HaveOccurred()) + + Expect(lifecycle.DeletePod(ctx, pod.Name)).To(Succeed()) + Expect(lifecycle.DeletePod(ctx, pod.Name)).To(Succeed(), "second delete must be a no-op") + + gone := &corev1.Pod{} + err = intClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: intNamespace}, gone) + Expect(err).To(HaveOccurred()) + }) + + It("WaitReady succeeds on a status-patched ready pod", func() { + lifecycle := newIntLifecycle() + req := intRequest("sess-int-3") + pod, err := lifecycle.EnsurePod(ctx, req) + Expect(err).NotTo(HaveOccurred()) + + // envtest has no kubelet; patch status to simulate scheduling+ready. + latest := &corev1.Pod{} + Expect(intClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: intNamespace}, latest)).To(Succeed()) + latest.Status.PodIP = "10.9.9.9" + latest.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + Expect(intClient.Status().Update(ctx, latest)).To(Succeed()) + + Expect(lifecycle.WaitReady(ctx, pod, 10*time.Second)).To(Succeed()) + }) + + It("two racing EnsurePod calls create exactly one pod and take exactly one quota slot", func() { + lifecycle := newIntLifecycleWithQuota(10) + req := intRequest("sess-int-race") + + // Two independent lifecycle instances (two vMCP replicas) race the same + // (user, session, server) tuple: only the deterministic pod name + // coordinates them (real apiserver AlreadyExists semantics — the fake + // client cannot reproduce this). + other := newIntLifecycleWithQuota(10) + + const racers = 2 + results := make(chan error, racers) + for _, lc := range []untrusted.PodLifecycle{lifecycle, other} { + go func(lc untrusted.PodLifecycle) { + _, err := lc.EnsurePod(ctx, req) + results <- err + }(lc) + } + for range racers { + Expect(<-results).NotTo(HaveOccurred()) + } + + // Exactly one pod exists for the tuple. + pods := &corev1.PodList{} + Expect(intClient.List(ctx, pods, + client.InNamespace(intNamespace), + client.MatchingLabels{untrusted.LabelUntrusted: "true", untrusted.LabelMCPServerUID: intMCPServerUID}, + )).To(Succeed()) + Expect(pods.Items).To(HaveLen(1)) + Expect(pods.Items[0].Annotations[untrusted.AnnotationSessionID]).To(Equal("sess-int-race")) + + // Exactly one quota slot was taken: the loser adopted, it did not + // double-count. + userKey, err := binding.Format(req.Session.Issuer, req.Session.Subject) + Expect(err).NotTo(HaveOccurred()) + rc := redis.NewClient(&redis.Options{Addr: intMR.Addr()}) + sum := sha256.Sum256([]byte(userKey)) + quotaKey := "thv:vmcp:session:untrusted:userquota:" + hex.EncodeToString(sum[:])[:40] + Expect(rc.Get(ctx, quotaKey).Val()).To(Equal("1")) + }) + + It("fail-closed: hand-built secret-backed template is rejected, no pod created", func() { + sts := intSTS() + sts.Name = "int-backend-sts-secret" + sts.Labels[untrusted.LabelMCPServerUID] = "uid-secret" + sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "creds", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "backend-creds"}, + }, + }) + Expect(intClient.Create(ctx, sts)).To(Succeed()) + + lifecycle := newIntLifecycle() + req := intRequest("sess-int-4") + req.MCPServerUID = "uid-secret" + req.OwnerRefUID = types.UID("uid-secret") + + _, err := lifecycle.EnsurePod(ctx, req) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("untrusted pod clone")) + + // Nothing may have been created. + podName := untrusted.PodNameFor("uid-secret", req.Session.UserKey(), "sess-int-4") + gone := &corev1.Pod{} + getErr := intClient.Get(ctx, types.NamespacedName{Name: podName, Namespace: intNamespace}, gone) + Expect(getErr).To(HaveOccurred()) + }) +}) diff --git a/pkg/vmcp/session/untrusted/lifecycle_test.go b/pkg/vmcp/session/untrusted/lifecycle_test.go new file mode 100644 index 0000000000..2e0af19cac --- /dev/null +++ b/pkg/vmcp/session/untrusted/lifecycle_test.go @@ -0,0 +1,384 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func newScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, appsv1.AddToScheme(scheme)) + return scheme +} + +// backendSTS builds a backend StatefulSet in the test namespace ("toolhive"). +// +//nolint:unparam // namespace kept as a parameter for readability; all callers use the test namespace +func backendSTS(name, namespace, mcpserverUID string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{LabelMCPServerUID: mcpserverUID, "toolhive": "true", LabelApp: "backend-app"}, + }, + Spec: appsv1.StatefulSetSpec{ + Template: *validTemplate(), + }, + } +} + +//nolint:unparam // miniredis handle returned for tests that kill Redis mid-test +func newLifecycle(t *testing.T, objs ...runtime.Object) (PodLifecycle, *redisStore, *miniredis.Miniredis) { + t.Helper() + scheme := newScheme(t) + builder := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objs...) + mr := miniredis.RunT(t) + store, err := newRedisStore(redis.NewClient(&redis.Options{Addr: mr.Addr()}), "thv:vmcp:session:", "toolhive") + require.NoError(t, err) + lc, err := NewK8sPodLifecycle(builder.Build(), store, "vmcp-1", 30*time.Minute, 2*time.Minute, defaultPerUserPodQuota) + require.NoError(t, err) + return lc, store, mr +} + +func TestEnsurePod(t *testing.T) { + t.Parallel() + ctx := context.Background() + + t.Run("creates pod from STS template and records bookkeeping", func(t *testing.T) { + t.Parallel() + req := testRequest() + lc, store, _ := newLifecycle(t, backendSTS("backend-sts", "toolhive", req.MCPServerUID)) + + pod, err := lc.EnsurePod(ctx, req) + require.NoError(t, err) + assert.Equal(t, PodNameFor(req.MCPServerUID, req.Session.UserKey(), req.Session.SessionID), pod.Name) + assert.Equal(t, "true", pod.Labels[LabelUntrusted]) + + // Bookkeeping: quota counter, set membership, TTL lease. + assert.Equal(t, "1", store.client.Get(ctx, store.userQuotaKey(req.Session.UserKey())).Val()) + assert.True(t, store.client.SIsMember(ctx, store.podsSetKey(), pod.Name).Val()) + assert.True(t, store.client.SIsMember(ctx, store.serverPodsSetKey(req.MCPServerUID), pod.Name).Val()) + assert.Equal(t, int64(1), store.client.Exists(ctx, store.podTTLKey(pod.Name)).Val()) + }) + + t.Run("idempotent: second EnsurePod reuses the pod and does not invoke OnNewPod", func(t *testing.T) { + t.Parallel() + req := testRequest() + lc, store, _ := newLifecycle(t, backendSTS("backend-sts", "toolhive", req.MCPServerUID)) + + newPodCalls := 0 + req.OnNewPod = func(string) { newPodCalls++ } + + first, err := lc.EnsurePod(ctx, req) + require.NoError(t, err) + assert.Equal(t, 1, newPodCalls) + + second, err := lc.EnsurePod(ctx, req) + require.NoError(t, err) + assert.Equal(t, first.Name, second.Name) + assert.Equal(t, 1, newPodCalls, "OnNewPod must fire only on fresh create") + + // Reuse does not double-count quota. + assert.True(t, store.client.SIsMember(ctx, store.podsSetKey(), first.Name).Val()) + }) + + t.Run("soft-fails when no STS matches", func(t *testing.T) { + t.Parallel() + req := testRequest() + lc, _, _ := newLifecycle(t) + _, err := lc.EnsurePod(ctx, req) + require.Error(t, err) + assert.Contains(t, err.Error(), "exactly one backend StatefulSet") + }) + + t.Run("soft-fails when multiple STS match", func(t *testing.T) { + t.Parallel() + req := testRequest() + lc, _, _ := newLifecycle(t, + backendSTS("sts-a", "toolhive", req.MCPServerUID), + backendSTS("sts-b", "toolhive", req.MCPServerUID)) + _, err := lc.EnsurePod(ctx, req) + require.Error(t, err) + assert.Contains(t, err.Error(), "found 2") + }) + + t.Run("fail-closed: secret-backed template is rejected, no pod created", func(t *testing.T) { + t.Parallel() + req := testRequest() + sts := backendSTS("backend-sts", "toolhive", req.MCPServerUID) + sts.Spec.Template.Spec.Containers[0].Env = append(sts.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{Name: "TOKEN", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "creds"}, Key: "token", + }, + }}) + lc, _, _ := newLifecycle(t, sts) + _, err := lc.EnsurePod(ctx, req) + require.Error(t, err) + assert.Contains(t, err.Error(), "untrusted pod clone") + }) + + t.Run("compensating delete when Redis bookkeeping fails", func(t *testing.T) { + t.Parallel() + req := testRequest() + k8sClient := fake.NewClientBuilder().WithScheme(newScheme(t)). + WithRuntimeObjects(backendSTS("backend-sts", "toolhive", req.MCPServerUID)).Build() + mr := miniredis.RunT(t) + store, err := newRedisStore(redis.NewClient(&redis.Options{Addr: mr.Addr()}), "thv:vmcp:session:", "toolhive") + require.NoError(t, err) + lc, err := NewK8sPodLifecycle(k8sClient, store, "vmcp-1", 30*time.Minute, 2*time.Minute, defaultPerUserPodQuota) + require.NoError(t, err) + + mr.Close() // Redis down: bookkeeping must fail and the created pod must be removed. + _, err = lc.EnsurePod(ctx, req) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to record") + + // The compensating delete must have removed the created pod from the + // same apiserver: a fresh EnsurePod (Redis back up) must create anew. + mr2 := miniredis.RunT(t) + store2, err := newRedisStore(redis.NewClient(&redis.Options{Addr: mr2.Addr()}), "thv:vmcp:session:", "toolhive") + require.NoError(t, err) + lc2, err := NewK8sPodLifecycle(k8sClient, store2, "vmcp-1", 30*time.Minute, 2*time.Minute, defaultPerUserPodQuota) + require.NoError(t, err) + fired := false + req.OnNewPod = func(string) { fired = true } + _, err = lc2.EnsurePod(ctx, req) + require.NoError(t, err) + assert.True(t, fired, "compensating delete must have removed the failed-create pod") + }) + + t.Run("lifecycle enforces the per-user quota: over-cap create is denied and removed", func(t *testing.T) { + t.Parallel() + req := testRequest() + k8sClient := fake.NewClientBuilder().WithScheme(newScheme(t)). + WithRuntimeObjects(backendSTS("backend-sts", "toolhive", req.MCPServerUID)).Build() + mr := miniredis.RunT(t) + store, err := newRedisStore(redis.NewClient(&redis.Options{Addr: mr.Addr()}), "thv:vmcp:session:", "toolhive") + require.NoError(t, err) + lc, err := NewK8sPodLifecycle(k8sClient, store, "vmcp-1", 30*time.Minute, 2*time.Minute, 2) + require.NoError(t, err) + + // Two pods (distinct sessions of the same user) fit the cap of 2. + for _, sessID := range []string{"sess-q1", "sess-q2"} { + r := req + r.Session.SessionID = sessID + _, err := lc.EnsurePod(ctx, r) + require.NoError(t, err) + } + assert.Equal(t, "2", store.client.Get(ctx, store.userQuotaKey(req.Session.UserKey())).Val()) + + // The third create must be denied AND the just-created pod must be + // compensated away (no over-quota pod survives). + r := req + r.Session.SessionID = "sess-q3" + deniedName := PodNameFor(r.MCPServerUID, r.Session.UserKey(), r.Session.SessionID) + _, err = lc.EnsurePod(ctx, r) + require.ErrorIs(t, err, ErrQuotaExceeded) + gone := &corev1.Pod{} + getErr := k8sClient.Get(ctx, types.NamespacedName{Name: deniedName, Namespace: "toolhive"}, gone) + require.Error(t, getErr, "over-quota pod must be deleted by the compensating path") + // The counter rolled back: still 2, never 3. + assert.Equal(t, "2", store.client.Get(ctx, store.userQuotaKey(req.Session.UserKey())).Val()) + }) + + t.Run("quota slot is reusable after DeletePod", func(t *testing.T) { + t.Parallel() + req := testRequest() + k8sClient := fake.NewClientBuilder().WithScheme(newScheme(t)). + WithRuntimeObjects(backendSTS("backend-sts", "toolhive", req.MCPServerUID)).Build() + mr := miniredis.RunT(t) + store, err := newRedisStore(redis.NewClient(&redis.Options{Addr: mr.Addr()}), "thv:vmcp:session:", "toolhive") + require.NoError(t, err) + lc, err := NewK8sPodLifecycle(k8sClient, store, "vmcp-1", 30*time.Minute, 2*time.Minute, 1) + require.NoError(t, err) + + pod, err := lc.EnsurePod(ctx, req) + require.NoError(t, err) + require.NoError(t, lc.DeletePod(ctx, pod.Name)) + assert.Equal(t, int64(0), store.client.Exists(ctx, store.userQuotaKey(req.Session.UserKey())).Val(), + "counter must be zero after release") + + // A different session of the same user fits immediately (no backoff). + r := req + r.Session.SessionID = "sess-q-reuse" + fired := false + r.OnNewPod = func(string) { fired = true } + _, err = lc.EnsurePod(ctx, r) + require.NoError(t, err) + assert.True(t, fired) + }) + + t.Run("constructor rejects a non-positive quota (fail loudly)", func(t *testing.T) { + t.Parallel() + _, store, _ := newLifecycle(t) + _, err := NewK8sPodLifecycle(fake.NewClientBuilder().WithScheme(newScheme(t)).Build(), + store, "vmcp-1", 30*time.Minute, 2*time.Minute, 0) + require.Error(t, err) + }) + + t.Run("refuses to attach to a pod bound to another session", func(t *testing.T) { + t.Parallel() + req := testRequest() + podName := PodNameFor(req.MCPServerUID, req.Session.UserKey(), req.Session.SessionID) + foreign := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: "toolhive", + Labels: map[string]string{ + LabelUntrusted: "true", LabelMCPServerUID: req.MCPServerUID, + }, + Annotations: map[string]string{AnnotationSessionID: "someone-elses-session"}, + }, + } + lc, _, _ := newLifecycle(t, foreign) + _, err := lc.EnsurePod(ctx, req) + require.Error(t, err) + assert.Contains(t, err.Error(), "bound to a different session") + }) +} + +func TestDeletePod(t *testing.T) { + t.Parallel() + ctx := context.Background() + + t.Run("deletes pod and cleans bookkeeping", func(t *testing.T) { + t.Parallel() + req := testRequest() + lc, store, _ := newLifecycle(t, backendSTS("backend-sts", "toolhive", req.MCPServerUID)) + pod, err := lc.EnsurePod(ctx, req) + require.NoError(t, err) + + require.NoError(t, lc.DeletePod(ctx, pod.Name)) + assert.Equal(t, int64(0), store.client.Exists(ctx, store.podTTLKey(pod.Name)).Val()) + assert.False(t, store.client.SIsMember(ctx, store.podsSetKey(), pod.Name).Val()) + + // The quota slot is freed immediately (the key disappears at zero). + assert.Equal(t, int64(0), store.client.Exists(ctx, store.userQuotaKey(req.Session.UserKey())).Val()) + }) + + t.Run("NotFound is not an error", func(t *testing.T) { + t.Parallel() + lc, _, _ := newLifecycle(t) + require.NoError(t, lc.DeletePod(ctx, "never-existed")) + }) +} + +func TestWaitReady(t *testing.T) { + t.Parallel() + ctx := context.Background() + + readyPod := func(name string, uid types.UID) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "toolhive", UID: uid}, + Status: corev1.PodStatus{ + PodIP: "10.0.0.5", + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + } + } + + t.Run("returns when pod has IP and is Ready with matching UID", func(t *testing.T) { + t.Parallel() + p := readyPod("pod-a", "uid-a") + lc, _, _ := newLifecycle(t, p) + require.NoError(t, lc.WaitReady(ctx, p, 5*time.Second)) + }) + + t.Run("budget expires for never-ready pod", func(t *testing.T) { + t.Parallel() + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod-b", Namespace: "toolhive", UID: "uid-b"}, + } + lc, _, _ := newLifecycle(t, p) + start := time.Now() + err := lc.WaitReady(ctx, p, 1200*time.Millisecond) + require.Error(t, err) + assert.Contains(t, err.Error(), "not ready within") + assert.GreaterOrEqual(t, time.Since(start), 1200*time.Millisecond) + }) + + t.Run("NotFound pod errors immediately", func(t *testing.T) { + t.Parallel() + lc, _, _ := newLifecycle(t) + ghost := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "ghost", Namespace: "toolhive", UID: "uid-g"}} + err := lc.WaitReady(ctx, ghost, 5*time.Second) + require.Error(t, err) + assert.Contains(t, err.Error(), "disappeared") + }) + + t.Run("UID mismatch waits (IP not trusted)", func(t *testing.T) { + t.Parallel() + // Live pod is ready but under a DIFFERENT UID than the one we created. + live := readyPod("pod-c", "uid-other") + lc, _, _ := newLifecycle(t, live) + mine := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod-c", Namespace: "toolhive", UID: "uid-mine"}} + err := lc.WaitReady(ctx, mine, 600*time.Millisecond) + require.Error(t, err, "must not trust an IP from a pod whose UID differs") + }) + + t.Run("rejects nil pod", func(t *testing.T) { + t.Parallel() + lc, _, _ := newLifecycle(t) + require.Error(t, lc.WaitReady(ctx, nil, time.Second)) + }) +} + +func TestResolveTemplateAppLabelGuard(t *testing.T) { + t.Parallel() + ctx := context.Background() + + t.Run("STS without the app label is a hard error, never a silent empty string", func(t *testing.T) { + t.Parallel() + req := testRequest() + sts := backendSTS("backend-sts", "toolhive", req.MCPServerUID) + delete(sts.Labels, LabelApp) + lc, _, _ := newLifecycle(t, sts) + _, err := lc.EnsurePod(ctx, req) + require.Error(t, err) + assert.Contains(t, err.Error(), LabelApp) + }) + + t.Run("cached STS path also enforces the app-label guard", func(t *testing.T) { + t.Parallel() + req := testRequest() + lc, _, _ := newLifecycle(t, backendSTS("backend-sts", "toolhive", req.MCPServerUID)) + _, err := lc.EnsurePod(ctx, req) + require.NoError(t, err) + + // Replace the STS (same name = cached UID path) with one missing the + // label; the cache must not bypass the guard. + k8sClient := fake.NewClientBuilder().WithScheme(newScheme(t)).Build() + bad := backendSTS("backend-sts", "toolhive", req.MCPServerUID) + delete(bad.Labels, LabelApp) + require.NoError(t, k8sClient.Create(ctx, bad)) + _ = lc // cache is per-instance; assert via the store-backed instance below + store2, err := newRedisStore( + redis.NewClient(&redis.Options{Addr: miniredis.RunT(t).Addr()}), "thv:vmcp:session:", "toolhive") + require.NoError(t, err) + lc2, err := NewK8sPodLifecycle(k8sClient, store2, "vmcp-1", 30*time.Minute, 2*time.Minute, defaultPerUserPodQuota) + require.NoError(t, err) + _, err = lc2.EnsurePod(ctx, req) + require.Error(t, err) + assert.Contains(t, err.Error(), LabelApp) + }) +} diff --git a/pkg/vmcp/session/untrusted/metrics.go b/pkg/vmcp/session/untrusted/metrics.go new file mode 100644 index 0000000000..b649b63cef --- /dev/null +++ b/pkg/vmcp/session/untrusted/metrics.go @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// instrumentationName matches the vMCP-wide OTel instrumentation scope used by +// the other vMCP meters (pkg/vmcp/internal/backendtelemetry, +// pkg/vmcp/server/sessionmanager). +const instrumentationName = "github.com/stacklok/toolhive/pkg/vmcp" + +// untrustedMetrics holds the untrusted-mode instruments. Labels are kept +// low-cardinality per ADR D11: the MCPServer name and the admission result — +// never a user identifier. +type untrustedMetrics struct { + // backendPods gauges live untrusted backend pods per MCPServer, refreshed + // by the reaper from the authoritative pod LIST each tick. + backendPods metric.Int64Gauge + // admissions counts pod provisioning admission decisions by result. + admissions metric.Int64Counter +} + +// newUntrustedMetrics registers the untrusted-mode instruments on the vMCP +// meter. Registration failures are startup-fatal (fail loudly): a silently +// uninstrumented untrusted mode hides the DoS controls' effectiveness. +func newUntrustedMetrics(meterProvider metric.MeterProvider) (*untrustedMetrics, error) { + meter := meterProvider.Meter(instrumentationName) + + backendPods, err := meter.Int64Gauge( + "untrusted_backend_pods", + metric.WithDescription("Number of live untrusted per-session backend pods"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create untrusted_backend_pods gauge: %w", err) + } + admissions, err := meter.Int64Counter( + "untrusted_pod_admissions_total", + metric.WithDescription("Total untrusted pod provisioning admission decisions"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create untrusted_pod_admissions_total counter: %w", err) + } + return &untrustedMetrics{backendPods: backendPods, admissions: admissions}, nil +} + +// recordAdmission records one admission decision. result is a small fixed +// vocabulary ("admitted", "quota_exceeded", "error") — never a user or pod. +func (m *untrustedMetrics) recordAdmission(ctx context.Context, result string) { + if m == nil { + return + } + m.admissions.Add(ctx, 1, metric.WithAttributes(attribute.String("result", result))) +} + +// recordPodCounts records the per-MCPServer live pod gauge from the reaper's +// authoritative pod list. mcpserverName is low-cardinality (one per untrusted +// MCPServer CR). +func (m *untrustedMetrics) recordPodCounts(ctx context.Context, mcpserverName string, count int) { + if m == nil { + return + } + m.backendPods.Record(ctx, int64(count), metric.WithAttributes(attribute.String("mcpserver", mcpserverName))) +} diff --git a/pkg/vmcp/session/untrusted/metrics_test.go b/pkg/vmcp/session/untrusted/metrics_test.go new file mode 100644 index 0000000000..30e2a4711a --- /dev/null +++ b/pkg/vmcp/session/untrusted/metrics_test.go @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/noop" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// newTestMeterProvider returns an SDK meter provider backed by a manual reader +// so tests can collect and assert on recorded measurements. +func newTestMeterProvider(t *testing.T) (metric.MeterProvider, *sdkmetric.ManualReader) { + t.Helper() + reader := sdkmetric.NewManualReader() + return sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)), reader +} + +// collectMetrics gathers all recorded metrics from the reader into a name→data +// map for assertions. +func collectMetrics(t *testing.T, reader *sdkmetric.ManualReader) map[string]metricdata.Metrics { + t.Helper() + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + out := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + out[m.Name] = m + } + } + return out +} + +func TestNewUntrustedMetrics_Registration(t *testing.T) { + t.Parallel() + + t.Run("instruments register without error on a real provider", func(t *testing.T) { + t.Parallel() + mp, reader := newTestMeterProvider(t) + m, err := newUntrustedMetrics(mp) + require.NoError(t, err) + require.NotNil(t, m) + + // Record one of each so the instruments produce data. + ctx := context.Background() + m.recordAdmission(ctx, "admitted") + m.recordPodCounts(ctx, "github-mcp", 3) + + got := collectMetrics(t, reader) + assert.Contains(t, got, "untrusted_backend_pods") + assert.Contains(t, got, "untrusted_pod_admissions_total") + }) + + t.Run("nil metrics are a no-op (never panic)", func(t *testing.T) { + t.Parallel() + var m *untrustedMetrics + ctx := context.Background() + assert.NotPanics(t, func() { + m.recordAdmission(ctx, "admitted") + m.recordPodCounts(ctx, "github-mcp", 1) + }) + }) + + t.Run("noop provider registers instruments (metrics disabled path)", func(t *testing.T) { + t.Parallel() + m, err := newUntrustedMetrics(noop.NewMeterProvider()) + require.NoError(t, err) + require.NotNil(t, m) + }) +} + +func TestAdmissionResultLabel(t *testing.T) { + t.Parallel() + + mp, reader := newTestMeterProvider(t) + m, err := newUntrustedMetrics(mp) + require.NoError(t, err) + ctx := context.Background() + + // The admitted path records the literal "admitted" at the call site; + // admissionResult maps the two error vocabularies. + m.recordAdmission(ctx, "admitted") + m.recordAdmission(ctx, admissionResult(ErrQuotaExceeded)) // quota + m.recordAdmission(ctx, admissionResult(assert.AnError)) // generic error + + got := collectMetrics(t, reader) + counter, ok := got["untrusted_pod_admissions_total"] + require.True(t, ok, "counter must be registered") + + sum, ok := counter.Data.(metricdata.Sum[int64]) + require.True(t, ok, "admissions must be a Sum counter") + + results := map[string]int64{} + for _, dp := range sum.DataPoints { + for _, attr := range dp.Attributes.ToSlice() { + if attr.Key == "result" { + results[attr.Value.AsString()] = dp.Value + } + } + } + assert.Equal(t, int64(1), results["admitted"]) + assert.Equal(t, int64(1), results["quota_exceeded"]) + assert.Equal(t, int64(1), results["error"]) + + // Low-cardinality guard: the only attribute on any datapoint is "result". + for _, dp := range sum.DataPoints { + for _, attr := range dp.Attributes.ToSlice() { + assert.Equal(t, "result", string(attr.Key), "no user/pod/server labels allowed (ADR D11)") + } + } +} + +func TestReaperRecordsPodGauge(t *testing.T) { + t.Parallel() + + mp, reader := newTestMeterProvider(t) + m, err := newUntrustedMetrics(mp) + require.NoError(t, err) + + // Two servers, one with two pods, one with one, one pod with no name + // annotation (grouped under "unknown"). + ctx := context.Background() + m.recordPodCounts(ctx, "github-mcp", 2) + m.recordPodCounts(ctx, "gitlab-mcp", 1) + m.recordPodCounts(ctx, "unknown", 1) + + got := collectMetrics(t, reader) + gauge, ok := got["untrusted_backend_pods"] + require.True(t, ok) + + g, ok := gauge.Data.(metricdata.Gauge[int64]) + require.True(t, ok, "backend pods must be a gauge") + + counts := map[string]int64{} + for _, dp := range g.DataPoints { + for _, attr := range dp.Attributes.ToSlice() { + if attr.Key == "mcpserver" { + counts[attr.Value.AsString()] = dp.Value + } + } + } + assert.Equal(t, int64(2), counts["github-mcp"]) + assert.Equal(t, int64(1), counts["gitlab-mcp"]) + assert.Equal(t, int64(1), counts["unknown"]) + + // Low-cardinality guard: only the mcpserver label is present. + for _, dp := range g.DataPoints { + for _, attr := range dp.Attributes.ToSlice() { + assert.Equal(t, "mcpserver", string(attr.Key)) + } + } +} + +// TestReaperMetricsDisabled proves a nil *untrustedMetrics leaves the reaper's +// metrics disabled (the production "no MeterProvider" path). +func TestReaperMetricsDisabled(t *testing.T) { + t.Parallel() + store, _ := newTestStore(t) + k8sClient := fake.NewClientBuilder().WithScheme(newScheme(t)).Build() + reaper, err := NewReaper(k8sClient, store, ReaperConfig{IdleTTL: time.Minute}, "vmcp-1", nil) + require.NoError(t, err) + assert.Nil(t, reaper.metrics) +} diff --git a/pkg/vmcp/session/untrusted/mocks/mock_lifecycle.go b/pkg/vmcp/session/untrusted/mocks/mock_lifecycle.go new file mode 100644 index 0000000000..cb5c3e8993 --- /dev/null +++ b/pkg/vmcp/session/untrusted/mocks/mock_lifecycle.go @@ -0,0 +1,126 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/stacklok/toolhive/pkg/vmcp/session/untrusted (interfaces: PodLifecycle,BackendAddressResolver) +// +// Generated by this command: +// +// mockgen -destination=mocks/mock_lifecycle.go -package=mocks github.com/stacklok/toolhive/pkg/vmcp/session/untrusted PodLifecycle,BackendAddressResolver +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + time "time" + + vmcp "github.com/stacklok/toolhive/pkg/vmcp" + untrusted "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" + gomock "go.uber.org/mock/gomock" + v1 "k8s.io/api/core/v1" +) + +// MockPodLifecycle is a mock of PodLifecycle interface. +type MockPodLifecycle struct { + ctrl *gomock.Controller + recorder *MockPodLifecycleMockRecorder + isgomock struct{} +} + +// MockPodLifecycleMockRecorder is the mock recorder for MockPodLifecycle. +type MockPodLifecycleMockRecorder struct { + mock *MockPodLifecycle +} + +// NewMockPodLifecycle creates a new mock instance. +func NewMockPodLifecycle(ctrl *gomock.Controller) *MockPodLifecycle { + mock := &MockPodLifecycle{ctrl: ctrl} + mock.recorder = &MockPodLifecycleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPodLifecycle) EXPECT() *MockPodLifecycleMockRecorder { + return m.recorder +} + +// DeletePod mocks base method. +func (m *MockPodLifecycle) DeletePod(ctx context.Context, name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeletePod", ctx, name) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeletePod indicates an expected call of DeletePod. +func (mr *MockPodLifecycleMockRecorder) DeletePod(ctx, name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletePod", reflect.TypeOf((*MockPodLifecycle)(nil).DeletePod), ctx, name) +} + +// EnsurePod mocks base method. +func (m *MockPodLifecycle) EnsurePod(ctx context.Context, req untrusted.EnsurePodRequest) (*v1.Pod, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsurePod", ctx, req) + ret0, _ := ret[0].(*v1.Pod) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EnsurePod indicates an expected call of EnsurePod. +func (mr *MockPodLifecycleMockRecorder) EnsurePod(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePod", reflect.TypeOf((*MockPodLifecycle)(nil).EnsurePod), ctx, req) +} + +// WaitReady mocks base method. +func (m *MockPodLifecycle) WaitReady(ctx context.Context, pod *v1.Pod, budget time.Duration) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitReady", ctx, pod, budget) + ret0, _ := ret[0].(error) + return ret0 +} + +// WaitReady indicates an expected call of WaitReady. +func (mr *MockPodLifecycleMockRecorder) WaitReady(ctx, pod, budget any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitReady", reflect.TypeOf((*MockPodLifecycle)(nil).WaitReady), ctx, pod, budget) +} + +// MockBackendAddressResolver is a mock of BackendAddressResolver interface. +type MockBackendAddressResolver struct { + ctrl *gomock.Controller + recorder *MockBackendAddressResolverMockRecorder + isgomock struct{} +} + +// MockBackendAddressResolverMockRecorder is the mock recorder for MockBackendAddressResolver. +type MockBackendAddressResolverMockRecorder struct { + mock *MockBackendAddressResolver +} + +// NewMockBackendAddressResolver creates a new mock instance. +func NewMockBackendAddressResolver(ctrl *gomock.Controller) *MockBackendAddressResolver { + mock := &MockBackendAddressResolver{ctrl: ctrl} + mock.recorder = &MockBackendAddressResolverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBackendAddressResolver) EXPECT() *MockBackendAddressResolverMockRecorder { + return m.recorder +} + +// ResolveTargets mocks base method. +func (m *MockBackendAddressResolver) ResolveTargets(ctx context.Context, sess untrusted.SessionRef, backends []*vmcp.Backend, onNewPod func(string)) []*vmcp.Backend { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveTargets", ctx, sess, backends, onNewPod) + ret0, _ := ret[0].([]*vmcp.Backend) + return ret0 +} + +// ResolveTargets indicates an expected call of ResolveTargets. +func (mr *MockBackendAddressResolverMockRecorder) ResolveTargets(ctx, sess, backends, onNewPod any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveTargets", reflect.TypeOf((*MockBackendAddressResolver)(nil).ResolveTargets), ctx, sess, backends, onNewPod) +} diff --git a/pkg/vmcp/session/untrusted/naming.go b/pkg/vmcp/session/untrusted/naming.go new file mode 100644 index 0000000000..e066000cec --- /dev/null +++ b/pkg/vmcp/session/untrusted/naming.go @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package untrusted implements the session-scoped single-tenant backend +// lifecycle for untrusted MCPServers behind a vMCP (ADR-0001 D2/D4). +// +// For each (user, session, untrusted MCPServer) tuple the vMCP data plane +// provisions one bare Pod cloned from the operator-built backend StatefulSet's +// pod template, rewrites the per-session backend target BaseURL to the pod IP, +// and reaps pods whose session has gone idle (in-process reaper goroutine). +// +// Security invariants enforced here: +// - Pods are only ever created by cloning the live backend StatefulSet +// template (never from caller-supplied spec), with a fail-closed +// re-verification that no Secret/ConfigMap-sourced env or volumes reach +// the backend container. +// - Raw user identifiers never appear in labels, annotations, or logs +// beyond a documented sha256 prefix (PII). +package untrusted + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + + "github.com/stacklok/toolhive/pkg/egressbroker" + "github.com/stacklok/toolhive/pkg/vmcp/session/binding" +) + +const ( + // LabelUntrusted marks a pod as an untrusted per-session backend pod. + LabelUntrusted = "toolhive.stacklok.dev/untrusted" + + // LabelUntrustedUser carries sha256(UserKey)[:40] — never the raw sub (PII). + LabelUntrustedUser = "toolhive.stacklok.dev/untrusted-user" + + // LabelUntrustedSession carries a short hash of the session ID. + LabelUntrustedSession = "toolhive.stacklok.dev/untrusted-session" + + // LabelMCPServerUID carries the MCPServer CR UID (rename-safe). + LabelMCPServerUID = "toolhive.stacklok.dev/mcpserver-uid" + + // LabelApp is inherited from the backend StatefulSet so per-session pods + // match the same NetworkPolicy selectors as the shared backend pods. + LabelApp = "app" + + // AnnotationIssuer carries the raw OIDC issuer for the Wave-3 sidecar's + // downward-API registry (public, non-sensitive). Re-exported from + // pkg/egressbroker — the shared owner of the pod-annotation ↔ broker-env + // contract (this package is the writer, the broker the reader). + AnnotationIssuer = egressbroker.AnnotationIssuer + + // AnnotationSubjectHash carries sha256(sub)[:40] — never the raw sub. + AnnotationSubjectHash = "toolhive.stacklok.dev/sub" + + // AnnotationSubjectRaw carries the raw OIDC subject (see + // egressbroker.AnnotationSubjectRaw for the full rationale). + AnnotationSubjectRaw = egressbroker.AnnotationSubjectRaw + + // AnnotationMCPServerName carries the MCPServer CR name. + AnnotationMCPServerName = egressbroker.AnnotationMCPServerName + + // AnnotationSessionID carries the vMCP session ID this pod serves. + // Checked on pod reuse: a mismatch means a hash collision or a + // cross-session pod — fail closed, never attach. + AnnotationSessionID = egressbroker.AnnotationSessionID + + // AnnotationVMCPUid identifies the vMCP instance that provisioned the pod + // (heartbeat identity for the reaper's zombie rule). + AnnotationVMCPUid = "toolhive.stacklok.dev/vmcp-uid" + + // AnnotationCAGeneration is stamped by the OPERATOR on the backend + // StatefulSet's pod template: the bump-CA generation (cert hash) session + // pods must mount. The lifecycle reads it when cloning so a mid-rotation + // clone gets one consistent cert/key pair. The operator keeps its own + // mirror constant (module boundary); the contract is pinned by tests. + AnnotationCAGeneration = "toolhive.stacklok.dev/bump-ca-generation" +) + +// PodNameFor returns the deterministic pod name for the (mcpserverUID, +// userKey, sessionID) tuple: "untrusted--". +// The components bound the name to at most 10+13+1+24 = 48 characters, always +// within the 63-char DNS-1123 label limit (the trailing truncate is a defensive +// no-op). Deterministic naming makes EnsurePod an idempotent get-then-create +// (the name is the idempotency key) and keeps cross-vMCP-pod session restore +// collision-free. +func PodNameFor(mcpserverUID, userKey, sessionID string) string { + uid := mcpserverUID + if len(uid) > 13 { + uid = uid[:13] + } + sum := sha256.Sum256([]byte(userKey + "|" + sessionID)) + name := fmt.Sprintf("untrusted-%s-%s", uid, hex.EncodeToString(sum[:])[:24]) + if len(name) > 63 { + name = name[:63] + } + return name +} + +// userHash is the canonical hashed user identity used in labels, annotations, +// and Redis keys: sha256(UserKey)[:40]. +func userHash(userKey string) string { + sum := sha256.Sum256([]byte(userKey)) + return hex.EncodeToString(sum[:])[:40] +} + +// subjectHash hashes only the OIDC subject for the sub annotation (Wave-3 +// downward-API registry); the issuer is public and stored raw. +func subjectHash(sub string) string { + sum := sha256.Sum256([]byte(sub)) + return hex.EncodeToString(sum[:])[:40] +} + +// sessionHash is the short session-ID hash used in the untrusted-session label. +func sessionHash(sessionID string) string { + sum := sha256.Sum256([]byte(sessionID)) + return hex.EncodeToString(sum[:])[:40] +} + +// SessionRef is the identity tuple a pod is provisioned for. +type SessionRef struct { + SessionID string + Issuer string + Subject string + Namespace string // vMCP's own namespace (via k8s.GetCurrentNamespace) + // CAGeneration is the bump-CA generation (from the backend StatefulSet + // template's AnnotationCAGeneration) the cloned pod must mount. The + // lifecycle sets it while resolving the clone template; required by the + // egress sidecar wiring (fail closed when absent). + CAGeneration string +} + +// UserKey is the canonical registry identity. It uses binding.Format (NUL-joined +// iss,sub) so the registry key and the on-the-wire session binding are the same +// canonical form. Callers must only construct SessionRef with non-empty +// Issuer/Subject; an empty half yields the empty string, which untrusted +// sessions must never carry (anonymous sessions soft-fail untrusted backends). +func (s SessionRef) UserKey() string { + b, err := binding.Format(s.Issuer, s.Subject) + if err != nil { + return "" + } + return b +} diff --git a/pkg/vmcp/session/untrusted/naming_test.go b/pkg/vmcp/session/untrusted/naming_test.go new file mode 100644 index 0000000000..3f5dc08265 --- /dev/null +++ b/pkg/vmcp/session/untrusted/naming_test.go @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/vmcp/session/binding" +) + +var dns1123Name = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +func TestPodNameFor(t *testing.T) { + t.Parallel() + + userKey, err := binding.Format("https://issuer.example.com", "user-123") + require.NoError(t, err) + + t.Run("deterministic", func(t *testing.T) { + t.Parallel() + a := PodNameFor("uid-abc", userKey, "session-1") + b := PodNameFor("uid-abc", userKey, "session-1") + assert.Equal(t, a, b) + }) + + t.Run("63 chars and DNS-1123 safe", func(t *testing.T) { + t.Parallel() + name := PodNameFor("a-very-long-mcpserver-uid-that-exceeds-thirteen-chars", userKey, "session-1") + assert.LessOrEqual(t, len(name), 63) + assert.Regexp(t, dns1123Name, name) + }) + + t.Run("distinct per session", func(t *testing.T) { + t.Parallel() + a := PodNameFor("uid-abc", userKey, "session-1") + b := PodNameFor("uid-abc", userKey, "session-2") + assert.NotEqual(t, a, b) + }) + + t.Run("distinct per user", func(t *testing.T) { + t.Parallel() + otherKey, err := binding.Format("https://issuer.example.com", "user-456") + require.NoError(t, err) + a := PodNameFor("uid-abc", userKey, "session-1") + b := PodNameFor("uid-abc", otherKey, "session-1") + assert.NotEqual(t, a, b) + }) + + t.Run("distinct per mcpserver uid prefix", func(t *testing.T) { + t.Parallel() + a := PodNameFor("uid-abc", userKey, "session-1") + b := PodNameFor("uid-xyz", userKey, "session-1") + assert.NotEqual(t, a, b) + }) +} + +func TestSessionRefUserKey(t *testing.T) { + t.Parallel() + + t.Run("equals binding.Format output", func(t *testing.T) { + t.Parallel() + ref := SessionRef{Issuer: "https://iss", Subject: "sub"} + want, err := binding.Format("https://iss", "sub") + require.NoError(t, err) + assert.Equal(t, want, ref.UserKey()) + }) + + t.Run("empty halves yield empty key", func(t *testing.T) { + t.Parallel() + assert.Empty(t, SessionRef{}.UserKey()) + assert.Empty(t, SessionRef{Issuer: "iss"}.UserKey()) + assert.Empty(t, SessionRef{Subject: "sub"}.UserKey()) + }) +} + +func TestHashes(t *testing.T) { + t.Parallel() + + t.Run("hashes never contain raw identifiers", func(t *testing.T) { + t.Parallel() + userKey, err := binding.Format("https://issuer.example.com", "sensitive-sub") + require.NoError(t, err) + assert.NotContains(t, userHash(userKey), "sensitive-sub") + assert.NotContains(t, subjectHash("sensitive-sub"), "sensitive-sub") + assert.NotContains(t, sessionHash("session-id-value"), "session-id-value") + }) + + t.Run("hash lengths", func(t *testing.T) { + t.Parallel() + assert.Len(t, userHash("k"), 40) + assert.Len(t, subjectHash("s"), 40) + assert.Len(t, sessionHash("s"), 40) + }) +} diff --git a/pkg/vmcp/session/untrusted/reaper.go b/pkg/vmcp/session/untrusted/reaper.go new file mode 100644 index 0000000000..214ca514c8 --- /dev/null +++ b/pkg/vmcp/session/untrusted/reaper.go @@ -0,0 +1,343 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "context" + "fmt" + "log/slog" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ReaperConfig tunes the reaper goroutine. Zero values fall back to the +// documented defaults; the constructor validates bounds. +type ReaperConfig struct { + // TickInterval is the sweep cadence. Default 60s. + TickInterval time.Duration + // IdleTTL is the pod liveness lease enforced by the idle rule. + // Default 30m. + IdleTTL time.Duration + // HeartbeatInterval is how often this vMCP writes its liveness key. + // Default 30s. + HeartbeatInterval time.Duration + // HeartbeatTTL is the expiry of the liveness key. Default 5m. + HeartbeatTTL time.Duration +} + +const ( + defaultTickInterval = 60 * time.Second + defaultIdleTTL = 30 * time.Minute + defaultHeartbeatInterval = 30 * time.Second + defaultHeartbeatTTL = 5 * time.Minute +) + +func (c ReaperConfig) resolved() ReaperConfig { + if c.TickInterval <= 0 { + c.TickInterval = defaultTickInterval + } + if c.IdleTTL <= 0 { + c.IdleTTL = defaultIdleTTL + } + if c.HeartbeatInterval <= 0 { + c.HeartbeatInterval = defaultHeartbeatInterval + } + if c.HeartbeatTTL <= 0 { + c.HeartbeatTTL = defaultHeartbeatTTL + } + return c +} + +// Reaper is the single owner of untrusted pod garbage collection. One +// goroutine per vMCP pod; every replica runs the same loop against Redis plus +// the pod list, so teardown survives any single vMCP crash (overlapping +// deletes are harmless — Delete is idempotent, NotFound ignored). +// +// Rules enforced each tick: +// - readiness timeout: pod older than 120s and not Ready → delete +// (failed cold start); +// - idle TTL: podttl lease absent → delete (session gone); +// - zombie: pod's vMCP heartbeat is absent and has been for the grace +// period → delete (owning vMCP died, no replica adopted the pod); +// - registry reconciliation: admission sets/counters rebuilt from the +// authoritative pod LIST (self-heals drift from crashed creates/deletes); +// - refresh: pods whose session metadata key still exists get their podttl +// lease renewed (sliding window). +// +// Redis down at tick → the tick is skipped entirely (fail safe: no deletions +// without Redis evidence). +type Reaper struct { + k8s client.Client + store *redisStore + cfg ReaperConfig + vmcpUID string + ns string + now func() time.Time + + // metrics, when non-nil, records the untrusted_backend_pods gauge from the + // authoritative pod LIST each tick. + metrics *untrustedMetrics + + mu sync.Mutex + zombies map[string]time.Time // pod name -> when its heartbeat absence was first observed +} + +// NewReaper constructs the reaper. metrics may be nil (metrics disabled). +func NewReaper( + k8sClient client.Client, + store *redisStore, + cfg ReaperConfig, + vmcpUID string, + metrics *untrustedMetrics, +) (*Reaper, error) { + return &Reaper{ + k8s: k8sClient, + store: store, + cfg: cfg.resolved(), + vmcpUID: vmcpUID, + ns: store.namespace, + now: time.Now, + metrics: metrics, + zombies: make(map[string]time.Time), + }, nil +} + +// Run starts the heartbeat and sweep loops until ctx is cancelled. It returns +// when the context is done; both tickers are stopped. sessionExists reports +// whether a vMCP session's metadata key is alive in shared storage (supplied +// by the session manager, which owns the session store — the reaper never +// re-derives the session key shape); it must fail false on storage errors and +// must not be nil (fail loudly at startup, not on the first sweep). +func (r *Reaper) Run(ctx context.Context, sessionExists func(ctx context.Context, sessionID string) bool) { + if sessionExists == nil { + slog.Error("untrusted reaper: sessionExists probe is nil; refusing to run") + return + } + heartbeat := time.NewTicker(r.cfg.HeartbeatInterval) + sweep := time.NewTicker(r.cfg.TickInterval) + defer heartbeat.Stop() + defer sweep.Stop() + + r.writeHeartbeat(ctx) + for { + select { + case <-ctx.Done(): + return + case <-heartbeat.C: + r.writeHeartbeat(ctx) + case <-sweep.C: + r.tick(ctx, sessionExists) + } + } +} + +func (r *Reaper) writeHeartbeat(_ context.Context) { + // Heartbeats must outlive request-scope cancellation: a cancelled sweep + // context must not stop this vMCP from proving liveness, so the write uses + // its own bounded background context. + hbCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := r.store.writeHeartbeat(hbCtx, r.vmcpUID, r.cfg.HeartbeatTTL); err != nil { + slog.Warn("untrusted reaper: failed to write vMCP heartbeat", "error", err) + } +} + +// tick performs one sweep. Any Redis failure aborts the tick without any +// deletion (fail safe). sessionExists is the refresh-rule probe (see Run). +func (r *Reaper) tick(ctx context.Context, sessionExists func(ctx context.Context, sessionID string) bool) { + tickCtx, cancel := context.WithTimeout(ctx, r.cfg.TickInterval/2) + defer cancel() + + pods := &corev1.PodList{} + if err := r.k8s.List(tickCtx, pods, + client.InNamespace(r.ns), + client.MatchingLabels{LabelUntrusted: "true"}, + ); err != nil { + slog.Warn("untrusted reaper: pod list failed; skipping tick", "error", err) + return + } + + live := make(map[string]*corev1.Pod, len(pods.Items)) + for i := range pods.Items { + p := &pods.Items[i] + live[p.Name] = p + if err := r.evaluatePod(tickCtx, p, sessionExists); err != nil { + // Fail safe: Redis evidence is unavailable, so make no further + // deletion decisions this tick. + slog.Warn("untrusted reaper: Redis unavailable; skipping tick", "pod", p.Name, "error", err) + return + } + } + + if err := r.reconcile(tickCtx, live); err != nil { + slog.Warn("untrusted reaper: registry reconciliation failed; will retry next tick", "error", err) + } +} + +// evaluatePod applies the delete/refresh rules to one pod. A returned error +// means Redis evidence could not be read — the caller must abort the tick. +// Deletion failures are logged and swallowed (retried next tick). +// sessionExists is the refresh-rule probe (see Run). +func (r *Reaper) evaluatePod( + ctx context.Context, pod *corev1.Pod, sessionExists func(ctx context.Context, sessionID string) bool, +) error { + // Rule 1: readiness timeout (failed cold start). Pure-K8s evidence; no + // Redis read needed. + age := r.now().Sub(pod.CreationTimestamp.Time) + if age > readinessTimeout && !isPodReady(pod) { + r.delete(ctx, pod, "readiness timeout") + return nil + } + + // Rule 2: idle TTL. The podttl lease is the liveness contract. + leaseAlive, err := r.store.podLeaseExists(ctx, pod.Name) + if err != nil { + return fmt.Errorf("failed to read pod lease: %w", err) + } + if !leaseAlive { + r.delete(ctx, pod, "idle TTL expired") + return nil + } + + // Rule 3: zombie (owning vMCP dead, no replica adopted the pod). Two + // consecutive observations grace-period apart, so a single missed + // heartbeat window cannot reap a live vMCP's pods. + ownerUID := pod.Annotations[AnnotationVMCPUid] + if ownerUID != "" && ownerUID != r.vmcpUID { + alive, err := r.store.heartbeatExists(ctx, ownerUID) + if err != nil { + return fmt.Errorf("failed to read vMCP heartbeat: %w", err) + } + if !alive { + firstSeen := r.markZombie(pod.Name) + if r.now().Sub(firstSeen) >= zombieHeartbeatGrace { + r.delete(ctx, pod, "owning vMCP heartbeat gone (zombie)") + return nil + } + } else { + r.clearZombie(pod.Name) + } + } else { + r.clearZombie(pod.Name) + } + + // Rule 5 (refresh): renew the lease while the owning session is alive. + sessionID := pod.Annotations[AnnotationSessionID] + if sessionID != "" && sessionExists(ctx, sessionID) { + if err := r.store.touchPodLease(ctx, pod.Name, r.cfg.IdleTTL); err != nil { + return fmt.Errorf("failed to refresh pod lease: %w", err) + } + } + return nil +} + +// reconcile rebuilds admission state from the authoritative pod list and +// prunes orphaned quota counters. +func (r *Reaper) reconcile(ctx context.Context, live map[string]*corev1.Pod) error { + names := make([]string, 0, len(live)) + userCounts := make(map[string]int) + serverUIDs := make(map[string]struct{}) + serverCounts := make(map[string]int) + for _, p := range live { + names = append(names, p.Name) + if h := p.Labels[LabelUntrustedUser]; h != "" { + userCounts[h]++ + } + if uid := p.Labels[LabelMCPServerUID]; uid != "" { + serverUIDs[uid] = struct{}{} + serverCounts[uid]++ + } + } + + // Refresh the per-server pod gauge from the authoritative list. The label + // is the MCPServer name (low-cardinality, one per untrusted CR); pods + // missing the name annotation are grouped under "unknown" rather than + // dropped so the gauge accounts for every live pod. + if r.metrics != nil { + nameCounts := make(map[string]int) + for _, p := range live { + name := p.Annotations[AnnotationMCPServerName] + if name == "" { + name = "unknown" + } + nameCounts[name]++ + } + for name, count := range nameCounts { + r.metrics.recordPodCounts(ctx, name, count) + } + } + + // Per-user live counts for the quota-counter correction. Pairs are parsed + // positionally inside the script (from the end), so no separator can + // collide with a hash. + joined := "|" + for h, count := range userCounts { + joined += fmt.Sprintf("%s=%d|", h, count) + } + + // Rebuild the global set and every per-server set present in the + // authoritative list. Per-server sets for servers with zero live pods + // empty out naturally via DEL. + for uid := range serverUIDs { + members := make([]any, 0, len(names)) + for _, p := range live { + if p.Labels[LabelMCPServerUID] == uid { + members = append(members, p.Name) + } + } + if err := rebuildServerSetScript.Run(ctx, r.store.client, + []string{r.store.serverPodsSetKey(uid)}, members...).Err(); err != nil { + return fmt.Errorf("failed to rebuild per-server pod set: %w", err) + } + } + + // Rebuild the global set and correct the per-user quota counters. + args := make([]any, 0, len(names)+1) + args = append(args, joined) + for _, n := range names { + args = append(args, n) + } + if err := reconcileScript.Run(ctx, r.store.client, []string{ + r.store.podsSetKey(), + r.store.quotaKeysRegistryKey(), + }, args...).Err(); err != nil { + return fmt.Errorf("failed to rebuild pod registry: %w", err) + } + + return nil +} + +func (r *Reaper) delete(ctx context.Context, pod *corev1.Pod, reason string) { + if err := r.k8s.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + slog.Warn("untrusted reaper: pod delete failed; will retry next tick", + "pod", pod.Name, "reason", reason, "error", err) + return + } + if err := r.store.recordPodDelete(ctx, pod.Name, pod.Labels); err != nil { + slog.Warn("untrusted reaper: bookkeeping cleanup failed; reconciliation will heal", + "pod", pod.Name, "error", err) + } + slog.Info("untrusted reaper: pod deleted", "pod", pod.Name, "reason", reason) + r.clearZombie(pod.Name) +} + +func (r *Reaper) markZombie(podName string) time.Time { + r.mu.Lock() + defer r.mu.Unlock() + if t, ok := r.zombies[podName]; ok { + return t + } + r.zombies[podName] = r.now() + return r.zombies[podName] +} + +func (r *Reaper) clearZombie(podName string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.zombies, podName) +} diff --git a/pkg/vmcp/session/untrusted/reaper_test.go b/pkg/vmcp/session/untrusted/reaper_test.go new file mode 100644 index 0000000000..b2b211976f --- /dev/null +++ b/pkg/vmcp/session/untrusted/reaper_test.go @@ -0,0 +1,231 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func reaperPod(name string, age time.Duration, ready bool, annotations map[string]string) *corev1.Pod { + status := corev1.PodStatus{} + if ready { + status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + } + labels := map[string]string{ + LabelUntrusted: "true", + LabelUntrustedUser: userHash("iss\x00sub"), + LabelMCPServerUID: "uid-abc", + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "toolhive", + Labels: labels, + Annotations: annotations, + CreationTimestamp: metav1.NewTime(time.Now().Add(-age)), + }, + Status: status, + } +} + +func newReaper( + t *testing.T, sessionExists bool, objs ...client.Object, +) (*Reaper, func(context.Context, string) bool, *redisStore, *miniredis.Miniredis, client.Client) { + t.Helper() + scheme := newScheme(t) + k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + mr := miniredis.RunT(t) + store, err := newRedisStore(redis.NewClient(&redis.Options{Addr: mr.Addr()}), "thv:vmcp:session:", "toolhive") + require.NoError(t, err) + r, err := NewReaper(k8sClient, store, ReaperConfig{IdleTTL: time.Minute}, "vmcp-self", nil) + require.NoError(t, err) + exists := func(context.Context, string) bool { return sessionExists } + return r, exists, store, mr, k8sClient +} + +func podExists(t *testing.T, c client.Client, name string) bool { + t.Helper() + p := &corev1.Pod{} + err := c.Get(context.Background(), client.ObjectKey{Name: name, Namespace: "toolhive"}, p) + return err == nil +} + +func TestReaperTick(t *testing.T) { + t.Parallel() + ctx := context.Background() + + t.Run("idle pod (no TTL lease) is deleted", func(t *testing.T) { + t.Parallel() + p := reaperPod("pod-idle", 10*time.Minute, true, map[string]string{AnnotationVMCPUid: "vmcp-self"}) + r, exists, _, _, c := newReaper(t, false, p) + + r.tick(ctx, exists) + assert.False(t, podExists(t, c, "pod-idle")) + }) + + t.Run("pod with live lease and live session is kept and lease refreshed", func(t *testing.T) { + t.Parallel() + ann := map[string]string{AnnotationVMCPUid: "vmcp-self", AnnotationSessionID: "sess-1"} + p := reaperPod("pod-live", 10*time.Minute, true, ann) + r, exists, store, _, c := newReaper(t, true, p) + require.NoError(t, store.client.Set(ctx, store.podTTLKey("pod-live"), "1", 10*time.Second).Err()) + + r.tick(ctx, exists) + assert.True(t, podExists(t, c, "pod-live")) + ttl := store.client.PTTL(ctx, store.podTTLKey("pod-live")).Val() + assert.Greater(t, ttl, 30*time.Second, "lease must be refreshed to idle TTL") + }) + + t.Run("pod with live lease but dead session keeps lease until it lapses", func(t *testing.T) { + t.Parallel() + ann := map[string]string{AnnotationVMCPUid: "vmcp-self", AnnotationSessionID: "sess-gone"} + p := reaperPod("pod-dying", time.Minute, true, ann) + r, exists, store, _, c := newReaper(t, false, p) + require.NoError(t, store.client.Set(ctx, store.podTTLKey("pod-dying"), "1", time.Minute).Err()) + + r.tick(ctx, exists) + assert.True(t, podExists(t, c, "pod-dying"), "lease still alive: refresh simply stops; deletion happens at lapse") + }) + + t.Run("readiness timeout deletes old not-ready pod even with lease", func(t *testing.T) { + t.Parallel() + p := reaperPod("pod-stuck", 5*time.Minute, false, map[string]string{AnnotationVMCPUid: "vmcp-self"}) + r, exists, store, _, c := newReaper(t, false, p) + require.NoError(t, store.client.Set(ctx, store.podTTLKey("pod-stuck"), "1", time.Hour).Err()) + + r.tick(ctx, exists) + assert.False(t, podExists(t, c, "pod-stuck")) + }) + + t.Run("young not-ready pod is kept", func(t *testing.T) { + t.Parallel() + p := reaperPod("pod-cold", 30*time.Second, false, map[string]string{AnnotationVMCPUid: "vmcp-self"}) + r, exists, store, _, c := newReaper(t, false, p) + require.NoError(t, store.client.Set(ctx, store.podTTLKey("pod-cold"), "1", time.Hour).Err()) + + r.tick(ctx, exists) + assert.True(t, podExists(t, c, "pod-cold")) + }) + + t.Run("zombie pod deleted only after heartbeat grace", func(t *testing.T) { + t.Parallel() + ann := map[string]string{AnnotationVMCPUid: "vmcp-dead", AnnotationSessionID: "sess-1"} + p := reaperPod("pod-zombie", 10*time.Minute, true, ann) + r, exists, store, _, c := newReaper(t, true, p) + require.NoError(t, store.client.Set(ctx, store.podTTLKey("pod-zombie"), "1", time.Hour).Err()) + + // First tick: absence observed, grace starts — pod kept. + r.tick(ctx, exists) + assert.True(t, podExists(t, c, "pod-zombie")) + + // Simulate the grace period having elapsed since first observation. + r.mu.Lock() + r.zombies["pod-zombie"] = r.now().Add(-zombieHeartbeatGrace - time.Second) + r.mu.Unlock() + r.tick(ctx, exists) + assert.False(t, podExists(t, c, "pod-zombie")) + }) + + t.Run("zombie clears when heartbeat returns", func(t *testing.T) { + t.Parallel() + ann := map[string]string{AnnotationVMCPUid: "vmcp-back", AnnotationSessionID: "sess-1"} + p := reaperPod("pod-revived", 10*time.Minute, true, ann) + r, exists, store, _, c := newReaper(t, true, p) + require.NoError(t, store.client.Set(ctx, store.podTTLKey("pod-revived"), "1", time.Hour).Err()) + require.NoError(t, store.writeHeartbeat(ctx, "vmcp-back", 5*time.Minute)) + + r.tick(ctx, exists) + assert.True(t, podExists(t, c, "pod-revived")) + r.mu.Lock() + _, marked := r.zombies["pod-revived"] + r.mu.Unlock() + assert.False(t, marked) + }) + + t.Run("Redis down: tick skipped, no deletions", func(t *testing.T) { + t.Parallel() + p := reaperPod("pod-idle", 10*time.Minute, true, map[string]string{AnnotationVMCPUid: "vmcp-self"}) + r, exists, _, mr, c := newReaper(t, false, p) + mr.Close() + + r.tick(ctx, exists) + assert.True(t, podExists(t, c, "pod-idle"), "fail safe: no deletions without Redis evidence") + }) + + t.Run("reconciliation heals set drift and prunes orphan quota keys", func(t *testing.T) { + t.Parallel() + ann := map[string]string{AnnotationVMCPUid: "vmcp-self", AnnotationSessionID: "sess-1"} + p := reaperPod("pod-live", time.Minute, true, ann) + r, exists, store, _, _ := newReaper(t, true, p) + require.NoError(t, store.client.Set(ctx, store.podTTLKey("pod-live"), "1", time.Hour).Err()) + + // Inject drift: a stale pod in the sets and an orphaned quota key. + require.NoError(t, store.client.SAdd(ctx, store.podsSetKey(), "pod-stale").Err()) + require.NoError(t, store.client.SAdd(ctx, store.serverPodsSetKey("uid-abc"), "pod-stale").Err()) + orphanQuotaKey := store.quotaKeyForUserHash("deaduserhash") + require.NoError(t, store.client.Set(ctx, orphanQuotaKey, 1, time.Hour).Err()) + require.NoError(t, store.client.SAdd(ctx, store.quotaKeysRegistryKey(), orphanQuotaKey).Err()) + + r.tick(ctx, exists) + + assert.False(t, store.client.SIsMember(ctx, store.podsSetKey(), "pod-stale").Val()) + assert.True(t, store.client.SIsMember(ctx, store.podsSetKey(), "pod-live").Val()) + assert.False(t, store.client.SIsMember(ctx, store.serverPodsSetKey("uid-abc"), "pod-stale").Val()) + assert.Equal(t, int64(0), store.client.Exists(ctx, orphanQuotaKey).Val()) + }) + + t.Run("reconciliation corrects a drifted quota counter to the live pod count", func(t *testing.T) { + t.Parallel() + ann := map[string]string{AnnotationVMCPUid: "vmcp-self", AnnotationSessionID: "sess-1"} + p := reaperPod("pod-live", time.Minute, true, ann) + r, exists, store, _, _ := newReaper(t, true, p) + require.NoError(t, store.client.Set(ctx, store.podTTLKey("pod-live"), "1", time.Hour).Err()) + + // Drift: the user's counter claims 9 pods, only 1 is live. + quotaKey := store.userQuotaKey("iss\x00sub") + require.NoError(t, store.client.Set(ctx, quotaKey, 9, time.Hour).Err()) + require.NoError(t, store.client.SAdd(ctx, store.quotaKeysRegistryKey(), quotaKey).Err()) + + r.tick(ctx, exists) + + assert.Equal(t, "1", store.client.Get(ctx, quotaKey).Val(), + "reaper must correct the counter to the authoritative pod LIST count") + + // Second tick keeps it corrected (idempotent). + r.tick(ctx, exists) + assert.Equal(t, "1", store.client.Get(ctx, quotaKey).Val()) + }) + + t.Run("reconciliation deletes the counter when the user has no live pods", func(t *testing.T) { + t.Parallel() + r, exists, store, _, _ := newReaper(t, false) + + quotaKey := store.userQuotaKey("iss\x00sub") + require.NoError(t, store.client.Set(ctx, quotaKey, 4, time.Hour).Err()) + require.NoError(t, store.client.SAdd(ctx, store.quotaKeysRegistryKey(), quotaKey).Err()) + + r.tick(ctx, exists) + assert.Equal(t, int64(0), store.client.Exists(ctx, quotaKey).Val()) + }) +} + +func TestNewReaperValidation(t *testing.T) { + t.Parallel() + store, _ := newTestStore(t) + k8sClient := fake.NewClientBuilder().WithScheme(newScheme(t)).Build() + r, err := NewReaper(k8sClient, store, ReaperConfig{}, "vmcp-1", nil) + require.NoError(t, err) + require.NotNil(t, r) +} diff --git a/pkg/vmcp/session/untrusted/rediskeys.go b/pkg/vmcp/session/untrusted/rediskeys.go new file mode 100644 index 0000000000..6a3cb224e5 --- /dev/null +++ b/pkg/vmcp/session/untrusted/rediskeys.go @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "fmt" + "time" +) + +// Redis keys for untrusted pod lifecycle state. All are single-key operations +// under the tenant's ':'-terminated session-storage prefix (validated the same +// way as pkg/transport/session/storage_redis.go). The pod→user mapping lives +// only in pod labels/annotations (downward-API registry); Redis holds only +// counters and TTLs. +// +// Key shapes (prefix = tenant prefix, e.g. "thv:vmcp:session:"): +// +// untrusted:podttl: string idle TTL pod liveness lease +// untrusted:pods set none (rebuilt) admission global capacity +// untrusted:pods: set none (rebuilt) admission per-server cap +// untrusted:userquota: counter idle TTL per-user concurrent pods (INCR/DECR ledger) +// untrusted:quotakeys set none registry of quota keys for reaper correction +// untrusted:ratelimit:: counter 2m per-user create rate +// untrusted:heartbeat: string 5m vMCP liveness for zombie rule +func (r *redisStore) podTTLKey(podName string) string { + return r.prefix + "untrusted:podttl:" + podName +} + +func (r *redisStore) podsSetKey() string { + return r.prefix + "untrusted:pods" +} + +func (r *redisStore) serverPodsSetKey(mcpserverUID string) string { + return r.prefix + "untrusted:pods:" + mcpserverUID +} + +func (r *redisStore) userQuotaKey(userKey string) string { + return r.prefix + "untrusted:userquota:" + userHash(userKey) +} + +func (r *redisStore) rateLimitKey(userKey string, t time.Time) string { + return fmt.Sprintf("%suntrusted:ratelimit:%s:%d", r.prefix, userHash(userKey), t.Unix()/60) +} + +func (r *redisStore) heartbeatKey(vmcpUID string) string { + return r.prefix + "untrusted:heartbeat:" + vmcpUID +} diff --git a/pkg/vmcp/session/untrusted/template_clone.go b/pkg/vmcp/session/untrusted/template_clone.go new file mode 100644 index 0000000000..5e5fb77782 --- /dev/null +++ b/pkg/vmcp/session/untrusted/template_clone.go @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// mcpContainerName is the backend container name in operator-built pod +// templates. It mirrors the operator's constant; the packages must not import +// each other (cmd/thv-operator is a separate module boundary). +const mcpContainerName = "mcp" + +// MCPServer ownerReference identity. Kept as constants because the untrusted +// package must not import the operator API package; the group/version/kind is +// stable CRD identity. +const ( + mcpserverAPIVersion = "toolhive.stacklok.dev/v1beta1" + mcpserverKind = "MCPServer" +) + +// clonePodFromTemplate builds the per-session untrusted backend pod from the +// backend StatefulSet's pod template. The operator's builder output is the pod +// spec of record (ADR D4): cloning it inherits every Wave-0/1 security +// invariant by construction (sentinel env literals, no secret env, security +// context) — duplicating the builder here would create a second source of +// truth for exactly the surface those waves harden. +// +// The template is re-verified fail-closed before the pod is returned: any +// Secret/ConfigMap-sourced env or volume reaching the backend container (or a +// missing mcp container) is a hard error — this is defense-in-depth behind the +// operator-side gates, not a replacement for them. +func clonePodFromTemplate( + template *corev1.PodTemplateSpec, + appLabel string, + req EnsurePodRequest, + vmcpUID string, +) (*corev1.Pod, error) { + if err := reverifyNoSecretMaterial(template); err != nil { + return nil, err + } + + userKey := req.Session.UserKey() + labels := map[string]string{ + LabelApp: appLabel, + LabelUntrusted: "true", + LabelUntrustedUser: userHash(userKey), + LabelUntrustedSession: sessionHash(req.Session.SessionID), + LabelMCPServerUID: req.MCPServerUID, + } + annotations := map[string]string{ + AnnotationIssuer: req.Session.Issuer, + AnnotationSubjectHash: subjectHash(req.Session.Subject), + AnnotationSubjectRaw: req.Session.Subject, + AnnotationMCPServerName: req.MCPServerName, + AnnotationSessionID: req.Session.SessionID, + AnnotationVMCPUid: vmcpUID, + } + + spec := *template.Spec.DeepCopy() + // Standalone pods need an explicit restart policy; StatefulSet templates + // always carry Always, so this is a belt-and-braces normalization. + spec.RestartPolicy = corev1.RestartPolicyAlways + spec.NodeName = "" + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: PodNameFor(req.MCPServerUID, userKey, req.Session.SessionID), + Namespace: req.Session.Namespace, + Labels: labels, + Annotations: annotations, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: mcpserverAPIVersion, + Kind: mcpserverKind, + Name: req.MCPServerName, + UID: req.OwnerRefUID, + // Non-controller ownerRef: cascade-deletes session pods when the + // MCPServer is deleted. No finalizer, no controller flag. + }}, + }, + Spec: spec, + } + + // Wave 3: append the egress-broker sidecar data plane (init-container, + // Envoy, broker, proxy/CA env on the backend). This runs AFTER + // reverifyNoSecretMaterial: the gate is the authority on the operator + // template, and the sidecar volumes it adds are operator-managed. + if err := applyEgressBrokerSidecar(pod, req); err != nil { + return nil, err + } + + return pod, nil +} + +// reverifyNoSecretMaterial mirrors the operator-side +// controllerutil.ValidateNoSecretEnvForUntrusted for a pod template about to +// be cloned (the two packages must not import each other). It rejects: +// - env ValueFrom SecretKeyRef / ConfigMapKeyRef on the backend container, +// - envFrom secretRef / configMapRef on the backend container, +// - Secret-, ConfigMap-, projected (Secret/ConfigMap source), and CSI +// secretProvider volumes. +// +// FieldRef/ResourceFieldRef env sources expose pod metadata, not credentials, +// and are allowed. Errors name the source object metadata only, never values. +func reverifyNoSecretMaterial(template *corev1.PodTemplateSpec) error { + if template == nil { + return fmt.Errorf("untrusted pod clone: pod template is nil") + } + + found := false + for i := range template.Spec.Containers { + c := &template.Spec.Containers[i] + if c.Name != mcpContainerName { + continue + } + found = true + if err := reverifyContainerEnv(c); err != nil { + return err + } + } + if !found { + return fmt.Errorf("untrusted pod clone: pod template has no %q container", mcpContainerName) + } + + for i := range template.Spec.Volumes { + if err := reverifyVolume(&template.Spec.Volumes[i]); err != nil { + return err + } + } + return nil +} + +func reverifyContainerEnv(c *corev1.Container) error { + for _, env := range c.Env { + if env.ValueFrom == nil { + continue + } + if ref := env.ValueFrom.SecretKeyRef; ref != nil { + return fmt.Errorf( + "untrusted pod clone: env var %q on container %q sources from Secret %q (key %q); refusing to create pod", + env.Name, c.Name, ref.Name, ref.Key) + } + if ref := env.ValueFrom.ConfigMapKeyRef; ref != nil { + return fmt.Errorf( + "untrusted pod clone: env var %q on container %q sources from ConfigMap %q (key %q); refusing to create pod", + env.Name, c.Name, ref.Name, ref.Key) + } + } + for _, envFrom := range c.EnvFrom { + if ref := envFrom.SecretRef; ref != nil { + return fmt.Errorf( + "untrusted pod clone: envFrom on container %q sources from Secret %q; refusing to create pod", + c.Name, ref.Name) + } + if ref := envFrom.ConfigMapRef; ref != nil { + return fmt.Errorf( + "untrusted pod clone: envFrom on container %q sources from ConfigMap %q; refusing to create pod", + c.Name, ref.Name) + } + } + return nil +} + +func reverifyVolume(v *corev1.Volume) error { + if v.Secret != nil { + return fmt.Errorf( + "untrusted pod clone: volume %q sources from Secret %q; refusing to create pod", + v.Name, v.Secret.SecretName) + } + if v.ConfigMap != nil { + return fmt.Errorf( + "untrusted pod clone: volume %q sources from ConfigMap %q; refusing to create pod", + v.Name, v.ConfigMap.Name) + } + if csi := v.CSI; csi != nil && csi.VolumeAttributes["secretProviderClass"] != "" { + return fmt.Errorf( + "untrusted pod clone: volume %q sources from CSI SecretProviderClass %q; refusing to create pod", + v.Name, csi.VolumeAttributes["secretProviderClass"]) + } + if projected := v.Projected; projected != nil { + for _, source := range projected.Sources { + if source.Secret != nil { + return fmt.Errorf( + "untrusted pod clone: volume %q projects Secret %q; refusing to create pod", + v.Name, source.Secret.Name) + } + if source.ConfigMap != nil { + return fmt.Errorf( + "untrusted pod clone: volume %q projects ConfigMap %q; refusing to create pod", + v.Name, source.ConfigMap.Name) + } + } + } + return nil +} diff --git a/pkg/vmcp/session/untrusted/template_clone_test.go b/pkg/vmcp/session/untrusted/template_clone_test.go new file mode 100644 index 0000000000..7cbdadd72a --- /dev/null +++ b/pkg/vmcp/session/untrusted/template_clone_test.go @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/stacklok/toolhive/pkg/vmcp/session/binding" +) + +func testRequest() EnsurePodRequest { + return EnsurePodRequest{ + Session: SessionRef{ + SessionID: "session-abc", + Issuer: "https://issuer.example.com", + Subject: "user-123", + Namespace: "toolhive", + CAGeneration: "gen-aaa111", + }, + MCPServerUID: "uid-1234567890abcdef", + MCPServerName: "github-mcp", + Port: 8080, + OwnerRefUID: types.UID("uid-1234567890abcdef"), + } +} + +func validTemplate() *corev1.PodTemplateSpec { + return &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{AnnotationCAGeneration: "gen-aaa111"}, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "backend-sa", + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: boolPtr(true)}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "pull-secret"}}, + Containers: []corev1.Container{{ + Name: "mcp", + Image: "ghcr.io/example/backend:1.0", + Env: []corev1.EnvVar{ + {Name: "SENTINEL", Value: "literal-ok"}, + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }}, + }, + }}, + Volumes: []corev1.Volume{ + {Name: "scratch", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + }, + }, + } +} + +func boolPtr(b bool) *bool { return &b } + +// findContainer returns the named container from the pod or fails the test. +func findContainer(t *testing.T, pod *corev1.Pod, name string) *corev1.Container { + t.Helper() + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == name { + return &pod.Spec.Containers[i] + } + } + t.Fatalf("pod has no %q container", name) + return nil +} + +func TestClonePodFromTemplate(t *testing.T) { + t.Parallel() + + t.Run("clones backend container and pod-level settings", func(t *testing.T) { + t.Parallel() + pod, err := clonePodFromTemplate(validTemplate(), "backend-app", testRequest(), "vmcp-1") + require.NoError(t, err) + + backend := findContainer(t, pod, "mcp") + assert.Equal(t, "ghcr.io/example/backend:1.0", backend.Image) + assert.Equal(t, "backend-sa", pod.Spec.ServiceAccountName) + require.NotNil(t, pod.Spec.SecurityContext) + assert.Equal(t, corev1.RestartPolicyAlways, pod.Spec.RestartPolicy) + assert.Empty(t, pod.Spec.NodeName) + }) + + t.Run("deterministic name, ownerRef to MCPServer", func(t *testing.T) { + t.Parallel() + req := testRequest() + pod, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.NoError(t, err) + + userKey, err := binding.Format(req.Session.Issuer, req.Session.Subject) + require.NoError(t, err) + assert.Equal(t, PodNameFor(req.MCPServerUID, userKey, req.Session.SessionID), pod.Name) + assert.Equal(t, "toolhive", pod.Namespace) + + require.Len(t, pod.OwnerReferences, 1) + or := pod.OwnerReferences[0] + assert.Equal(t, "toolhive.stacklok.dev/v1beta1", or.APIVersion) + assert.Equal(t, "MCPServer", or.Kind) + assert.Equal(t, "github-mcp", or.Name) + assert.Equal(t, req.OwnerRefUID, or.UID) + assert.Nil(t, or.Controller, "ownerRef must be non-controller") + }) + + t.Run("labels carry hashed identity, never raw sub", func(t *testing.T) { + t.Parallel() + req := testRequest() + pod, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.NoError(t, err) + + assert.Equal(t, "backend-app", pod.Labels[LabelApp]) + assert.Equal(t, "true", pod.Labels[LabelUntrusted]) + assert.Equal(t, req.MCPServerUID, pod.Labels[LabelMCPServerUID]) + assert.Len(t, pod.Labels[LabelUntrustedUser], 40) + assert.Len(t, pod.Labels[LabelUntrustedSession], 40) + for k, v := range pod.Labels { + assert.NotContains(t, v, "user-123", "label %s leaks raw sub", k) + } + }) + + t.Run("annotations: raw iss, hashed sub, session id, vmcp uid", func(t *testing.T) { + t.Parallel() + req := testRequest() + pod, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.NoError(t, err) + + assert.Equal(t, "https://issuer.example.com", pod.Annotations[AnnotationIssuer]) + assert.Equal(t, subjectHash("user-123"), pod.Annotations[AnnotationSubjectHash]) + assert.NotContains(t, pod.Annotations[AnnotationSubjectHash], "user-123") + assert.Equal(t, "user-123", pod.Annotations[AnnotationSubjectRaw], + "raw sub must be carried in the sub-raw annotation (Wave-2/3 contract)") + assert.Equal(t, "github-mcp", pod.Annotations[AnnotationMCPServerName]) + assert.Equal(t, "session-abc", pod.Annotations[AnnotationSessionID]) + assert.Equal(t, "vmcp-1", pod.Annotations[AnnotationVMCPUid]) + }) + + t.Run("does not mutate the source template", func(t *testing.T) { + t.Parallel() + tmpl := validTemplate() + _, err := clonePodFromTemplate(tmpl, "backend-app", testRequest(), "vmcp-1") + require.NoError(t, err) + assert.Empty(t, tmpl.Spec.RestartPolicy, "template was mutated") + }) +} + +func TestClonePodFromTemplate_FailClosed(t *testing.T) { + t.Parallel() + + withContainer := func(mut func(c *corev1.Container)) *corev1.PodTemplateSpec { + tmpl := validTemplate() + mut(&tmpl.Spec.Containers[0]) + return tmpl + } + withVolume := func(v corev1.Volume) *corev1.PodTemplateSpec { + tmpl := validTemplate() + tmpl.Spec.Volumes = append(tmpl.Spec.Volumes, v) + return tmpl + } + + cases := []struct { + name string + template *corev1.PodTemplateSpec + wantErr string + }{ + { + name: "nil template", + template: nil, + wantErr: "pod template is nil", + }, + { + name: "secretKeyRef env", + template: withContainer(func(c *corev1.Container) { + c.Env = append(c.Env, corev1.EnvVar{Name: "TOKEN", ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "creds"}, Key: "token", + }, + }}) + }), + wantErr: `sources from Secret "creds"`, + }, + { + name: "configMapKeyRef env", + template: withContainer(func(c *corev1.Container) { + c.Env = append(c.Env, corev1.EnvVar{Name: "CFG", ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "cfg"}, Key: "k", + }, + }}) + }), + wantErr: `sources from ConfigMap "cfg"`, + }, + { + name: "envFrom secretRef", + template: withContainer(func(c *corev1.Container) { + c.EnvFrom = append(c.EnvFrom, corev1.EnvFromSource{ + SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "creds"}}, + }) + }), + wantErr: `envFrom on container "mcp" sources from Secret "creds"`, + }, + { + name: "envFrom configMapRef", + template: withContainer(func(c *corev1.Container) { + c.EnvFrom = append(c.EnvFrom, corev1.EnvFromSource{ + ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "cfg"}}, + }) + }), + wantErr: `envFrom on container "mcp" sources from ConfigMap "cfg"`, + }, + { + name: "secret volume", + template: withVolume(corev1.Volume{Name: "sec", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "creds"}, + }}), + wantErr: `volume "sec" sources from Secret "creds"`, + }, + { + name: "configmap volume", + template: withVolume(corev1.Volume{Name: "cm", VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "cfg"}}, + }}), + wantErr: `volume "cm" sources from ConfigMap "cfg"`, + }, + { + name: "projected secret", + template: withVolume(corev1.Volume{Name: "proj", VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{ + Secret: &corev1.SecretProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "creds"}}, + }}}, + }}), + wantErr: `volume "proj" projects Secret "creds"`, + }, + { + name: "projected configmap", + template: withVolume(corev1.Volume{Name: "proj", VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{ + ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "cfg"}}, + }}}, + }}), + wantErr: `volume "proj" projects ConfigMap "cfg"`, + }, + { + name: "csi secretProviderClass", + template: withVolume(corev1.Volume{Name: "csi", VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{Driver: "secrets-store.csi.k8s.io", + VolumeAttributes: map[string]string{"secretProviderClass": "spc"}}, + }}), + wantErr: `CSI SecretProviderClass "spc"`, + }, + { + name: "missing mcp container", + template: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "sidecar", Image: "img"}}, + }}, + wantErr: `no "mcp" container`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + _, err := clonePodFromTemplate(tc.template, "app", testRequest(), "vmcp-1") + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + assert.Contains(t, err.Error(), "untrusted pod clone") + }) + } +} + +func TestReverifyAllowsFieldRefEnv(t *testing.T) { + t.Parallel() + // FieldRef exposes pod metadata, not credentials — allowed by the gate. + tmpl := validTemplate() + require.NoError(t, reverifyNoSecretMaterial(tmpl)) +} diff --git a/pkg/vmcp/session/untrusted/tokenstore.go b/pkg/vmcp/session/untrusted/tokenstore.go new file mode 100644 index 0000000000..2e334a4dcb --- /dev/null +++ b/pkg/vmcp/session/untrusted/tokenstore.go @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "fmt" +) + +// TokenStoreConfig carries the coordinates the egress-broker sidecar needs to +// reach the auth-server's upstream token store, injected into the cloned pod at +// clone time (pkg/vmcp/session/untrusted/egress.go). Only non-secret +// coordinates live here — credentials are never carried on this struct. +// +// The KeyPrefix is derived by the vMCP composition root from its own identity +// (the auth server it embeds owns the token rows) via +// storage.DeriveKeyPrefix(vmcpNamespace, vmcpName), so the sidecar reads the +// exact same per-tenant prefix the auth server writes under. RedisAddr is the +// one coordinate the vMCP cannot compute and is supplied explicitly by the +// operator / deployment. +type TokenStoreConfig struct { + // RedisAddr is the auth-server Redis address (host:port). Required. + RedisAddr string + // KeyPrefix is the auth-server per-tenant key prefix (e.g. + // "thv:auth:{ns:name}:"). Required; must end with ':'. + KeyPrefix string + // KEKSecretRef, when non-nil, names the Secret + key holding the base64 + // token-encryption KEK (32 bytes decoded). The sidecar mounts it as an env + // SecretKeyRef — the KEK value is never placed in a ConfigMap or pod env + // literal. Nil means token rows are read unencrypted (legacy plaintext). + KEKSecretRef *SecretKeyRef +} + +// SecretKeyRef is a minimal (name, key) reference into a Secret, mirroring the +// subset of corev1.SecretKeySelector the wiring needs without coupling the +// config to a specific source. +type SecretKeyRef struct { + // Name is the Secret name. Required. + Name string + // Key is the data key within the Secret. Required. + Key string +} + +// validate enforces the fail-closed contract: the sidecar must never be given +// partial token-store coordinates (it would crash-loop on a malformed prefix +// or dial a default address). An entirely-empty config is valid and means +// "no token store wired" (the broker is then expected to be absent). +func (c *TokenStoreConfig) validate() error { + if c == nil { + return fmt.Errorf("untrusted token store: config must not be nil") + } + if c.RedisAddr == "" { + return fmt.Errorf("untrusted token store: RedisAddr must not be empty") + } + if c.KeyPrefix == "" || c.KeyPrefix[len(c.KeyPrefix)-1] != ':' { + return fmt.Errorf("untrusted token store: KeyPrefix must be non-empty and end with ':'") + } + if c.KEKSecretRef != nil { + if c.KEKSecretRef.Name == "" || c.KEKSecretRef.Key == "" { + return fmt.Errorf("untrusted token store: KEKSecretRef requires both Name and Key") + } + } + return nil +} diff --git a/pkg/vmcp/session/untrusted/wiring.go b/pkg/vmcp/session/untrusted/wiring.go new file mode 100644 index 0000000000..2e3f5e5a26 --- /dev/null +++ b/pkg/vmcp/session/untrusted/wiring.go @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "fmt" + "time" + + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/metric" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Stack bundles the untrusted-mode runtime components wired at the vMCP +// composition root (session manager / CLI): the address resolver installed on +// the session factory, the pod lifecycle (DeletePod on session Terminate), and +// the reaper owning pod GC. +type Stack struct { + Resolver BackendAddressResolver + Lifecycle PodLifecycle + Reaper *Reaper +} + +// WiringConfig carries the fully-resolved untrusted-mode parameters. +// RedisClient must be the same client (and KeyPrefix the same per-tenant +// ':'-terminated prefix) as the session metadata store — untrusted pod +// admission reuses the session-storage Redis. +type WiringConfig struct { + // K8sClient is vMCP's in-cluster controller-runtime client. Required. + K8sClient client.Client + // RedisClient backs admission counters and pod leases. Required; Redis + // unavailability fails admission closed. + RedisClient redis.UniversalClient + // KeyPrefix is the session-storage key prefix (e.g. "thv:vmcp:session:"). + // Required, must end with ':'. + KeyPrefix string + // Namespace is vMCP's own namespace. Required. + Namespace string + // VMCPUId identifies this vMCP instance for heartbeats/zombie detection. + // Required. + VMCPUId string + // Admission tunes the DoS controls. + Admission AdmissionConfig + // Reaper tunes the GC loop. + Reaper ReaperConfig + // ReadyBudget bounds per-pod cold start. Default DefaultReadyBudget. + ReadyBudget time.Duration + // AdoptTTL is the bounded lease written when adopting a pre-existing pod. + // Default 2 minutes. + AdoptTTL time.Duration + // TokenStore, when non-nil, wires the egress-broker sidecar's auth-server + // token-store coordinates into every provisioned pod. Nil leaves the broker + // without a token store (it fails closed at startup). + TokenStore *TokenStoreConfig + // MeterProvider, when non-nil, registers the untrusted-mode OTel instruments + // (untrusted_backend_pods gauge, untrusted_pod_admissions_total counter). + // Nil disables metrics. + MeterProvider metric.MeterProvider +} + +// defaultAdoptTTL bounds the lease written on pod adoption. +const defaultAdoptTTL = 2 * time.Minute + +// NewK8sPodLifecycleFromParts builds a PodLifecycle directly from the raw +// client/prefix parts, bypassing admission and the reaper. Exported for +// envtest/integration tests that exercise the lifecycle in isolation; +// production wiring goes through NewStack. perUserQuota is the lifecycle's +// authoritative per-user cap (same value the admission gate pre-flights). +func NewK8sPodLifecycleFromParts( + k8sClient client.Client, + redisClient redis.UniversalClient, + keyPrefix, namespace, vmcpUID string, + idleTTL, adoptTTL time.Duration, + perUserQuota int, +) (PodLifecycle, error) { + store, err := newRedisStore(redisClient, keyPrefix, namespace) + if err != nil { + return nil, err + } + return NewK8sPodLifecycle(k8sClient, store, vmcpUID, idleTTL, adoptTTL, perUserQuota) +} + +// NewStack wires the full untrusted-mode stack. The reaper's session-liveness +// probe is NOT taken here: it comes from the session manager (the owner of +// session storage), which does not exist until the server is built — the +// composition root hands it to Reaper.Run. +func NewStack(cfg WiringConfig) (*Stack, error) { + store, err := newRedisStore(cfg.RedisClient, cfg.KeyPrefix, cfg.Namespace) + if err != nil { + return nil, err + } + if cfg.K8sClient == nil { + return nil, fmt.Errorf("untrusted wiring: K8sClient must not be nil") + } + if cfg.VMCPUId == "" { + return nil, fmt.Errorf("untrusted wiring: VMCPUId must not be empty") + } + adoptTTL := cfg.AdoptTTL + if adoptTTL <= 0 { + adoptTTL = defaultAdoptTTL + } + idleTTL := cfg.Reaper.resolved().IdleTTL + + if cfg.TokenStore != nil { + if err := cfg.TokenStore.validate(); err != nil { + return nil, err + } + } + var metrics *untrustedMetrics + if cfg.MeterProvider != nil { + metrics, err = newUntrustedMetrics(cfg.MeterProvider) + if err != nil { + return nil, err + } + } + adm, err := newAdmission(store, cfg.Admission, time.Now) + if err != nil { + return nil, err + } + // The lifecycle enforces the same per-user quota the admission gate + // pre-flights; resolved() has already applied the default. + quota := adm.cfg.PerUserPodQuota + lifecycle, err := NewK8sPodLifecycle(cfg.K8sClient, store, cfg.VMCPUId, idleTTL, adoptTTL, quota) + if err != nil { + return nil, err + } + resolver := podAddressResolverWithTokenStore(lifecycle, adm, cfg.ReadyBudget, cfg.TokenStore, metrics) + reaper, err := NewReaper(cfg.K8sClient, store, cfg.Reaper, cfg.VMCPUId, metrics) + if err != nil { + return nil, err + } + return &Stack{Resolver: resolver, Lifecycle: lifecycle, Reaper: reaper}, nil +} diff --git a/pkg/vmcp/workloads/k8s.go b/pkg/vmcp/workloads/k8s.go index 94b92eacc1..50b40907a5 100644 --- a/pkg/vmcp/workloads/k8s.go +++ b/pkg/vmcp/workloads/k8s.go @@ -34,6 +34,12 @@ const ( metadataKeyWorkloadStatus = "workload_status" metadataKeyNamespace = "namespace" metadataKeyRemoteURL = "remote_url" + + // metadataKeyUntrusted mirrors untrusted.MetadataKeyUntrusted (the untrusted + // session package must not be imported here; the values must stay in sync). + metadataKeyUntrusted = "toolhive.stacklok.dev/untrusted" + // metadataKeyMCPServerUID mirrors untrusted.MetadataKeyMCPServerUID. + metadataKeyMCPServerUID = "toolhive.stacklok.dev/mcpserver-uid" ) // k8sDiscoverer is a direct implementation of Discoverer for Kubernetes workloads. @@ -286,6 +292,14 @@ func (d *k8sDiscoverer) mcpServerToBackend(ctx context.Context, mcpServer *mcpv1 backend.Metadata[metadataKeyNamespace] = mcpServer.Namespace } + // Mark untrusted backends for the per-session address resolver. The UID is + // rename-safe identity: pods/StatefulSets are keyed on it (handoff §4), so + // an MCPServer rename cannot strand a session pod's ownership. + if mcpServer.Spec.Untrusted { + backend.Metadata[metadataKeyUntrusted] = "true" + backend.Metadata[metadataKeyMCPServerUID] = string(mcpServer.UID) + } + // Discover and populate authentication configuration from MCPServer if err := d.discoverAuthConfig(ctx, mcpServer, backend); err != nil { // If auth discovery fails, we must fail - don't silently allow unauthorized access diff --git a/pkg/vmcp/workloads/k8s_test.go b/pkg/vmcp/workloads/k8s_test.go index 3c73067a86..f0d1d703bc 100644 --- a/pkg/vmcp/workloads/k8s_test.go +++ b/pkg/vmcp/workloads/k8s_test.go @@ -376,6 +376,54 @@ func TestMCPServerToBackend_BasicFields(t *testing.T) { assert.Equal(t, namespace, backend.Metadata["namespace"]) } +func TestMCPServerToBackend_UntrustedMetadata(t *testing.T) { + t.Parallel() + + namespace := testNamespace + + newServer := func(untrusted bool) *mcpv1beta1.MCPServer { + return &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-server", + Namespace: namespace, + UID: "uid-abc-123", + }, + Spec: mcpv1beta1.MCPServerSpec{ + Image: "test-image:latest", + Transport: "streamable-http", + ProxyPort: 8080, + Untrusted: untrusted, + }, + Status: mcpv1beta1.MCPServerStatus{ + Phase: mcpv1beta1.MCPServerPhaseReady, + URL: "http://localhost:8080", + }, + } + } + + t.Run("untrusted server is marked with UID", func(t *testing.T) { + t.Parallel() + srv := newServer(true) + discoverer := NewK8SDiscovererWithClient(setupTestClient(t, srv), namespace).(*k8sDiscoverer) + backend := discoverer.mcpServerToBackend(context.Background(), srv) + require.NotNil(t, backend) + assert.Equal(t, "true", backend.Metadata["toolhive.stacklok.dev/untrusted"]) + assert.Equal(t, "uid-abc-123", backend.Metadata["toolhive.stacklok.dev/mcpserver-uid"]) + }) + + t.Run("trusted server carries no untrusted metadata", func(t *testing.T) { + t.Parallel() + srv := newServer(false) + discoverer := NewK8SDiscovererWithClient(setupTestClient(t, srv), namespace).(*k8sDiscoverer) + backend := discoverer.mcpServerToBackend(context.Background(), srv) + require.NotNil(t, backend) + _, hasUntrusted := backend.Metadata["toolhive.stacklok.dev/untrusted"] + _, hasUID := backend.Metadata["toolhive.stacklok.dev/mcpserver-uid"] + assert.False(t, hasUntrusted) + assert.False(t, hasUID) + }) +} + func TestMCPServerToBackend_StdioTransport(t *testing.T) { t.Parallel() diff --git a/test/e2e/chainsaw/operator/untrusted-egress/chainsaw-test.yaml b/test/e2e/chainsaw/operator/untrusted-egress/chainsaw-test.yaml new file mode 100644 index 0000000000..fc50638183 --- /dev/null +++ b/test/e2e/chainsaw/operator/untrusted-egress/chainsaw-test.yaml @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# Wave-3 untrusted egress data plane: the operator must materialize the +# per-tenant bump CA, the egress-policy ConfigMap, and the egress-lockdown +# NetworkPolicy for an untrusted MCPServer — and delete them when the server +# flips back to trusted. +# +# The MCPServer intentionally uses a groupRef-free fixture shape: the +# data-plane resources are reconciled independent of vMCP wiring, so this +# scenario creates the group it references to satisfy CEL R2. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: mcpserver-untrusted-egress-resources +spec: + description: | + Untrusted MCPServer → bump-CA Secret + bundle ConfigMap + egress-policy + ConfigMap + egress NetworkPolicy exist with ownerReferences; flipping + untrusted to false deletes them. The policy ConfigMap content matches the + CRD EgressPolicy. The NetworkPolicy must select the untrusted session + pods and permit only loopback, DNS, and the allowlisted destination. + steps: + - name: create-group-and-untrusted-server + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPGroup + metadata: + name: untrusted-egress-group + namespace: default + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-egress-srv + namespace: default + spec: + image: ghcr.io/stacklok/toolhive/test-mcp-server:latest + untrusted: true + groupRef: + name: untrusted-egress-group + egressPolicy: + providers: + - provider: github + allowedHosts: + - api.github.com + allowedMethods: + - GET + allowedPathPrefixes: + - /repos/ + + - name: bump-ca-secret-exists-with-ownerref + try: + # The Secret is generation-named (-bump-ca-): + # cert+key of ONE CA generation live in one object so a pod cloned + # mid-rotation mounts a consistent pair. Assert by label + prefix. + - script: + timeout: 2m + content: | + set -eu + for i in $(seq 1 24); do + NAME=$(kubectl get secrets -n default \ + -l toolhive.stacklok.dev/untrusted-resource=true \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + | grep '^untrusted-egress-srv-bump-ca-' || true) + COUNT=$(printf '%s\n' "$NAME" | grep -c . || true) + if [ "$COUNT" = "1" ]; then break; fi + sleep 5 + done + [ "$COUNT" = "1" ] || { echo "expected exactly one generation-named CA Secret, got: $NAME"; exit 1; } + GEN=${NAME##untrusted-egress-srv-bump-ca-} + [ -n "$GEN" ] || { echo "CA Secret $NAME is not generation-named"; exit 1; } + kubectl get secret "$NAME" -n default -o jsonpath='{.data.ca\.crt}' | grep -q . || { + echo "CA Secret $NAME has no ca.crt"; exit 1; } + kubectl get secret "$NAME" -n default -o jsonpath='{.data.ca\.key}' | grep -q . || { + echo "CA Secret $NAME has no ca.key"; exit 1; } + OWNER=$(kubectl get secret "$NAME" -n default -o jsonpath='{.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}') + [ "$OWNER" = "MCPServer/untrusted-egress-srv" ] || { echo "bad ownerRef: $OWNER"; exit 1; } + echo "$GEN" > /tmp/untrusted-egress-gen.txt + + - name: bump-ca-bundle-has-public-cert-only + try: + # The bundle is generation-named like the Secret and carries the + # public cert of the same generation (never the key). + - script: + timeout: 2m + content: | + set -eu + GEN=$(cat /tmp/untrusted-egress-gen.txt) + NAME="untrusted-egress-srv-bump-ca-bundle-$GEN" + for i in $(seq 1 24); do + kubectl get configmap "$NAME" -n default >/dev/null 2>&1 && break + sleep 5 + done + CERT_SECRET=$(kubectl get secret "untrusted-egress-srv-bump-ca-$GEN" -n default -o jsonpath='{.data.ca\.crt}') + CERT_BUNDLE=$(kubectl get configmap "$NAME" -n default -o jsonpath='{.data.ca\.crt}') + [ -n "$CERT_BUNDLE" ] || { echo "bundle $NAME has no ca.crt"; exit 1; } + [ "$CERT_SECRET" = "$CERT_BUNDLE" ] || { + echo "bundle $NAME cert does not match the same generation's Secret cert"; exit 1; } + KEY=$(kubectl get configmap "$NAME" -n default -o jsonpath='{.data.ca\.key}' || true) + [ -z "$KEY" ] || { echo "CA private key must never appear in the bundle"; exit 1; } + OWNER=$(kubectl get configmap "$NAME" -n default -o jsonpath='{.metadata.ownerReferences[0].kind}') + [ "$OWNER" = "MCPServer" ] || { echo "bad ownerRef: $OWNER"; exit 1; } + + - name: egress-policy-configmap-matches-crd + try: + - assert: + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + name: untrusted-egress-srv-egress-policy + namespace: default + (metadata.ownerReferences[0].kind): MCPServer + (contains(data['policy.yaml'], 'provider: github')): true + (contains(data['policy.yaml'], 'api.github.com')): true + (contains(data['policy.yaml'], '/repos/')): true + + - name: egress-networkpolicy-locks-down-session-pods + try: + - assert: + resource: + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: untrusted-egress-srv-egress + namespace: default + (metadata.ownerReferences[0].kind): MCPServer + (spec.podSelector.matchLabels['toolhive.stacklok.dev/untrusted']): "true" + (contains(spec.policyTypes, 'Egress')): true + + - name: flip-to-trusted + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-egress-srv + namespace: default + spec: + image: ghcr.io/stacklok/toolhive/test-mcp-server:latest + groupRef: + name: untrusted-egress-group + + - name: untrusted-resources-are-deleted-on-flip + try: + # Every generation-named CA Secret and bundle goes (listed by label); + # the fixed-name policy ConfigMap and NetworkPolicy go by name. + - script: + timeout: 2m + content: | + set -eu + for i in $(seq 1 24); do + LEFT=$(kubectl get secrets,configmaps -n default \ + -l toolhive.stacklok.dev/untrusted-resource=true \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + | grep -c '^untrusted-egress-srv-' || true) + [ "$LEFT" = "0" ] && break + sleep 5 + done + [ "$LEFT" = "0" ] || { echo "untrusted CA generations still present after flip"; exit 1; } + - error: + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + name: untrusted-egress-srv-egress-policy + namespace: default + - error: + resource: + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: untrusted-egress-srv-egress + namespace: default + + - name: cleanup + try: + - delete: + ref: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-egress-srv + namespace: default + - delete: + ref: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPGroup + metadata: + name: untrusted-egress-group + namespace: default diff --git a/test/e2e/chainsaw/operator/validation/mcpserver-untrusted/chainsaw-test.yaml b/test/e2e/chainsaw/operator/validation/mcpserver-untrusted/chainsaw-test.yaml new file mode 100644 index 0000000000..c64ef31d0e --- /dev/null +++ b/test/e2e/chainsaw/operator/validation/mcpserver-untrusted/chainsaw-test.yaml @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# Test CEL validation rules R1-R6 for MCPServer untrusted mode. +# These validations happen at the API server level and reject invalid specs immediately. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: mcpserver-untrusted-cel-validation +spec: + description: | + Test CEL validation rules for MCPServer untrusted mode (ADR-0001): + R1 egressPolicy required, R2 MCPGroup membership, R3 no spec.secrets, + R4 no podTemplateSpec, R5 sessionAffinity ClientIP, R6 no backendReplicas. + steps: + # R1: untrusted without egressPolicy should be rejected + - name: reject-untrusted-missing-egress-policy + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-r1-violation + namespace: default + spec: + image: example/mcp-server:latest + untrusted: true + groupRef: + name: test-group + expect: + - check: + ($error != null): true + ($error.message): "?* egressPolicy with at least one provider is required when untrusted is true *" + + # R2: untrusted without groupRef should be rejected + - name: reject-untrusted-missing-group-ref + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-r2-violation + namespace: default + spec: + image: example/mcp-server:latest + untrusted: true + egressPolicy: + providers: + - provider: github + allowedHosts: + - api.github.com + expect: + - check: + ($error != null): true + ($error.message): "?* untrusted workloads must belong to an MCPGroup fronted by a VirtualMCPServer *" + + # R3: untrusted with spec.secrets should be rejected + - name: reject-untrusted-with-secrets + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-r3-violation + namespace: default + spec: + image: example/mcp-server:latest + untrusted: true + groupRef: + name: test-group + egressPolicy: + providers: + - provider: github + allowedHosts: + - api.github.com + secrets: + - name: backend-creds + key: token + targetEnvName: API_TOKEN + expect: + - check: + ($error != null): true + ($error.message): "?* spec.secrets is forbidden when untrusted is true *" + + # R4: untrusted with podTemplateSpec should be rejected + - name: reject-untrusted-with-pod-template-spec + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-r4-violation + namespace: default + spec: + image: example/mcp-server:latest + untrusted: true + groupRef: + name: test-group + egressPolicy: + providers: + - provider: github + allowedHosts: + - api.github.com + podTemplateSpec: + spec: + containers: + - name: mcp + env: + - name: A + value: b + expect: + - check: + ($error != null): true + ($error.message): "?* podTemplateSpec is forbidden when untrusted is true *" + + # R5: untrusted with sessionAffinity None should be rejected + - name: reject-untrusted-session-affinity-none + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-r5-violation + namespace: default + spec: + image: example/mcp-server:latest + untrusted: true + sessionAffinity: None + groupRef: + name: test-group + egressPolicy: + providers: + - provider: github + allowedHosts: + - api.github.com + expect: + - check: + ($error != null): true + ($error.message): "?* untrusted workloads require sessionAffinity ClientIP *" + + # R6: untrusted with backendReplicas should be rejected + - name: reject-untrusted-with-backend-replicas + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-r6-violation + namespace: default + spec: + image: example/mcp-server:latest + untrusted: true + groupRef: + name: test-group + backendReplicas: 2 + egressPolicy: + providers: + - provider: github + allowedHosts: + - api.github.com + expect: + - check: + ($error != null): true + ($error.message): "?* backendReplicas is managed by the untrusted-mode session lifecycle and must not be set *" + + # Valid untrusted spec should be accepted + - name: accept-valid-untrusted + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-valid + namespace: default + spec: + image: example/mcp-server:latest + untrusted: true + groupRef: + name: test-group + egressPolicy: + providers: + - provider: github + allowedHosts: + - api.github.com + - "*.githubusercontent.com" + allowedMethods: + - GET + credentialEnvName: GITHUB_TOKEN + - assert: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: untrusted-valid + spec: + untrusted: true + + # Trusted workloads are inert for all rules at once + - name: accept-trusted-with-all-otherwise-forbidden-fields + try: + - apply: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: trusted-inert + namespace: default + spec: + image: example/mcp-server:latest + untrusted: false + sessionAffinity: None + backendReplicas: 2 + secrets: + - name: backend-creds + key: token + targetEnvName: API_TOKEN + podTemplateSpec: + spec: + containers: + - name: mcp + env: + - name: A + value: b + - assert: + resource: + apiVersion: toolhive.stacklok.dev/v1beta1 + kind: MCPServer + metadata: + name: trusted-inert + spec: + untrusted: false From 4af587b4225fca74041c39c56bd92d9e874e2515 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 22 Jul 2026 10:42:27 +0300 Subject: [PATCH 03/12] Add consent-on-demand for upstream providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a user has not consented an upstream provider, all three surfaces that can fail now produce an actionable response carrying the provider name and authorize endpoint, completing the ADR-0001 D8 requirement in both subsystems. Proxy path: the upstreamswap 401 keeps its RFC 6750 challenge and gains a JSON body (error, provider, authorize_url) clients can render as a connect-your-account prompt. vMCP path: upstream_inject returns a typed ConsentRequiredError that unwraps to ErrUpstreamTokenNotFound (classification preserved, errors.Join keeps ErrAuthenticationFailed), fails fast before any backend dispatch, and surfaces to the agent as a tool-result error with a machine-readable UPSTREAM_CONSENT_REQUIRED marker plus JSON payload. Post-consent retry is client-driven: the next request's per-request identity enrichment picks up the new token, no server-side watch. The sidecar's 403 deny text deliberately carries no URL — consent guidance never traverses the untrusted server. --- pkg/auth/upstreamswap/middleware.go | 44 ++++- pkg/auth/upstreamswap/middleware_test.go | 124 ++++++++++++-- pkg/egressbroker/injector.go | 7 + pkg/egressbroker/server_test.go | 10 ++ pkg/vmcp/auth/strategies/upstream_inject.go | 13 +- .../auth/strategies/upstream_inject_test.go | 49 ++++++ pkg/vmcp/auth/types/consent_test.go | 97 +++++++++++ pkg/vmcp/auth/types/types.go | 69 +++++++- pkg/vmcp/client/client.go | 72 ++++++--- pkg/vmcp/client/client_test.go | 42 +++++ pkg/vmcp/client/consent_integration_test.go | 153 ++++++++++++++++++ pkg/vmcp/server/adapter/handler_factory.go | 12 ++ .../server/adapter/handler_factory_test.go | 84 ++++++++++ 13 files changed, 735 insertions(+), 41 deletions(-) create mode 100644 pkg/vmcp/auth/types/consent_test.go create mode 100644 pkg/vmcp/client/consent_integration_test.go diff --git a/pkg/auth/upstreamswap/middleware.go b/pkg/auth/upstreamswap/middleware.go index 99c34221d7..3195c7a5fd 100644 --- a/pkg/auth/upstreamswap/middleware.go +++ b/pkg/auth/upstreamswap/middleware.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "github.com/stacklok/toolhive/pkg/auth" "github.com/stacklok/toolhive/pkg/transport/types" @@ -36,6 +37,14 @@ type Config struct { // ProviderName identifies which upstream provider's tokens to retrieve for injection. // This is required and must match a configured upstream provider name. ProviderName string `json:"provider_name" yaml:"provider_name"` + + // AuthorizeURL is the ToolHive authorization server's authorize-endpoint URL + // ({issuer}/oauth/authorize). When set, the 401 consent response includes it + // so clients can direct the user to consent. It is only an endpoint: the + // client merges its own client_id, redirect_uri, and PKCE parameters. It + // must never be derived from the request Host header (attacker-controlled). + // Optional; when empty the 401 body omits authorize_url. + AuthorizeURL string `json:"authorize_url,omitempty" yaml:"authorize_url,omitempty"` } // MiddlewareParams represents the JSON parameters for the middleware factory. @@ -106,16 +115,43 @@ func validateConfig(cfg *Config) error { return fmt.Errorf("custom_header_name must be specified when header_strategy is '%s'", HeaderStrategyCustom) } + // AuthorizeURL, when set, must be an absolute https URL. It is emitted to + // clients in the 401 consent response, so it must point at the configured + // authorization server — never a relative or plain-HTTP address. + if cfg.AuthorizeURL != "" { + u, err := url.Parse(cfg.AuthorizeURL) + if err != nil || u.Scheme != "https" || u.Host == "" { + return fmt.Errorf("authorize_url must be an absolute https:// URL") + } + } + return nil } +// consentRequiredBody is the JSON body of the 401 consent response. +type consentRequiredBody struct { + Error string `json:"error"` + Provider string `json:"provider"` + AuthorizeURL string `json:"authorize_url,omitempty"` +} + // writeUpstreamAuthRequired writes a 401 response with a WWW-Authenticate Bearer // challenge per RFC 6750 Section 3.1, signalling that the caller must re-authenticate -// with the upstream IdP. -func writeUpstreamAuthRequired(w http.ResponseWriter) { +// with the upstream IdP, plus a JSON body carrying the provider and (when +// configured) the authorize endpoint so the client can render an actionable +// "connect your account" prompt. The body never contains token material. +func writeUpstreamAuthRequired(w http.ResponseWriter, provider, authorizeURL string) { w.Header().Set("WWW-Authenticate", `Bearer error="invalid_token", error_description="upstream token is no longer valid; re-authentication required"`) - http.Error(w, "upstream authentication required", http.StatusUnauthorized) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + // Marshal errors are impossible for this fixed-shape struct; on failure the + // client still has the RFC 6750 challenge header to act on. + _ = json.NewEncoder(w).Encode(consentRequiredBody{ + Error: "upstream_consent_required", + Provider: provider, + AuthorizeURL: authorizeURL, + }) } // injectionFunc is a function that injects a token into an HTTP request. @@ -178,7 +214,7 @@ func createMiddlewareFunc(cfg *Config) types.MiddlewareFunction { token, exists := identity.UpstreamTokens[cfg.ProviderName] if !exists || token == "" { - writeUpstreamAuthRequired(w) + writeUpstreamAuthRequired(w, cfg.ProviderName, cfg.AuthorizeURL) return } diff --git a/pkg/auth/upstreamswap/middleware_test.go b/pkg/auth/upstreamswap/middleware_test.go index ee10914861..56eb8e49f5 100644 --- a/pkg/auth/upstreamswap/middleware_test.go +++ b/pkg/auth/upstreamswap/middleware_test.go @@ -84,6 +84,48 @@ func TestValidateConfig(t *testing.T) { wantErr: true, errMsg: "custom_header_name must be specified", }, + { + name: "valid https authorize URL", + cfg: &Config{ + ProviderName: "default", + AuthorizeURL: "https://thv.example.com/oauth/authorize", + }, + wantErr: false, + }, + { + name: "empty authorize URL allowed", + cfg: &Config{ + ProviderName: "default", + }, + wantErr: false, + }, + { + name: "non-https authorize URL rejected", + cfg: &Config{ + ProviderName: "default", + AuthorizeURL: "http://thv.example.com/oauth/authorize", + }, + wantErr: true, + errMsg: "authorize_url must be an absolute https:// URL", + }, + { + name: "relative authorize URL rejected", + cfg: &Config{ + ProviderName: "default", + AuthorizeURL: "/oauth/authorize", + }, + wantErr: true, + errMsg: "authorize_url must be an absolute https:// URL", + }, + { + name: "malformed authorize URL rejected", + cfg: &Config{ + ProviderName: "default", + AuthorizeURL: "://not-a-url", + }, + wantErr: true, + errMsg: "authorize_url must be an absolute https:// URL", + }, } for _, tt := range tests { @@ -122,6 +164,11 @@ func TestMiddleware_NoIdentity(t *testing.T) { assert.Equal(t, http.StatusOK, rr.Code) } +// expectedBearerChallenge is the RFC 6750 Section 3.1 challenge the 401 +// consent response must carry unchanged (mcp-go's authorization-required +// detection depends on it). +const expectedBearerChallenge = `Bearer error="invalid_token", error_description="upstream token is no longer valid; re-authentication required"` + func TestMiddleware_NilUpstreamTokens(t *testing.T) { t.Parallel() @@ -146,26 +193,76 @@ func TestMiddleware_NilUpstreamTokens(t *testing.T) { handler.ServeHTTP(rr, req) assert.Equal(t, http.StatusUnauthorized, rr.Code) - assert.Contains(t, rr.Header().Get("WWW-Authenticate"), `error="invalid_token"`) + assert.Equal(t, expectedBearerChallenge, rr.Header().Get("WWW-Authenticate")) } +// TestMiddleware_ProviderMissing_Returns401 covers the structured consent +// body emitted when the provider token is absent, with and without a +// configured authorize URL. func TestMiddleware_ProviderMissing_Returns401(t *testing.T) { t.Parallel() - cfg := &Config{ProviderName: "atlassian"} - middleware := createMiddlewareFunc(cfg) + tests := []struct { + name string + cfg *Config + wantAuthURL string + wantHasURL bool + wantProvider string + }{ + { + name: "authorize URL configured", + cfg: &Config{ + ProviderName: "atlassian", + AuthorizeURL: "https://thv.example.com/oauth/authorize", + }, + wantAuthURL: "https://thv.example.com/oauth/authorize", + wantHasURL: true, + wantProvider: "atlassian", + }, + { + name: "authorize URL empty omits the key", + cfg: &Config{ProviderName: "atlassian"}, + wantHasURL: false, + wantProvider: "atlassian", + }, + } - nextHandler := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { - t.Error("next handler should NOT be called when provider is missing") - }) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() - handler := middleware(nextHandler) - req := requestWithIdentity("gh-token") // has github but not atlassian + middleware := createMiddlewareFunc(tt.cfg) - rr := httptest.NewRecorder() - handler.ServeHTTP(rr, req) + nextHandler := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + t.Error("next handler should NOT be called when provider is missing") + }) - assert.Equal(t, http.StatusUnauthorized, rr.Code) + handler := middleware(nextHandler) + req := requestWithIdentity("gh-token") // has github but not atlassian + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusUnauthorized, rr.Code) + assert.Equal(t, expectedBearerChallenge, rr.Header().Get("WWW-Authenticate")) + assert.Equal(t, "application/json", rr.Header().Get("Content-Type")) + + var body map[string]any + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &body), + "401 body must be JSON, got: %q", rr.Body.String()) + assert.Equal(t, "upstream_consent_required", body["error"]) + assert.Equal(t, tt.wantProvider, body["provider"]) + if tt.wantHasURL { + assert.Equal(t, tt.wantAuthURL, body["authorize_url"]) + } else { + _, hasURL := body["authorize_url"] + assert.False(t, hasURL, "authorize_url key must be absent when not configured") + } + + // The body must never contain token material. + assert.NotContains(t, rr.Body.String(), "gh-token") + }) + } } func TestMiddleware_EmptyToken_Returns401(t *testing.T) { @@ -185,6 +282,11 @@ func TestMiddleware_EmptyToken_Returns401(t *testing.T) { handler.ServeHTTP(rr, req) assert.Equal(t, http.StatusUnauthorized, rr.Code) + + var body map[string]any + require.NoError(t, json.Unmarshal(rr.Body.Bytes(), &body)) + assert.Equal(t, "upstream_consent_required", body["error"]) + assert.Equal(t, "github", body["provider"]) } func TestMiddleware_SuccessfulSwap_ReplaceStrategy(t *testing.T) { diff --git a/pkg/egressbroker/injector.go b/pkg/egressbroker/injector.go index 3000e6e2f2..6629b90241 100644 --- a/pkg/egressbroker/injector.go +++ b/pkg/egressbroker/injector.go @@ -105,6 +105,13 @@ func (i *CredentialInjector) Evaluate(ctx context.Context, dest Destination) Dec } cred, ok := creds[provider] if !ok || slices.Contains(failed, provider) { + // Re-consent denial: this 403 is the security enforcement backstop for + // the case where vMCP's fail-fast consent check is bypassed (e.g. a + // malicious server attempting egress anyway). The agent-facing consent + // UX (provider + authorize URL) comes from the vMCP upstream_inject + // path, never from here: the denial surfaces on the server's own + // outbound HTTPS call, not on the vMCP channel, so no consent URL can + // or should be threaded through this response. return deny("upstream credential unavailable; re-consent required") } if cred.AccessToken == "" { diff --git a/pkg/egressbroker/server_test.go b/pkg/egressbroker/server_test.go index da1868ef8d..a760085808 100644 --- a/pkg/egressbroker/server_test.go +++ b/pkg/egressbroker/server_test.go @@ -104,6 +104,16 @@ func TestAuthorizationServerCheck(t *testing.T) { require.NoError(t, err) assert.Equal(t, int32(code.Code_PERMISSION_DENIED), resp.GetStatus().GetCode()) assert.Nil(t, resp.GetOkResponse()) + + // Regression guard (Wave 4 consent-on-demand): the sidecar denial text + // stays exactly "re-consent required" with no consent URL threaded + // through. The agent-facing consent UX (provider + authorize URL) comes + // from the vMCP upstream_inject path, never from the sidecar. + denied := resp.GetDeniedResponse() + require.NotNil(t, denied) + assert.Contains(t, denied.GetBody(), "re-consent required") + assert.NotContains(t, denied.GetBody(), "authorize") + assert.NotContains(t, denied.GetBody(), "http") }) t.Run("missing destination → deny (fail closed)", func(t *testing.T) { diff --git a/pkg/vmcp/auth/strategies/upstream_inject.go b/pkg/vmcp/auth/strategies/upstream_inject.go index 519216cc04..f8dee5d168 100644 --- a/pkg/vmcp/auth/strategies/upstream_inject.go +++ b/pkg/vmcp/auth/strategies/upstream_inject.go @@ -24,6 +24,11 @@ import ( // - ProviderName: The upstream provider name matching an entry in AuthServer.Upstreams. // The token for this provider must be present in the identity's UpstreamTokens map. // +// Optional configuration fields: +// - AuthorizeURL: The authorization server's authorize-endpoint URL, carried +// in the ConsentRequiredError returned when the provider token is absent +// so clients can direct the user to consent. +// // This strategy is appropriate when: // - The backend requires a user-specific upstream IDP token for authentication // - The embedded authorization server has been configured to obtain tokens from @@ -79,7 +84,13 @@ func (*UpstreamInjectStrategy) Authenticate( token := identity.UpstreamTokens[providerName] if token == "" { - return fmt.Errorf("provider %q: %w", providerName, authtypes.ErrUpstreamTokenNotFound) + // ConsentRequiredError unwraps to ErrUpstreamTokenNotFound, so existing + // errors.Is classification is preserved while errors.As lets the + // tool-call layer render an actionable consent message. + return fmt.Errorf("provider %q: %w", providerName, &authtypes.ConsentRequiredError{ + Provider: providerName, + AuthorizeURL: strategy.UpstreamInject.AuthorizeURL, + }) } req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) diff --git a/pkg/vmcp/auth/strategies/upstream_inject_test.go b/pkg/vmcp/auth/strategies/upstream_inject_test.go index d77d8e75e7..512c89a806 100644 --- a/pkg/vmcp/auth/strategies/upstream_inject_test.go +++ b/pkg/vmcp/auth/strategies/upstream_inject_test.go @@ -34,6 +34,7 @@ func TestUpstreamInjectStrategy_Authenticate(t *testing.T) { expectError bool errorContains string checkSentinel bool + checkConsent *authtypes.ConsentRequiredError checkHeader func(t *testing.T, req *http.Request) }{ { @@ -116,6 +117,47 @@ func TestUpstreamInjectStrategy_Authenticate(t *testing.T) { errorContains: "github", checkSentinel: true, }, + { + name: "missing token carries ConsentRequiredError with authorize URL", + setupCtx: func() context.Context { + return createContextWithUpstreamTokens("user1", "incoming-token", map[string]string{ + "other": "tok", + }) + }, + strategy: &authtypes.BackendAuthStrategy{ + Type: authtypes.StrategyTypeUpstreamInject, + UpstreamInject: &authtypes.UpstreamInjectConfig{ + ProviderName: "github", + AuthorizeURL: "https://thv.example.com/oauth/authorize", + }, + }, + expectError: true, + errorContains: "github", + checkSentinel: true, + checkConsent: &authtypes.ConsentRequiredError{ + Provider: "github", + AuthorizeURL: "https://thv.example.com/oauth/authorize", + }, + }, + { + name: "missing token without authorize URL carries provider only", + setupCtx: func() context.Context { + return createContextWithUpstreamTokens("user1", "incoming-token", nil) + }, + strategy: &authtypes.BackendAuthStrategy{ + Type: authtypes.StrategyTypeUpstreamInject, + UpstreamInject: &authtypes.UpstreamInjectConfig{ + ProviderName: "github", + }, + }, + expectError: true, + errorContains: "github", + checkSentinel: true, + checkConsent: &authtypes.ConsentRequiredError{ + Provider: "github", + AuthorizeURL: "", + }, + }, { name: "health check bypass", setupCtx: func() context.Context { return healthcontext.WithHealthCheckMarker(context.Background()) }, @@ -193,6 +235,13 @@ func TestUpstreamInjectStrategy_Authenticate(t *testing.T) { assert.True(t, errors.Is(err, authtypes.ErrUpstreamTokenNotFound), "expected error to wrap ErrUpstreamTokenNotFound, got: %v", err) } + if tt.checkConsent != nil { + var consentErr *authtypes.ConsentRequiredError + require.ErrorAs(t, err, &consentErr, + "expected error to carry a ConsentRequiredError, got: %v", err) + assert.Equal(t, tt.checkConsent.Provider, consentErr.Provider) + assert.Equal(t, tt.checkConsent.AuthorizeURL, consentErr.AuthorizeURL) + } return } diff --git a/pkg/vmcp/auth/types/consent_test.go b/pkg/vmcp/auth/types/consent_test.go new file mode 100644 index 0000000000..2bf7fb636e --- /dev/null +++ b/pkg/vmcp/auth/types/consent_test.go @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConsentRequiredError(t *testing.T) { + t.Parallel() + + t.Run("Unwrap preserves ErrUpstreamTokenNotFound classification", func(t *testing.T) { + t.Parallel() + err := &ConsentRequiredError{Provider: "github", AuthorizeURL: "https://thv.example.com/oauth/authorize"} + assert.ErrorIs(t, err, ErrUpstreamTokenNotFound) + }) + + t.Run("errors.Is and errors.As traverse a fmt.Errorf wrap", func(t *testing.T) { + t.Parallel() + wrapped := fmt.Errorf("provider %q: %w", "github", + &ConsentRequiredError{Provider: "github", AuthorizeURL: "https://thv.example.com/oauth/authorize"}) + var consentErr *ConsentRequiredError + require.ErrorAs(t, wrapped, &consentErr) + assert.Equal(t, "github", consentErr.Provider) + assert.Equal(t, "https://thv.example.com/oauth/authorize", consentErr.AuthorizeURL) + assert.ErrorIs(t, wrapped, ErrUpstreamTokenNotFound) + }) + + t.Run("Error contains provider and never token material", func(t *testing.T) { + t.Parallel() + err := &ConsentRequiredError{Provider: "github", AuthorizeURL: "https://thv.example.com/oauth/authorize"} + msg := err.Error() + assert.Contains(t, msg, "github") + assert.Contains(t, msg, "https://thv.example.com/oauth/authorize") + assert.NotContains(t, msg, "gho_") + assert.NotContains(t, msg, "Bearer") + + bare := &ConsentRequiredError{Provider: "github"} + assert.Contains(t, bare.Error(), "github") + assert.NotContains(t, bare.Error(), "http") + }) +} + +func TestFormatConsentRequired(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + provider string + authorizeURL string + wantHasURL bool + }{ + { + name: "with authorize URL", + provider: "github", + authorizeURL: "https://thv.example.com/oauth/authorize", + wantHasURL: true, + }, + { + name: "without authorize URL omits the key", + provider: "github", + authorizeURL: "", + wantHasURL: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + out := FormatConsentRequired(tt.provider, tt.authorizeURL) + + require.True(t, strings.HasPrefix(out, ConsentRequiredMarker+" "), + "output must start with the marker followed by a space, got: %q", out) + + payloadJSON := strings.TrimPrefix(out, ConsentRequiredMarker+" ") + var payload map[string]any + require.NoError(t, json.Unmarshal([]byte(payloadJSON), &payload), + "trailing payload must parse as JSON: %q", payloadJSON) + + assert.Equal(t, tt.provider, payload["provider"]) + if tt.wantHasURL { + assert.Equal(t, tt.authorizeURL, payload["authorize_url"]) + } else { + _, hasURL := payload["authorize_url"] + assert.False(t, hasURL, "authorize_url key must be absent when empty") + } + }) + } +} diff --git a/pkg/vmcp/auth/types/types.go b/pkg/vmcp/auth/types/types.go index 6c0d028b41..7fae571898 100644 --- a/pkg/vmcp/auth/types/types.go +++ b/pkg/vmcp/auth/types/types.go @@ -12,12 +12,70 @@ // - Backend auth configuration structs (BackendAuthStrategy, etc.) package types -import "errors" +import ( + "encoding/json" + "errors" + "fmt" +) // ErrUpstreamTokenNotFound is returned when a required upstream provider token // is not present in the identity's UpstreamTokens map. var ErrUpstreamTokenNotFound = errors.New("upstream token not found") +// ConsentRequiredMarker is the machine-detectable prefix of the tool-result +// text emitted when a tool call fails because the user has not consented an +// upstream provider. The marker is followed by a single space and a JSON +// payload (consentRequiredPayload). It is a convention, not a wire-protocol +// change: it travels in the existing IsError/text channel that untrusted +// servers and stock MCP clients already carry untouched. +const ConsentRequiredMarker = "UPSTREAM_CONSENT_REQUIRED" + +// ConsentRequiredError carries the provider and authorize endpoint for a +// missing upstream consent, so the tool-call layer can render an actionable +// IsError result (see FormatConsentRequired). AuthorizeURL may be empty when +// no authorization server issuer is configured. +type ConsentRequiredError struct { + Provider string + AuthorizeURL string +} + +// Error returns a human-readable message. It contains the provider name and +// authorize endpoint only — never token material. +func (e *ConsentRequiredError) Error() string { + if e.AuthorizeURL != "" { + return fmt.Sprintf("consent required for upstream provider %q (authorize at %s)", + e.Provider, e.AuthorizeURL) + } + return fmt.Sprintf("consent required for upstream provider %q", e.Provider) +} + +// Unwrap returns ErrUpstreamTokenNotFound so existing errors.Is classification +// (health monitors, pre-check middleware) keeps working while errors.As +// extracts the structured payload. +func (*ConsentRequiredError) Unwrap() error { + return ErrUpstreamTokenNotFound +} + +// consentRequiredPayload is the JSON object following ConsentRequiredMarker. +type consentRequiredPayload struct { + Provider string `json:"provider"` + AuthorizeURL string `json:"authorize_url,omitempty"` +} + +// FormatConsentRequired renders the consent-required tool-result text: +// the marker, a space, then a JSON payload carrying the provider and (when +// configured) the authorize endpoint the client should open to consent. +// The payload never contains token material. +func FormatConsentRequired(provider, authorizeURL string) string { + payload, err := json.Marshal(consentRequiredPayload{Provider: provider, AuthorizeURL: authorizeURL}) + if err != nil { + // Only non-UTF8 control characters could fail marshaling; fall back to + // the marker plus provider so the message is still detectable. + return fmt.Sprintf("%s {\"provider\":%q}", ConsentRequiredMarker, provider) + } + return fmt.Sprintf("%s %s", ConsentRequiredMarker, payload) +} + // Strategy type identifiers used to identify authentication strategies. const ( // StrategyTypeUnauthenticated identifies the unauthenticated strategy. @@ -162,6 +220,15 @@ type UpstreamInjectConfig struct { // ProviderName is the name of the upstream provider configured in the // embedded authorization server. Must match an entry in AuthServer.Upstreams. ProviderName string `json:"providerName" yaml:"providerName"` + + // AuthorizeURL is the ToolHive authorization server's authorize-endpoint + // URL ({issuer}/oauth/authorize). When set, it is carried in the + // ConsentRequiredError returned when the provider token is absent, so + // clients can direct the user to consent. The URL cannot be a complete + // one-click link: the client must merge its own client_id, redirect_uri, + // and PKCE parameters. Optional; when empty the error carries no URL. + // +optional + AuthorizeURL string `json:"authorizeUrl,omitempty" yaml:"authorizeUrl,omitempty"` } // RoleMapping defines a rule for mapping JWT claims to IAM roles. diff --git a/pkg/vmcp/client/client.go b/pkg/vmcp/client/client.go index c33826e0ac..4ddabb0012 100644 --- a/pkg/vmcp/client/client.go +++ b/pkg/vmcp/client/client.go @@ -646,6 +646,36 @@ func isAuthorizationRequired(err error) bool { errors.Is(err, transport.ErrOAuthAuthorizationRequired) } +// wrapTransportSentinelError maps mcp-go transport sentinel errors to vmcp +// sentinel errors, returning nil when err matches none of them. Extracted to +// keep wrapBackendError within the cyclomatic complexity limit. +func wrapTransportSentinelError(err error, backendID string, operation string) error { + if errors.Is(err, transport.ErrUnauthorized) { + return fmt.Errorf("%w: failed to %s for backend %s: %v", + vmcp.ErrAuthenticationFailed, operation, backendID, err) + } + // transport.ErrAuthorizationRequired is returned (wrapped in *transport.Error + // and *transport.AuthorizationRequiredError) for 401 responses with a + // WWW-Authenticate header. transport.ErrOAuthAuthorizationRequired is the + // companion sentinel from the OAuth-handler path. Both must map to + // ErrAuthenticationFailed so health monitoring engages the auth-aware + // branch (#4935) instead of treating the probe as unhealthy (#5223). + if isAuthorizationRequired(err) { + return fmt.Errorf("%w: failed to %s for backend %s: %v", + vmcp.ErrAuthenticationFailed, operation, backendID, err) + } + // ErrLegacySSEServer is returned for any 4xx (except 401) on initialize POST. + // This includes 403 (auth rejection) and 404/405 (endpoint not found/method not allowed). + // We cannot distinguish auth failures from routing errors without the raw status code, + // so we surface a clear message and classify as backend unavailable to allow recovery. + if errors.Is(err, transport.ErrLegacySSEServer) { + const legacyMsg = "server rejected MCP initialize — possible auth rejection or legacy SSE-only server" + return fmt.Errorf("%w: failed to %s for backend %s (%s): %v", + vmcp.ErrBackendUnavailable, operation, backendID, legacyMsg, err) + } + return nil +} + // wrapBackendError wraps an error with the appropriate sentinel error based on error type. // This enables type-safe error checking with errors.Is() instead of string matching. // @@ -659,8 +689,10 @@ func isAuthorizationRequired(err error) bool { // - errors.Is() works for checking the sentinel error (e.g., errors.Is(err, vmcp.ErrTimeout)) // - errors.As() cannot access the underlying original error type // This is a deliberate trade-off due to Go's limitation of one %w per fmt.Errorf call. -// If access to the underlying error type is needed in the future, consider implementing -// a custom error type with multiple Unwrap() methods (Go 1.20+). +// The single exception is ConsentRequiredError: it is wrapped via errors.Join +// together with ErrAuthenticationFailed, so errors.As extracts the typed +// consent payload (provider + authorize URL) while errors.Is keeps the +// health-monitor sentinel contract. func wrapBackendError(err error, backendID string, operation string) error { if err == nil { return nil @@ -693,28 +725,20 @@ func wrapBackendError(err error, backendID string, operation string) error { // 4. mcp-go transport sentinel errors: check before string-based fallbacks // to ensure accurate classification of protocol-level errors. - if errors.Is(err, transport.ErrUnauthorized) { - return fmt.Errorf("%w: failed to %s for backend %s: %v", - vmcp.ErrAuthenticationFailed, operation, backendID, err) - } - // transport.ErrAuthorizationRequired is returned (wrapped in *transport.Error - // and *transport.AuthorizationRequiredError) for 401 responses with a - // WWW-Authenticate header. transport.ErrOAuthAuthorizationRequired is the - // companion sentinel from the OAuth-handler path. Both must map to - // ErrAuthenticationFailed so health monitoring engages the auth-aware - // branch (#4935) instead of treating the probe as unhealthy (#5223). - if isAuthorizationRequired(err) { - return fmt.Errorf("%w: failed to %s for backend %s: %v", - vmcp.ErrAuthenticationFailed, operation, backendID, err) - } - // ErrLegacySSEServer is returned for any 4xx (except 401) on initialize POST. - // This includes 403 (auth rejection) and 404/405 (endpoint not found/method not allowed). - // We cannot distinguish auth failures from routing errors without the raw status code, - // so we surface a clear message and classify as backend unavailable to allow recovery. - if errors.Is(err, transport.ErrLegacySSEServer) { - const legacyMsg = "server rejected MCP initialize — possible auth rejection or legacy SSE-only server" - return fmt.Errorf("%w: failed to %s for backend %s (%s): %v", - vmcp.ErrBackendUnavailable, operation, backendID, legacyMsg, err) + if wrapped := wrapTransportSentinelError(err, backendID, operation); wrapped != nil { + return wrapped + } + + // 5. ConsentRequiredError: upstream provider token missing and the error + // carries consent context (provider + authorize URL). Checked before the + // plain sentinel branch so errors.As can extract the typed payload through + // the wrap; errors.Join preserves both ErrAuthenticationFailed (health + // monitors) and the ConsentRequiredError chain (tool-result rendering). + var consentErr *authtypes.ConsentRequiredError + if errors.As(err, &consentErr) { + return errors.Join(vmcp.ErrAuthenticationFailed, + fmt.Errorf("failed to %s for backend %s (upstream consent required): %w", + operation, backendID, err)) } // 5. ErrUpstreamTokenNotFound: upstream provider token missing from identity. diff --git a/pkg/vmcp/client/client_test.go b/pkg/vmcp/client/client_test.go index edea258f7b..4c26ed43bc 100644 --- a/pkg/vmcp/client/client_test.go +++ b/pkg/vmcp/client/client_test.go @@ -986,6 +986,8 @@ func TestWrapBackendError(t *testing.T) { err error wantSentinel error wantMsgContains string + wantConsent *authtypes.ConsentRequiredError + wantNoConsent bool }{ { name: "nil error returns nil", @@ -1069,6 +1071,34 @@ func TestWrapBackendError(t *testing.T) { wantSentinel: vmcp.ErrAuthenticationFailed, wantMsgContains: "upstream token missing", }, + { + // ConsentRequiredError (upstream_inject with consent context) must + // still classify as ErrAuthenticationFailed for the health-monitor + // contract, AND the typed payload must survive wrapBackendError via + // errors.As so the tool-call layer can render the consent marker. + name: "ConsentRequiredError maps to ErrAuthenticationFailed and stays extractable", + err: fmt.Errorf("authentication failed for backend foo: provider %q: %w", + "github", &authtypes.ConsentRequiredError{ + Provider: "github", + AuthorizeURL: "https://thv.example.com/oauth/authorize", + }), + wantSentinel: vmcp.ErrAuthenticationFailed, + wantMsgContains: "upstream consent required", + wantConsent: &authtypes.ConsentRequiredError{ + Provider: "github", + AuthorizeURL: "https://thv.example.com/oauth/authorize", + }, + }, + { + // Bare ErrUpstreamTokenNotFound (no ConsentRequiredError — the shape + // returned by tokenexchange/aws_sts/xaa) must keep the existing + // mapping and must NOT extract as a ConsentRequiredError. + name: "bare ErrUpstreamTokenNotFound does not extract as ConsentRequiredError", + err: authtypes.ErrUpstreamTokenNotFound, + wantSentinel: vmcp.ErrAuthenticationFailed, + wantMsgContains: "upstream token missing", + wantNoConsent: true, + }, } for _, tt := range tests { @@ -1087,6 +1117,18 @@ func TestWrapBackendError(t *testing.T) { if tt.wantMsgContains != "" { assert.Contains(t, result.Error(), tt.wantMsgContains) } + if tt.wantConsent != nil { + var consentErr *authtypes.ConsentRequiredError + require.ErrorAs(t, result, &consentErr, + "ConsentRequiredError must survive wrapBackendError") + assert.Equal(t, tt.wantConsent.Provider, consentErr.Provider) + assert.Equal(t, tt.wantConsent.AuthorizeURL, consentErr.AuthorizeURL) + } + if tt.wantNoConsent { + var consentErr *authtypes.ConsentRequiredError + assert.False(t, errors.As(result, &consentErr), + "bare sentinel must not extract as ConsentRequiredError") + } }) } } diff --git a/pkg/vmcp/client/consent_integration_test.go b/pkg/vmcp/client/consent_integration_test.go new file mode 100644 index 0000000000..0b31eb2391 --- /dev/null +++ b/pkg/vmcp/client/consent_integration_test.go @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package client + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive-core/mcpcompat/mcp" + mcpserver "github.com/stacklok/toolhive-core/mcpcompat/server" + pkgauth "github.com/stacklok/toolhive/pkg/auth" + "github.com/stacklok/toolhive/pkg/vmcp" + "github.com/stacklok/toolhive/pkg/vmcp/auth" + "github.com/stacklok/toolhive/pkg/vmcp/auth/strategies" + authtypes "github.com/stacklok/toolhive/pkg/vmcp/auth/types" +) + +// newUpstreamInjectClient builds a backend client whose only outgoing auth +// strategy is upstream_inject for the given provider/authorize URL. +func newUpstreamInjectClient(t *testing.T) vmcp.BackendClient { + t.Helper() + registry := auth.NewDefaultOutgoingAuthRegistry() + require.NoError(t, registry.RegisterStrategy( + authtypes.StrategyTypeUpstreamInject, strategies.NewUpstreamInjectStrategy())) + backendClient, err := NewHTTPBackendClient(registry) + require.NoError(t, err) + return backendClient +} + +// upstreamInjectTarget builds a BackendTarget pointing at srv with an +// upstream_inject auth strategy for the given provider. +func upstreamInjectTarget(baseURL, provider, authorizeURL string) *vmcp.BackendTarget { + return &vmcp.BackendTarget{ + WorkloadID: "untrusted-backend", + WorkloadName: "Untrusted Backend", + BaseURL: baseURL + "/mcp", + TransportType: "streamable-http", + AuthConfig: &authtypes.BackendAuthStrategy{ + Type: authtypes.StrategyTypeUpstreamInject, + UpstreamInject: &authtypes.UpstreamInjectConfig{ + ProviderName: provider, + AuthorizeURL: authorizeURL, + }, + }, + } +} + +// identityContext returns a context carrying an identity whose upstream-token +// map is exactly tokens. +func identityContext(tokens map[string]string) context.Context { + return pkgauth.WithIdentity(context.Background(), &pkgauth.Identity{ + PrincipalInfo: pkgauth.PrincipalInfo{Subject: "user1"}, + UpstreamTokens: tokens, + }) +} + +// TestCallTool_ConsentRequired_FailsBeforeDispatch is the Wave 4 fail-fast +// end-to-end: a tool call to an untrusted backend whose upstream provider +// token is absent from the identity must fail with a ConsentRequiredError — +// carrying the provider and authorize URL, still classified as +// ErrAuthenticationFailed for health monitors — BEFORE any backend HTTP +// request is attempted. +func TestCallTool_ConsentRequired_FailsBeforeDispatch(t *testing.T) { + t.Parallel() + + // Count every HTTP request hitting the "backend": the fail-fast consent + // check must fire before the initialize request is ever sent. + var backendRequests atomic.Int64 + mcpServer := mcpserver.NewMCPServer("untrusted-backend", "1.0.0") + mux := http.NewServeMux() + mux.Handle("/mcp", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + backendRequests.Add(1) + mcpserver.NewStreamableHTTPServer(mcpServer).ServeHTTP(w, r) + })) + ts := httptest.NewServer(mux) + t.Cleanup(ts.Close) + + backendClient := newUpstreamInjectClient(t) + target := upstreamInjectTarget(ts.URL, "github", "https://thv.example.com/oauth/authorize") + + ctx, cancel := context.WithTimeout(identityContext(map[string]string{}), 10*time.Second) + defer cancel() + + result, err := backendClient.CallTool(ctx, target, "create_issue", map[string]any{}, nil) + + require.Error(t, err) + assert.Nil(t, result) + + // Health-monitor contract: still classified as authentication failure. + assert.ErrorIs(t, err, vmcp.ErrAuthenticationFailed) + + // Tool-result rendering contract: typed payload survives the wrap. + var consentErr *authtypes.ConsentRequiredError + require.ErrorAs(t, err, &consentErr) + assert.Equal(t, "github", consentErr.Provider) + assert.Equal(t, "https://thv.example.com/oauth/authorize", consentErr.AuthorizeURL) + + // Fail-fast: no backend HTTP request may have been attempted. + assert.Equal(t, int64(0), backendRequests.Load(), + "consent check must fire before any backend HTTP request") + + // No token material anywhere in the error. + assert.NotContains(t, err.Error(), "gho_") +} + +// TestCallTool_PostConsentRetrySucceeds is the Wave 4 retry end-to-end: after +// consent completes out of band, the next request's identity carries the new +// upstream token (TokenValidator.Middleware reloads upstream tokens on every +// request), so the retried tool call succeeds with no server-side watch, no +// cache invalidation, and no session/pod churn. +func TestCallTool_PostConsentRetrySucceeds(t *testing.T) { + t.Parallel() + + mcpServer := mcpserver.NewMCPServer("untrusted-backend", "1.0.0") + mcpServer.AddTool( + mcp.NewTool("create_issue", mcp.WithDescription("Create an issue")), + func(_ context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + return &mcp.CallToolResult{ + Content: []mcp.Content{mcp.NewTextContent("issue created")}, + }, nil + }, + ) + ts := httptest.NewServer(mcpserver.NewStreamableHTTPServer(mcpServer)) + t.Cleanup(ts.Close) + + backendClient := newUpstreamInjectClient(t) + target := upstreamInjectTarget(ts.URL, "github", "") + + // Pre-consent: token absent → typed consent error. + preConsentCtx, cancel := context.WithTimeout(identityContext(map[string]string{}), 10*time.Second) + defer cancel() + _, err := backendClient.CallTool(preConsentCtx, target, "create_issue", map[string]any{}, nil) + require.Error(t, err) + assert.ErrorIs(t, err, vmcp.ErrAuthenticationFailed) + + // Post-consent: the retried request carries the freshly loaded token and + // succeeds against the same client/target — nothing was invalidated. + postConsentCtx, cancel2 := context.WithTimeout( + identityContext(map[string]string{"github": "gho_consented"}), 10*time.Second) + defer cancel2() + result, err := backendClient.CallTool(postConsentCtx, target, "create_issue", map[string]any{}, nil) + require.NoError(t, err) + require.NotNil(t, result) + assert.False(t, result.IsError) +} diff --git a/pkg/vmcp/server/adapter/handler_factory.go b/pkg/vmcp/server/adapter/handler_factory.go index f8a3dfb7a5..1af695ade2 100644 --- a/pkg/vmcp/server/adapter/handler_factory.go +++ b/pkg/vmcp/server/adapter/handler_factory.go @@ -15,6 +15,7 @@ import ( "github.com/stacklok/toolhive-core/mcpcompat/mcp" "github.com/stacklok/toolhive/pkg/vmcp" + authtypes "github.com/stacklok/toolhive/pkg/vmcp/auth/types" "github.com/stacklok/toolhive/pkg/vmcp/conversion" "github.com/stacklok/toolhive/pkg/vmcp/internal/compositetools" "github.com/stacklok/toolhive/pkg/vmcp/router" @@ -96,6 +97,17 @@ func (f *DefaultHandlerFactory) CreateToolHandler( result, err := f.backendClient.CallTool(ctx, target, toolName, args, meta) if err != nil { // Only actual network/transport errors reach here now (IsError=true is handled in result) + var consentErr *authtypes.ConsentRequiredError + if errors.As(err, &consentErr) { + // Missing upstream consent: render the machine-detectable marker + // + JSON payload so the client can prompt the user to authorize + // and retry. This is fail-fast UX; the egress-broker sidecar + // remains the security enforcement point. + slog.Debug("upstream consent required for tool", + "tool", toolName, "provider", consentErr.Provider) + return mcp.NewToolResultError( + authtypes.FormatConsentRequired(consentErr.Provider, consentErr.AuthorizeURL)), nil + } if errors.Is(err, vmcp.ErrBackendUnavailable) { slog.Warn("backend unavailable for tool", "tool", toolName, "error", err) return mcp.NewToolResultError(fmt.Sprintf("Backend unavailable: %v", err)), nil diff --git a/pkg/vmcp/server/adapter/handler_factory_test.go b/pkg/vmcp/server/adapter/handler_factory_test.go index d2d36856fb..9df6c3417d 100644 --- a/pkg/vmcp/server/adapter/handler_factory_test.go +++ b/pkg/vmcp/server/adapter/handler_factory_test.go @@ -5,7 +5,9 @@ package adapter_test import ( "context" + "encoding/json" "errors" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -14,6 +16,7 @@ import ( "github.com/stacklok/toolhive-core/mcpcompat/mcp" "github.com/stacklok/toolhive/pkg/vmcp" + authtypes "github.com/stacklok/toolhive/pkg/vmcp/auth/types" vmcpmocks "github.com/stacklok/toolhive/pkg/vmcp/mocks" "github.com/stacklok/toolhive/pkg/vmcp/router" routermocks "github.com/stacklok/toolhive/pkg/vmcp/router/mocks" @@ -251,6 +254,87 @@ func TestDefaultHandlerFactory_CreateToolHandler(t *testing.T) { assert.Contains(t, result.Content[0].(mcp.TextContent).Text, "Tool call failed") }, }, + { + name: "consent required returns marker tool result", + toolName: "test_tool", + setupMocks: func(mockRouter *routermocks.MockRouter, mockClient *vmcpmocks.MockBackendClient) { + target := &vmcp.BackendTarget{ + WorkloadID: "backend1", + } + + mockRouter.EXPECT(). + RouteTool(gomock.Any(), "test_tool"). + Return(target, nil) + + // Production shape: wrapBackendError joins ErrAuthenticationFailed + // with the wrapped ConsentRequiredError chain. + consentErr := &authtypes.ConsentRequiredError{ + Provider: "github", + AuthorizeURL: "https://thv.example.com/oauth/authorize", + } + mockClient.EXPECT(). + CallTool(gomock.Any(), target, "test_tool", map[string]any{"input": "test"}, gomock.Any()). + Return(nil, errors.Join(vmcp.ErrAuthenticationFailed, consentErr)) + }, + request: mcp.CallToolRequest{ + Params: mcp.CallToolParams{ + Name: "test_tool", + Arguments: map[string]any{"input": "test"}, + }, + }, + wantErr: false, + checkResult: func(t *testing.T, result *mcp.CallToolResult) { + t.Helper() + require.True(t, result.IsError) + require.NotEmpty(t, result.Content) + text := result.Content[0].(mcp.TextContent).Text + require.True(t, strings.HasPrefix(text, authtypes.ConsentRequiredMarker+" "), + "text must start with the consent marker, got: %q", text) + + var payload map[string]any + require.NoError(t, json.Unmarshal( + []byte(strings.TrimPrefix(text, authtypes.ConsentRequiredMarker+" ")), &payload)) + assert.Equal(t, "github", payload["provider"]) + assert.Equal(t, "https://thv.example.com/oauth/authorize", payload["authorize_url"]) + }, + }, + { + name: "consent required without authorize URL omits the key", + toolName: "test_tool", + setupMocks: func(mockRouter *routermocks.MockRouter, mockClient *vmcpmocks.MockBackendClient) { + target := &vmcp.BackendTarget{ + WorkloadID: "backend1", + } + + mockRouter.EXPECT(). + RouteTool(gomock.Any(), "test_tool"). + Return(target, nil) + + mockClient.EXPECT(). + CallTool(gomock.Any(), target, "test_tool", map[string]any{"input": "test"}, gomock.Any()). + Return(nil, errors.Join(vmcp.ErrAuthenticationFailed, + &authtypes.ConsentRequiredError{Provider: "github"})) + }, + request: mcp.CallToolRequest{ + Params: mcp.CallToolParams{ + Name: "test_tool", + Arguments: map[string]any{"input": "test"}, + }, + }, + wantErr: false, + checkResult: func(t *testing.T, result *mcp.CallToolResult) { + t.Helper() + require.True(t, result.IsError) + text := result.Content[0].(mcp.TextContent).Text + require.True(t, strings.HasPrefix(text, authtypes.ConsentRequiredMarker+" ")) + var payload map[string]any + require.NoError(t, json.Unmarshal( + []byte(strings.TrimPrefix(text, authtypes.ConsentRequiredMarker+" ")), &payload)) + assert.Equal(t, "github", payload["provider"]) + _, hasURL := payload["authorize_url"] + assert.False(t, hasURL, "authorize_url key must be absent when not configured") + }, + }, { name: "name translation for conflict resolution", toolName: "backend1_fetch", From 5b1097e9dd1ecf495f0a303d5856fb8d81c0ea59 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 22 Jul 2026 14:15:52 +0300 Subject: [PATCH 04/12] Add untrusted-mode production readiness Supply chain: Envoy sidecar pinned to envoyproxy/envoy:v1.36.2 with a real sha256 digest; broker image pinned without :latest; image overrides (THV_UNTRUSTED_ENVOY_IMAGE/BROKER_IMAGE) resolve once at the composition root, with non-digest overrides warned and :latest overrides fatal. Pod hardening: sidecar containers get bounded resource requests/limits (multiplier overrides NaN/Inf/>100 rejected), and the broker exposes a loopback-only /healthz (Redis reachable, policy loaded, CA within validity) driving readiness/liveness probes. Operations: lifecycle tunables (idle TTL, per-user quota, per-server cap, global ratio, readiness timeout) surface as startup-validated THV_UNTRUSTED_* env vars. Token encryption: VirtualMCPServer authServerConfig gains a tokenEncryption field (activeKeyId + keySecretRef, CEL-validated to require redis storage and a non-empty ref). The operator resolves all data keys in the referenced Secret, renders one SecretKeyRef env per key so rotation keeps retired keys decryptable, watches the Secret for rotation rollout, and warns when tokenEncryption is set with Sentinel storage (unsupported). The full key set flows to the egress broker so its keyring tracks the auth server's, closing the key-ID drift that would have denied all injections on rotation. Also guards reserved env prefixes (THV_UNTRUSTED_*, TOOLHIVE_AUTHSERVER_*) against user overrides via vMCP podTemplateSpec. --- cmd/thv-egressbroker/health_test.go | 93 ++++++ cmd/thv-egressbroker/main.go | 179 +++++++++-- cmd/thv-egressbroker/main_test.go | 149 ++++++++++ .../v1beta1/mcpexternalauthconfig_types.go | 47 +++ .../api/v1beta1/zz_generated.deepcopy.go | 21 ++ .../virtualmcpserver_controller.go | 21 +- .../virtualmcpserver_deployment.go | 130 +++++++- .../controllers/virtualmcpserver_keksecret.go | 138 +++++++++ .../virtualmcpserver_rbac_untrusted_test.go | 185 ++++++++++-- .../pkg/controllerutil/authserver.go | 256 +++++++++++++++- .../pkg/controllerutil/authserver_test.go | 245 +++++++++++++++- .../controllerutil/podtemplatespec_builder.go | 7 + cmd/thv-operator/pkg/vmcpconfig/converter.go | 13 +- .../token_encryption_validation_test.go | 132 +++++++++ ...e.stacklok.dev_mcpexternalauthconfigs.yaml | 82 ++++++ ...olhive.stacklok.dev_virtualmcpservers.yaml | 118 ++++++++ ...e.stacklok.dev_mcpexternalauthconfigs.yaml | 82 ++++++ ...olhive.stacklok.dev_virtualmcpservers.yaml | 118 ++++++++ docs/operator/crd-api.md | 26 ++ pkg/vmcp/cli/untrusted.go | 277 +++++++++++++++++- pkg/vmcp/cli/untrusted_test.go | 142 ++++++++- pkg/vmcp/session/untrusted/addressing.go | 27 +- pkg/vmcp/session/untrusted/egress.go | 214 ++++++++++++-- pkg/vmcp/session/untrusted/egress_test.go | 177 ++++++++++- pkg/vmcp/session/untrusted/lifecycle.go | 11 +- pkg/vmcp/session/untrusted/reaper.go | 12 +- pkg/vmcp/session/untrusted/tokenstore.go | 67 +++-- pkg/vmcp/session/untrusted/wiring.go | 13 +- pkg/vmcp/session/untrusted/wiring_test.go | 171 +++++++++++ 29 files changed, 2993 insertions(+), 160 deletions(-) create mode 100644 cmd/thv-egressbroker/health_test.go create mode 100644 cmd/thv-egressbroker/main_test.go create mode 100644 cmd/thv-operator/controllers/virtualmcpserver_keksecret.go create mode 100644 cmd/thv-operator/test-integration/mcp-external-auth/token_encryption_validation_test.go create mode 100644 pkg/vmcp/session/untrusted/wiring_test.go diff --git a/cmd/thv-egressbroker/health_test.go b/cmd/thv-egressbroker/health_test.go new file mode 100644 index 0000000000..a6640a472e --- /dev/null +++ b/cmd/thv-egressbroker/health_test.go @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +// healthFixture builds a health handler around a freshly generated CA and a +// configurable ping. The CA is valid (not rotation-due) at test time. +func healthFixture(t *testing.T, ping redisPinger) (*healthServer, *atomic.Bool) { + t.Helper() + ca, err := egressbroker.GenerateBumpCA("test", time.Now()) + require.NoError(t, err) + var loaded atomic.Bool + loaded.Store(true) + if ping == nil { + ping = func(context.Context) error { return nil } + } + return &healthServer{ca: ca, policyLoaded: &loaded, ping: ping}, &loaded +} + +func TestHealthz(t *testing.T) { + t.Parallel() + + t.Run("healthy when CA valid, policy loaded, Redis reachable", func(t *testing.T) { + t.Parallel() + h, _ := healthFixture(t, nil) + rec := httptest.NewRecorder() + h.handle(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + assert.Equal(t, http.StatusOK, rec.Code) + }) + + t.Run("503 when Redis unreachable", func(t *testing.T) { + t.Parallel() + h, _ := healthFixture(t, func(context.Context) error { return errors.New("dial refused") }) + rec := httptest.NewRecorder() + h.handle(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Contains(t, rec.Body.String(), "redis") + }) + + t.Run("503 when policy not loaded", func(t *testing.T) { + t.Parallel() + h, loaded := healthFixture(t, nil) + loaded.Store(false) + rec := httptest.NewRecorder() + h.handle(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Contains(t, rec.Body.String(), "policy") + }) + + t.Run("503 when bump CA is past rotation-due", func(t *testing.T) { + t.Parallel() + h, _ := healthFixture(t, nil) + // NeedsRotation fires at 50% of validity, so now+half is always past due. + ca, err := egressbroker.GenerateBumpCA("test", time.Now().Add(-egressbroker.CAValidity)) + require.NoError(t, err) + h.ca = ca + rec := httptest.NewRecorder() + h.handle(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + assert.Equal(t, http.StatusServiceUnavailable, rec.Code) + assert.Contains(t, rec.Body.String(), "rotation") + }) + + t.Run("non-GET is rejected", func(t *testing.T) { + t.Parallel() + h, _ := healthFixture(t, nil) + rec := httptest.NewRecorder() + h.handle(rec, httptest.NewRequest(http.MethodPost, "/healthz", nil)) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) + }) + + t.Run("health listener is loopback-only on the pinned port", func(t *testing.T) { + t.Parallel() + h, _ := healthFixture(t, nil) + var loadedFlag atomic.Bool + srv := newHealthServer(h.ca, &loadedFlag, h.ping) + assert.Equal(t, "127.0.0.1:15083", srv.Addr) + }) +} diff --git a/cmd/thv-egressbroker/main.go b/cmd/thv-egressbroker/main.go index 194d396d27..17ade51db1 100644 --- a/cmd/thv-egressbroker/main.go +++ b/cmd/thv-egressbroker/main.go @@ -11,12 +11,17 @@ package main import ( "context" "encoding/base64" + "errors" "fmt" "log/slog" + "net" + "net/http" "os" "os/signal" "strings" + "sync/atomic" "syscall" + "time" goredis "github.com/redis/go-redis/v9" @@ -25,6 +30,7 @@ import ( "github.com/stacklok/toolhive/pkg/authserver/tokenenc" "github.com/stacklok/toolhive/pkg/egressbroker" vmcpconfig "github.com/stacklok/toolhive/pkg/vmcp/config" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) // Environment contract for the token store (injected at clone time by the @@ -35,17 +41,25 @@ const ( // envRedisKeyPrefix is the auth-server per-tenant key prefix the upstream // token rows live under (e.g. "thv:auth:{ns}:{name}:"). envRedisKeyPrefix = "THV_EGRESSBROKER_REDIS_KEY_PREFIX" - // envKEKBase64 is the base64 token-encryption KEK (32 bytes decoded). - // When unset, token rows are read unencrypted (legacy plaintext storage); - // encrypted rows then fail closed (Open rejects non-legacy values). - envKEKBase64 = "THV_EGRESSBROKER_KEK" + // envKEKActiveID carries the key ID the auth server encrypts NEW writes + // under (THV_EGRESSBROKER_KEK_ID). It must match the operator's + // activeKeyId — the clone wiring passes it through verbatim; a drift here + // would deny every injection. + envKEKActiveID = "THV_EGRESSBROKER_KEK_ID" + // envKEKPrefix is the per-key-ID env prefix: THV_EGRESSBROKER_KEK_ + // holds the base64 32-byte KEK for that ID (active + retired, so key + // rotation never orphans rows sealed under a retired ID). When no + // THV_EGRESSBROKER_KEK_* env is present, token rows are read unencrypted + // (legacy plaintext storage); encrypted rows then fail closed (Open + // rejects non-legacy values). + envKEKPrefix = "THV_EGRESSBROKER_KEK_" // envEnvoyBootstrapOut is the path the broker renders the Envoy bootstrap // into (a shared emptyDir the Envoy container mounts read-only). When // unset, bootstrap rendering is skipped (the Envoy config is managed // externally). envEnvoyBootstrapOut = "THV_EGRESSBROKER_ENVOY_BOOTSTRAP_OUT" - // defaultKEKID is the key ID for the single static KEK. - defaultKEKID = "kek-1" + // healthPingTimeout bounds the Redis reachability check per probe. + healthPingTimeout = 2 * time.Second ) func main() { @@ -88,12 +102,12 @@ func run() error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - tokenReader, err := buildTokenReader(ctx, dialGuard) + tokens, err := buildTokenReader(ctx, dialGuard) if err != nil { return err } - injector, err := egressbroker.NewCredentialInjector(resolver.PodIdentity(), policy, tokenReader) + injector, err := egressbroker.NewCredentialInjector(resolver.PodIdentity(), policy, tokens) if err != nil { return err } @@ -109,9 +123,98 @@ func run() error { if err != nil { return err } + + // Health listener (readiness + liveness probes): answers 200 only when + // the broker can actually serve — Redis reachable AND policy loaded AND + // the bump CA not past rotation-due. Everything past this point is + // already loaded, so policyLoaded flips exactly once at startup. + var policyLoaded atomic.Bool + policyLoaded.Store(true) + healthSrv := newHealthServer(ca, &policyLoaded, tokens.ping) + go func() { + if err := runHealthServer(ctx, healthSrv); err != nil { + // A wedged health listener wedges the pod: probes start failing, + // the pod never goes Ready, and the untrusted reaper deletes it + // once it is older than the readiness timeout. Log at Error so the + // failure is visible before that teardown. + slog.Error("egressbroker: health listener stopped; readiness/liveness probes will now fail", + "error", err) + } + }() + return server.Run(ctx) } +// redisPinger abstracts the Redis reachability check for tests. +type redisPinger func(ctx context.Context) error + +// healthServer is the /healthz handler. 200 iff the bump CA is not past +// rotation-due AND the policy is loaded AND Redis answers PING within +// healthPingTimeout; otherwise 503 with a coarse reason (never credential +// material). +type healthServer struct { + ca *egressbroker.BumpCA + policyLoaded *atomic.Bool + ping redisPinger +} + +// newHealthServer builds the loopback-only HTTP server bound to the health +// port. The handler is exported through the returned server's Handler for +// tests. +func newHealthServer(ca *egressbroker.BumpCA, policyLoaded *atomic.Bool, ping redisPinger) *http.Server { + h := &healthServer{ca: ca, policyLoaded: policyLoaded, ping: ping} + mux := http.NewServeMux() + mux.HandleFunc("/healthz", h.handle) + return &http.Server{ + Addr: net.JoinHostPort("127.0.0.1", fmt.Sprintf("%d", untrusted.BrokerHealthPort)), + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } +} + +// runHealthServer serves until ctx is cancelled, then shuts down gracefully. +func runHealthServer(ctx context.Context, srv *http.Server) error { + errCh := make(chan error, 1) + go func() { errCh <- srv.ListenAndServe() }() + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + return fmt.Errorf("egressbroker: health server shutdown failed: %w", err) + } + return nil + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + } +} + +func (h *healthServer) handle(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if h.ca.NeedsRotation(time.Now()) { + http.Error(w, "bump CA past rotation-due", http.StatusServiceUnavailable) + return + } + if !h.policyLoaded.Load() { + http.Error(w, "policy not loaded", http.StatusServiceUnavailable) + return + } + pingCtx, cancel := context.WithTimeout(r.Context(), healthPingTimeout) + defer cancel() + if err := h.ping(pingCtx); err != nil { + http.Error(w, "redis unreachable", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) +} + // renderEnvoyBootstrap writes the Envoy bootstrap for this policy into the // shared config emptyDir, when configured. The bootstrap's route table // carries exactly the policy's host allowlist (D6b); the ext_authz cluster @@ -130,13 +233,20 @@ func renderEnvoyBootstrap(cfg *egressbroker.Config, policy *egressbroker.EgressP }) } +// tokenStore bundles the upstream-token reader with its Redis reachability +// probe (the health endpoint's Redis check). +type tokenStore struct { + upstreamtoken.TokenReader + ping redisPinger +} + // buildTokenReader constructs the upstream-token reader against the vMCP's // session-storage Redis. The Redis dials go through the D7 guard: the Redis // address is operator-injected config, and the guard keeps the broker from // ever dialing outside the policy's destination set. No refresh is wired (the // broker holds no OAuth client material): expired rows surface on the failed // list and deny with "re-consent required" (Wave 4 consent UX). -func buildTokenReader(_ context.Context, dialGuard *egressbroker.IPAllowlist) (upstreamtoken.TokenReader, error) { +func buildTokenReader(_ context.Context, dialGuard *egressbroker.IPAllowlist) (*tokenStore, error) { addr := strings.TrimSpace(os.Getenv(envRedisAddr)) keyPrefix := strings.TrimSpace(os.Getenv(envRedisKeyPrefix)) if addr == "" || keyPrefix == "" { @@ -155,24 +265,51 @@ func buildTokenReader(_ context.Context, dialGuard *egressbroker.IPAllowlist) (u Dialer: dialGuard.DialContext, }) stor := storage.NewRedisStorageWithClient(client, keyPrefix, opts...) - return upstreamtoken.NewInProcessService(stor, nil), nil + return &tokenStore{ + TokenReader: upstreamtoken.NewInProcessService(stor, nil), + ping: func(ctx context.Context) error { return client.Ping(ctx).Err() }, + }, nil } -// tokenEncOption builds the token-encryption option from the KEK env. A -// malformed KEK is a startup error (fail closed); an absent KEK means -// plaintext legacy rows only. +// tokenEncOption builds the token-encryption option from the KEK env set. +// THV_EGRESSBROKER_KEK_ entries supply the keyring (active + retired +// IDs); THV_EGRESSBROKER_KEK_ID names the active one (the auth server's +// activeKeyId, passed through verbatim by the clone wiring). Any malformed +// coordinate — no keys, a missing/unknown active ID, bad base64, a +// wrong-length key — is a startup error (fail closed); an entirely absent KEK +// env set means plaintext legacy rows only. func tokenEncOption() ([]storage.RedisStorageOption, error) { - raw := strings.TrimSpace(os.Getenv(envKEKBase64)) - if raw == "" { - return nil, nil + keys := make(map[string][]byte) + for _, kv := range os.Environ() { + name, value, found := strings.Cut(kv, "=") + if !found || !strings.HasPrefix(name, envKEKPrefix) { + continue + } + id := strings.TrimPrefix(name, envKEKPrefix) + if id == "ID" { + // THV_EGRESSBROKER_KEK_ID is the active-ID coordinate, not a key. + continue + } + raw := strings.TrimSpace(value) + if raw == "" { + return nil, fmt.Errorf("egressbroker: %s%s is empty", envKEKPrefix, id) + } + kek, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, fmt.Errorf("egressbroker: %s%s is not valid base64", envKEKPrefix, id) + } + keys[id] = kek } - kek, err := base64.StdEncoding.DecodeString(raw) - if err != nil { - return nil, fmt.Errorf("egressbroker: %s is not valid base64", envKEKBase64) + if len(keys) == 0 { + if activeID := strings.TrimSpace(os.Getenv(envKEKActiveID)); activeID != "" { + return nil, fmt.Errorf("egressbroker: %s is set but no %s key entries exist", envKEKActiveID, envKEKPrefix) + } + return nil, nil } - keyring, err := tokenenc.NewStaticKeyring(defaultKEKID, map[string][]byte{defaultKEKID: kek}) + activeID := strings.TrimSpace(os.Getenv(envKEKActiveID)) + keyring, err := tokenenc.NewStaticKeyring(activeID, keys) if err != nil { - return nil, fmt.Errorf("egressbroker: invalid token-encryption key: %w", err) + return nil, fmt.Errorf("egressbroker: invalid token-encryption keys: %w", err) } return []storage.RedisStorageOption{storage.WithTokenEncryption(keyring)}, nil } diff --git a/cmd/thv-egressbroker/main_test.go b/cmd/thv-egressbroker/main_test.go new file mode 100644 index 0000000000..6d832d5dd6 --- /dev/null +++ b/cmd/thv-egressbroker/main_test.go @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/base64" + "fmt" + "net" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/egressbroker" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" +) + +// TestBrokerPortContract pins the clone-time ↔ broker port contract: the +// health listener binds the port the operator-rendered probes target +// (untrusted.BrokerHealthPort), and the Envoy explicit-proxy port the +// backend's HTTP(S)_PROXY env points at equals the broker-rendered bootstrap +// default. +func TestBrokerPortContract(t *testing.T) { + t.Parallel() + + assert.Equal(t, 15083, untrusted.BrokerHealthPort, + "the health listener port must match the clone-time probe wiring") + assert.Equal(t, 15001, egressbroker.DefaultProxyPort, + "the Envoy proxy port must match the backend HTTP(S)_PROXY env the clone wiring renders") +} + +// TestHealthListenerLoopbackBind performs a real loopback bind on the pinned +// health port: newHealthServer must bind 127.0.0.1 (never a pod-external +// interface — the handler answers unauthenticated) and serve /healthz. +// The bind is the assertion; port conflicts skip rather than fail (another +// listener on 15083 says nothing about where this server would have bound). +func TestHealthListenerLoopbackBind(t *testing.T) { + t.Parallel() + + h, _ := healthFixture(t, nil) + var loadedFlag atomic.Bool + loadedFlag.Store(true) + srv := newHealthServer(h.ca, &loadedFlag, h.ping) + + ln, err := net.Listen("tcp", srv.Addr) + if err != nil { + t.Skipf("health port %s already in use; cannot assert the bind", srv.Addr) + } + defer ln.Close() + + // The bound address must be loopback-only. + host, _, err := net.SplitHostPort(ln.Addr().String()) + require.NoError(t, err) + ip := net.ParseIP(host) + require.NotNil(t, ip, "bound host %q is not an IP", host) + assert.True(t, ip.IsLoopback(), "health listener must bind loopback only, got %s", host) + + // Serve one real request through the bound listener. + go func() { _ = srv.Serve(ln) }() + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = srv.Shutdown(shutdownCtx) + }() + + resp, err := http.Get(fmt.Sprintf("http://%s/healthz", ln.Addr().String())) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + +//nolint:paralleltest // t.Setenv modifies the process environment; subtests cannot run in parallel. +func TestTokenEncOption(t *testing.T) { + kek32 := base64.StdEncoding.EncodeToString([]byte("0123456789abcdef0123456789abcdef")) + kekShort := base64.StdEncoding.EncodeToString([]byte("too-short")) + + t.Run("no KEK env at all means plaintext legacy rows (nil options)", func(t *testing.T) { + opts, err := tokenEncOption() + require.NoError(t, err) + assert.Nil(t, opts) + }) + + t.Run("active ID without key entries is startup-fatal", func(t *testing.T) { + t.Setenv(envKEKActiveID, "kek-1") + _, err := tokenEncOption() + require.Error(t, err) + assert.Contains(t, err.Error(), envKEKActiveID) + }) + + t.Run("key entries without an active ID are startup-fatal", func(t *testing.T) { + t.Setenv(envKEKPrefix+"kek-1", kek32) + _, err := tokenEncOption() + require.Error(t, err, "an empty active key ID must fail the keyring constructor") + }) + + t.Run("active ID absent from the key set is startup-fatal", func(t *testing.T) { + t.Setenv(envKEKActiveID, "kek-9") + t.Setenv(envKEKPrefix+"kek-1", kek32) + _, err := tokenEncOption() + require.Error(t, err) + assert.Contains(t, err.Error(), "kek-9") + }) + + t.Run("non-base64 key is startup-fatal", func(t *testing.T) { + t.Setenv(envKEKActiveID, "kek-1") + t.Setenv(envKEKPrefix+"kek-1", "!!!not-base64!!!") + _, err := tokenEncOption() + require.Error(t, err) + assert.Contains(t, err.Error(), "base64") + }) + + t.Run("empty key value is startup-fatal", func(t *testing.T) { + t.Setenv(envKEKActiveID, "kek-1") + t.Setenv(envKEKPrefix+"kek-1", " ") + _, err := tokenEncOption() + require.Error(t, err) + }) + + t.Run("wrong-length key is startup-fatal", func(t *testing.T) { + t.Setenv(envKEKActiveID, "kek-1") + t.Setenv(envKEKPrefix+"kek-1", kekShort) + _, err := tokenEncOption() + require.Error(t, err) + }) + + t.Run("KEK_ID alone is not mistaken for a key entry", func(t *testing.T) { + // Only THV_EGRESSBROKER_KEK_ID set: the active-ID coordinate must not + // be parsed as a per-ID key named "ID" (that would make the "active + // ID without keys" error branch unreachable). + t.Setenv(envKEKActiveID, "kek-1") + _, err := tokenEncOption() + require.Error(t, err) + assert.Contains(t, err.Error(), "no "+envKEKPrefix+" key entries") + }) + + t.Run("multi-key set (active + retired) builds the keyring", func(t *testing.T) { + t.Setenv(envKEKActiveID, "kek-2") + t.Setenv(envKEKPrefix+"kek-1", kek32) + t.Setenv(envKEKPrefix+"kek-2", kek32) + opts, err := tokenEncOption() + require.NoError(t, err) + require.Len(t, opts, 1) + }) +} diff --git a/cmd/thv-operator/api/v1beta1/mcpexternalauthconfig_types.go b/cmd/thv-operator/api/v1beta1/mcpexternalauthconfig_types.go index 8266fa8ab2..5765f64c9f 100644 --- a/cmd/thv-operator/api/v1beta1/mcpexternalauthconfig_types.go +++ b/cmd/thv-operator/api/v1beta1/mcpexternalauthconfig_types.go @@ -7,6 +7,7 @@ import ( "fmt" "sort" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/stacklok/toolhive/pkg/authserver/oauthparams" @@ -1017,6 +1018,10 @@ const ( type AuthServerStorageType string // AuthServerStorageConfig configures the storage backend for the embedded auth server. +// +// +kubebuilder:validation:XValidation:rule="!has(self.tokenEncryption) || self.type == 'redis'",message="tokenEncryption requires storage type 'redis'" +// +//nolint:lll // CEL validation rules exceed line length limit type AuthServerStorageConfig struct { // Type specifies the storage backend type. // Valid values: "memory" (default), "redis". @@ -1028,6 +1033,48 @@ type AuthServerStorageConfig struct { // Required when type is "redis". // +optional Redis *RedisStorageConfig `json:"redis,omitempty"` + + // TokenEncryption enables envelope encryption of upstream tokens at rest. + // Requires redis storage: the RunConfig shape only supports token + // encryption on the Redis backend, and untrusted-mode egress (the primary + // consumer) requires Redis-backed token storage. ActiveKeyID names the + // key used for new writes; KeySecretRef references a Secret whose data + // keys are key IDs and values are base64 32-byte KEKs. + // +optional + TokenEncryption *TokenEncryptionConfig `json:"tokenEncryption,omitempty"` +} + +// GetTokenEncryption returns the token-encryption config, or nil when unset +// (nil-safe accessor for the operator's wiring paths). +func (c *AuthServerStorageConfig) GetTokenEncryption() *TokenEncryptionConfig { + if c == nil { + return nil + } + return c.TokenEncryption +} + +// TokenEncryptionConfig configures AES-256-GCM envelope encryption of +// upstream OAuth token values stored in Redis. The Secret referenced by +// KeySecretRef holds one data entry per key ID (value = base64 32-byte KEK); +// the operator mounts every data key (active + retired) as a SecretKeyRef env +// entry on the vMCP container and clones the same references into untrusted +// egress-broker sidecars. Key material never appears in the CRD, a ConfigMap, +// or a pod env literal. +// +// +kubebuilder:validation:XValidation:rule="self.keySecretRef.name.size() > 0",message="keySecretRef.name must not be empty" +// +//nolint:lll // CEL validation rules exceed line length limit +type TokenEncryptionConfig struct { + // ActiveKeyID identifies the KEK used to encrypt new writes. Must match a + // data key of the Secret referenced by KeySecretRef. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + ActiveKeyID string `json:"activeKeyId"` + + // KeySecretRef references the Secret holding the key-encryption keys. + // Its data keys are key IDs; values are base64 32-byte KEKs. + // +kubebuilder:validation:Required + KeySecretRef corev1.LocalObjectReference `json:"keySecretRef"` } // RedisStorageConfig configures Redis connection for auth server storage. diff --git a/cmd/thv-operator/api/v1beta1/zz_generated.deepcopy.go b/cmd/thv-operator/api/v1beta1/zz_generated.deepcopy.go index e954ec39cc..ad7aa04770 100644 --- a/cmd/thv-operator/api/v1beta1/zz_generated.deepcopy.go +++ b/cmd/thv-operator/api/v1beta1/zz_generated.deepcopy.go @@ -93,6 +93,11 @@ func (in *AuthServerStorageConfig) DeepCopyInto(out *AuthServerStorageConfig) { *out = new(RedisStorageConfig) (*in).DeepCopyInto(*out) } + if in.TokenEncryption != nil { + in, out := &in.TokenEncryption, &out.TokenEncryption + *out = new(TokenEncryptionConfig) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthServerStorageConfig. @@ -2859,6 +2864,22 @@ func (in *SessionStorageConfig) DeepCopy() *SessionStorageConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TokenEncryptionConfig) DeepCopyInto(out *TokenEncryptionConfig) { + *out = *in + out.KeySecretRef = in.KeySecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TokenEncryptionConfig. +func (in *TokenEncryptionConfig) DeepCopy() *TokenEncryptionConfig { + if in == nil { + return nil + } + out := new(TokenEncryptionConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenExchangeConfig) DeepCopyInto(out *TokenExchangeConfig) { *out = *in diff --git a/cmd/thv-operator/controllers/virtualmcpserver_controller.go b/cmd/thv-operator/controllers/virtualmcpserver_controller.go index c599c61d7a..2879067fd1 100644 --- a/cmd/thv-operator/controllers/virtualmcpserver_controller.go +++ b/cmd/thv-operator/controllers/virtualmcpserver_controller.go @@ -992,7 +992,12 @@ func (r *VirtualMCPServerReconciler) validateAndUpdatePodTemplateStatus( return true } - _, err := ctrlutil.NewPodTemplateSpecBuilder(vmcp.Spec.PodTemplateSpec, "vmcp") + parsed, err := ctrlutil.ParsePodTemplateSpec(vmcp.Spec.PodTemplateSpec) + if err == nil { + // Reserved-prefix guard: user templates must not override the + // operator-owned env wiring on the vmcp container. + err = validateNoReservedVMCPEnvVars(parsed) + } if err != nil { // Record event for invalid PodTemplateSpec if r.Recorder != nil { @@ -1709,7 +1714,9 @@ func (r *VirtualMCPServerReconciler) containerNeedsUpdate( return true } - // Check if environment variables have changed + // Check if environment variables have changed. The status manager is nil + // here: conditions are collected by the resource-ensuring path, not the + // drift check — and errors must still surface as "needs update" below. expectedEnv, err := r.buildEnvVarsForVmcp(ctx, vmcp, telemetryCfg, typedWorkloads) if err != nil { return true // Trigger update to surface the error @@ -2798,6 +2805,16 @@ func (r *VirtualMCPServerReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(r.mapAuthzConfigMapToVirtualMCPServer), builder.WithPredicates(configMapDataChangedPredicate()), ). + // Watch Secrets referenced via + // spec.authServerConfig.storage.tokenEncryption.keySecretRef so that a + // KEK rotation (key-ID set change) rolls the vMCP pods onto the new env. + // The predicate filters out metadata-only updates; the mapper narrows to + // VirtualMCPServers whose rendered pod env would actually change. + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.mapKEKSecretToVirtualMCPServer), + builder.WithPredicates(secretDataChangedPredicate()), + ). Complete(r) } diff --git a/cmd/thv-operator/controllers/virtualmcpserver_deployment.go b/cmd/thv-operator/controllers/virtualmcpserver_deployment.go index 67bcbafaa0..19a809f46d 100644 --- a/cmd/thv-operator/controllers/virtualmcpserver_deployment.go +++ b/cmd/thv-operator/controllers/virtualmcpserver_deployment.go @@ -84,6 +84,26 @@ const ( // vMCP, which clones it into untrusted egress-broker sidecars. The value is // the non-secret host:port from spec.authServerConfig.storage.redis.addr. untrustedTokenStoreAddrEnvVar = "THV_UNTRUSTED_TOKEN_STORE_REDIS_ADDR" // #nosec G101 -- env var name, not a credential + + // untrustedTokenStoreKEK*EnvVars carry the token-encryption KEK + // coordinates (Secret name + active key ID + the full key-ID set, all + // non-secret references) to the vMCP, which renders one SecretKeyRef env + // per ID on every cloned egress-broker sidecar. The KEK value itself + // never transits an env literal. The IDs env is the ordered union of the + // KEK Secret's data keys (active + retired) so key rotation never orphans + // sidecar-decrypted rows. pkg/vmcp/cli/untrusted.go mirrors these names; + // the contract is pinned by tests on both sides. + untrustedTokenStoreKEKSecretEnvVar = "THV_UNTRUSTED_TOKEN_STORE_KEK_SECRET" // #nosec G101 -- env var name, not a credential + untrustedTokenStoreKEKKeyEnvVar = "THV_UNTRUSTED_TOKEN_STORE_KEK_KEY" // #nosec G101 -- env var name, not a credential + untrustedTokenStoreKEKIDsEnvVar = "THV_UNTRUSTED_TOKEN_STORE_KEK_IDS" // #nosec G101 -- env var name, not a credential + + // reservedVMCPEnvPrefixes are the env-var prefixes the operator owns on + // the vmcp container. A user PodTemplateSpec must not set them: they + // carry operator-injected wiring (untrusted egress coordinates, embedded + // auth-server credential indirection) whose override could silently + // re-point credential flows. + reservedVMCPEnvPrefixUntrusted = "THV_UNTRUSTED_" + reservedVMCPEnvPrefixAuthServer = "TOOLHIVE_AUTHSERVER_" ) // RBAC rules for VirtualMCPServer service account in inline mode @@ -411,42 +431,89 @@ func (r *VirtualMCPServerReconciler) buildEnvVarsForVmcp( // does not, reflect.DeepEqual never matches, and the operator updates the // Deployment on every reconcile (see #5616). if vmcp.Spec.AuthServerConfig != nil { - env = append(env, ctrlutil.GenerateAuthServerEnvVars(vmcp.Spec.AuthServerConfig)...) + authServerEnv, err := ctrlutil.GenerateAuthServerEnvVars(ctx, r.Client, vmcp.Namespace, vmcp.Spec.AuthServerConfig) + if err != nil { + return nil, fmt.Errorf("failed to build embedded auth server env vars: %w", err) + } + env = append(env, authServerEnv...) } // Mount the auth-server token-store coordinates for untrusted-mode egress // (Wave-3): the vMCP forwards these to the egress-broker sidecar at pod-clone // time. The address is non-secret; the vMCP derives the per-tenant key prefix // from its own identity (VMCP_NAME/VMCP_NAMESPACE) via DeriveKeyPrefix. - env = append(env, buildUntrustedTokenStoreEnvVars(vmcp)...) + tokenStoreEnv, err := r.buildUntrustedTokenStoreEnvVars(ctx, vmcp) + if err != nil { + return nil, fmt.Errorf("failed to build untrusted token-store env vars: %w", err) + } + env = append(env, tokenStoreEnv...) return ctrlutil.EnsureRequiredEnvVars(ctx, env), nil } -// buildUntrustedTokenStoreEnvVars returns the auth-server Redis address env var -// the vMCP clones into untrusted egress-broker sidecars (THV_EGRESSBROKER_*). -// Empty when the embedded auth server does not use Redis storage — the broker -// then fails closed at startup, matching the untrusted-mode posture that -// upstream-token injection requires Redis-backed token storage. The KEK is -// deliberately NOT injected here: the CRD has no token-encryption field, and -// when encryption is enabled (enterprise RunConfig path) the sidecar's KEK is -// supplied via its own Secret env reference, never a ConfigMap. -func buildUntrustedTokenStoreEnvVars(vmcp *mcpv1beta1.VirtualMCPServer) []corev1.EnvVar { +// buildUntrustedTokenStoreEnvVars returns the auth-server token-store +// coordinate env vars the vMCP clones into untrusted egress-broker sidecars +// (THV_UNTRUSTED_TOKEN_STORE_*). Empty when the embedded auth server does not +// use Redis storage — the broker then fails closed at startup, matching the +// untrusted-mode posture that upstream-token injection requires Redis-backed +// token storage. When spec.authServerConfig.storage.tokenEncryption is set, +// the KEK coordinates (Secret name + active key ID + the full key-ID set, +// read from the referenced Secret) ride along as non-secret literals; the +// vMCP turns them into one SecretKeyRef env per key ID on each cloned +// sidecar, so the KEK values never appear in a pod spec or ConfigMap and a +// key rotation never orphans ciphertext sealed under a retired ID. +func (r *VirtualMCPServerReconciler) buildUntrustedTokenStoreEnvVars( + ctx context.Context, + vmcp *mcpv1beta1.VirtualMCPServer, +) ([]corev1.EnvVar, error) { as := vmcp.Spec.AuthServerConfig if as == nil || as.Storage == nil || as.Storage.Type != mcpv1beta1.AuthServerStorageTypeRedis || as.Storage.Redis == nil { - return nil + return nil, nil } // Sentinel-based auth-server Redis has no single address the sidecar can // dial directly; untrusted egress requires a standalone/cluster addr. if as.Storage.Redis.Addr == "" { - return nil + // A configured tokenEncryption is silently unusable for untrusted + // egress here — never drop the KEK coordinates without a trace (the + // broker would fail closed on encrypted rows with no operator-visible + // reason). The deployment builder is on every write path (create, + // drift-check rebuild), so the Warning fires on each reconcile — a + // standing misconfiguration that deserves the repetition. + if as.Storage.GetTokenEncryption() != nil { + log.FromContext(ctx).Info("tokenEncryption set but Redis storage uses Sentinel; "+ + "untrusted egress-broker sidecars cannot decrypt upstream tokens", + "vmcp", vmcp.Name, "namespace", vmcp.Namespace) + if r.Recorder != nil { + r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "TokenEncryptionNotSupportedForUntrusted", + "UntrustedTokenStore", "tokenEncryption requires standalone/cluster Redis storage; "+ + "Sentinel-backed auth-server Redis leaves untrusted egress-broker sidecars unable to decrypt upstream tokens") + } + } + return nil, nil } - return []corev1.EnvVar{{ + env := []corev1.EnvVar{{ Name: untrustedTokenStoreAddrEnvVar, Value: as.Storage.Redis.Addr, }} + if te := as.Storage.GetTokenEncryption(); te != nil { + envByID, err := ctrlutil.ResolveKEKKeySet(ctx, r.Client, vmcp.Namespace, as) + if err != nil { + return nil, err + } + ids := make([]string, 0, len(envByID)) + for id := range envByID { + ids = append(ids, id) + } + sort.Strings(ids) + env = append(env, + corev1.EnvVar{Name: untrustedTokenStoreKEKSecretEnvVar, Value: te.KeySecretRef.Name}, + corev1.EnvVar{Name: untrustedTokenStoreKEKKeyEnvVar, Value: te.ActiveKeyID}, + corev1.EnvVar{Name: untrustedTokenStoreKEKIDsEnvVar, Value: strings.Join(ids, ",")}, + ) + } + return env, nil } // buildOIDCEnvVars builds environment variables for OIDC client secret mounting. @@ -1312,9 +1379,13 @@ func (*VirtualMCPServerReconciler) applyPodTemplateSpecToDeployment( // only enumerates a subset of PodSpec fields and would skip the patch for // fields like runtimeClassName or topologySpreadConstraints. Strategic merge // patch is a no-op for `{}` anyway, so always running it is safe. - if _, err := ctrlutil.NewPodTemplateSpecBuilder(vmcp.Spec.PodTemplateSpec, "vmcp"); err != nil { + parsed, err := ctrlutil.ParsePodTemplateSpec(vmcp.Spec.PodTemplateSpec) + if err != nil { return fmt.Errorf("failed to build PodTemplateSpec: %w", err) } + if err := validateNoReservedVMCPEnvVars(parsed); err != nil { + return err + } merged, err := ctrlutil.ApplyPodTemplateSpecPatch(deployment.Spec.Template, vmcp.Spec.PodTemplateSpec.Raw) if err != nil { @@ -1335,6 +1406,35 @@ const ( caBundleBasePath = "/etc/toolhive/ca-bundles" ) +// validateNoReservedVMCPEnvVars rejects user PodTemplateSpecs that set env +// vars with operator-reserved prefixes on the vmcp container +// (THV_UNTRUSTED_* / TOOLHIVE_AUTHSERVER_*). Those env vars carry +// operator-injected wiring — untrusted egress token-store coordinates and +// embedded auth-server credential indirection — and letting a user template +// override them would silently re-point credential flows. Only the container +// named "vmcp" is checked; sidecar containers the user adds are theirs. +// Returns nil when parsed is nil (no user template). +func validateNoReservedVMCPEnvVars(parsed *corev1.PodTemplateSpec) error { + if parsed == nil { + return nil + } + for i := range parsed.Spec.Containers { + c := &parsed.Spec.Containers[i] + if c.Name != "vmcp" { + continue + } + for _, e := range c.Env { + if strings.HasPrefix(e.Name, reservedVMCPEnvPrefixUntrusted) || + strings.HasPrefix(e.Name, reservedVMCPEnvPrefixAuthServer) { + return fmt.Errorf("PodTemplateSpec must not set env var %q on the vmcp container: "+ + "the %s*/%s* prefixes are reserved for operator-injected wiring", + e.Name, reservedVMCPEnvPrefixUntrusted, reservedVMCPEnvPrefixAuthServer) + } + } + } + return nil +} + // caBundleMountPath returns the mount path for a CA bundle ConfigMap for a given entry name. // The key defaults to "ca.crt" if not specified in the CABundleSource. func caBundleMountPath(entryName string, caBundleRef *mcpv1beta1.CABundleSource) string { diff --git a/cmd/thv-operator/controllers/virtualmcpserver_keksecret.go b/cmd/thv-operator/controllers/virtualmcpserver_keksecret.go new file mode 100644 index 0000000000..588a416d08 --- /dev/null +++ b/cmd/thv-operator/controllers/virtualmcpserver_keksecret.go @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "context" + "reflect" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" + ctrlutil "github.com/stacklok/toolhive/cmd/thv-operator/pkg/controllerutil" +) + +// mapKEKSecretToVirtualMCPServer maps KEK Secret changes to the +// VirtualMCPServers whose spec.authServerConfig.storage.tokenEncryption +// references the Secret by name. The operator renders one pod env var per +// Secret data key, so a rotation that adds or removes a key ID changes the +// desired pod env and must trigger reconciliation (the Deployment drift check +// then rolls the vMCP pods). Value-only updates under an unchanged key set +// are already picked up by kubelet's env refresh; the mapper still enqueues +// (cheap) and the reconcile is a no-op. +// +// Key IDs are compared via the deterministic env list the deployment builder +// renders (ctrlutil.TokenEncryptionEnvVars), so the mapper fires exactly when +// the Secret-driven pod env would change. +func (r *VirtualMCPServerReconciler) mapKEKSecretToVirtualMCPServer( + ctx context.Context, + obj client.Object, +) []reconcile.Request { + secret, ok := obj.(*corev1.Secret) + if !ok { + return nil + } + + vmcpList := &mcpv1beta1.VirtualMCPServerList{} + if err := r.List(ctx, vmcpList, client.InNamespace(secret.Namespace)); err != nil { + log.FromContext(ctx).Error(err, "Failed to list VirtualMCPServers for KEK Secret watch") + return nil + } + + var requests []reconcile.Request + for i := range vmcpList.Items { + vmcp := &vmcpList.Items[i] + as := vmcp.Spec.AuthServerConfig + if as == nil || as.Storage == nil { + continue + } + te := as.Storage.GetTokenEncryption() + if te == nil || te.KeySecretRef.Name != secret.Name { + continue + } + envByID, err := ctrlutil.ResolveKEKKeySet(ctx, r.Client, vmcp.Namespace, as) + if err != nil { + // The reconcile surfaces the same error on the Valid condition; + // enqueue so the diagnostic refreshes. + log.FromContext(ctx).V(1).Info("KEK Secret watch: key set unresolvable; enqueueing for diagnosis", + "virtualmcpserver", vmcp.Name, "secret", secret.Name, "error", err) + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: vmcp.Name, Namespace: vmcp.Namespace, + }}) + continue + } + podEnv := ctrlutil.TokenEncryptionEnvVars(te.KeySecretRef.Name, envByID) + if r.vmcpKEKEnvChanged(ctx, vmcp, podEnv) { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: vmcp.Name, Namespace: vmcp.Namespace, + }}) + } + } + return requests +} + +// vmcpKEKEnvChanged reports whether the token-encryption env set on the +// vMCP's live Deployment differs from the one derived from the current KEK +// Secret (the signal a rotation changed the key-ID set). A missing Deployment +// means creation will pick up the current set — no reconcile needed from the +// watch. +func (r *VirtualMCPServerReconciler) vmcpKEKEnvChanged( + ctx context.Context, + vmcp *mcpv1beta1.VirtualMCPServer, + want []corev1.EnvVar, +) bool { + dep := &appsv1.Deployment{} + if err := r.Get(ctx, types.NamespacedName{Name: vmcp.Name, Namespace: vmcp.Namespace}, dep); err != nil { + return false + } + if len(dep.Spec.Template.Spec.Containers) == 0 { + return false + } + prefix := ctrlutil.TokenEncryptionKEKEnvVarPrefix + "_" + live := make(map[string]corev1.EnvVar) + for _, e := range dep.Spec.Template.Spec.Containers[0].Env { + if strings.HasPrefix(e.Name, prefix) { + live[e.Name] = e + } + } + if len(live) != len(want) { + return true + } + for _, e := range want { + if !reflect.DeepEqual(live[e.Name], e) { + return true + } + } + return false +} + +// secretDataChangedPredicate admits Secret events that may affect the KEK +// key-ID set rendered into vMCP pod env. Update events are admitted only when +// .Data actually changes, so metadata-only churn (annotation rewrites, +// resourceVersion bumps) does not fan out reconciles. Create and Delete pass +// through so a rotated-into-existence or removed Secret is observed. +// Mirrors configMapDataChangedPredicate. +func secretDataChangedPredicate() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldSecret, okOld := e.ObjectOld.(*corev1.Secret) + newSecret, okNew := e.ObjectNew.(*corev1.Secret) + if !okOld || !okNew { + return false + } + return !reflect.DeepEqual(oldSecret.Data, newSecret.Data) + }, + CreateFunc: func(_ event.CreateEvent) bool { return true }, + DeleteFunc: func(_ event.DeleteEvent) bool { return true }, + GenericFunc: func(_ event.GenericEvent) bool { return false }, + } +} diff --git a/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go b/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go index 40ee9af45a..105fe07586 100644 --- a/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go +++ b/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go @@ -4,14 +4,20 @@ package controllers import ( + "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client/fake" mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1/v1beta1test" + "github.com/stacklok/toolhive/cmd/thv-operator/internal/testutil" ) // TestVmcpDiscoveredRBACRules_PodsRule pins the untrusted-mode pods grant: @@ -75,29 +81,46 @@ func TestVmcpDiscoveredRBACRules_StatefulSetsRule(t *testing.T) { assert.NotContains(t, rule.Verbs, "delete", "read-only: vMCP never writes StatefulSets") } +// untrustedRedisStorage is the shared fixture for the token-store env tests: +// standalone/cluster Redis storage with the given address and token-encryption +// config. +func untrustedRedisStorage(addr string, te *mcpv1beta1.TokenEncryptionConfig) *mcpv1beta1.EmbeddedAuthServerConfig { + return &mcpv1beta1.EmbeddedAuthServerConfig{ + Storage: &mcpv1beta1.AuthServerStorageConfig{ + Type: mcpv1beta1.AuthServerStorageTypeRedis, + Redis: &mcpv1beta1.RedisStorageConfig{ + Addr: addr, + ACLUserConfig: &mcpv1beta1.RedisACLUserConfig{ + PasswordSecretRef: &mcpv1beta1.SecretKeyRef{Name: "redis-creds", Key: "password"}, + }, + }, + TokenEncryption: te, + }, + } +} + +// newTokenStoreTestReconciler builds a reconciler over a fake client seeded +// with objects. The status manager is nil: none of the address-only paths +// touch it (only the Sentinel+tokenEncryption condition does, and that test +// supplies a mock). +func newTokenStoreTestReconciler(t *testing.T, objects ...*corev1.Secret) *VirtualMCPServerReconciler { + t.Helper() + scheme := testutil.NewScheme(t) + builder := fake.NewClientBuilder().WithScheme(scheme) + for _, o := range objects { + builder = builder.WithObjects(o) + } + return &VirtualMCPServerReconciler{Client: builder.Build(), Scheme: scheme} +} + // TestBuildUntrustedTokenStoreEnvVars pins the Wave-3 token-store coordinate // injection: the vMCP Deployment gets the (non-secret) auth-server Redis // address when the embedded auth server uses standalone/cluster Redis storage, -// and nothing otherwise. The KEK is never injected here — it is a sidecar-only -// Secret reference resolved at clone time. +// and nothing otherwise. The KEK values are never injected here — they stay +// Secret-only and reach sidecars as SecretKeyRef envs resolved at clone time. func TestBuildUntrustedTokenStoreEnvVars(t *testing.T) { t.Parallel() - redisStorage := func(addr string) *mcpv1beta1.EmbeddedAuthServerConfig { - cfg := &mcpv1beta1.EmbeddedAuthServerConfig{ - Storage: &mcpv1beta1.AuthServerStorageConfig{ - Type: mcpv1beta1.AuthServerStorageTypeRedis, - Redis: &mcpv1beta1.RedisStorageConfig{ - Addr: addr, - ACLUserConfig: &mcpv1beta1.RedisACLUserConfig{ - PasswordSecretRef: &mcpv1beta1.SecretKeyRef{Name: "redis-creds", Key: "password"}, - }, - }, - }, - } - return cfg - } - tests := []struct { name string authCfg *mcpv1beta1.EmbeddedAuthServerConfig @@ -118,7 +141,7 @@ func TestBuildUntrustedTokenStoreEnvVars(t *testing.T) { }, { name: "redis storage with addr produces the address env var", - authCfg: redisStorage("redis.auth:6379"), + authCfg: untrustedRedisStorage("redis.auth:6379", nil), wantAddr: "redis.auth:6379", wantEmpty: false, }, @@ -139,7 +162,9 @@ func TestBuildUntrustedTokenStoreEnvVars(t *testing.T) { t.Parallel() vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", v1beta1test.WithVMCPAuthServerConfig(tc.authCfg)) - env := buildUntrustedTokenStoreEnvVars(vmcp) + r := newTokenStoreTestReconciler(t) + env, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) + require.NoError(t, err) if tc.wantEmpty { assert.Empty(t, env) return @@ -151,3 +176,125 @@ func TestBuildUntrustedTokenStoreEnvVars(t *testing.T) { }) } } + +// TestBuildUntrustedTokenStoreEnvVars_KEK pins the Wave-5 KEK wiring: when the +// auth-server storage carries tokenEncryption, the vMCP Deployment gets the +// KEK Secret coordinates (Secret name + active key ID + the full key-ID set +// read from the Secret) as plain literals — the vMCP turns them into one +// SecretKeyRef env per key ID on every cloned sidecar, so the KEK values +// themselves never appear in any pod spec. +func TestBuildUntrustedTokenStoreEnvVars_KEK(t *testing.T) { + t.Parallel() + + redisStorage := func(te *mcpv1beta1.TokenEncryptionConfig) *mcpv1beta1.EmbeddedAuthServerConfig { + return untrustedRedisStorage("redis.auth:6379", te) + } + + t.Run("tokenEncryption set emits the KEK coordinate env vars (full key set)", func(t *testing.T) { + t.Parallel() + cfg := redisStorage(&mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-2", + KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, + }) + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", + v1beta1test.WithVMCPAuthServerConfig(cfg)) + r := newTokenStoreTestReconciler(t, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "my-vmcp-kek", Namespace: "default"}, + Data: map[string][]byte{ + "kek-1": []byte("retired-key-bytes"), + "kek-2": []byte("active-key-bytes"), + }, + }) + env, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) + require.NoError(t, err) + require.Len(t, env, 4) + + byName := map[string]corev1.EnvVar{} + for _, e := range env { + byName[e.Name] = e + } + assert.Equal(t, "redis.auth:6379", byName[untrustedTokenStoreAddrEnvVar].Value) + assert.Equal(t, "my-vmcp-kek", byName[untrustedTokenStoreKEKSecretEnvVar].Value) + assert.Equal(t, "kek-2", byName[untrustedTokenStoreKEKKeyEnvVar].Value) + assert.Equal(t, "kek-1,kek-2", byName[untrustedTokenStoreKEKIDsEnvVar].Value, + "the full key-ID set (active + retired) rides along so rotation never orphans ciphertext") + for _, e := range env { + assert.Nil(t, e.ValueFrom, "%s must be a literal coordinate, never a Secret ref", e.Name) + } + }) + + t.Run("tokenEncryption nil emits only the address env var", func(t *testing.T) { + t.Parallel() + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", + v1beta1test.WithVMCPAuthServerConfig(redisStorage(nil))) + r := newTokenStoreTestReconciler(t) + env, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) + require.NoError(t, err) + require.Len(t, env, 1) + assert.Equal(t, untrustedTokenStoreAddrEnvVar, env[0].Name) + }) + + t.Run("memory storage with tokenEncryption emits nothing (CEL guards admission)", func(t *testing.T) { + t.Parallel() + // The CEL rule rejects this at admission; the builder must still not + // emit KEK coordinates for non-Redis storage (defense in depth). + cfg := &mcpv1beta1.EmbeddedAuthServerConfig{ + Storage: &mcpv1beta1.AuthServerStorageConfig{ + Type: mcpv1beta1.AuthServerStorageTypeMemory, + TokenEncryption: &mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-1", + KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, + }, + }, + } + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", + v1beta1test.WithVMCPAuthServerConfig(cfg)) + r := newTokenStoreTestReconciler(t) + env, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) + require.NoError(t, err) + assert.Empty(t, env) + }) + + t.Run("unresolvable KEK Secret is an error (fail closed, coordinates dropped)", func(t *testing.T) { + t.Parallel() + cfg := redisStorage(&mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-1", + KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, + }) + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", + v1beta1test.WithVMCPAuthServerConfig(cfg)) + // No KEK Secret seeded — the resolution must fail. + r := newTokenStoreTestReconciler(t) + _, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) + require.Error(t, err) + }) + + t.Run("sentinel storage with tokenEncryption emits a Warning event and no env", func(t *testing.T) { + t.Parallel() + cfg := untrustedRedisStorage("", &mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-1", + KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, + }) + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", + v1beta1test.WithVMCPAuthServerConfig(cfg)) + + recorder := events.NewFakeRecorder(10) + scheme := testutil.NewScheme(t) + r := &VirtualMCPServerReconciler{ + Client: fake.NewClientBuilder().WithScheme(scheme).Build(), + Scheme: scheme, + Recorder: recorder, + } + + env, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) + require.NoError(t, err, "the Sentinel misconfiguration must not error the reconcile") + assert.Empty(t, env, "no token-store env for Sentinel-backed storage") + + select { + case event := <-recorder.Events: + assert.Contains(t, event, "TokenEncryptionNotSupportedForUntrusted") + default: + t.Fatal("expected a Warning event for Sentinel-backed storage with tokenEncryption") + } + }) +} diff --git a/cmd/thv-operator/pkg/controllerutil/authserver.go b/cmd/thv-operator/pkg/controllerutil/authserver.go index 97a3692b96..a99f912075 100644 --- a/cmd/thv-operator/pkg/controllerutil/authserver.go +++ b/cmd/thv-operator/pkg/controllerutil/authserver.go @@ -6,10 +6,12 @@ package controllerutil import ( "context" "fmt" + "sort" "strings" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" k8sptr "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -71,8 +73,102 @@ const ( // DefaultSentinelPort is the default Redis Sentinel port DefaultSentinelPort = 26379 + + // TokenEncryptionKEKEnvVarPrefix is the prefix for the env var names + // carrying the base64 token-encryption KEKs to the embedded auth server: + // one SecretKeyRef env entry per data key of the referenced Secret, named + // _. The auth-server RunConfig references them by + // NAME only (TokenEncryptionRunConfig.Keys) — key values are sourced from + // the Secret at pod start. Active AND retired IDs are rendered so a + // rotation can never orphan ciphertext sealed under the old key. + TokenEncryptionKEKEnvVarPrefix = "TOOLHIVE_AUTHSERVER_TOKEN_ENCRYPTION_KEK" // #nosec G101 -- env var name, not a credential + + // maxTokenEncryptionKeys bounds the KEK key IDs rendered per auth server. + // Secret data keys are operator-managed; the bound keeps a runaway Secret + // from inflating the pod env beyond apiserver limits. + maxTokenEncryptionKeys = 16 ) +// tokenEncryptionKEKEnvVarName returns the env var name carrying the KEK for +// the given key ID. The ID is uppercased and non-[A-Z0-9_] characters become +// '_' so the name is env-var-safe (C_IDENTIFIER). +func tokenEncryptionKEKEnvVarName(keyID string) string { + var b strings.Builder + b.WriteString(TokenEncryptionKEKEnvVarPrefix) + b.WriteByte('_') + for _, r := range strings.ToUpper(keyID) { + if (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' { + b.WriteRune(r) + } else { + b.WriteByte('_') + } + } + return b.String() +} + +// resolveTokenEncryptionKeyIDs reads the Secret referenced by +// TokenEncryptionConfig and returns the validated KEK key-ID → env-var-name +// map (deterministically ordered IDs). The active ID must be a data key of +// the Secret; IDs are validated for uniqueness after env-var sanitization so +// two IDs can never collide onto one env var name (which would silently hand +// one KEK to two key IDs). Key VALUES are never inspected — only the data key +// names are used. +func resolveTokenEncryptionKeyIDs( + ctx context.Context, + c client.Client, + namespace string, + te *mcpv1beta1.TokenEncryptionConfig, +) (map[string]string, error) { + if te.KeySecretRef.Name == "" { + // CEL (XValidation on TokenEncryptionConfig) rejects this at admission; + // this is the reconcile-time backstop for objects that predate the rule. + return nil, fmt.Errorf("token encryption: keySecretRef.name must not be empty") + } + if te.ActiveKeyID == "" { + return nil, fmt.Errorf("token encryption: activeKeyId must not be empty") + } + + secret := &corev1.Secret{} + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: te.KeySecretRef.Name}, secret); err != nil { + return nil, fmt.Errorf("token encryption: failed to get KEK secret %s/%s: %w", + namespace, te.KeySecretRef.Name, err) + } + if len(secret.Data) == 0 { + return nil, fmt.Errorf("token encryption: KEK secret %s/%s has no data keys", + namespace, te.KeySecretRef.Name) + } + if len(secret.Data) > maxTokenEncryptionKeys { + return nil, fmt.Errorf("token encryption: KEK secret %s/%s has %d data keys (max %d)", + namespace, te.KeySecretRef.Name, len(secret.Data), maxTokenEncryptionKeys) + } + + envByID := make(map[string]string, len(secret.Data)) + ids := make([]string, 0, len(secret.Data)) + for id := range secret.Data { + ids = append(ids, id) + } + sort.Strings(ids) + if _, ok := secret.Data[te.ActiveKeyID]; !ok { + return nil, fmt.Errorf("token encryption: activeKeyId %q is not a data key of KEK secret %s/%s", + te.ActiveKeyID, namespace, te.KeySecretRef.Name) + } + seenEnv := make(map[string]string, len(ids)) + for _, id := range ids { + if id == "" { + return nil, fmt.Errorf("token encryption: KEK secret %s/%s carries an empty key ID", + namespace, te.KeySecretRef.Name) + } + env := tokenEncryptionKEKEnvVarName(id) + if other, dup := seenEnv[env]; dup { + return nil, fmt.Errorf("token encryption: key IDs %q and %q both sanitize to env var %q; "+ + "rename one so the IDs are unique after sanitization", other, id, env) + } + seenEnv[env] = id + envByID[id] = env + } + return envByID, nil +} + // upstreamSecretBinding binds an upstream provider to the env var names for // the secrets it owns (client secret and, optionally, the DCR initial access // token). Both GenerateAuthServerEnvVars (Pod env) and buildUpstreamRunConfig @@ -216,7 +312,10 @@ func GenerateAuthServerConfigByName( } volumes, volumeMounts := GenerateAuthServerVolumes(authServerConfig) - envVars := GenerateAuthServerEnvVars(authServerConfig) + envVars, err := GenerateAuthServerEnvVars(ctx, c, namespace, authServerConfig) + if err != nil { + return nil, nil, nil, err + } return volumes, volumeMounts, envVars, nil } @@ -352,12 +451,21 @@ func GenerateAuthServerVolumes( // provider that has a client secret reference configured, where PROVIDER is the // provider name uppercased with hyphens replaced by underscores. // +// When storage.tokenEncryption is configured, the referenced KEK Secret is read +// (data key NAMES only — never values) and one SecretKeyRef env entry is rendered +// per key ID (active + retired) so key rotation can never orphan ciphertext. +// A failure to read or validate the Secret is an error: rendering only the +// active key would break decryption of rows sealed under retired keys. +// // Returns nil slice if authConfig is nil or if no client secrets are configured. func GenerateAuthServerEnvVars( + ctx context.Context, + c client.Client, + namespace string, authConfig *mcpv1beta1.EmbeddedAuthServerConfig, -) []corev1.EnvVar { +) ([]corev1.EnvVar, error) { if authConfig == nil { - return nil + return nil, nil } var envVars []corev1.EnvVar @@ -403,7 +511,49 @@ func GenerateAuthServerEnvVars( } } - return envVars + // Generate one SecretKeyRef env entry per KEK data key when token + // encryption is configured. The env var NAMES (derived from the key IDs) + // are the indirection the auth-server RunConfig references + // (buildTokenEncryptionRunConfig uses the same derivation); the VALUES come + // from the operator-referenced Secret at pod start — KEKs never land in + // the CRD, a ConfigMap, or an env literal. + if te := authConfig.Storage.GetTokenEncryption(); te != nil { + envByID, err := resolveTokenEncryptionKeyIDs(ctx, c, namespace, te) + if err != nil { + return nil, err + } + envVars = append(envVars, TokenEncryptionEnvVars(te.KeySecretRef.Name, envByID)...) + } + + return envVars, nil +} + +// TokenEncryptionEnvVars renders the deterministic, sorted SecretKeyRef env +// entries for a KEK key-ID → env-var-name set (one per data key of the named +// Secret). Exported for the VirtualMCPServer reconciler, which re-derives the +// same env list during its KEK-Secret watch mapping — the pod env and the +// watch mapping must agree exactly, so both go through this function. +func TokenEncryptionEnvVars(secretName string, envByID map[string]string) []corev1.EnvVar { + ids := make([]string, 0, len(envByID)) + for id := range envByID { + ids = append(ids, id) + } + sort.Strings(ids) + out := make([]corev1.EnvVar, 0, len(ids)) + for _, id := range ids { + out = append(out, corev1.EnvVar{ + Name: envByID[id], + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + Key: id, + }, + }, + }) + } + return out } // AddEmbeddedAuthServerConfigOptions adds embedded auth server configuration to @@ -455,11 +605,19 @@ func AddEmbeddedAuthServerConfigOptions( return err } + // Resolve the KEK key set (active + retired) so the RunConfig decrypts + // rows sealed under retired keys, matching the env vars + // GenerateAuthServerEnvVars renders on the same pod. + kekEnvByID, err := ResolveKEKKeySet(ctx, c, namespace, authServerConfig) + if err != nil { + return err + } + // Build the embedded auth server config for runner embeddedConfig, err := BuildAuthServerRunConfig( namespace, mcpServerName, authServerConfig, []string{oidcConfig.ResourceURL}, oidcConfig.Scopes, - oidcConfig.ResourceURL, + oidcConfig.ResourceURL, kekEnvByID, ) if err != nil { return fmt.Errorf("failed to build embedded auth server config: %w", err) @@ -514,6 +672,14 @@ func validateOIDCConfigForEmbeddedAuthServer(oidcConfig *oidc.OIDCConfig) error // controllers derive them from different sources (MCPServer uses oidcConfig.ResourceURL/Scopes; // VirtualMCPServer derives from the resolved vmcp Config). // +// kekEnvByID maps KEK key ID → env var name for storage.tokenEncryption. Callers +// that can read the KEK Secret should pass the map resolved by +// resolveTokenEncryptionKeyIDs (active + retired IDs, so rotation never orphans +// ciphertext); callers that cannot (e.g. offline renderers with no API access) +// pass nil, which renders only the ACTIVE key ID via the same deterministic env +// name derivation — retired keys are then decrypt-unavailable. Nil is also +// correct when tokenEncryption is unset. +// // resourceURL is used to default the RedirectURI on upstream providers when not explicitly set. // The default is {resourceURL}/oauth/callback as documented in the MCPExternalAuthConfig CRD. func BuildAuthServerRunConfig( @@ -523,6 +689,7 @@ func BuildAuthServerRunConfig( allowedAudiences []string, scopesSupported []string, resourceURL string, + kekEnvByID map[string]string, ) (*authserver.RunConfig, error) { config := &authserver.RunConfig{ SchemaVersion: authserver.CurrentSchemaVersion, @@ -576,7 +743,7 @@ func BuildAuthServerRunConfig( } // Build storage configuration - storageCfg, err := buildStorageRunConfig(namespace, name, authConfig) + storageCfg, err := buildStorageRunConfig(namespace, name, authConfig, kekEnvByID) if err != nil { return nil, fmt.Errorf("failed to build storage config: %w", err) } @@ -604,10 +771,13 @@ func BuildAuthServerRunConfig( // buildStorageRunConfig converts CRD AuthServerStorageConfig to storage.RunConfig. // Returns nil (memory storage default) if no storage config is specified. +// +//nolint:gocyclo // sequential fail-loud config validation is clearest linear. func buildStorageRunConfig( namespace string, mcpServerName string, authConfig *mcpv1beta1.EmbeddedAuthServerConfig, + kekEnvByID map[string]string, ) (*storage.RunConfig, error) { if authConfig.Storage == nil || authConfig.Storage.Type == mcpv1beta1.AuthServerStorageTypeMemory { return nil, nil @@ -656,6 +826,17 @@ func buildStorageRunConfig( TLS: convertRedisTLSConfig(redisConfig.TLS, false), } + // Token encryption (Wave-5): the RunConfig references each KEK by env var + // NAME (resolved at pod start from the SecretKeyRef env entries rendered by + // GenerateAuthServerEnvVars) — the CRD never carries key material. + if te := authConfig.Storage.GetTokenEncryption(); te != nil { + teRc, err := buildTokenEncryptionRunConfig(te, kekEnvByID) + if err != nil { + return nil, err + } + rc.TokenEncryption = teRc + } + if redisConfig.SentinelConfig != nil { // Resolve Sentinel addresses (static or via Kubernetes Service discovery) sentinelAddrs, err := resolveSentinelAddrs(redisConfig.SentinelConfig, namespace) @@ -676,6 +857,59 @@ func buildStorageRunConfig( }, nil } +// buildTokenEncryptionRunConfig builds the serializable token-encryption +// config: every key ID in kekEnvByID (active + retired, resolved from the KEK +// Secret) becomes a decrypt-capable key; the active ID encrypts new writes. +// A nil kekEnvByID falls back to rendering only the active ID via the same +// deterministic env name derivation resolveTokenEncryptionKeyIDs uses — the +// fallback exists for callers without API access (offline renderers); when +// the KEK Secret holds retired IDs those rows then fail decryption at startup +// reads, which is the fail-closed direction. +func buildTokenEncryptionRunConfig( + te *mcpv1beta1.TokenEncryptionConfig, + kekEnvByID map[string]string, +) (*storage.TokenEncryptionRunConfig, error) { + if len(kekEnvByID) == 0 { + if te.ActiveKeyID == "" { + return nil, fmt.Errorf("token encryption: activeKeyId must not be empty") + } + return &storage.TokenEncryptionRunConfig{ + ActiveKeyID: te.ActiveKeyID, + Keys: map[string]string{te.ActiveKeyID: tokenEncryptionKEKEnvVarName(te.ActiveKeyID)}, + }, nil + } + if _, ok := kekEnvByID[te.ActiveKeyID]; !ok { + return nil, fmt.Errorf("token encryption: active key ID %q not present in resolved KEK key set", te.ActiveKeyID) + } + return &storage.TokenEncryptionRunConfig{ + ActiveKeyID: te.ActiveKeyID, + Keys: kekEnvByID, + }, nil +} + +// ResolveKEKKeySet resolves the full KEK key-ID → env-var-name set for +// storage.tokenEncryption (active + retired) so key rotation never orphans +// ciphertext. Returns nil when token encryption is unset. Used by every +// controller path that builds the auth-server RunConfig AND renders the +// matching pod env — the two must agree on the env var names, so both go +// through the same resolution. Exported for the vMCP deployment builder, +// which carries the ID set (names only) to the untrusted egress sidecars. +func ResolveKEKKeySet( + ctx context.Context, + c client.Client, + namespace string, + authConfig *mcpv1beta1.EmbeddedAuthServerConfig, +) (map[string]string, error) { + if authConfig == nil || authConfig.Storage == nil { + return nil, nil + } + te := authConfig.Storage.GetTokenEncryption() + if te == nil { + return nil, nil + } + return resolveTokenEncryptionKeyIDs(ctx, c, namespace, te) +} + // convertRedisTLSConfig converts CRD RedisTLSConfig to RunConfig. // isSentinel determines which mount path to use for the CA cert file. func convertRedisTLSConfig(cfg *mcpv1beta1.RedisTLSConfig, isSentinel bool) *storage.RedisTLSRunConfig { @@ -988,11 +1222,19 @@ func AddAuthServerRefOptions( return err } + // Resolve the KEK key set (active + retired) so the RunConfig decrypts + // rows sealed under retired keys, matching the env vars + // GenerateAuthServerEnvVars renders on the same pod. + kekEnvByID, err := ResolveKEKKeySet(ctx, c, namespace, authServerConfig) + if err != nil { + return err + } + // Build the embedded auth server config for runner embeddedConfig, err := BuildAuthServerRunConfig( namespace, mcpServerName, authServerConfig, []string{oidcConfig.ResourceURL}, oidcConfig.Scopes, - oidcConfig.ResourceURL, + oidcConfig.ResourceURL, kekEnvByID, ) if err != nil { return fmt.Errorf("failed to build embedded auth server config: %w", err) diff --git a/cmd/thv-operator/pkg/controllerutil/authserver_test.go b/cmd/thv-operator/pkg/controllerutil/authserver_test.go index b79d01584b..c591af6016 100644 --- a/cmd/thv-operator/pkg/controllerutil/authserver_test.go +++ b/cmd/thv-operator/pkg/controllerutil/authserver_test.go @@ -611,7 +611,8 @@ func TestGenerateAuthServerEnvVars(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - envVars := GenerateAuthServerEnvVars(tt.authConfig) + envVars, err := GenerateAuthServerEnvVars(context.Background(), nil, "default", tt.authConfig) + require.NoError(t, err) if len(tt.wantEnvNames) == 0 { assert.Empty(t, envVars) @@ -1650,7 +1651,8 @@ func TestBuildAuthServerRunConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - config, err := BuildAuthServerRunConfig("default", "test-server", tt.authConfig, tt.allowedAudiences, tt.scopesSupported, tt.resourceURL) + config, err := BuildAuthServerRunConfig( + "default", "test-server", tt.authConfig, tt.allowedAudiences, tt.scopesSupported, tt.resourceURL, nil) require.NoError(t, err) require.NotNil(t, config) @@ -1702,6 +1704,7 @@ func TestBuildAuthServerRunConfig_InvalidDCR(t *testing.T) { []string{"http://test-server.default.svc.cluster.local:8080"}, []string{"openid", "offline_access"}, "http://test-server.default.svc.cluster.local:8080", + nil, ) require.Error(t, err, "expected BuildAuthServerRunConfig to fail on invalid DCR pairing") @@ -1973,7 +1976,8 @@ func TestGenerateAuthServerEnvVars_RedisCredentials(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - envVars := GenerateAuthServerEnvVars(tt.authConfig) + envVars, err := GenerateAuthServerEnvVars(context.Background(), nil, "default", tt.authConfig) + require.NoError(t, err) assert.Len(t, envVars, tt.wantEnvVarLen) envMap := make(map[string]corev1.EnvVar) @@ -2011,6 +2015,237 @@ func TestGenerateAuthServerEnvVars_RedisCredentials(t *testing.T) { } } +// TestGenerateAuthServerEnvVars_TokenEncryption pins the Wave-5 KEK wiring: +// when authServerConfig.storage.tokenEncryption is set, the vmcp container +// gets ONE SecretKeyRef env entry per data key of the referenced KEK Secret +// (active + retired, so rotation never orphans ciphertext). Each entry is +// named TokenEncryptionKEKEnvVarPrefix_ and sources the key from +// the Secret — a KEK value never appears as a literal. The KEK Secret must be +// readable and consistent (active ID a data key, no empty or +// sanitization-colliding IDs) or rendering is an error (fail closed). +func TestGenerateAuthServerEnvVars_TokenEncryption(t *testing.T) { + t.Parallel() + + redisStorageWithTE := func(te *mcpv1beta1.TokenEncryptionConfig) *mcpv1beta1.EmbeddedAuthServerConfig { + return &mcpv1beta1.EmbeddedAuthServerConfig{ + Issuer: "https://auth.example.com", + UpstreamProviders: []mcpv1beta1.UpstreamProviderConfig{}, + Storage: &mcpv1beta1.AuthServerStorageConfig{ + Type: mcpv1beta1.AuthServerStorageTypeRedis, + Redis: &mcpv1beta1.RedisStorageConfig{ + Addr: "redis.auth:6379", + ACLUserConfig: &mcpv1beta1.RedisACLUserConfig{ + PasswordSecretRef: &mcpv1beta1.SecretKeyRef{Name: "redis-creds", Key: "password"}, + }, + }, + TokenEncryption: te, + }, + } + } + + kekSecret := func(data map[string][]byte) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "my-vmcp-kek", Namespace: "default"}, + Data: data, + } + } + // Placeholder values only — the renderer must never inspect key VALUES, + // and the assertion below pins that the values never leave the Secret. + kekSecretClient := func(data map[string][]byte) client.Client { + return fake.NewClientBuilder().WithScheme(testutil.NewScheme(t)). + WithObjects(kekSecret(data)).Build() + } + + t.Run("tokenEncryption renders one SecretKeyRef env entry per Secret data key", func(t *testing.T) { + t.Parallel() + cfg := redisStorageWithTE(&mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-2", + KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, + }) + c := kekSecretClient(map[string][]byte{ + "kek-1": []byte("retired-key-bytes"), + "kek-2": []byte("active-key-bytes"), + }) + envVars, err := GenerateAuthServerEnvVars(context.Background(), c, "default", cfg) + require.NoError(t, err) + + envMap := make(map[string]corev1.EnvVar) + for _, ev := range envVars { + envMap[ev.Name] = ev + } + for _, id := range []string{"kek-1", "kek-2"} { + name := TokenEncryptionKEKEnvVarPrefix + "_" + strings.ToUpper(strings.ReplaceAll(id, "-", "_")) + ev, ok := envMap[name] + require.True(t, ok, "expected the token-encryption KEK env var for key ID %s", id) + assert.Empty(t, ev.Value, "KEK value must never be an env literal") + require.NotNil(t, ev.ValueFrom) + require.NotNil(t, ev.ValueFrom.SecretKeyRef) + assert.Equal(t, "my-vmcp-kek", ev.ValueFrom.SecretKeyRef.Name) + assert.Equal(t, id, ev.ValueFrom.SecretKeyRef.Key, + "the Secret data key is the key ID") + } + }) + + t.Run("missing KEK Secret is an error (fail closed)", func(t *testing.T) { + t.Parallel() + cfg := redisStorageWithTE(&mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-1", + KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, + }) + c := fake.NewClientBuilder().WithScheme(testutil.NewScheme(t)).Build() + _, err := GenerateAuthServerEnvVars(context.Background(), c, "default", cfg) + require.Error(t, err) + }) + + t.Run("active key ID absent from the Secret is an error", func(t *testing.T) { + t.Parallel() + cfg := redisStorageWithTE(&mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-9", + KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, + }) + c := kekSecretClient(map[string][]byte{"kek-1": []byte("x")}) + _, err := GenerateAuthServerEnvVars(context.Background(), c, "default", cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "kek-9") + }) + + t.Run("key IDs colliding after sanitization are an error", func(t *testing.T) { + t.Parallel() + cfg := redisStorageWithTE(&mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-1", + KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, + }) + c := kekSecretClient(map[string][]byte{ + "kek-1": []byte("x"), + "kek.1": []byte("y"), + }) + _, err := GenerateAuthServerEnvVars(context.Background(), c, "default", cfg) + require.Error(t, err) + }) + + t.Run("nil tokenEncryption renders no KEK env entry", func(t *testing.T) { + t.Parallel() + envVars, err := GenerateAuthServerEnvVars(context.Background(), nil, "default", redisStorageWithTE(nil)) + require.NoError(t, err) + for _, ev := range envVars { + assert.NotContains(t, ev.Name, TokenEncryptionKEKEnvVarPrefix) + } + }) + + t.Run("nil storage renders no KEK env entry", func(t *testing.T) { + t.Parallel() + cfg := &mcpv1beta1.EmbeddedAuthServerConfig{ + Issuer: "https://auth.example.com", + UpstreamProviders: []mcpv1beta1.UpstreamProviderConfig{}, + } + envVars, err := GenerateAuthServerEnvVars(context.Background(), nil, "default", cfg) + require.NoError(t, err) + for _, ev := range envVars { + assert.NotContains(t, ev.Name, TokenEncryptionKEKEnvVarPrefix) + } + }) + + t.Run("nil auth config renders nothing", func(t *testing.T) { + t.Parallel() + envVars, err := GenerateAuthServerEnvVars(context.Background(), nil, "default", nil) + require.NoError(t, err) + assert.Empty(t, envVars) + }) +} + +// TestBuildAuthServerRunConfig_TokenEncryption pins the RunConfig side of the +// Wave-5 KEK wiring: storage.tokenEncryption becomes a +// TokenEncryptionRunConfig whose Keys map references every KEK by env var NAME +// (the indirection the pod env resolves at start). Callers pass the key set +// resolved from the KEK Secret (active + retired, so rotation never orphans +// ciphertext); the nil fallback renders only the active ID via the same +// deterministic env-name derivation. +func TestBuildAuthServerRunConfig_TokenEncryption(t *testing.T) { + t.Parallel() + + baseAuthConfig := func() *mcpv1beta1.EmbeddedAuthServerConfig { + return &mcpv1beta1.EmbeddedAuthServerConfig{ + Issuer: "https://auth.example.com", + UpstreamProviders: []mcpv1beta1.UpstreamProviderConfig{ + { + Name: "okta", + Type: mcpv1beta1.UpstreamProviderTypeOIDC, + OIDCConfig: &mcpv1beta1.OIDCUpstreamConfig{ + IssuerURL: "https://okta.example.com", + ClientID: "client-id", + }, + }, + }, + Storage: &mcpv1beta1.AuthServerStorageConfig{ + Type: mcpv1beta1.AuthServerStorageTypeRedis, + Redis: &mcpv1beta1.RedisStorageConfig{ + Addr: "redis.auth:6379", + ACLUserConfig: &mcpv1beta1.RedisACLUserConfig{ + PasswordSecretRef: &mcpv1beta1.SecretKeyRef{Name: "redis-creds", Key: "password"}, + }, + }, + }, + } + } + withTokenEncryption := func(activeID string) *mcpv1beta1.EmbeddedAuthServerConfig { + cfg := baseAuthConfig() + cfg.Storage.TokenEncryption = &mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: activeID, + KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, + } + return cfg + } + + t.Run("resolved key set carries active and retired keys into the RunConfig", func(t *testing.T) { + t.Parallel() + kekEnvByID := map[string]string{ + "kek-1": TokenEncryptionKEKEnvVarPrefix + "_KEK_1", + "kek-2": TokenEncryptionKEKEnvVarPrefix + "_KEK_2", + } + rc, err := BuildAuthServerRunConfig("default", "test-vmcp", withTokenEncryption("kek-2"), + []string{"https://vmcp.example.com"}, []string{"openid"}, "https://vmcp.example.com", kekEnvByID) + require.NoError(t, err) + require.NotNil(t, rc.Storage) + require.NotNil(t, rc.Storage.RedisConfig) + require.NotNil(t, rc.Storage.RedisConfig.TokenEncryption) + assert.Equal(t, "kek-2", rc.Storage.RedisConfig.TokenEncryption.ActiveKeyID) + assert.Equal(t, kekEnvByID, rc.Storage.RedisConfig.TokenEncryption.Keys, + "the RunConfig carries the full key set by env var name, never by value") + }) + + t.Run("nil key set falls back to the active ID only (offline renderers)", func(t *testing.T) { + t.Parallel() + rc, err := BuildAuthServerRunConfig("default", "test-vmcp", withTokenEncryption("kek-1"), + []string{"https://vmcp.example.com"}, []string{"openid"}, "https://vmcp.example.com", nil) + require.NoError(t, err) + require.NotNil(t, rc.Storage.RedisConfig.TokenEncryption) + assert.Equal(t, "kek-1", rc.Storage.RedisConfig.TokenEncryption.ActiveKeyID) + assert.Equal(t, + map[string]string{"kek-1": TokenEncryptionKEKEnvVarPrefix + "_KEK_1"}, + rc.Storage.RedisConfig.TokenEncryption.Keys, + "the fallback derives the active ID's env name deterministically") + }) + + t.Run("active ID absent from the resolved key set is an error", func(t *testing.T) { + t.Parallel() + _, err := BuildAuthServerRunConfig("default", "test-vmcp", withTokenEncryption("kek-9"), + []string{"https://vmcp.example.com"}, []string{"openid"}, "https://vmcp.example.com", + map[string]string{"kek-1": TokenEncryptionKEKEnvVarPrefix + "_KEK_1"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "kek-9") + }) + + t.Run("nil tokenEncryption leaves the RunConfig encryption-free", func(t *testing.T) { + t.Parallel() + rc, err := BuildAuthServerRunConfig("default", "test-vmcp", baseAuthConfig(), + []string{"https://vmcp.example.com"}, []string{"openid"}, "https://vmcp.example.com", nil) + require.NoError(t, err) + require.NotNil(t, rc.Storage) + require.NotNil(t, rc.Storage.RedisConfig) + assert.Nil(t, rc.Storage.RedisConfig.TokenEncryption) + }) +} + func TestResolveSentinelAddrs(t *testing.T) { t.Parallel() @@ -2340,7 +2575,7 @@ func TestBuildStorageRunConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - cfg, err := buildStorageRunConfig("default", "test-server", tt.authConfig) + cfg, err := buildStorageRunConfig("default", "test-server", tt.authConfig, nil) if tt.wantErr { require.Error(t, err) @@ -2396,6 +2631,7 @@ func TestBuildAuthServerRunConfig_WithRedisStorage(t *testing.T) { []string{"http://test-server.default.svc.cluster.local:8080"}, []string{"openid"}, "http://test-server.default.svc.cluster.local:8080", + nil, ) require.NoError(t, err) @@ -2849,6 +3085,7 @@ func TestBuildAuthServerRunConfig_CIMD(t *testing.T) { baseAuthConfig(tt.cimd), defaultAudiences, defaultScopes, "https://mcp.example.com", + nil, ) if tt.wantErr { diff --git a/cmd/thv-operator/pkg/controllerutil/podtemplatespec_builder.go b/cmd/thv-operator/pkg/controllerutil/podtemplatespec_builder.go index e54792397f..6d026404ff 100644 --- a/cmd/thv-operator/pkg/controllerutil/podtemplatespec_builder.go +++ b/cmd/thv-operator/pkg/controllerutil/podtemplatespec_builder.go @@ -155,3 +155,10 @@ func parsePodTemplateSpec(raw *runtime.RawExtension) (*corev1.PodTemplateSpec, e return spec.DeepCopy(), nil } + +// ParsePodTemplateSpec is the exported form of parsePodTemplateSpec for +// controller call sites that need the parsed user template itself (e.g. to +// run additional field-level validation on it before applying the patch). +func ParsePodTemplateSpec(raw *runtime.RawExtension) (*corev1.PodTemplateSpec, error) { + return parsePodTemplateSpec(raw) +} diff --git a/cmd/thv-operator/pkg/vmcpconfig/converter.go b/cmd/thv-operator/pkg/vmcpconfig/converter.go index 9e9fce5f19..717410c795 100644 --- a/cmd/thv-operator/pkg/vmcpconfig/converter.go +++ b/cmd/thv-operator/pkg/vmcpconfig/converter.go @@ -157,7 +157,7 @@ func (c *Converter) Convert( var authServerRC *authserver.RunConfig // Convert inline AuthServerConfig if specified. if vmcp.Spec.AuthServerConfig != nil { - rc, err := c.convertAuthServerConfig(vmcp, config) + rc, err := c.convertAuthServerConfig(ctx, vmcp, config) if err != nil { return nil, nil, fmt.Errorf("failed to convert auth server config: %w", err) } @@ -524,19 +524,28 @@ func convertSessionStorage(vmcp *mcpv1beta1.VirtualMCPServer) *vmcpconfig.Sessio // convertAuthServerConfig converts the inline EmbeddedAuthServerConfig from the // VirtualMCPServer spec into an authserver.RunConfig using the shared builder in // controllerutil. AllowedAudiences is derived from the resolved incoming OIDC config. -func (*Converter) convertAuthServerConfig( +// The KEK key set (active + retired) is resolved from the referenced Secret so +// the RunConfig can decrypt rows sealed under retired keys, matching the env +// vars the deployment builder renders on the same pod. +func (c *Converter) convertAuthServerConfig( + ctx context.Context, vmcp *mcpv1beta1.VirtualMCPServer, config *vmcpconfig.Config, ) (*authserver.RunConfig, error) { if vmcp.Spec.AuthServerConfig == nil { return nil, nil } + kekEnvByID, err := controllerutil.ResolveKEKKeySet(ctx, c.k8sClient, vmcp.Namespace, vmcp.Spec.AuthServerConfig) + if err != nil { + return nil, fmt.Errorf("failed to resolve token-encryption KEK key set: %w", err) + } return controllerutil.BuildAuthServerRunConfig( vmcp.Namespace, vmcp.Name, vmcp.Spec.AuthServerConfig, deriveAllowedAudiences(config), deriveScopesSupported(config), deriveResourceURL(config), + kekEnvByID, ) } diff --git a/cmd/thv-operator/test-integration/mcp-external-auth/token_encryption_validation_test.go b/cmd/thv-operator/test-integration/mcp-external-auth/token_encryption_validation_test.go new file mode 100644 index 0000000000..f13dee8a4e --- /dev/null +++ b/cmd/thv-operator/test-integration/mcp-external-auth/token_encryption_validation_test.go @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" +) + +// These tests exercise the CEL XValidation rules on TokenEncryptionConfig and +// the storage-type gate through the real apiserver (envtest): +// - keySecretRef.name must not be empty (the reconcile-time resolver fails +// closed on an empty name; admission must reject it first); +// - tokenEncryption requires storage type 'redis' (the RunConfig shape and +// untrusted-mode egress only support the Redis backend). +var _ = Describe("MCPExternalAuthConfig tokenEncryption CEL validation", func() { + const namespace = "default" + + // makeAuthConfig returns a minimum-valid embeddedAuthServer config whose + // only varying piece is the storage block. + makeAuthConfig := func(name string, storage *mcpv1beta1.AuthServerStorageConfig) *mcpv1beta1.MCPExternalAuthConfig { + return &mcpv1beta1.MCPExternalAuthConfig{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: mcpv1beta1.MCPExternalAuthConfigSpec{ + Type: "embeddedAuthServer", + EmbeddedAuthServer: &mcpv1beta1.EmbeddedAuthServerConfig{ + Issuer: "https://auth.example.com", + UpstreamProviders: []mcpv1beta1.UpstreamProviderConfig{{ + Name: "github", + Type: mcpv1beta1.UpstreamProviderTypeOAuth2, + OAuth2Config: &mcpv1beta1.OAuth2UpstreamConfig{ + AuthorizationEndpoint: "https://github.com/login/oauth/authorize", + TokenEndpoint: "https://github.com/login/oauth/access_token", + ClientID: "test-client-id", + }, + }}, + Storage: storage, + }, + }, + } + } + + redisStorage := func(te *mcpv1beta1.TokenEncryptionConfig) *mcpv1beta1.AuthServerStorageConfig { + return &mcpv1beta1.AuthServerStorageConfig{ + Type: mcpv1beta1.AuthServerStorageTypeRedis, + Redis: &mcpv1beta1.RedisStorageConfig{ + Addr: "redis.example.com:6379", + ACLUserConfig: &mcpv1beta1.RedisACLUserConfig{ + PasswordSecretRef: &mcpv1beta1.SecretKeyRef{ + Name: "redis-credentials", + Key: "password", + }, + }, + }, + TokenEncryption: te, + } + } + + BeforeEach(func() { + _ = k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}) + }) + + type validationCase struct { + name string + storage *mcpv1beta1.AuthServerStorageConfig + shouldAdmit bool + errMatch string // substring expected on rejection; ignored when shouldAdmit is true + } + + cases := []validationCase{ + { + name: "tokenEncryption on redis storage with a named keySecretRef", + storage: redisStorage(&mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-1", + KeySecretRef: corev1.LocalObjectReference{Name: "my-kek-secret"}, + }), + shouldAdmit: true, + }, + { + name: "no tokenEncryption on redis storage", + storage: redisStorage(nil), + shouldAdmit: true, + }, + { + name: "empty keySecretRef.name is rejected", + storage: redisStorage(&mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-1", + KeySecretRef: corev1.LocalObjectReference{Name: ""}, + }), + shouldAdmit: false, + errMatch: "keySecretRef.name must not be empty", + }, + { + name: "tokenEncryption on memory storage is rejected", + storage: &mcpv1beta1.AuthServerStorageConfig{ + Type: mcpv1beta1.AuthServerStorageTypeMemory, + TokenEncryption: &mcpv1beta1.TokenEncryptionConfig{ + ActiveKeyID: "kek-1", + KeySecretRef: corev1.LocalObjectReference{Name: "my-kek-secret"}, + }, + }, + shouldAdmit: false, + errMatch: "tokenEncryption requires storage type 'redis'", + }, + } + + for i, c := range cases { + name := fmt.Sprintf("token-encryption-validation-%d", i) + It(c.name, func() { + cfg := makeAuthConfig(name, c.storage) + err := k8sClient.Create(ctx, cfg) + if c.shouldAdmit { + Expect(err).NotTo(HaveOccurred(), + "expected apiserver to admit config: %s", c.name) + DeferCleanup(func() { + Expect(k8sClient.Delete(ctx, cfg)).To(Succeed()) + }) + return + } + Expect(err).To(HaveOccurred(), + "expected apiserver to reject config: %s", c.name) + Expect(err.Error()).To(ContainSubstring(c.errMatch)) + }) + } +}) diff --git a/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpexternalauthconfigs.yaml b/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpexternalauthconfigs.yaml index d47545b4d6..048e4dd1ee 100644 --- a/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpexternalauthconfigs.yaml +++ b/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpexternalauthconfigs.yaml @@ -562,6 +562,44 @@ spec: - message: clusterMode requires addr to be set rule: '!(has(self.clusterMode) && self.clusterMode) || (has(self.addr) && self.addr.size() > 0)' + tokenEncryption: + description: |- + TokenEncryption enables envelope encryption of upstream tokens at rest. + Requires redis storage: the RunConfig shape only supports token + encryption on the Redis backend, and untrusted-mode egress (the primary + consumer) requires Redis-backed token storage. ActiveKeyID names the + key used for new writes; KeySecretRef references a Secret whose data + keys are key IDs and values are base64 32-byte KEKs. + properties: + activeKeyId: + description: |- + ActiveKeyID identifies the KEK used to encrypt new writes. Must match a + data key of the Secret referenced by KeySecretRef. + minLength: 1 + type: string + keySecretRef: + description: |- + KeySecretRef references the Secret holding the key-encryption keys. + Its data keys are key IDs; values are base64 32-byte KEKs. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKeyId + - keySecretRef + type: object + x-kubernetes-validations: + - message: keySecretRef.name must not be empty + rule: self.keySecretRef.name.size() > 0 type: default: memory description: |- @@ -572,6 +610,9 @@ spec: - redis type: string type: object + x-kubernetes-validations: + - message: tokenEncryption requires storage type 'redis' + rule: '!has(self.tokenEncryption) || self.type == ''redis''' tokenLifespans: description: |- TokenLifespans configures the duration that various tokens are valid. @@ -2164,6 +2205,44 @@ spec: - message: clusterMode requires addr to be set rule: '!(has(self.clusterMode) && self.clusterMode) || (has(self.addr) && self.addr.size() > 0)' + tokenEncryption: + description: |- + TokenEncryption enables envelope encryption of upstream tokens at rest. + Requires redis storage: the RunConfig shape only supports token + encryption on the Redis backend, and untrusted-mode egress (the primary + consumer) requires Redis-backed token storage. ActiveKeyID names the + key used for new writes; KeySecretRef references a Secret whose data + keys are key IDs and values are base64 32-byte KEKs. + properties: + activeKeyId: + description: |- + ActiveKeyID identifies the KEK used to encrypt new writes. Must match a + data key of the Secret referenced by KeySecretRef. + minLength: 1 + type: string + keySecretRef: + description: |- + KeySecretRef references the Secret holding the key-encryption keys. + Its data keys are key IDs; values are base64 32-byte KEKs. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKeyId + - keySecretRef + type: object + x-kubernetes-validations: + - message: keySecretRef.name must not be empty + rule: self.keySecretRef.name.size() > 0 type: default: memory description: |- @@ -2174,6 +2253,9 @@ spec: - redis type: string type: object + x-kubernetes-validations: + - message: tokenEncryption requires storage type 'redis' + rule: '!has(self.tokenEncryption) || self.type == ''redis''' tokenLifespans: description: |- TokenLifespans configures the duration that various tokens are valid. diff --git a/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_virtualmcpservers.yaml b/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_virtualmcpservers.yaml index 17c372dbc5..7167c1f5aa 100644 --- a/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_virtualmcpservers.yaml +++ b/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_virtualmcpservers.yaml @@ -435,6 +435,44 @@ spec: - message: clusterMode requires addr to be set rule: '!(has(self.clusterMode) && self.clusterMode) || (has(self.addr) && self.addr.size() > 0)' + tokenEncryption: + description: |- + TokenEncryption enables envelope encryption of upstream tokens at rest. + Requires redis storage: the RunConfig shape only supports token + encryption on the Redis backend, and untrusted-mode egress (the primary + consumer) requires Redis-backed token storage. ActiveKeyID names the + key used for new writes; KeySecretRef references a Secret whose data + keys are key IDs and values are base64 32-byte KEKs. + properties: + activeKeyId: + description: |- + ActiveKeyID identifies the KEK used to encrypt new writes. Must match a + data key of the Secret referenced by KeySecretRef. + minLength: 1 + type: string + keySecretRef: + description: |- + KeySecretRef references the Secret holding the key-encryption keys. + Its data keys are key IDs; values are base64 32-byte KEKs. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKeyId + - keySecretRef + type: object + x-kubernetes-validations: + - message: keySecretRef.name must not be empty + rule: self.keySecretRef.name.size() > 0 type: default: memory description: |- @@ -445,6 +483,9 @@ spec: - redis type: string type: object + x-kubernetes-validations: + - message: tokenEncryption requires storage type 'redis' + rule: '!has(self.tokenEncryption) || self.type == ''redis''' tokenLifespans: description: |- TokenLifespans configures the duration that various tokens are valid. @@ -2136,6 +2177,15 @@ spec: UpstreamInject contains configuration for upstream inject auth strategy. Used when Type = "upstream_inject". properties: + authorizeUrl: + description: |- + AuthorizeURL is the ToolHive authorization server's authorize-endpoint + URL ({issuer}/oauth/authorize). When set, it is carried in the + ConsentRequiredError returned when the provider token is absent, so + clients can direct the user to consent. The URL cannot be a complete + one-click link: the client must merge its own client_id, redirect_uri, + and PKCE parameters. Optional; when empty the error carries no URL. + type: string providerName: description: |- ProviderName is the name of the upstream provider configured in the @@ -2440,6 +2490,15 @@ spec: UpstreamInject contains configuration for upstream inject auth strategy. Used when Type = "upstream_inject". properties: + authorizeUrl: + description: |- + AuthorizeURL is the ToolHive authorization server's authorize-endpoint + URL ({issuer}/oauth/authorize). When set, it is carried in the + ConsentRequiredError returned when the provider token is absent, so + clients can direct the user to consent. The URL cannot be a complete + one-click link: the client must merge its own client_id, redirect_uri, + and PKCE parameters. Optional; when empty the error carries no URL. + type: string providerName: description: |- ProviderName is the name of the upstream provider configured in the @@ -3867,6 +3926,44 @@ spec: - message: clusterMode requires addr to be set rule: '!(has(self.clusterMode) && self.clusterMode) || (has(self.addr) && self.addr.size() > 0)' + tokenEncryption: + description: |- + TokenEncryption enables envelope encryption of upstream tokens at rest. + Requires redis storage: the RunConfig shape only supports token + encryption on the Redis backend, and untrusted-mode egress (the primary + consumer) requires Redis-backed token storage. ActiveKeyID names the + key used for new writes; KeySecretRef references a Secret whose data + keys are key IDs and values are base64 32-byte KEKs. + properties: + activeKeyId: + description: |- + ActiveKeyID identifies the KEK used to encrypt new writes. Must match a + data key of the Secret referenced by KeySecretRef. + minLength: 1 + type: string + keySecretRef: + description: |- + KeySecretRef references the Secret holding the key-encryption keys. + Its data keys are key IDs; values are base64 32-byte KEKs. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKeyId + - keySecretRef + type: object + x-kubernetes-validations: + - message: keySecretRef.name must not be empty + rule: self.keySecretRef.name.size() > 0 type: default: memory description: |- @@ -3877,6 +3974,9 @@ spec: - redis type: string type: object + x-kubernetes-validations: + - message: tokenEncryption requires storage type 'redis' + rule: '!has(self.tokenEncryption) || self.type == ''redis''' tokenLifespans: description: |- TokenLifespans configures the duration that various tokens are valid. @@ -5568,6 +5668,15 @@ spec: UpstreamInject contains configuration for upstream inject auth strategy. Used when Type = "upstream_inject". properties: + authorizeUrl: + description: |- + AuthorizeURL is the ToolHive authorization server's authorize-endpoint + URL ({issuer}/oauth/authorize). When set, it is carried in the + ConsentRequiredError returned when the provider token is absent, so + clients can direct the user to consent. The URL cannot be a complete + one-click link: the client must merge its own client_id, redirect_uri, + and PKCE parameters. Optional; when empty the error carries no URL. + type: string providerName: description: |- ProviderName is the name of the upstream provider configured in the @@ -5872,6 +5981,15 @@ spec: UpstreamInject contains configuration for upstream inject auth strategy. Used when Type = "upstream_inject". properties: + authorizeUrl: + description: |- + AuthorizeURL is the ToolHive authorization server's authorize-endpoint + URL ({issuer}/oauth/authorize). When set, it is carried in the + ConsentRequiredError returned when the provider token is absent, so + clients can direct the user to consent. The URL cannot be a complete + one-click link: the client must merge its own client_id, redirect_uri, + and PKCE parameters. Optional; when empty the error carries no URL. + type: string providerName: description: |- ProviderName is the name of the upstream provider configured in the diff --git a/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpexternalauthconfigs.yaml b/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpexternalauthconfigs.yaml index b73e66445b..49192d56f7 100644 --- a/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpexternalauthconfigs.yaml +++ b/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpexternalauthconfigs.yaml @@ -565,6 +565,44 @@ spec: - message: clusterMode requires addr to be set rule: '!(has(self.clusterMode) && self.clusterMode) || (has(self.addr) && self.addr.size() > 0)' + tokenEncryption: + description: |- + TokenEncryption enables envelope encryption of upstream tokens at rest. + Requires redis storage: the RunConfig shape only supports token + encryption on the Redis backend, and untrusted-mode egress (the primary + consumer) requires Redis-backed token storage. ActiveKeyID names the + key used for new writes; KeySecretRef references a Secret whose data + keys are key IDs and values are base64 32-byte KEKs. + properties: + activeKeyId: + description: |- + ActiveKeyID identifies the KEK used to encrypt new writes. Must match a + data key of the Secret referenced by KeySecretRef. + minLength: 1 + type: string + keySecretRef: + description: |- + KeySecretRef references the Secret holding the key-encryption keys. + Its data keys are key IDs; values are base64 32-byte KEKs. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKeyId + - keySecretRef + type: object + x-kubernetes-validations: + - message: keySecretRef.name must not be empty + rule: self.keySecretRef.name.size() > 0 type: default: memory description: |- @@ -575,6 +613,9 @@ spec: - redis type: string type: object + x-kubernetes-validations: + - message: tokenEncryption requires storage type 'redis' + rule: '!has(self.tokenEncryption) || self.type == ''redis''' tokenLifespans: description: |- TokenLifespans configures the duration that various tokens are valid. @@ -2167,6 +2208,44 @@ spec: - message: clusterMode requires addr to be set rule: '!(has(self.clusterMode) && self.clusterMode) || (has(self.addr) && self.addr.size() > 0)' + tokenEncryption: + description: |- + TokenEncryption enables envelope encryption of upstream tokens at rest. + Requires redis storage: the RunConfig shape only supports token + encryption on the Redis backend, and untrusted-mode egress (the primary + consumer) requires Redis-backed token storage. ActiveKeyID names the + key used for new writes; KeySecretRef references a Secret whose data + keys are key IDs and values are base64 32-byte KEKs. + properties: + activeKeyId: + description: |- + ActiveKeyID identifies the KEK used to encrypt new writes. Must match a + data key of the Secret referenced by KeySecretRef. + minLength: 1 + type: string + keySecretRef: + description: |- + KeySecretRef references the Secret holding the key-encryption keys. + Its data keys are key IDs; values are base64 32-byte KEKs. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKeyId + - keySecretRef + type: object + x-kubernetes-validations: + - message: keySecretRef.name must not be empty + rule: self.keySecretRef.name.size() > 0 type: default: memory description: |- @@ -2177,6 +2256,9 @@ spec: - redis type: string type: object + x-kubernetes-validations: + - message: tokenEncryption requires storage type 'redis' + rule: '!has(self.tokenEncryption) || self.type == ''redis''' tokenLifespans: description: |- TokenLifespans configures the duration that various tokens are valid. diff --git a/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_virtualmcpservers.yaml b/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_virtualmcpservers.yaml index de1f36d54a..5c7cc9f05b 100644 --- a/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_virtualmcpservers.yaml +++ b/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_virtualmcpservers.yaml @@ -438,6 +438,44 @@ spec: - message: clusterMode requires addr to be set rule: '!(has(self.clusterMode) && self.clusterMode) || (has(self.addr) && self.addr.size() > 0)' + tokenEncryption: + description: |- + TokenEncryption enables envelope encryption of upstream tokens at rest. + Requires redis storage: the RunConfig shape only supports token + encryption on the Redis backend, and untrusted-mode egress (the primary + consumer) requires Redis-backed token storage. ActiveKeyID names the + key used for new writes; KeySecretRef references a Secret whose data + keys are key IDs and values are base64 32-byte KEKs. + properties: + activeKeyId: + description: |- + ActiveKeyID identifies the KEK used to encrypt new writes. Must match a + data key of the Secret referenced by KeySecretRef. + minLength: 1 + type: string + keySecretRef: + description: |- + KeySecretRef references the Secret holding the key-encryption keys. + Its data keys are key IDs; values are base64 32-byte KEKs. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKeyId + - keySecretRef + type: object + x-kubernetes-validations: + - message: keySecretRef.name must not be empty + rule: self.keySecretRef.name.size() > 0 type: default: memory description: |- @@ -448,6 +486,9 @@ spec: - redis type: string type: object + x-kubernetes-validations: + - message: tokenEncryption requires storage type 'redis' + rule: '!has(self.tokenEncryption) || self.type == ''redis''' tokenLifespans: description: |- TokenLifespans configures the duration that various tokens are valid. @@ -2139,6 +2180,15 @@ spec: UpstreamInject contains configuration for upstream inject auth strategy. Used when Type = "upstream_inject". properties: + authorizeUrl: + description: |- + AuthorizeURL is the ToolHive authorization server's authorize-endpoint + URL ({issuer}/oauth/authorize). When set, it is carried in the + ConsentRequiredError returned when the provider token is absent, so + clients can direct the user to consent. The URL cannot be a complete + one-click link: the client must merge its own client_id, redirect_uri, + and PKCE parameters. Optional; when empty the error carries no URL. + type: string providerName: description: |- ProviderName is the name of the upstream provider configured in the @@ -2443,6 +2493,15 @@ spec: UpstreamInject contains configuration for upstream inject auth strategy. Used when Type = "upstream_inject". properties: + authorizeUrl: + description: |- + AuthorizeURL is the ToolHive authorization server's authorize-endpoint + URL ({issuer}/oauth/authorize). When set, it is carried in the + ConsentRequiredError returned when the provider token is absent, so + clients can direct the user to consent. The URL cannot be a complete + one-click link: the client must merge its own client_id, redirect_uri, + and PKCE parameters. Optional; when empty the error carries no URL. + type: string providerName: description: |- ProviderName is the name of the upstream provider configured in the @@ -3870,6 +3929,44 @@ spec: - message: clusterMode requires addr to be set rule: '!(has(self.clusterMode) && self.clusterMode) || (has(self.addr) && self.addr.size() > 0)' + tokenEncryption: + description: |- + TokenEncryption enables envelope encryption of upstream tokens at rest. + Requires redis storage: the RunConfig shape only supports token + encryption on the Redis backend, and untrusted-mode egress (the primary + consumer) requires Redis-backed token storage. ActiveKeyID names the + key used for new writes; KeySecretRef references a Secret whose data + keys are key IDs and values are base64 32-byte KEKs. + properties: + activeKeyId: + description: |- + ActiveKeyID identifies the KEK used to encrypt new writes. Must match a + data key of the Secret referenced by KeySecretRef. + minLength: 1 + type: string + keySecretRef: + description: |- + KeySecretRef references the Secret holding the key-encryption keys. + Its data keys are key IDs; values are base64 32-byte KEKs. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - activeKeyId + - keySecretRef + type: object + x-kubernetes-validations: + - message: keySecretRef.name must not be empty + rule: self.keySecretRef.name.size() > 0 type: default: memory description: |- @@ -3880,6 +3977,9 @@ spec: - redis type: string type: object + x-kubernetes-validations: + - message: tokenEncryption requires storage type 'redis' + rule: '!has(self.tokenEncryption) || self.type == ''redis''' tokenLifespans: description: |- TokenLifespans configures the duration that various tokens are valid. @@ -5571,6 +5671,15 @@ spec: UpstreamInject contains configuration for upstream inject auth strategy. Used when Type = "upstream_inject". properties: + authorizeUrl: + description: |- + AuthorizeURL is the ToolHive authorization server's authorize-endpoint + URL ({issuer}/oauth/authorize). When set, it is carried in the + ConsentRequiredError returned when the provider token is absent, so + clients can direct the user to consent. The URL cannot be a complete + one-click link: the client must merge its own client_id, redirect_uri, + and PKCE parameters. Optional; when empty the error carries no URL. + type: string providerName: description: |- ProviderName is the name of the upstream provider configured in the @@ -5875,6 +5984,15 @@ spec: UpstreamInject contains configuration for upstream inject auth strategy. Used when Type = "upstream_inject". properties: + authorizeUrl: + description: |- + AuthorizeURL is the ToolHive authorization server's authorize-endpoint + URL ({issuer}/oauth/authorize). When set, it is carried in the + ConsentRequiredError returned when the provider token is absent, so + clients can direct the user to consent. The URL cannot be a complete + one-click link: the client must merge its own client_id, redirect_uri, + and PKCE parameters. Optional; when empty the error carries no URL. + type: string providerName: description: |- ProviderName is the name of the upstream provider configured in the diff --git a/docs/operator/crd-api.md b/docs/operator/crd-api.md index e4e67c465b..712b1a0ee3 100644 --- a/docs/operator/crd-api.md +++ b/docs/operator/crd-api.md @@ -108,6 +108,8 @@ _Appears in:_ | `xaa` _[auth.types.XAAConfig](#authtypesxaaconfig)_ | XAA contains configuration for XAA (Cross-Application Access) auth strategy.
Used when Type = "xaa". | | | + + #### auth.types.HeaderInjectionConfig @@ -1715,6 +1717,7 @@ _Appears in:_ | --- | --- | --- | --- | | `type` _[api.v1beta1.AuthServerStorageType](#apiv1beta1authserverstoragetype)_ | Type specifies the storage backend type.
Valid values: "memory" (default), "redis". | memory | Enum: [memory redis]
| | `redis` _[api.v1beta1.RedisStorageConfig](#apiv1beta1redisstorageconfig)_ | Redis configures the Redis storage backend.
Required when type is "redis". | | Optional: \{\}
| +| `tokenEncryption` _[api.v1beta1.TokenEncryptionConfig](#apiv1beta1tokenencryptionconfig)_ | TokenEncryption enables envelope encryption of upstream tokens at rest.
Requires redis storage: the RunConfig shape only supports token
encryption on the Redis backend, and untrusted-mode egress (the primary
consumer) requires Redis-backed token storage. ActiveKeyID names the
key used for new writes; KeySecretRef references a Secret whose data
keys are key IDs and values are base64 32-byte KEKs. | | Optional: \{\}
| #### api.v1beta1.AuthServerStorageType @@ -4156,6 +4159,29 @@ _Appears in:_ | `passwordRef` _[api.v1beta1.SecretKeyRef](#apiv1beta1secretkeyref)_ | PasswordRef is a reference to a Secret key containing the Redis password | | Optional: \{\}
| +#### api.v1beta1.TokenEncryptionConfig + + + +TokenEncryptionConfig configures AES-256-GCM envelope encryption of +upstream OAuth token values stored in Redis. The Secret referenced by +KeySecretRef holds one data entry per key ID (value = base64 32-byte KEK); +the operator mounts the active key as a SecretKeyRef env entry on the vMCP +container and clones the same reference into untrusted egress-broker +sidecars. Key material never appears in the CRD, a ConfigMap, or a pod env +literal. + + + +_Appears in:_ +- [api.v1beta1.AuthServerStorageConfig](#apiv1beta1authserverstorageconfig) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `activeKeyId` _string_ | ActiveKeyID identifies the KEK used to encrypt new writes. Must match a
data key of the Secret referenced by KeySecretRef. | | MinLength: 1
Required: \{\}
| +| `keySecretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#localobjectreference-v1-core)_ | KeySecretRef references the Secret holding the key-encryption keys.
Its data keys are key IDs; values are base64 32-byte KEKs. | | Required: \{\}
| + + #### api.v1beta1.TokenExchangeConfig diff --git a/pkg/vmcp/cli/untrusted.go b/pkg/vmcp/cli/untrusted.go index 19308afdf2..956a2e5287 100644 --- a/pkg/vmcp/cli/untrusted.go +++ b/pkg/vmcp/cli/untrusted.go @@ -7,7 +7,12 @@ import ( "context" "fmt" "log/slog" + "math" "os" + "slices" + "strconv" + "strings" + "time" "github.com/google/uuid" "github.com/redis/go-redis/v9" @@ -31,6 +36,49 @@ import ( // packages must not import each other; the contract is pinned by tests. const untrustedTokenStoreAddrEnvVar = "THV_UNTRUSTED_TOKEN_STORE_REDIS_ADDR" // #nosec G101 -- env var name, not a credential +// untrustedTokenStoreKEK*EnvVars mirror the operator-side env vars carrying +// the token-encryption KEK coordinates (see +// cmd/thv-operator/controllers/virtualmcpserver_deployment.go +// buildUntrustedTokenStoreEnvVars). +const ( + untrustedTokenStoreKEKSecretEnvVar = "THV_UNTRUSTED_TOKEN_STORE_KEK_SECRET" // #nosec G101 -- env var name, not a credential + untrustedTokenStoreKEKKeyEnvVar = "THV_UNTRUSTED_TOKEN_STORE_KEK_KEY" // #nosec G101 -- env var name, not a credential + untrustedTokenStoreKEKIDsEnvVar = "THV_UNTRUSTED_TOKEN_STORE_KEK_IDS" // #nosec G101 -- env var name, not a credential +) + +// Platform-operator tunables (Wave-5 spec §3.1/§3.2/§4), resolved ONCE here at +// the composition root — never hot-reloaded. Every one fails startup on an +// unparseable/zero/negative value (fail loud). +const ( + // envUntrustedEnvoyImage overrides the pinned Envoy data-plane image + // (air-gapped mirrors). + envUntrustedEnvoyImage = "THV_UNTRUSTED_ENVOY_IMAGE" + // envUntrustedBrokerImage overrides the pinned broker sidecar image. + envUntrustedBrokerImage = "THV_UNTRUSTED_BROKER_IMAGE" + // envUntrustedSidecarCPU scales the envoy/broker sidecar CPU + // requests/limits (multiplier, 1.0 = defaults). + envUntrustedSidecarCPU = "THV_UNTRUSTED_SIDECAR_CPU" + // envUntrustedSidecarMem scales the envoy/broker sidecar memory + // requests/limits (multiplier, 1.0 = defaults). + envUntrustedSidecarMem = "THV_UNTRUSTED_SIDECAR_MEM" + + // envUntrustedIdleTTL is the pod liveness lease the reaper enforces. + // Default 30m. + envUntrustedIdleTTL = "THV_UNTRUSTED_IDLE_TTL" + // envUntrustedPerUserQuota caps concurrent untrusted pods per user. + // Default 10. + envUntrustedPerUserQuota = "THV_UNTRUSTED_PER_USER_QUOTA" + // envUntrustedPerServerCap caps concurrent untrusted pods per MCPServer. + // Default 200. + envUntrustedPerServerCap = "THV_UNTRUSTED_PER_SERVER_CAP" + // envUntrustedGlobalCapRatio is the fraction of the session cache + // capacity bounding total untrusted pods. Default 0.8. + envUntrustedGlobalCapRatio = "THV_UNTRUSTED_GLOBAL_CAP_RATIO" + // envUntrustedReadinessTimeout is the failed-cold-start threshold + // (resolver wait budget AND reaper sweep rule). Default 120s. + envUntrustedReadinessTimeout = "THV_UNTRUSTED_READINESS_TIMEOUT" +) + // defaultSessionKeyPrefix mirrors the fallback in server.buildSessionDataStorage. const defaultSessionKeyPrefix = "thv:vmcp:session:" @@ -107,6 +155,15 @@ func buildUntrustedStack( if keyPrefix == "" { keyPrefix = defaultSessionKeyPrefix } + + // Resolve the platform tunables once, before any dependency construction, + // so a malformed knob fails startup with a clean error (no half-built + // stack). + tunables, err := resolveUntrustedTunables() + if err != nil { + return nil, err + } + redisClient, err := tcredis.NewClient(ctx, &tcredis.Config{ Addr: cfg.SessionStorage.Address, Password: os.Getenv(config.RedisPasswordEnvVar), @@ -123,14 +180,26 @@ func buildUntrustedStack( } stack, err := untrusted.NewStack(untrusted.WiringConfig{ - K8sClient: k8sClient, - RedisClient: redisClient, - KeyPrefix: keyPrefix, - Namespace: namespace, - VMCPUId: untrustedVMCPUId(), - Admission: untrusted.AdmissionConfig{CacheCapacity: untrustedCacheCapacity()}, - TokenStore: resolveTokenStoreConfig(namespace, vmcpName), - MeterProvider: meterProvider, + K8sClient: k8sClient, + RedisClient: redisClient, + KeyPrefix: keyPrefix, + Namespace: namespace, + VMCPUId: untrustedVMCPUId(), + Admission: untrusted.AdmissionConfig{ + PerUserPodQuota: tunables.perUserQuota, + PerMCPServerCap: tunables.perServerCap, + GlobalCapFactor: tunables.globalCapRatio, + CacheCapacity: untrustedCacheCapacity(), + }, + Reaper: untrusted.ReaperConfig{ + IdleTTL: tunables.idleTTL, + ReadinessTimeout: tunables.readinessTimeout, + }, + ReadyBudget: tunables.readinessTimeout, + TokenStore: resolveTokenStoreConfig(namespace, vmcpName), + Images: tunables.images, + SidecarResources: tunables.sidecarResources, + MeterProvider: meterProvider, }) if err != nil { _ = redisClient.Close() @@ -145,7 +214,11 @@ func buildUntrustedStack( // resolveTokenStoreConfig builds the egress-broker token-store coordinates from // the vMCP's identity and the operator-injected Redis address. Returns nil -// (broker fails closed) when the address is absent. +// (broker fails closed) when the address is absent. When the operator wired +// token encryption (THV_UNTRUSTED_TOKEN_STORE_KEK_SECRET/_KEY/_IDS), the KEK +// Secret name, the ACTIVE key ID, and the full key-ID set are carried so every +// cloned sidecar reads the keys from the Secret — never literals — and its +// keyring knows retired IDs too (rotation never orphans ciphertext). func resolveTokenStoreConfig(namespace, vmcpName string) *untrusted.TokenStoreConfig { addr := os.Getenv(untrustedTokenStoreAddrEnvVar) if addr == "" { @@ -154,10 +227,194 @@ func resolveTokenStoreConfig(namespace, vmcpName string) *untrusted.TokenStoreCo "env_var", untrustedTokenStoreAddrEnvVar) return nil } - return &untrusted.TokenStoreConfig{ + ts := &untrusted.TokenStoreConfig{ RedisAddr: addr, KeyPrefix: authstorage.DeriveKeyPrefix(namespace, vmcpName), } + secretName := os.Getenv(untrustedTokenStoreKEKSecretEnvVar) + activeID := os.Getenv(untrustedTokenStoreKEKKeyEnvVar) + idsRaw := os.Getenv(untrustedTokenStoreKEKIDsEnvVar) + // All-or-nothing: partial KEK coordinates (a hand-edited Deployment or an + // operator/vMCP version skew) leave the sidecar keyring under-specified — + // warn loudly and render no KEK config (encryption off on the sidecar; the + // broker fails closed on any encrypted row). + if secretName == "" && activeID == "" && idsRaw == "" { + return ts + } + ids := splitNonEmpty(idsRaw, ",") + if secretName == "" || activeID == "" || len(ids) == 0 || !slices.Contains(ids, activeID) { + slog.Warn("untrusted token-store KEK coordinates are incomplete "+ + "(secret name, active key ID, and the key-ID set are required together); "+ + "egress-broker sidecars run without token decryption", + "secret_set", secretName != "", + "active_id_set", activeID != "", + "ids_set", len(ids) > 0) + return ts + } + ts.KEKSecret = secretName + ts.KEKActiveID = activeID + ts.KEKIDs = ids + return ts +} + +// splitNonEmpty splits s on sep and drops empty elements (so a trailing or +// leading separator in the operator-rendered list cannot produce an empty ID). +func splitNonEmpty(s, sep string) []string { + var out []string + for _, part := range strings.Split(s, sep) { + if part != "" { + out = append(out, part) + } + } + return out +} + +// untrustedTunables carries the resolved Wave-5 platform knobs into +// buildUntrustedStack. All values come from env vars (see the const block +// above) or their documented defaults. +type untrustedTunables struct { + images *untrusted.SidecarImages + sidecarResources *untrusted.SidecarResourceOverride + idleTTL time.Duration + readinessTimeout time.Duration + perUserQuota int + perServerCap int + globalCapRatio float64 +} + +// resolveUntrustedTunables reads the THV_UNTRUSTED_* env vars once. Absent +// vars take defaults; an unparseable or non-positive value is a startup-fatal +// error (fail loud — a silently-defaulted quota or timeout is a misconfig the +// operator cannot see). +func resolveUntrustedTunables() (*untrustedTunables, error) { + t := &untrustedTunables{} + + var err error + if t.idleTTL, err = parseEnvDuration(envUntrustedIdleTTL, 30*time.Minute); err != nil { + return nil, err + } + if t.readinessTimeout, err = parseEnvDuration(envUntrustedReadinessTimeout, untrusted.DefaultReadyBudget); err != nil { + return nil, err + } + if t.perUserQuota, err = parseEnvPositiveInt(envUntrustedPerUserQuota, 10); err != nil { + return nil, err + } + if t.perServerCap, err = parseEnvPositiveInt(envUntrustedPerServerCap, 200); err != nil { + return nil, err + } + if t.globalCapRatio, err = parseEnvPositiveFloat(envUntrustedGlobalCapRatio, 0.8); err != nil { + return nil, err + } + + // Image overrides (supply-chain): absent = the pinned defaults. Overrides + // that are not digest-pinned are honored (air-gapped mirrors may retag) + // but warned about: a floating tag can silently re-point the untrusted + // data plane at a different image on the next pull. A ":latest" tag is + // rejected outright — it re-points on EVERY pull and bypasses the + // release-pinned broker binary contract. + t.images = &untrusted.SidecarImages{ + EnvoyProxy: os.Getenv(envUntrustedEnvoyImage), + EgressBroker: os.Getenv(envUntrustedBrokerImage), + } + for _, override := range []struct { + envVar string + image string + }{ + {envUntrustedEnvoyImage, t.images.EnvoyProxy}, + {envUntrustedBrokerImage, t.images.EgressBroker}, + } { + if override.image == "" { + continue + } + if strings.HasSuffix(override.image, ":latest") { + return nil, fmt.Errorf("%s value %q must not use the :latest tag; "+ + "pin the untrusted sidecar image by digest (preferred) or an immutable tag", + override.envVar, override.image) + } + if !strings.Contains(override.image, "@sha256:") { + slog.Warn("untrusted sidecar image override is not digest-pinned; "+ + "a floating tag can silently re-point the untrusted data plane on the next pull", + "env_var", override.envVar, "image", override.image) + } + } + + cpuMult, err := parseEnvMultiplier(envUntrustedSidecarCPU, 1.0) + if err != nil { + return nil, err + } + memMult, err := parseEnvMultiplier(envUntrustedSidecarMem, 1.0) + if err != nil { + return nil, err + } + t.sidecarResources = &untrusted.SidecarResourceOverride{ + CPUMultiplier: cpuMult, + MemoryMultiplier: memMult, + } + return t, nil +} + +// parseEnvDuration resolves a duration env var: absent = def, otherwise the +// value must parse to a positive duration. +func parseEnvDuration(envVar string, def time.Duration) (time.Duration, error) { + raw := os.Getenv(envVar) + if raw == "" { + return def, nil + } + d, err := time.ParseDuration(raw) + if err != nil || d <= 0 { + return 0, fmt.Errorf("%s value %q must be a positive Go duration (e.g. %q)", envVar, raw, def.String()) + } + return d, nil +} + +// parseEnvPositiveInt resolves an integer env var: absent = def, otherwise +// the value must be a positive integer. +func parseEnvPositiveInt(envVar string, def int) (int, error) { + raw := os.Getenv(envVar) + if raw == "" { + return def, nil + } + n, err := strconv.Atoi(raw) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%s value %q must be a positive integer (default %d)", envVar, raw, def) + } + return n, nil +} + +// parseEnvPositiveFloat resolves a float env var: absent = def, otherwise the +// value must be a finite positive number (NaN/Inf are not config). +func parseEnvPositiveFloat(envVar string, def float64) (float64, error) { + raw := os.Getenv(envVar) + if raw == "" { + return def, nil + } + f, err := strconv.ParseFloat(raw, 64) + if err != nil || f <= 0 || math.IsNaN(f) || math.IsInf(f, 0) { + return 0, fmt.Errorf("%s value %q must be a finite positive number (default %v)", envVar, raw, def) + } + return f, nil +} + +// maxSidecarMultiplier bounds the sidecar CPU/memory multipliers: a larger +// factor is a misconfiguration (a stray zero or a units mix-up), never an +// intent — fail loud instead of requesting thousands of CPUs. +const maxSidecarMultiplier = 100.0 + +// parseEnvMultiplier resolves a resource-multiplier env var: absent = def, +// otherwise the value must be a finite number in (0, maxSidecarMultiplier]. +// NaN/Inf and absurd factors are startup-fatal (a silently-clamped multiplier +// is a misconfig the operator cannot see). +func parseEnvMultiplier(envVar string, def float64) (float64, error) { + raw := os.Getenv(envVar) + if raw == "" { + return def, nil + } + f, err := strconv.ParseFloat(raw, 64) + if err != nil || f <= 0 || math.IsNaN(f) || math.IsInf(f, 0) || f > maxSidecarMultiplier { + return 0, fmt.Errorf("%s value %q must be a finite number in (0, %v] (default %v)", + envVar, raw, maxSidecarMultiplier, def) + } + return f, nil } // untrustedK8sClient builds the in-cluster controller-runtime client the pod diff --git a/pkg/vmcp/cli/untrusted_test.go b/pkg/vmcp/cli/untrusted_test.go index 164d6bb193..c7cc638335 100644 --- a/pkg/vmcp/cli/untrusted_test.go +++ b/pkg/vmcp/cli/untrusted_test.go @@ -5,6 +5,7 @@ package cli import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,7 +54,146 @@ func TestResolveTokenStoreConfig(t *testing.T) { assert.Equal(t, "redis.auth:6379", ts.RedisAddr) // DeriveKeyPrefix(ns, name) — the same prefix the embedded auth server uses. assert.Equal(t, "thv:auth:{toolhive:my-vmcp}:", ts.KeyPrefix) - assert.Nil(t, ts.KEKSecretRef, "no KEK unless encryption is configured") + assert.Empty(t, ts.KEKSecret, "no KEK unless encryption is configured") + assert.Empty(t, ts.KEKActiveID) + assert.Empty(t, ts.KEKIDs) + }) + + t.Run("KEK coordinates populate the multi-key set the sidecar clones", func(t *testing.T) { + t.Setenv(untrustedTokenStoreAddrEnvVar, "redis.auth:6379") + t.Setenv(untrustedTokenStoreKEKSecretEnvVar, "my-vmcp-kek") + t.Setenv(untrustedTokenStoreKEKKeyEnvVar, "kek-2") + t.Setenv(untrustedTokenStoreKEKIDsEnvVar, "kek-1,kek-2") + ts := resolveTokenStoreConfig("toolhive", "my-vmcp") + require.NotNil(t, ts) + assert.Equal(t, "my-vmcp-kek", ts.KEKSecret) + assert.Equal(t, "kek-2", ts.KEKActiveID) + assert.Equal(t, []string{"kek-1", "kek-2"}, ts.KEKIDs) + }) + + t.Run("partial KEK coordinates render no KEK config", func(t *testing.T) { + t.Setenv(untrustedTokenStoreAddrEnvVar, "redis.auth:6379") + t.Setenv(untrustedTokenStoreKEKSecretEnvVar, "my-vmcp-kek") + ts := resolveTokenStoreConfig("toolhive", "my-vmcp") + require.NotNil(t, ts) + assert.Empty(t, ts.KEKSecret) + assert.Empty(t, ts.KEKActiveID) + assert.Empty(t, ts.KEKIDs) + }) + + t.Run("active ID missing from the ID set renders no KEK config", func(t *testing.T) { + t.Setenv(untrustedTokenStoreAddrEnvVar, "redis.auth:6379") + t.Setenv(untrustedTokenStoreKEKSecretEnvVar, "my-vmcp-kek") + t.Setenv(untrustedTokenStoreKEKKeyEnvVar, "kek-9") + t.Setenv(untrustedTokenStoreKEKIDsEnvVar, "kek-1,kek-2") + ts := resolveTokenStoreConfig("toolhive", "my-vmcp") + require.NotNil(t, ts) + assert.Empty(t, ts.KEKSecret) + assert.Empty(t, ts.KEKActiveID) + assert.Empty(t, ts.KEKIDs) + }) +} + +// TestResolveUntrustedTunables pins the Wave-5 platform knobs: defaults when +// unset, overrides honored, and startup-fatal on unparseable/zero/negative +// values. t.Setenv serializes subtests (no t.Parallel). +// +//nolint:paralleltest // t.Setenv modifies the process environment. +func TestResolveUntrustedTunables(t *testing.T) { + t.Run("defaults when no env is set", func(t *testing.T) { + tb, err := resolveUntrustedTunables() + require.NoError(t, err) + assert.Equal(t, 30*time.Minute, tb.idleTTL) + assert.Equal(t, 120*time.Second, tb.readinessTimeout) + assert.Equal(t, 10, tb.perUserQuota) + assert.Equal(t, 200, tb.perServerCap) + assert.InDelta(t, 0.8, tb.globalCapRatio, 1e-9) + assert.Empty(t, tb.images.EnvoyProxy) + assert.Empty(t, tb.images.EgressBroker) + assert.Equal(t, 1.0, tb.sidecarResources.CPUMultiplier) + assert.Equal(t, 1.0, tb.sidecarResources.MemoryMultiplier) + }) + + t.Run("overrides are honored", func(t *testing.T) { + t.Setenv(envUntrustedIdleTTL, "15m") + t.Setenv(envUntrustedReadinessTimeout, "60s") + t.Setenv(envUntrustedPerUserQuota, "5") + t.Setenv(envUntrustedPerServerCap, "50") + t.Setenv(envUntrustedGlobalCapRatio, "0.5") + t.Setenv(envUntrustedEnvoyImage, "mirror.local/envoy:v1") + t.Setenv(envUntrustedBrokerImage, "mirror.local/broker:v2") + t.Setenv(envUntrustedSidecarCPU, "2") + t.Setenv(envUntrustedSidecarMem, "1.5") + + tb, err := resolveUntrustedTunables() + require.NoError(t, err) + assert.Equal(t, 15*time.Minute, tb.idleTTL) + assert.Equal(t, 60*time.Second, tb.readinessTimeout) + assert.Equal(t, 5, tb.perUserQuota) + assert.Equal(t, 50, tb.perServerCap) + assert.InDelta(t, 0.5, tb.globalCapRatio, 1e-9) + assert.Equal(t, "mirror.local/envoy:v1", tb.images.EnvoyProxy) + assert.Equal(t, "mirror.local/broker:v2", tb.images.EgressBroker) + assert.Equal(t, 2.0, tb.sidecarResources.CPUMultiplier) + assert.Equal(t, 1.5, tb.sidecarResources.MemoryMultiplier) + }) + + t.Run("partial multiplier override leaves the other dimension at the default", func(t *testing.T) { + t.Setenv(envUntrustedSidecarCPU, "2") + tb, err := resolveUntrustedTunables() + require.NoError(t, err) + assert.Equal(t, 2.0, tb.sidecarResources.CPUMultiplier) + assert.Equal(t, 1.0, tb.sidecarResources.MemoryMultiplier) + }) + + t.Run("partial image override leaves the other image pinned (empty = default)", func(t *testing.T) { + t.Setenv(envUntrustedBrokerImage, "mirror.local/broker@sha256:abc") + tb, err := resolveUntrustedTunables() + require.NoError(t, err) + assert.Empty(t, tb.images.EnvoyProxy, "unset envoy image resolves to the pinned default at clone time") + assert.Equal(t, "mirror.local/broker@sha256:abc", tb.images.EgressBroker) + }) + + t.Run("invalid values are startup-fatal", func(t *testing.T) { + cases := []struct { + name string + env map[string]string + }{ + {"unparseable idle TTL", map[string]string{envUntrustedIdleTTL: "not-a-duration"}}, + {"zero idle TTL", map[string]string{envUntrustedIdleTTL: "0s"}}, + {"negative readiness timeout", map[string]string{envUntrustedReadinessTimeout: "-5s"}}, + {"zero per-user quota", map[string]string{envUntrustedPerUserQuota: "0"}}, + {"negative per-server cap", map[string]string{envUntrustedPerServerCap: "-3"}}, + {"unparseable per-server cap", map[string]string{envUntrustedPerServerCap: "abc"}}, + {"zero global cap ratio", map[string]string{envUntrustedGlobalCapRatio: "0"}}, + {"negative global cap ratio", map[string]string{envUntrustedGlobalCapRatio: "-0.5"}}, + {"NaN global cap ratio", map[string]string{envUntrustedGlobalCapRatio: "NaN"}}, + {"negative sidecar cpu", map[string]string{envUntrustedSidecarCPU: "-1"}}, + {"unparseable sidecar mem", map[string]string{envUntrustedSidecarMem: "lots"}}, + {"NaN sidecar cpu", map[string]string{envUntrustedSidecarCPU: "NaN"}}, + {"+Inf sidecar mem", map[string]string{envUntrustedSidecarMem: "+Inf"}}, + {"-Inf sidecar cpu", map[string]string{envUntrustedSidecarCPU: "-Inf"}}, + {"sidecar cpu above the bound", map[string]string{envUntrustedSidecarCPU: "101"}}, + {"sidecar mem absurdly large", map[string]string{envUntrustedSidecarMem: "1e9"}}, + {"broker image pinned to :latest", map[string]string{envUntrustedBrokerImage: "mirror.local/broker:latest"}}, + {"envoy image pinned to :latest", map[string]string{envUntrustedEnvoyImage: "envoyproxy/envoy:latest"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.env { + t.Setenv(k, v) + } + _, err := resolveUntrustedTunables() + require.Error(t, err) + }) + } + }) + + t.Run("multiplier at the bound is admitted", func(t *testing.T) { + t.Setenv(envUntrustedSidecarCPU, "100") + tb, err := resolveUntrustedTunables() + require.NoError(t, err) + assert.Equal(t, 100.0, tb.sidecarResources.CPUMultiplier) }) } diff --git a/pkg/vmcp/session/untrusted/addressing.go b/pkg/vmcp/session/untrusted/addressing.go index b08890852f..57ad3d2ff1 100644 --- a/pkg/vmcp/session/untrusted/addressing.go +++ b/pkg/vmcp/session/untrusted/addressing.go @@ -58,6 +58,8 @@ type podAddressResolver struct { admission Admission readyBudget time.Duration tokenStore *TokenStoreConfig + images *SidecarImages + sidecarRes *SidecarResourceOverride metrics *untrustedMetrics } @@ -71,15 +73,18 @@ func NewPodAddressResolver(lifecycle PodLifecycle, adm Admission, readyBudget ti } // podAddressResolverWithTokenStore is the internal constructor that also wires -// the egress-broker token-store coordinates and metrics into every provisioned -// pod. Kept unexported: production wiring goes through NewStack, which resolves -// the TokenStoreConfig and metrics once; NewPodAddressResolver stays the public -// seam for tests that exercise the resolver without a token store or metrics. +// the egress-broker token-store coordinates, sidecar images/resources, and +// metrics into every provisioned pod. Kept unexported: production wiring goes +// through NewStack, which resolves them once; NewPodAddressResolver stays the +// public seam for tests that exercise the resolver without them. func podAddressResolverWithTokenStore( lifecycle PodLifecycle, adm Admission, readyBudget time.Duration, ts *TokenStoreConfig, m *untrustedMetrics, + images *SidecarImages, sidecarRes *SidecarResourceOverride, ) *podAddressResolver { r := NewPodAddressResolver(lifecycle, adm, readyBudget).(*podAddressResolver) r.tokenStore = ts + r.images = images + r.sidecarRes = sidecarRes r.metrics = m return r } @@ -145,12 +150,14 @@ func (r *podAddressResolver) resolveOne( } pod, err := r.lifecycle.EnsurePod(ctx, EnsurePodRequest{ - Session: sess, - MCPServerUID: mcpserverUID, - MCPServerName: b.Name, - Port: port, - OwnerRefUID: types.UID(mcpserverUID), - TokenStore: r.tokenStore, + Session: sess, + MCPServerUID: mcpserverUID, + MCPServerName: b.Name, + Port: port, + OwnerRefUID: types.UID(mcpserverUID), + TokenStore: r.tokenStore, + Images: r.images, + SidecarResources: r.sidecarRes, OnNewPod: func(string) { if onNewPod != nil { onNewPod(b.ID) diff --git a/pkg/vmcp/session/untrusted/egress.go b/pkg/vmcp/session/untrusted/egress.go index f70f7c8d14..29099252fc 100644 --- a/pkg/vmcp/session/untrusted/egress.go +++ b/pkg/vmcp/session/untrusted/egress.go @@ -8,6 +8,8 @@ import ( "sort" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" "github.com/stacklok/toolhive/pkg/egressbroker" ) @@ -23,10 +25,26 @@ const ( // caKeyKey is the private-key data key of the CA Secret. caKeyKey = egressbroker.CAKeyKey - // EgressBrokerImage is the broker sidecar image (ext_authz + SDS). - EgressBrokerImage = "ghcr.io/stacklok/toolhive/egressbroker:latest" - // EnvoyProxyImage is the Envoy data-plane image. - EnvoyProxyImage = "envoyproxy/envoy:v1.36-latest" + // DefaultEgressBrokerImage is the default broker sidecar image + // (ext_authz + SDS). Bump with the ToolHive release that ships the + // broker binary; overridable per install via THV_UNTRUSTED_BROKER_IMAGE + // (resolved once in pkg/vmcp/cli/untrusted.go) for air-gapped mirrors. + DefaultEgressBrokerImage = "ghcr.io/stacklok/toolhive/egressbroker:v0.17.0" + // DefaultEnvoyProxyImage is the default Envoy data-plane image, pinned by + // tag AND digest — a floating tag would let the untrusted data plane run + // whatever upstream publishes next. Bump procedure: update tag+digest + // together (resolve the digest with `crane digest envoyproxy/envoy:`), + // then run `task operator-test`; the untrusted-egress chainsaw scenario + // must pass. Overridable per install via THV_UNTRUSTED_ENVOY_IMAGE for + // air-gapped mirrors. + DefaultEnvoyProxyImage = "envoyproxy/envoy:v1.36.2@sha256:4972515dd9a069b44beb43cebba7851596e72a8c61cd7a7c33d8f48efc5280ba" + + // BrokerHealthPort is the loopback-only health HTTP port (readiness + + // liveness probes). It is the single source of truth for the contract + // between the clone-time probes (here) and the broker's health listener: + // cmd/thv-egressbroker binds it directly (test-only import back into this + // package pins the two sides). + BrokerHealthPort = 15083 // brokerListenPort is the ext_authz/SDS gRPC port (loopback). brokerListenPort = 9001 @@ -63,6 +81,96 @@ const ( brokerContainerName = "thv-egressbroker" ) +// SidecarImages carries the egress data-plane images for one clone. Resolved +// once at the vMCP composition root (pkg/vmcp/cli/untrusted.go); a nil *Images +// field falls back to the pinned default. +type SidecarImages struct { + // EnvoyProxy is the Envoy data-plane image. Empty = DefaultEnvoyProxyImage. + EnvoyProxy string + // EgressBroker is the broker sidecar image (also the CA-seed init image). + // Empty = DefaultEgressBrokerImage. + EgressBroker string +} + +// resolved applies the pinned defaults. +func (s SidecarImages) resolved() SidecarImages { + if s.EnvoyProxy == "" { + s.EnvoyProxy = DefaultEnvoyProxyImage + } + if s.EgressBroker == "" { + s.EgressBroker = DefaultEgressBrokerImage + } + return s +} + +// SidecarResourceOverride tunes the envoy/broker sidecar container resources +// (the ca-seed init container stays fixed — it copies one file and exits). +// Multipliers scale the default requests and limits together; resolved once +// at the composition root from THV_UNTRUSTED_SIDECAR_CPU/MEM. +type SidecarResourceOverride struct { + // CPUMultiplier scales sidecar CPU requests/limits (1.0 = defaults). + // Must be > 0. + CPUMultiplier float64 + // MemoryMultiplier scales sidecar memory requests/limits (1.0 = defaults). + // Must be > 0. + MemoryMultiplier float64 +} + +// Default sidecar container resources (spec §3.2). Envoy's bump is +// connection-heavy; the Go broker is light; the CA-seed init copies one file. +var ( + defaultEnvoyResources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + } + defaultBrokerResources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("25m"), + corev1.ResourceMemory: resource.MustParse("32Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + // caSeedResources are fixed: the init container copies one file and exits. + caSeedResources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("32Mi"), + }, + } +) + +// scaledResources returns a copy of base with CPU quantities multiplied by +// cpuMult and memory quantities by memMult (1.0 = unchanged). +func scaledResources(base corev1.ResourceRequirements, cpuMult, memMult float64) corev1.ResourceRequirements { + out := *base.DeepCopy() + scale := func(list corev1.ResourceList, name corev1.ResourceName, mult float64) { + q, ok := list[name] + if !ok || mult == 1.0 { + return + } + q.SetMilli(int64(float64(q.MilliValue()) * mult)) + list[name] = q + } + for _, list := range []corev1.ResourceList{out.Requests, out.Limits} { + scale(list, corev1.ResourceCPU, cpuMult) + scale(list, corev1.ResourceMemory, memMult) + } + return out +} + // sidecarEnv is the deterministic (sorted) view of the shared annotation→env // contract (egressbroker.EnvToAnnotation): the broker's THV_UNTRUSTED_* env // vars are downward-API mirrors of the pod annotations this package stamps. @@ -108,12 +216,14 @@ var sidecarEnv = func() []struct { // ConfigMap) are operator-managed additions appended after verification. // // The broker's token-store env (THV_EGRESSBROKER_REDIS_ADDR / -// _REDIS_KEY_PREFIX, and the tokenenc KEK) is injected here from -// req.TokenStore. The address and key prefix are non-secret and are rendered -// as env literals; the KEK, when encryption is enabled, is referenced from a -// Secret via SecretKeyRef (never a ConfigMap or env literal). A nil TokenStore -// renders no token-store env at all — the broker then refuses to start -// (fail closed, per cmd/thv-egressbroker). +// _REDIS_KEY_PREFIX, and the tokenenc KEKs) is injected here from +// req.TokenStore. The address, key prefix, and active key ID are non-secret +// and are rendered as env literals; each KEK, when encryption is enabled, is +// referenced from a Secret via a per-ID SecretKeyRef (never a ConfigMap or env +// literal). A nil TokenStore renders no token-store env at all — the broker +// then refuses to start (fail closed, per cmd/thv-egressbroker). +// +//nolint:gocyclo // one flat pod-assembly walk: validate, volumes, init, envoy, broker, backend. func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { if pod == nil { return fmt.Errorf("untrusted egress wiring: pod must not be nil") @@ -127,6 +237,19 @@ func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { } } + images := SidecarImages{} + if req.Images != nil { + images = *req.Images + } + images = images.resolved() + res := SidecarResourceOverride{CPUMultiplier: 1.0, MemoryMultiplier: 1.0} + if req.SidecarResources != nil { + res = *req.SidecarResources + } + if res.CPUMultiplier <= 0 || res.MemoryMultiplier <= 0 { + return fmt.Errorf("untrusted egress wiring: sidecar resource multipliers must be positive") + } + // Generation-qualified CA objects: the pod mounts the exact CA generation // the operator published in the STS template annotation (cert+key in ONE // Secret), so a pod cloned mid-rotation can never get new-key+old-cert. @@ -164,11 +287,12 @@ func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { // gets a ConfigMap-backed path. pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{ Name: caSeedInitContainerName, - Image: EgressBrokerImage, + Image: images.EgressBroker, Command: []string{"sh", "-c"}, Args: []string{ fmt.Sprintf("cp /bundle/%s /ca/%s && chmod 0444 /ca/%s", caKeyCert, caFileName, caFileName), }, + Resources: caSeedResources, VolumeMounts: []corev1.VolumeMount{ {Name: caSharedVolumeName, MountPath: "/ca"}, {Name: caBundleVolumeName, MountPath: "/bundle", ReadOnly: true}, @@ -186,9 +310,10 @@ func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { // Envoy data plane: consumes the broker-rendered bootstrap. pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ - Name: envoyContainerName, - Image: EnvoyProxyImage, - Command: []string{"envoy", "-c", envoyConfigPath + "/envoy.yaml"}, + Name: envoyContainerName, + Image: images.EnvoyProxy, + Command: []string{"envoy", "-c", envoyConfigPath + "/envoy.yaml"}, + Resources: scaledResources(defaultEnvoyResources, res.CPUMultiplier, res.MemoryMultiplier), Ports: []corev1.ContainerPort{ {Name: "egress-proxy", ContainerPort: proxyPort}, }, @@ -219,32 +344,63 @@ func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { }) } // Token-store coordinates (Wave-3 deviation 1). Address/prefix are non-secret - // literals; the KEK comes from a Secret env reference so the key value never - // appears in the pod spec or a ConfigMap. A nil TokenStore renders none of - // these — the broker then fails closed at startup. + // literals; the KEKs come from per-ID Secret env references so the key values + // never appear in the pod spec or a ConfigMap. The broker keyring gets the + // FULL key-ID set (active + retired) so a key rotation never orphans rows + // sealed under the old ID. A nil TokenStore renders none of these — the + // broker then fails closed at startup. if req.TokenStore != nil { brokerEnv = append(brokerEnv, corev1.EnvVar{Name: "THV_EGRESSBROKER_REDIS_ADDR", Value: req.TokenStore.RedisAddr}, corev1.EnvVar{Name: "THV_EGRESSBROKER_REDIS_KEY_PREFIX", Value: req.TokenStore.KeyPrefix}, ) - if ref := req.TokenStore.KEKSecretRef; ref != nil { - brokerEnv = append(brokerEnv, corev1.EnvVar{ - Name: "THV_EGRESSBROKER_KEK", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: ref.Name}, - Key: ref.Key, + if req.TokenStore.KEKSecret != "" { + brokerEnv = append(brokerEnv, + corev1.EnvVar{Name: "THV_EGRESSBROKER_KEK_ID", Value: req.TokenStore.KEKActiveID}, + ) + for _, id := range req.TokenStore.KEKIDs { + brokerEnv = append(brokerEnv, corev1.EnvVar{ + Name: "THV_EGRESSBROKER_KEK_" + id, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: req.TokenStore.KEKSecret}, + Key: id, + }, }, - }, - }) + }) + } } } pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ - Name: brokerContainerName, - Image: EgressBrokerImage, - Env: brokerEnv, + Name: brokerContainerName, + Image: images.EgressBroker, + Env: brokerEnv, + Resources: scaledResources(defaultBrokerResources, res.CPUMultiplier, res.MemoryMultiplier), Ports: []corev1.ContainerPort{ {Name: "ext-authz", ContainerPort: brokerListenPort}, + {Name: "healthz", ContainerPort: BrokerHealthPort}, + }, + // Health contract: /healthz is 200 iff Redis is reachable AND the + // policy is loaded AND the bump CA is not past rotation-due. Readiness + // gates the pod; liveness (3 strikes) restarts a wedged broker. + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt32(BrokerHealthPort), + }, + }, + PeriodSeconds: 5, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt32(BrokerHealthPort), + }, + }, + PeriodSeconds: 5, + FailureThreshold: 3, }, VolumeMounts: []corev1.VolumeMount{ {Name: caSecretVolumeName, MountPath: "/ca-secret", ReadOnly: true}, diff --git a/pkg/vmcp/session/untrusted/egress_test.go b/pkg/vmcp/session/untrusted/egress_test.go index d9549769d2..c32ea8f7f0 100644 --- a/pkg/vmcp/session/untrusted/egress_test.go +++ b/pkg/vmcp/session/untrusted/egress_test.go @@ -205,12 +205,11 @@ func TestApplyEgressBrokerSidecar(t *testing.T) { t.Parallel() req := testRequest() req.TokenStore = &TokenStoreConfig{ - RedisAddr: "redis.auth:6379", - KeyPrefix: "thv:auth:{toolhive:my-vmcp}:", - KEKSecretRef: &SecretKeyRef{ - Name: "my-vmcp-token-kek", - Key: "kek", - }, + RedisAddr: "redis.auth:6379", + KeyPrefix: "thv:auth:{toolhive:my-vmcp}:", + KEKSecret: "my-vmcp-token-kek", + KEKActiveID: "kek-2", + KEKIDs: []string{"kek-1", "kek-2"}, } pod, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") require.NoError(t, err) @@ -223,13 +222,19 @@ func TestApplyEgressBrokerSidecar(t *testing.T) { assert.Equal(t, "redis.auth:6379", env["THV_EGRESSBROKER_REDIS_ADDR"].Value) assert.Equal(t, "thv:auth:{toolhive:my-vmcp}:", env["THV_EGRESSBROKER_REDIS_KEY_PREFIX"].Value) - // The KEK is a Secret env reference, never a literal or ConfigMap. - kek := env["THV_EGRESSBROKER_KEK"] - require.NotNil(t, kek.ValueFrom, "KEK must come from a Secret env reference") - require.NotNil(t, kek.ValueFrom.SecretKeyRef) - assert.Equal(t, "my-vmcp-token-kek", kek.ValueFrom.SecretKeyRef.Name) - assert.Equal(t, "kek", kek.ValueFrom.SecretKeyRef.Key) - assert.Empty(t, kek.Value, "KEK value must never be a pod-spec literal") + // The active key ID is a non-secret literal. + assert.Equal(t, "kek-2", env["THV_EGRESSBROKER_KEK_ID"].Value) + + // Every key ID (active + retired) is a per-ID Secret env reference, + // never a literal or ConfigMap. + for _, id := range []string{"kek-1", "kek-2"} { + kek := env["THV_EGRESSBROKER_KEK_"+id] + require.NotNil(t, kek.ValueFrom, "KEK %s must come from a Secret env reference", id) + require.NotNil(t, kek.ValueFrom.SecretKeyRef) + assert.Equal(t, "my-vmcp-token-kek", kek.ValueFrom.SecretKeyRef.Name) + assert.Equal(t, id, kek.ValueFrom.SecretKeyRef.Key) + assert.Empty(t, kek.Value, "KEK value must never be a pod-spec literal") + } }) t.Run("nil TokenStore renders no token-store env (broker fails closed)", func(t *testing.T) { @@ -241,8 +246,10 @@ func TestApplyEgressBrokerSidecar(t *testing.T) { assert.NotContains(t, []string{ "THV_EGRESSBROKER_REDIS_ADDR", "THV_EGRESSBROKER_REDIS_KEY_PREFIX", - "THV_EGRESSBROKER_KEK", + "THV_EGRESSBROKER_KEK_ID", }, e.Name, "no token-store env without a TokenStore config") + assert.NotContains(t, e.Name, "THV_EGRESSBROKER_KEK_", + "no per-ID KEK env without a TokenStore config") } }) @@ -254,9 +261,21 @@ func TestApplyEgressBrokerSidecar(t *testing.T) { }{ {"empty addr", &TokenStoreConfig{KeyPrefix: "thv:auth:{a:b}:"}}, {"prefix missing colon", &TokenStoreConfig{RedisAddr: "r:6379", KeyPrefix: "thv:auth"}}, - {"kek ref missing key", &TokenStoreConfig{ + {"kek coords partial", &TokenStoreConfig{ + RedisAddr: "r:6379", KeyPrefix: "thv:auth:{a:b}:", + KEKSecret: "s", + }}, + {"active ID not in key set", &TokenStoreConfig{ + RedisAddr: "r:6379", KeyPrefix: "thv:auth:{a:b}:", + KEKSecret: "s", KEKActiveID: "kek-9", KEKIDs: []string{"kek-1"}, + }}, + {"duplicate key ID", &TokenStoreConfig{ + RedisAddr: "r:6379", KeyPrefix: "thv:auth:{a:b}:", + KEKSecret: "s", KEKActiveID: "kek-1", KEKIDs: []string{"kek-1", "kek-1"}, + }}, + {"key ID unsafe as env suffix", &TokenStoreConfig{ RedisAddr: "r:6379", KeyPrefix: "thv:auth:{a:b}:", - KEKSecretRef: &SecretKeyRef{Name: "s"}, + KEKSecret: "s", KEKActiveID: "kek.1", KEKIDs: []string{"kek.1"}, }}, } { t.Run(tc.name, func(t *testing.T) { @@ -268,4 +287,130 @@ func TestApplyEgressBrokerSidecar(t *testing.T) { }) } }) + + t.Run("sidecar images are pinned by default and overridable per clone", func(t *testing.T) { + t.Parallel() + pod := newPod() + assert.Equal(t, DefaultEnvoyProxyImage, findContainer(t, pod, envoyContainerName).Image) + assert.Equal(t, DefaultEgressBrokerImage, findContainer(t, pod, brokerContainerName).Image) + assert.Contains(t, DefaultEnvoyProxyImage, "@sha256:", + "the default Envoy image must be digest-pinned (supply chain)") + var init *corev1.Container + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == caSeedInitContainerName { + init = &pod.Spec.InitContainers[i] + } + } + require.NotNil(t, init) + assert.Equal(t, DefaultEgressBrokerImage, init.Image) + + req := testRequest() + req.Images = &SidecarImages{EnvoyProxy: "mirror.local/envoy:v1", EgressBroker: "mirror.local/broker:v2"} + overridden, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.NoError(t, err) + assert.Equal(t, "mirror.local/envoy:v1", findContainer(t, overridden, envoyContainerName).Image) + assert.Equal(t, "mirror.local/broker:v2", findContainer(t, overridden, brokerContainerName).Image) + }) + + t.Run("sidecar containers carry resource requests and limits", func(t *testing.T) { + t.Parallel() + pod := newPod() + envoy := findContainer(t, pod, envoyContainerName) + broker := findContainer(t, pod, brokerContainerName) + + assert.Equal(t, "50m", envoy.Resources.Requests.Cpu().String()) + assert.Equal(t, "64Mi", envoy.Resources.Requests.Memory().String()) + assert.Equal(t, "500m", envoy.Resources.Limits.Cpu().String()) + assert.Equal(t, "256Mi", envoy.Resources.Limits.Memory().String()) + assert.Equal(t, "25m", broker.Resources.Requests.Cpu().String()) + assert.Equal(t, "32Mi", broker.Resources.Requests.Memory().String()) + assert.Equal(t, "250m", broker.Resources.Limits.Cpu().String()) + assert.Equal(t, "128Mi", broker.Resources.Limits.Memory().String()) + + var init *corev1.Container + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == caSeedInitContainerName { + init = &pod.Spec.InitContainers[i] + } + } + require.NotNil(t, init) + assert.Equal(t, "10m", init.Resources.Requests.Cpu().String()) + assert.Equal(t, "16Mi", init.Resources.Requests.Memory().String()) + assert.Equal(t, "50m", init.Resources.Limits.Cpu().String()) + assert.Equal(t, "32Mi", init.Resources.Limits.Memory().String()) + }) + + t.Run("resource multipliers scale envoy+broker, never the ca-seed init", func(t *testing.T) { + t.Parallel() + req := testRequest() + req.SidecarResources = &SidecarResourceOverride{CPUMultiplier: 2.0, MemoryMultiplier: 4.0} + pod, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.NoError(t, err) + + envoy := findContainer(t, pod, envoyContainerName) + assert.Equal(t, "100m", envoy.Resources.Requests.Cpu().String()) + assert.Equal(t, "256Mi", envoy.Resources.Requests.Memory().String()) + assert.Equal(t, "1", envoy.Resources.Limits.Cpu().String()) + broker := findContainer(t, pod, brokerContainerName) + assert.Equal(t, "50m", broker.Resources.Requests.Cpu().String()) + + var init *corev1.Container + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == caSeedInitContainerName { + init = &pod.Spec.InitContainers[i] + } + } + require.NotNil(t, init) + assert.Equal(t, "10m", init.Resources.Requests.Cpu().String(), "init resources stay fixed") + }) + + t.Run("partial multiplier scales only its dimension", func(t *testing.T) { + t.Parallel() + req := testRequest() + req.SidecarResources = &SidecarResourceOverride{CPUMultiplier: 2.0, MemoryMultiplier: 1.0} + pod, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.NoError(t, err) + + envoy := findContainer(t, pod, envoyContainerName) + assert.Equal(t, "100m", envoy.Resources.Requests.Cpu().String(), "CPU scaled 2x") + assert.Equal(t, "64Mi", envoy.Resources.Requests.Memory().String(), "memory untouched") + assert.Equal(t, "256Mi", envoy.Resources.Limits.Memory().String(), "memory limit untouched") + }) + + t.Run("non-positive resource multipliers fail loudly", func(t *testing.T) { + t.Parallel() + for _, mult := range []SidecarResourceOverride{ + {CPUMultiplier: 0, MemoryMultiplier: 1}, + {CPUMultiplier: 1, MemoryMultiplier: -1}, + } { + req := testRequest() + m := mult + req.SidecarResources = &m + _, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") + require.Error(t, err) + } + }) + + t.Run("broker container gets readiness and liveness probes on the health port", func(t *testing.T) { + t.Parallel() + pod := newPod() + broker := findContainer(t, pod, brokerContainerName) + + for name, probe := range map[string]*corev1.Probe{ + "readiness": broker.ReadinessProbe, + "liveness": broker.LivenessProbe, + } { + require.NotNil(t, probe, "%s probe must be set", name) + require.NotNil(t, probe.HTTPGet, "%s probe must be httpGet", name) + assert.Equal(t, "/healthz", probe.HTTPGet.Path) + assert.Equal(t, int32(BrokerHealthPort), probe.HTTPGet.Port.IntVal) + assert.Equal(t, int32(5), probe.PeriodSeconds, "%s probe period", name) + } + assert.Equal(t, int32(3), broker.LivenessProbe.FailureThreshold) + + // Envoy and the backend container get no probes (the broker's health + // is the pod's egress data-plane liveness contract). + assert.Nil(t, findContainer(t, pod, envoyContainerName).ReadinessProbe) + assert.Nil(t, findContainer(t, pod, "mcp").ReadinessProbe) + }) } diff --git a/pkg/vmcp/session/untrusted/lifecycle.go b/pkg/vmcp/session/untrusted/lifecycle.go index 837991768b..86c935e1de 100644 --- a/pkg/vmcp/session/untrusted/lifecycle.go +++ b/pkg/vmcp/session/untrusted/lifecycle.go @@ -55,6 +55,12 @@ type EnsurePodRequest struct { // broker without a token store (it fails closed at startup — only valid // where the broker is not expected to inject credentials). TokenStore *TokenStoreConfig + // Images overrides the egress data-plane images (Wave-5 supply-chain + // pinning). Nil = the pinned defaults in egress.go. + Images *SidecarImages + // SidecarResources overrides the envoy/broker sidecar resource + // multipliers (Wave-5). Nil = defaults. + SidecarResources *SidecarResourceOverride // OnNewPod, when non-nil, is invoked exactly once per fresh pod create // with the deterministic pod name (used by the resolver to drop stale // session hints). Not invoked when an existing pod is reused. @@ -70,8 +76,9 @@ const ( waitReadyInitialPoll = 500 * time.Millisecond waitReadyMaxPoll = 5 * time.Second - // readinessTimeout is the reaper's failed-cold-start threshold. - readinessTimeout = DefaultReadyBudget + // defaultReadinessTimeout is the reaper's failed-cold-start threshold + // (overridable via ReaperConfig.ReadinessTimeout). + defaultReadinessTimeout = DefaultReadyBudget // zombieHeartbeatGrace is how long a missing vMCP heartbeat is tolerated // before the zombie rule deletes a pod (2x the 5-minute heartbeat TTL). diff --git a/pkg/vmcp/session/untrusted/reaper.go b/pkg/vmcp/session/untrusted/reaper.go index 214ca514c8..d7294330bb 100644 --- a/pkg/vmcp/session/untrusted/reaper.go +++ b/pkg/vmcp/session/untrusted/reaper.go @@ -28,6 +28,9 @@ type ReaperConfig struct { HeartbeatInterval time.Duration // HeartbeatTTL is the expiry of the liveness key. Default 5m. HeartbeatTTL time.Duration + // ReadinessTimeout is the failed-cold-start threshold: a pod older than + // this and not Ready is deleted. Default 120s (DefaultReadyBudget). + ReadinessTimeout time.Duration } const ( @@ -50,6 +53,9 @@ func (c ReaperConfig) resolved() ReaperConfig { if c.HeartbeatTTL <= 0 { c.HeartbeatTTL = defaultHeartbeatTTL } + if c.ReadinessTimeout <= 0 { + c.ReadinessTimeout = defaultReadinessTimeout + } return c } @@ -59,8 +65,8 @@ func (c ReaperConfig) resolved() ReaperConfig { // deletes are harmless — Delete is idempotent, NotFound ignored). // // Rules enforced each tick: -// - readiness timeout: pod older than 120s and not Ready → delete -// (failed cold start); +// - readiness timeout: pod older than ReadinessTimeout (default 120s) and +// not Ready → delete (failed cold start); // - idle TTL: podttl lease absent → delete (session gone); // - zombie: pod's vMCP heartbeat is absent and has been for the grace // period → delete (owning vMCP died, no replica adopted the pod); @@ -189,7 +195,7 @@ func (r *Reaper) evaluatePod( // Rule 1: readiness timeout (failed cold start). Pure-K8s evidence; no // Redis read needed. age := r.now().Sub(pod.CreationTimestamp.Time) - if age > readinessTimeout && !isPodReady(pod) { + if age > r.cfg.ReadinessTimeout && !isPodReady(pod) { r.delete(ctx, pod, "readiness timeout") return nil } diff --git a/pkg/vmcp/session/untrusted/tokenstore.go b/pkg/vmcp/session/untrusted/tokenstore.go index 2e334a4dcb..889a109a5c 100644 --- a/pkg/vmcp/session/untrusted/tokenstore.go +++ b/pkg/vmcp/session/untrusted/tokenstore.go @@ -5,8 +5,14 @@ package untrusted import ( "fmt" + "regexp" ) +// kekIDPattern bounds key IDs to the charset that is safe as a broker env-var +// suffix (THV_EGRESSBROKER_KEK_) — the sidecar never sanitizes IDs, so a +// malformed ID must be rejected here at clone time. +var kekIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + // TokenStoreConfig carries the coordinates the egress-broker sidecar needs to // reach the auth-server's upstream token store, injected into the cloned pod at // clone time (pkg/vmcp/session/untrusted/egress.go). Only non-secret @@ -24,27 +30,30 @@ type TokenStoreConfig struct { // KeyPrefix is the auth-server per-tenant key prefix (e.g. // "thv:auth:{ns:name}:"). Required; must end with ':'. KeyPrefix string - // KEKSecretRef, when non-nil, names the Secret + key holding the base64 - // token-encryption KEK (32 bytes decoded). The sidecar mounts it as an env - // SecretKeyRef — the KEK value is never placed in a ConfigMap or pod env - // literal. Nil means token rows are read unencrypted (legacy plaintext). - KEKSecretRef *SecretKeyRef -} - -// SecretKeyRef is a minimal (name, key) reference into a Secret, mirroring the -// subset of corev1.SecretKeySelector the wiring needs without coupling the -// config to a specific source. -type SecretKeyRef struct { - // Name is the Secret name. Required. - Name string - // Key is the data key within the Secret. Required. - Key string + // KEKSecret, when non-empty, names the Secret holding the base64 + // token-encryption KEKs (one data entry per key ID, 32 bytes decoded). + // Every ID in KEKIDs is mounted as a per-ID SecretKeyRef env on the + // sidecar — the KEK values are never placed in a ConfigMap or pod env + // literal. Empty means token rows are read unencrypted (legacy + // plaintext). + KEKSecret string + // KEKActiveID is the key ID the auth server encrypts NEW writes under. + // It must be one of KEKIDs — if it drifts (e.g. the operator rotated + // activeKeyId) and the sidecar keyring only knew a hardcoded ID, every + // injection would deny. + KEKActiveID string + // KEKIDs is the full set of key IDs the sidecar keyring must know: the + // active ID plus every retired ID rows may still be sealed under. A + // single-ID set (no rotation history) is the common case. + KEKIDs []string } // validate enforces the fail-closed contract: the sidecar must never be given // partial token-store coordinates (it would crash-loop on a malformed prefix // or dial a default address). An entirely-empty config is valid and means // "no token store wired" (the broker is then expected to be absent). +// +//nolint:gocyclo // sequential fail-loud coordinate validation is clearest linear. func (c *TokenStoreConfig) validate() error { if c == nil { return fmt.Errorf("untrusted token store: config must not be nil") @@ -55,10 +64,32 @@ func (c *TokenStoreConfig) validate() error { if c.KeyPrefix == "" || c.KeyPrefix[len(c.KeyPrefix)-1] != ':' { return fmt.Errorf("untrusted token store: KeyPrefix must be non-empty and end with ':'") } - if c.KEKSecretRef != nil { - if c.KEKSecretRef.Name == "" || c.KEKSecretRef.Key == "" { - return fmt.Errorf("untrusted token store: KEKSecretRef requires both Name and Key") + // KEK coordinates are all-or-nothing: secret name + active ID + at least + // one key ID, with the active ID a member of the set. + if c.KEKSecret == "" && c.KEKActiveID == "" && len(c.KEKIDs) == 0 { + return nil + } + if c.KEKSecret == "" || c.KEKActiveID == "" || len(c.KEKIDs) == 0 { + return fmt.Errorf("untrusted token store: KEK coordinates are all-or-nothing " + + "(KEKSecret, KEKActiveID, and at least one KEKIDs entry are required together)") + } + seen := make(map[string]struct{}, len(c.KEKIDs)) + activeInSet := false + for _, id := range c.KEKIDs { + if !kekIDPattern.MatchString(id) { + return fmt.Errorf("untrusted token store: KEK key ID %q must match [A-Za-z0-9_-]+ "+ + "(it becomes a broker env-var suffix)", id) + } + if _, dup := seen[id]; dup { + return fmt.Errorf("untrusted token store: duplicate KEK key ID %q", id) } + seen[id] = struct{}{} + if id == c.KEKActiveID { + activeInSet = true + } + } + if !activeInSet { + return fmt.Errorf("untrusted token store: KEKActiveID %q must be one of KEKIDs", c.KEKActiveID) } return nil } diff --git a/pkg/vmcp/session/untrusted/wiring.go b/pkg/vmcp/session/untrusted/wiring.go index 2e3f5e5a26..176f6c84ff 100644 --- a/pkg/vmcp/session/untrusted/wiring.go +++ b/pkg/vmcp/session/untrusted/wiring.go @@ -53,6 +53,12 @@ type WiringConfig struct { // token-store coordinates into every provisioned pod. Nil leaves the broker // without a token store (it fails closed at startup). TokenStore *TokenStoreConfig + // Images overrides the egress data-plane images (supply-chain pinning, + // Wave-5). Nil = the pinned defaults in egress.go. + Images *SidecarImages + // SidecarResources overrides the envoy/broker sidecar resource + // multipliers. Nil = defaults. Values must be positive (fail loudly). + SidecarResources *SidecarResourceOverride // MeterProvider, when non-nil, registers the untrusted-mode OTel instruments // (untrusted_backend_pods gauge, untrusted_pod_admissions_total counter). // Nil disables metrics. @@ -107,6 +113,10 @@ func NewStack(cfg WiringConfig) (*Stack, error) { return nil, err } } + if cfg.SidecarResources != nil && + (cfg.SidecarResources.CPUMultiplier <= 0 || cfg.SidecarResources.MemoryMultiplier <= 0) { + return nil, fmt.Errorf("untrusted wiring: sidecar resource multipliers must be positive") + } var metrics *untrustedMetrics if cfg.MeterProvider != nil { metrics, err = newUntrustedMetrics(cfg.MeterProvider) @@ -125,7 +135,8 @@ func NewStack(cfg WiringConfig) (*Stack, error) { if err != nil { return nil, err } - resolver := podAddressResolverWithTokenStore(lifecycle, adm, cfg.ReadyBudget, cfg.TokenStore, metrics) + resolver := podAddressResolverWithTokenStore( + lifecycle, adm, cfg.ReadyBudget, cfg.TokenStore, metrics, cfg.Images, cfg.SidecarResources) reaper, err := NewReaper(cfg.K8sClient, store, cfg.Reaper, cfg.VMCPUId, metrics) if err != nil { return nil, err diff --git a/pkg/vmcp/session/untrusted/wiring_test.go b/pkg/vmcp/session/untrusted/wiring_test.go new file mode 100644 index 0000000000..0e1d5b5926 --- /dev/null +++ b/pkg/vmcp/session/untrusted/wiring_test.go @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/stacklok/toolhive/pkg/vmcp" +) + +// newWiringTestScheme builds the scheme the lifecycle's K8s client needs +// (core pods + the MCPServer CRD is unnecessary here — the mock lifecycle +// never touches the cluster, so an empty core scheme is enough). +func newWiringTestK8sClient(t *testing.T) *fake.ClientBuilder { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + return fake.NewClientBuilder().WithScheme(scheme) +} + +// TestNewStack_WiresTunables pins the Wave-5 tunables → WiringConfig → +// resolver flow: the images and sidecar-resource multipliers the composition +// root resolves from THV_UNTRUSTED_* env must reach the EnsurePodRequest the +// resolver issues for every provisioned pod (they are consumed by +// applyEgressBrokerSidecar at clone time). +func TestNewStack_WiresTunables(t *testing.T) { + t.Parallel() + + mr := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = redisClient.Close() }) + + k8sClient := newWiringTestK8sClient(t).Build() + images := &SidecarImages{EnvoyProxy: "mirror.local/envoy:v1", EgressBroker: "mirror.local/broker:v2"} + sidecarRes := &SidecarResourceOverride{CPUMultiplier: 2.0, MemoryMultiplier: 1.5} + tokenStore := &TokenStoreConfig{ + RedisAddr: "redis.auth:6379", + KeyPrefix: "thv:auth:{toolhive:my-vmcp}:", + KEKSecret: "my-vmcp-kek", + KEKActiveID: "kek-1", + KEKIDs: []string{"kek-1"}, + } + + stack, err := NewStack(WiringConfig{ + K8sClient: k8sClient, + RedisClient: redisClient, + KeyPrefix: "thv:vmcp:session:", + Namespace: "toolhive", + VMCPUId: "vmcp-1", + Admission: AdmissionConfig{CacheCapacity: 100}, + TokenStore: tokenStore, + Images: images, + SidecarResources: sidecarRes, + }) + require.NoError(t, err) + require.NotNil(t, stack) + + // The resolver must carry the tunables into the EnsurePodRequest. Swap in + // a capturing lifecycle to observe the request without a cluster. + resolver, ok := stack.Resolver.(*podAddressResolver) + require.True(t, ok, "NewStack must return the production podAddressResolver") + + var captured EnsurePodRequest + resolver.lifecycle = captureLifecycle{pod: &corev1.Pod{}, capture: &captured} + + backend := &vmcp.Backend{ + ID: "github-mcp", Name: "github-mcp", BaseURL: "http://svc:8080/mcp", + TransportType: "streamable-http", + Metadata: map[string]string{ + MetadataKeyUntrusted: "true", + MetadataKeyMCPServerUID: "uid-abc", + }, + } + sess := SessionRef{SessionID: "sess-1", Issuer: "https://iss", Subject: "sub", Namespace: "toolhive"} + out := resolver.ResolveTargets(context.Background(), sess, []*vmcp.Backend{backend}, nil) + require.Len(t, out, 1, "the untrusted backend must resolve through the capturing lifecycle") + + assert.Same(t, tokenStore, captured.TokenStore, "token-store coordinates must flow to the clone request") + assert.Same(t, images, captured.Images, "image overrides must flow to the clone request") + assert.Same(t, sidecarRes, captured.SidecarResources, "resource multipliers must flow to the clone request") +} + +// TestNewStack_RejectsInvalidTunables pins fail-loud wiring: a non-positive +// sidecar multiplier or an invalid TokenStore is a startup error, not a +// silently-defaulted clone. +func TestNewStack_RejectsInvalidTunables(t *testing.T) { + t.Parallel() + + base := func() WiringConfig { + mr := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + t.Cleanup(func() { _ = redisClient.Close() }) + return WiringConfig{ + K8sClient: newWiringTestK8sClient(t).Build(), + RedisClient: redisClient, + KeyPrefix: "thv:vmcp:session:", + Namespace: "toolhive", + VMCPUId: "vmcp-1", + Admission: AdmissionConfig{CacheCapacity: 100}, + } + } + + t.Run("zero CPU multiplier", func(t *testing.T) { + t.Parallel() + cfg := base() + cfg.SidecarResources = &SidecarResourceOverride{CPUMultiplier: 0, MemoryMultiplier: 1} + _, err := NewStack(cfg) + require.Error(t, err) + }) + + t.Run("negative memory multiplier", func(t *testing.T) { + t.Parallel() + cfg := base() + cfg.SidecarResources = &SidecarResourceOverride{CPUMultiplier: 1, MemoryMultiplier: -2} + _, err := NewStack(cfg) + require.Error(t, err) + }) + + t.Run("partial KEK coordinates", func(t *testing.T) { + t.Parallel() + cfg := base() + cfg.TokenStore = &TokenStoreConfig{ + RedisAddr: "redis.auth:6379", KeyPrefix: "thv:auth:{a:b}:", KEKSecret: "s", + } + _, err := NewStack(cfg) + require.Error(t, err) + }) + + t.Run("nil images and nil resources wire the pinned defaults downstream", func(t *testing.T) { + t.Parallel() + cfg := base() + stack, err := NewStack(cfg) + require.NoError(t, err) + resolver, ok := stack.Resolver.(*podAddressResolver) + require.True(t, ok) + assert.Nil(t, resolver.images, "nil images fall back to the pinned defaults at clone time") + assert.Nil(t, resolver.sidecarRes, "nil resources fall back to the defaults at clone time") + }) +} + +// captureLifecycle is a PodLifecycle stub that records the EnsurePodRequest +// and reports an immediately-ready pod. +type captureLifecycle struct { + pod *corev1.Pod + capture *EnsurePodRequest +} + +func (c captureLifecycle) EnsurePod(_ context.Context, req EnsurePodRequest) (*corev1.Pod, error) { + *c.capture = req + return c.pod, nil +} + +func (captureLifecycle) WaitReady(_ context.Context, _ *corev1.Pod, _ time.Duration) error { + return nil +} + +func (captureLifecycle) DeletePod(_ context.Context, _ string) error { return nil } + +var _ PodLifecycle = captureLifecycle{} From 0e2f7f3bd9654baec8c5d873c906ad2c3f90b17e Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 22 Jul 2026 16:29:43 +0300 Subject: [PATCH 05/12] Add response scanning and audit to the egress broker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the last credential channel (ADR-0001 D6c): an upstream response that echoes the injected credential back to the untrusted server is now suppressed. An Envoy ext_proc filter sends response headers and buffered bodies to the broker, which scans for the exact and base64 forms of the token it injected (correlated via Envoy-generated x-request-id through a Lua metadata filter) and replaces matches with a generic 502. Fail-open by default with a THV_EGRESSBROKER_SCAN_FAIL_CLOSED opt-in that also covers unknown request IDs and oversize bodies in-band. Byte-exact scanning is a tripwire, not a proof: encoding transforms evade it, and the ADR now says so — the destination allowlist remains the real boundary. The broker now emits structured audit events for every injection, denial (with reason), and suppressed leak — sub-hashed, never token material — plus low-cardinality OTel counters for scan results, denials, and injections (D11). Review-hardened: correlation metadata actually substituted (Lua filter, not literal %REQ% route metadata), fail-closed posture wired in-band, graceful-stop bounded with a hard-stop fallback, explicit OVERWRITE_IF_EXISTS_OR_ADD on credential injection, deny-reason split asserted, TokenMap race-tested under flood and same-ID contention, and the session-Redis password plumbed to the broker as a SecretKeyRef. --- cmd/thv-egressbroker/main.go | 96 +- .../virtualmcpserver_deployment.go | 33 + .../virtualmcpserver_rbac_untrusted_test.go | 54 +- .../adr/0001-untrusted-mcp-egress-broker.md | 6 + docs/server/docs.go | 244 ++--- docs/server/swagger.json | 244 ++--- docs/server/swagger.yaml | 213 ++-- pkg/egressbroker/audit.go | 183 ++++ pkg/egressbroker/audit_test.go | 188 ++++ pkg/egressbroker/config.go | 72 +- pkg/egressbroker/envoyconfig.go | 87 +- pkg/egressbroker/envoyconfig_test.go | 129 ++- pkg/egressbroker/extproc.go | 243 +++++ pkg/egressbroker/extproc_test.go | 923 ++++++++++++++++++ pkg/egressbroker/identity.go | 10 + pkg/egressbroker/injector.go | 111 ++- pkg/egressbroker/injector_test.go | 58 +- pkg/egressbroker/metrics.go | 139 +++ pkg/egressbroker/policy.go | 23 +- pkg/egressbroker/requestid.go | 53 + pkg/egressbroker/scanner.go | 123 +++ pkg/egressbroker/server.go | 115 ++- pkg/egressbroker/tokenmap.go | 176 ++++ pkg/vmcp/cli/untrusted.go | 28 + pkg/vmcp/cli/untrusted_test.go | 18 + pkg/vmcp/session/untrusted/egress.go | 23 + pkg/vmcp/session/untrusted/egress_test.go | 25 +- pkg/vmcp/session/untrusted/tokenstore.go | 13 + 28 files changed, 3215 insertions(+), 415 deletions(-) create mode 100644 pkg/egressbroker/audit.go create mode 100644 pkg/egressbroker/audit_test.go create mode 100644 pkg/egressbroker/extproc.go create mode 100644 pkg/egressbroker/extproc_test.go create mode 100644 pkg/egressbroker/metrics.go create mode 100644 pkg/egressbroker/requestid.go create mode 100644 pkg/egressbroker/scanner.go create mode 100644 pkg/egressbroker/tokenmap.go diff --git a/cmd/thv-egressbroker/main.go b/cmd/thv-egressbroker/main.go index 17ade51db1..717f2f0371 100644 --- a/cmd/thv-egressbroker/main.go +++ b/cmd/thv-egressbroker/main.go @@ -24,6 +24,7 @@ import ( "time" goredis "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel" "github.com/stacklok/toolhive/pkg/auth/upstreamtoken" "github.com/stacklok/toolhive/pkg/authserver/storage" @@ -60,6 +61,9 @@ const ( envEnvoyBootstrapOut = "THV_EGRESSBROKER_ENVOY_BOOTSTRAP_OUT" // healthPingTimeout bounds the Redis reachability check per probe. healthPingTimeout = 2 * time.Second + // scanEvictionSweep is the background TTL sweep cadence for the D6c + // scan-correlation map. + scanEvictionSweep = 30 * time.Second ) func main() { @@ -107,19 +111,7 @@ func run() error { return err } - injector, err := egressbroker.NewCredentialInjector(resolver.PodIdentity(), policy, tokens) - if err != nil { - return err - } - authz, err := egressbroker.NewAuthorizationServer(injector) - if err != nil { - return err - } - sds, err := egressbroker.NewSecretDiscoveryServer(ca, policy) - if err != nil { - return err - } - server, err := egressbroker.NewServer(cfg.ListenAddress, cfg.ListenPort, authz, sds) + server, err := buildGRPCServer(ctx, cfg, resolver, policy, ca, tokens) if err != nil { return err } @@ -145,6 +137,58 @@ func run() error { return server.Run(ctx) } +// buildGRPCServer wires the broker's gRPC surface: the D6c response-scan +// correlation map shared by ext_authz (record at injection) and ext_proc +// (scan on response), the D11 audit/metrics sinks, the injector, and the +// three Envoy services (ext_authz, SDS, ext_proc) on one loopback socket. +func buildGRPCServer( + ctx context.Context, + cfg *egressbroker.Config, + resolver *egressbroker.PodIdentityResolver, + policy *egressbroker.EgressPolicy, + ca *egressbroker.BumpCA, + tokens *tokenStore, +) (*egressbroker.Server, error) { + scanMap, err := egressbroker.NewTokenMap(egressbroker.ScanCorrelationTTL, egressbroker.ScanCorrelationMaxEntries) + if err != nil { + return nil, err + } + go scanMap.RunEvictionLoop(ctx, scanEvictionSweep) + auditLog := egressbroker.NewAuditLogger() + brokerMetrics, err := egressbroker.NewBrokerMetrics(otel.GetMeterProvider()) + if err != nil { + return nil, err + } + podName := resolver.PodName(os.Getenv) + + injector, err := egressbroker.NewCredentialInjector(resolver.PodIdentity(), policy, tokens) + if err != nil { + return nil, err + } + injector.WithScanCorrelation(scanMap).WithObservability(brokerMetrics, auditLog, podName) + authz, err := egressbroker.NewAuthorizationServer(injector) + if err != nil { + return nil, err + } + authz.WithObservability(auditLog, brokerMetrics, resolver.PodIdentity(), podName) + sds, err := egressbroker.NewSecretDiscoveryServer(ca, policy) + if err != nil { + return nil, err + } + extproc, err := egressbroker.NewExternalProcessorServer( + scanMap, + egressbroker.ScannerBounds{MaxBodyBytes: cfg.ScanMaxBodyBytes}, + !cfg.ScanFailClosed, // failOpen: the documented default (D6c), inverted config knob + resolver.PodIdentity(), + brokerMetrics, + auditLog, + ) + if err != nil { + return nil, err + } + return egressbroker.NewServer(cfg.ListenAddress, cfg.ListenPort, authz, sds, extproc) +} + // redisPinger abstracts the Redis reachability check for tests. type redisPinger func(ctx context.Context) error @@ -219,17 +263,20 @@ func (h *healthServer) handle(w http.ResponseWriter, r *http.Request) { // shared config emptyDir, when configured. The bootstrap's route table // carries exactly the policy's host allowlist (D6b); the ext_authz cluster // points at this process (loopback); no redirect-following filter exists -// (D6a). +// (D6a). The ext_proc scanner's failure_mode_allow is the inverse of the +// broker's fail-closed setting (D6c documented fail-open default). func renderEnvoyBootstrap(cfg *egressbroker.Config, policy *egressbroker.EgressPolicy) error { out := strings.TrimSpace(os.Getenv(envEnvoyBootstrapOut)) if out == "" { return nil } return egressbroker.WriteEnvoyBootstrap(out, egressbroker.EnvoyConfigParams{ - ExtAuthzAddress: cfg.ListenAddress, - ExtAuthzPort: cfg.ListenPort, - ProxyPort: egressbroker.DefaultProxyPort, - AllowedHosts: policy.HostAllowlist(), + ExtAuthzAddress: cfg.ListenAddress, + ExtAuthzPort: cfg.ListenPort, + ProxyPort: egressbroker.DefaultProxyPort, + AllowedHosts: policy.HostAllowlist(), + ScanFailOpen: !cfg.ScanFailClosed, + ScanMaxBodyBytes: cfg.ScanMaxBodyBytes, }) } @@ -246,6 +293,11 @@ type tokenStore struct { // ever dialing outside the policy's destination set. No refresh is wired (the // broker holds no OAuth client material): expired rows surface on the failed // list and deny with "re-consent required" (Wave 4 consent UX). +// +// The Redis ACL password arrives via vmcpconfig.RedisPasswordEnvVar, +// rendered at clone time as a SecretKeyRef env (never a literal). It is +// REQUIRED: without it every Redis call fails AUTH and every credential +// injection denies — fail loud at startup instead of crash-looping on 403s. func buildTokenReader(_ context.Context, dialGuard *egressbroker.IPAllowlist) (*tokenStore, error) { addr := strings.TrimSpace(os.Getenv(envRedisAddr)) keyPrefix := strings.TrimSpace(os.Getenv(envRedisKeyPrefix)) @@ -253,6 +305,12 @@ func buildTokenReader(_ context.Context, dialGuard *egressbroker.IPAllowlist) (* return nil, fmt.Errorf("egressbroker: %s and %s must be set (upstream token store)", envRedisAddr, envRedisKeyPrefix) } + password := os.Getenv(vmcpconfig.RedisPasswordEnvVar) + if password == "" { + return nil, fmt.Errorf("egressbroker: %s must be set (Redis ACL password for the upstream token store; "+ + "the clone wiring renders it from the auth-server storage ACL Secret)", + vmcpconfig.RedisPasswordEnvVar) + } opts, err := tokenEncOption() if err != nil { @@ -261,7 +319,7 @@ func buildTokenReader(_ context.Context, dialGuard *egressbroker.IPAllowlist) (* client := goredis.NewClient(&goredis.Options{ Addr: addr, - Password: os.Getenv(vmcpconfig.RedisPasswordEnvVar), + Password: password, Dialer: dialGuard.DialContext, }) stor := storage.NewRedisStorageWithClient(client, keyPrefix, opts...) diff --git a/cmd/thv-operator/controllers/virtualmcpserver_deployment.go b/cmd/thv-operator/controllers/virtualmcpserver_deployment.go index 19a809f46d..0f714e7b3c 100644 --- a/cmd/thv-operator/controllers/virtualmcpserver_deployment.go +++ b/cmd/thv-operator/controllers/virtualmcpserver_deployment.go @@ -97,6 +97,17 @@ const ( untrustedTokenStoreKEKKeyEnvVar = "THV_UNTRUSTED_TOKEN_STORE_KEK_KEY" // #nosec G101 -- env var name, not a credential untrustedTokenStoreKEKIDsEnvVar = "THV_UNTRUSTED_TOKEN_STORE_KEK_IDS" // #nosec G101 -- env var name, not a credential + // untrustedTokenStorePassword*EnvVars carry the auth-server Redis ACL + // password Secret COORDINATES (name + key, never the value) to the vMCP, + // which renders a SecretKeyRef env (THV_SESSION_REDIS_PASSWORD) on every + // cloned egress-broker sidecar. Without them the sidecar cannot + // authenticate against the token store and every injection denies. + // pkg/vmcp/cli/untrusted.go mirrors these names; the contract is pinned by + // tests on both sides. + // #nosec G101 -- env var names, not credentials. + untrustedTokenStorePasswordSecretEnvVar = "THV_UNTRUSTED_TOKEN_STORE_PASSWORD_SECRET" + untrustedTokenStorePasswordKeyEnvVar = "THV_UNTRUSTED_TOKEN_STORE_PASSWORD_KEY" // #nosec G101 + // reservedVMCPEnvPrefixes are the env-var prefixes the operator owns on // the vmcp container. A user PodTemplateSpec must not set them: they // carry operator-injected wiring (untrusted egress coordinates, embedded @@ -497,6 +508,28 @@ func (r *VirtualMCPServerReconciler) buildUntrustedTokenStoreEnvVars( Name: untrustedTokenStoreAddrEnvVar, Value: as.Storage.Redis.Addr, }} + // The token store's ACL password must reach the egress-broker sidecar or + // every credential injection denies (Redis AUTH failure). It is forwarded + // as Secret COORDINATES (never the value): the vMCP turns them into a + // SecretKeyRef env on each cloned sidecar (KEK pattern). A standalone + // Redis addr without the ACL coordinates is almost always a + // misconfiguration — RedisStorageConfig requires aclUserConfig — so flag + // it loudly instead of silently cloning sidecars that cannot authenticate. + if acl := as.Storage.Redis.ACLUserConfig; acl != nil && acl.PasswordSecretRef != nil { + env = append(env, + corev1.EnvVar{Name: untrustedTokenStorePasswordSecretEnvVar, Value: acl.PasswordSecretRef.Name}, + corev1.EnvVar{Name: untrustedTokenStorePasswordKeyEnvVar, Value: acl.PasswordSecretRef.Key}, + ) + } else { + log.FromContext(ctx).Info("auth-server Redis storage carries no ACL password coordinates; "+ + "untrusted egress-broker sidecars will fail to authenticate against the token store", + "vmcp", vmcp.Name, "namespace", vmcp.Namespace) + if r.Recorder != nil { + r.Recorder.Eventf(vmcp, nil, corev1.EventTypeWarning, "TokenStorePasswordMissing", + "UntrustedTokenStore", "auth-server Redis storage has no aclUserConfig.passwordSecretRef; "+ + "untrusted egress-broker sidecars cannot authenticate against the upstream token store") + } + } if te := as.Storage.GetTokenEncryption(); te != nil { envByID, err := ctrlutil.ResolveKEKKeySet(ctx, r.Client, vmcp.Namespace, as) if err != nil { diff --git a/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go b/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go index 105fe07586..48c70ab386 100644 --- a/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go +++ b/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go @@ -169,14 +169,58 @@ func TestBuildUntrustedTokenStoreEnvVars(t *testing.T) { assert.Empty(t, env) return } - require.Len(t, env, 1) + require.Len(t, env, 3, "addr + the Redis ACL password Secret coordinates") assert.Equal(t, untrustedTokenStoreAddrEnvVar, env[0].Name) assert.Equal(t, tc.wantAddr, env[0].Value) assert.Nil(t, env[0].ValueFrom, "address is non-secret; must be a plain literal, never a Secret ref") + byName := map[string]corev1.EnvVar{} + for _, e := range env { + byName[e.Name] = e + assert.Nil(t, e.ValueFrom, "%s must be a literal coordinate, never a Secret ref", e.Name) + } + assert.Equal(t, "redis-creds", byName[untrustedTokenStorePasswordSecretEnvVar].Value) + assert.Equal(t, "password", byName[untrustedTokenStorePasswordKeyEnvVar].Value) }) } } +// TestBuildUntrustedTokenStoreEnvVars_PasswordMissing pins the loud-failure +// seam: standalone Redis storage WITHOUT ACL password coordinates still +// renders the address (the vMCP/broker fail loud downstream), but flags the +// misconfiguration with a Warning event so it is never silently dropped. +func TestBuildUntrustedTokenStoreEnvVars_PasswordMissing(t *testing.T) { + t.Parallel() + + cfg := &mcpv1beta1.EmbeddedAuthServerConfig{ + Storage: &mcpv1beta1.AuthServerStorageConfig{ + Type: mcpv1beta1.AuthServerStorageTypeRedis, + Redis: &mcpv1beta1.RedisStorageConfig{Addr: "redis.auth:6379"}, + }, + } + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", + v1beta1test.WithVMCPAuthServerConfig(cfg)) + + recorder := events.NewFakeRecorder(10) + scheme := testutil.NewScheme(t) + r := &VirtualMCPServerReconciler{ + Client: fake.NewClientBuilder().WithScheme(scheme).Build(), + Scheme: scheme, + Recorder: recorder, + } + + env, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) + require.NoError(t, err, "the misconfiguration must not error the reconcile (the broker fails loud)") + require.Len(t, env, 1, "only the address renders — no password coordinates exist to forward") + assert.Equal(t, untrustedTokenStoreAddrEnvVar, env[0].Name) + + select { + case event := <-recorder.Events: + assert.Contains(t, event, "TokenStorePasswordMissing") + default: + t.Fatal("expected a Warning event for Redis storage without ACL password coordinates") + } +} + // TestBuildUntrustedTokenStoreEnvVars_KEK pins the Wave-5 KEK wiring: when the // auth-server storage carries tokenEncryption, the vMCP Deployment gets the // KEK Secret coordinates (Secret name + active key ID + the full key-ID set @@ -207,13 +251,15 @@ func TestBuildUntrustedTokenStoreEnvVars_KEK(t *testing.T) { }) env, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) require.NoError(t, err) - require.Len(t, env, 4) + require.Len(t, env, 6, "addr + password coordinates + KEK coordinates") byName := map[string]corev1.EnvVar{} for _, e := range env { byName[e.Name] = e } assert.Equal(t, "redis.auth:6379", byName[untrustedTokenStoreAddrEnvVar].Value) + assert.Equal(t, "redis-creds", byName[untrustedTokenStorePasswordSecretEnvVar].Value) + assert.Equal(t, "password", byName[untrustedTokenStorePasswordKeyEnvVar].Value) assert.Equal(t, "my-vmcp-kek", byName[untrustedTokenStoreKEKSecretEnvVar].Value) assert.Equal(t, "kek-2", byName[untrustedTokenStoreKEKKeyEnvVar].Value) assert.Equal(t, "kek-1,kek-2", byName[untrustedTokenStoreKEKIDsEnvVar].Value, @@ -223,14 +269,14 @@ func TestBuildUntrustedTokenStoreEnvVars_KEK(t *testing.T) { } }) - t.Run("tokenEncryption nil emits only the address env var", func(t *testing.T) { + t.Run("tokenEncryption nil emits the address and password coordinate env vars", func(t *testing.T) { t.Parallel() vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", v1beta1test.WithVMCPAuthServerConfig(redisStorage(nil))) r := newTokenStoreTestReconciler(t) env, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) require.NoError(t, err) - require.Len(t, env, 1) + require.Len(t, env, 3) assert.Equal(t, untrustedTokenStoreAddrEnvVar, env[0].Name) }) diff --git a/docs/arch/adr/0001-untrusted-mcp-egress-broker.md b/docs/arch/adr/0001-untrusted-mcp-egress-broker.md index a16c26ad7c..e88ee06595 100644 --- a/docs/arch/adr/0001-untrusted-mcp-egress-broker.md +++ b/docs/arch/adr/0001-untrusted-mcp-egress-broker.md @@ -305,6 +305,12 @@ never holds a token it can carry off, replay later, or use as another user. every injected call (D11) as the primary detection surface. 2. **Data exfiltration via MCP responses.** Anything the upstream API returns to the server can be embedded in tool responses to the agent. The credential never leaks; the data it fetches can. +3. **Transformed echoes of the credential.** The D6c response scanner matches the injected header + value byte-exactly (plus its standard base64 form). It is a tripwire, not a proof: a server that + transforms the echo — gzip/hex/base64-of-base64, splitting it across chunks or fields, or + smuggling it in response trailers (unscanned, SKIP mode) — evades the match. The real boundary + is the allowlist + destination binding (D5/D6b): the credential can only travel to consented + hosts in the first place, and every injection is audited (D11). Operators must be told both limits — here and in the Wave-5 user-facing doc — so "untrusted mode" is not over-trusted. diff --git a/docs/server/docs.go b/docs/server/docs.go index 7fc18c2f00..ce87fcd4a1 100644 --- a/docs/server/docs.go +++ b/docs/server/docs.go @@ -342,6 +342,10 @@ const docTemplate = `{ "github_com_stacklok_toolhive_pkg_auth_upstreamswap.Config": { "description": "UpstreamSwapConfig contains configuration for upstream token swap middleware.\nWhen set along with EmbeddedAuthServerConfig, this middleware exchanges ToolHive JWTs\nfor upstream IdP tokens before forwarding requests to the MCP server.", "properties": { + "authorize_url": { + "description": "AuthorizeURL is the ToolHive authorization server's authorize-endpoint URL\n({issuer}/oauth/authorize). When set, the 401 consent response includes it\nso clients can direct the user to consent. It is only an endpoint: the\nclient merges its own client_id, redirect_uri, and PKCE parameters. It\nmust never be derived from the request Host header (attacker-controlled).\nOptional; when empty the 401 body omits authorize_url.", + "type": "string" + }, "custom_header_name": { "description": "CustomHeaderName is the header name when HeaderStrategy is \"custom\".", "type": "string" @@ -611,7 +615,7 @@ const docTemplate = `{ "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver.SigningKeyRunConfig" }, "storage": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RunConfig" + "$ref": "#/components/schemas/storage.RunConfig" }, "token_lifespans": { "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver.TokenLifespanRunConfig" @@ -773,115 +777,6 @@ const docTemplate = `{ }, "type": "object" }, - "github_com_stacklok_toolhive_pkg_authserver_storage.ACLUserRunConfig": { - "description": "ACLUserConfig contains ACL user authentication configuration.", - "properties": { - "password_env_var": { - "description": "PasswordEnvVar is the environment variable containing the Redis password.", - "type": "string" - }, - "username_env_var": { - "description": "UsernameEnvVar is the environment variable containing the Redis username.", - "type": "string" - } - }, - "type": "object" - }, - "github_com_stacklok_toolhive_pkg_authserver_storage.RedisRunConfig": { - "description": "RedisConfig is the Redis-specific configuration when Type is \"redis\".", - "properties": { - "acl_user_config": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.ACLUserRunConfig" - }, - "addr": { - "description": "Addr is the Redis server address (host:port). Required for standalone and cluster modes.\nMutually exclusive with SentinelConfig.", - "type": "string" - }, - "auth_type": { - "description": "AuthType must be \"aclUser\" - only ACL user authentication is supported.", - "type": "string" - }, - "cluster_mode": { - "description": "ClusterMode enables the Redis Cluster protocol. Requires Addr to be set.", - "type": "boolean" - }, - "dial_timeout": { - "description": "DialTimeout is the timeout for establishing connections (e.g., \"5s\").", - "type": "string" - }, - "key_prefix": { - "description": "KeyPrefix for multi-tenancy, typically \"thv:auth:{ns}:{name}:\".", - "type": "string" - }, - "read_timeout": { - "description": "ReadTimeout is the timeout for read operations (e.g., \"3s\").", - "type": "string" - }, - "sentinel_config": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.SentinelRunConfig" - }, - "sentinel_tls": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RedisTLSRunConfig" - }, - "tls": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RedisTLSRunConfig" - }, - "write_timeout": { - "description": "WriteTimeout is the timeout for write operations (e.g., \"3s\").", - "type": "string" - } - }, - "type": "object" - }, - "github_com_stacklok_toolhive_pkg_authserver_storage.RedisTLSRunConfig": { - "description": "SentinelTLS configures TLS for Sentinel connections. Only applies when SentinelConfig is set.", - "properties": { - "ca_cert_file": { - "description": "CACertFile is the path to a PEM-encoded CA certificate file.", - "type": "string" - }, - "insecure_skip_verify": { - "description": "InsecureSkipVerify skips certificate verification.", - "type": "boolean" - } - }, - "type": "object" - }, - "github_com_stacklok_toolhive_pkg_authserver_storage.RunConfig": { - "description": "Storage configures the storage backend for the auth server.\nIf nil, defaults to in-memory storage.", - "properties": { - "redis_config": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RedisRunConfig" - }, - "type": { - "description": "Type specifies the storage backend type. Defaults to \"memory\".", - "type": "string" - } - }, - "type": "object" - }, - "github_com_stacklok_toolhive_pkg_authserver_storage.SentinelRunConfig": { - "description": "SentinelConfig contains Sentinel-specific configuration.\nMutually exclusive with Addr.", - "properties": { - "db": { - "description": "DB is the Redis database number (default: 0).", - "type": "integer" - }, - "master_name": { - "description": "MasterName is the name of the Redis Sentinel master.", - "type": "string" - }, - "sentinel_addrs": { - "description": "SentinelAddrs is the list of Sentinel addresses (host:port).", - "items": { - "type": "string" - }, - "type": "array", - "uniqueItems": false - } - }, - "type": "object" - }, "github_com_stacklok_toolhive_pkg_authz.Config": { "description": "DEPRECATED: Middleware configuration.\nAuthzConfig contains the authorization configuration", "properties": { @@ -4241,6 +4136,135 @@ const docTemplate = `{ }, "type": "object" }, + "storage.ACLUserRunConfig": { + "description": "ACLUserConfig contains ACL user authentication configuration.", + "properties": { + "password_env_var": { + "description": "PasswordEnvVar is the environment variable containing the Redis password.", + "type": "string" + }, + "username_env_var": { + "description": "UsernameEnvVar is the environment variable containing the Redis username.", + "type": "string" + } + }, + "type": "object" + }, + "storage.RedisRunConfig": { + "description": "RedisConfig is the Redis-specific configuration when Type is \"redis\".", + "properties": { + "acl_user_config": { + "$ref": "#/components/schemas/storage.ACLUserRunConfig" + }, + "addr": { + "description": "Addr is the Redis server address (host:port). Required for standalone and cluster modes.\nMutually exclusive with SentinelConfig.", + "type": "string" + }, + "auth_type": { + "description": "AuthType must be \"aclUser\" - only ACL user authentication is supported.", + "type": "string" + }, + "cluster_mode": { + "description": "ClusterMode enables the Redis Cluster protocol. Requires Addr to be set.", + "type": "boolean" + }, + "dial_timeout": { + "description": "DialTimeout is the timeout for establishing connections (e.g., \"5s\").", + "type": "string" + }, + "key_prefix": { + "description": "KeyPrefix for multi-tenancy, typically \"thv:auth:{ns}:{name}:\".", + "type": "string" + }, + "read_timeout": { + "description": "ReadTimeout is the timeout for read operations (e.g., \"3s\").", + "type": "string" + }, + "sentinel_config": { + "$ref": "#/components/schemas/storage.SentinelRunConfig" + }, + "sentinel_tls": { + "$ref": "#/components/schemas/storage.RedisTLSRunConfig" + }, + "tls": { + "$ref": "#/components/schemas/storage.RedisTLSRunConfig" + }, + "token_encryption": { + "$ref": "#/components/schemas/storage.TokenEncryptionRunConfig" + }, + "write_timeout": { + "description": "WriteTimeout is the timeout for write operations (e.g., \"3s\").", + "type": "string" + } + }, + "type": "object" + }, + "storage.RedisTLSRunConfig": { + "description": "SentinelTLS configures TLS for Sentinel connections. Only applies when SentinelConfig is set.", + "properties": { + "ca_cert_file": { + "description": "CACertFile is the path to a PEM-encoded CA certificate file.", + "type": "string" + }, + "insecure_skip_verify": { + "description": "InsecureSkipVerify skips certificate verification.", + "type": "boolean" + } + }, + "type": "object" + }, + "storage.RunConfig": { + "description": "Storage configures the storage backend for the auth server.\nIf nil, defaults to in-memory storage.", + "properties": { + "redis_config": { + "$ref": "#/components/schemas/storage.RedisRunConfig" + }, + "type": { + "description": "Type specifies the storage backend type. Defaults to \"memory\".", + "type": "string" + } + }, + "type": "object" + }, + "storage.SentinelRunConfig": { + "description": "SentinelConfig contains Sentinel-specific configuration.\nMutually exclusive with Addr.", + "properties": { + "db": { + "description": "DB is the Redis database number (default: 0).", + "type": "integer" + }, + "master_name": { + "description": "MasterName is the name of the Redis Sentinel master.", + "type": "string" + }, + "sentinel_addrs": { + "description": "SentinelAddrs is the list of Sentinel addresses (host:port).", + "items": { + "type": "string" + }, + "type": "array", + "uniqueItems": false + } + }, + "type": "object" + }, + "storage.TokenEncryptionRunConfig": { + "description": "TokenEncryption configures envelope encryption for upstream OAuth tokens\nat rest. Presence enables encryption; absence preserves the current\nplaintext behavior.", + "properties": { + "active_key_id": { + "description": "ActiveKeyID identifies the KEK used to encrypt new writes. Other IDs in\nKeys remain decrypt-only (retired), supporting lazy read-old/write-new\nrotation.", + "type": "string" + }, + "keys": { + "additionalProperties": { + "type": "string" + }, + "description": "Keys maps key ID → name of the environment variable holding the\nbase64-encoded 32-byte key-encryption key.", + "type": "object" + } + }, + "type": "object" + }, "v0.ServerJSON": { "properties": { "$schema": { diff --git a/docs/server/swagger.json b/docs/server/swagger.json index a43404b4eb..8a036240e8 100644 --- a/docs/server/swagger.json +++ b/docs/server/swagger.json @@ -335,6 +335,10 @@ "github_com_stacklok_toolhive_pkg_auth_upstreamswap.Config": { "description": "UpstreamSwapConfig contains configuration for upstream token swap middleware.\nWhen set along with EmbeddedAuthServerConfig, this middleware exchanges ToolHive JWTs\nfor upstream IdP tokens before forwarding requests to the MCP server.", "properties": { + "authorize_url": { + "description": "AuthorizeURL is the ToolHive authorization server's authorize-endpoint URL\n({issuer}/oauth/authorize). When set, the 401 consent response includes it\nso clients can direct the user to consent. It is only an endpoint: the\nclient merges its own client_id, redirect_uri, and PKCE parameters. It\nmust never be derived from the request Host header (attacker-controlled).\nOptional; when empty the 401 body omits authorize_url.", + "type": "string" + }, "custom_header_name": { "description": "CustomHeaderName is the header name when HeaderStrategy is \"custom\".", "type": "string" @@ -604,7 +608,7 @@ "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver.SigningKeyRunConfig" }, "storage": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RunConfig" + "$ref": "#/components/schemas/storage.RunConfig" }, "token_lifespans": { "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver.TokenLifespanRunConfig" @@ -766,115 +770,6 @@ }, "type": "object" }, - "github_com_stacklok_toolhive_pkg_authserver_storage.ACLUserRunConfig": { - "description": "ACLUserConfig contains ACL user authentication configuration.", - "properties": { - "password_env_var": { - "description": "PasswordEnvVar is the environment variable containing the Redis password.", - "type": "string" - }, - "username_env_var": { - "description": "UsernameEnvVar is the environment variable containing the Redis username.", - "type": "string" - } - }, - "type": "object" - }, - "github_com_stacklok_toolhive_pkg_authserver_storage.RedisRunConfig": { - "description": "RedisConfig is the Redis-specific configuration when Type is \"redis\".", - "properties": { - "acl_user_config": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.ACLUserRunConfig" - }, - "addr": { - "description": "Addr is the Redis server address (host:port). Required for standalone and cluster modes.\nMutually exclusive with SentinelConfig.", - "type": "string" - }, - "auth_type": { - "description": "AuthType must be \"aclUser\" - only ACL user authentication is supported.", - "type": "string" - }, - "cluster_mode": { - "description": "ClusterMode enables the Redis Cluster protocol. Requires Addr to be set.", - "type": "boolean" - }, - "dial_timeout": { - "description": "DialTimeout is the timeout for establishing connections (e.g., \"5s\").", - "type": "string" - }, - "key_prefix": { - "description": "KeyPrefix for multi-tenancy, typically \"thv:auth:{ns}:{name}:\".", - "type": "string" - }, - "read_timeout": { - "description": "ReadTimeout is the timeout for read operations (e.g., \"3s\").", - "type": "string" - }, - "sentinel_config": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.SentinelRunConfig" - }, - "sentinel_tls": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RedisTLSRunConfig" - }, - "tls": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RedisTLSRunConfig" - }, - "write_timeout": { - "description": "WriteTimeout is the timeout for write operations (e.g., \"3s\").", - "type": "string" - } - }, - "type": "object" - }, - "github_com_stacklok_toolhive_pkg_authserver_storage.RedisTLSRunConfig": { - "description": "SentinelTLS configures TLS for Sentinel connections. Only applies when SentinelConfig is set.", - "properties": { - "ca_cert_file": { - "description": "CACertFile is the path to a PEM-encoded CA certificate file.", - "type": "string" - }, - "insecure_skip_verify": { - "description": "InsecureSkipVerify skips certificate verification.", - "type": "boolean" - } - }, - "type": "object" - }, - "github_com_stacklok_toolhive_pkg_authserver_storage.RunConfig": { - "description": "Storage configures the storage backend for the auth server.\nIf nil, defaults to in-memory storage.", - "properties": { - "redis_config": { - "$ref": "#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RedisRunConfig" - }, - "type": { - "description": "Type specifies the storage backend type. Defaults to \"memory\".", - "type": "string" - } - }, - "type": "object" - }, - "github_com_stacklok_toolhive_pkg_authserver_storage.SentinelRunConfig": { - "description": "SentinelConfig contains Sentinel-specific configuration.\nMutually exclusive with Addr.", - "properties": { - "db": { - "description": "DB is the Redis database number (default: 0).", - "type": "integer" - }, - "master_name": { - "description": "MasterName is the name of the Redis Sentinel master.", - "type": "string" - }, - "sentinel_addrs": { - "description": "SentinelAddrs is the list of Sentinel addresses (host:port).", - "items": { - "type": "string" - }, - "type": "array", - "uniqueItems": false - } - }, - "type": "object" - }, "github_com_stacklok_toolhive_pkg_authz.Config": { "description": "DEPRECATED: Middleware configuration.\nAuthzConfig contains the authorization configuration", "properties": { @@ -4234,6 +4129,135 @@ }, "type": "object" }, + "storage.ACLUserRunConfig": { + "description": "ACLUserConfig contains ACL user authentication configuration.", + "properties": { + "password_env_var": { + "description": "PasswordEnvVar is the environment variable containing the Redis password.", + "type": "string" + }, + "username_env_var": { + "description": "UsernameEnvVar is the environment variable containing the Redis username.", + "type": "string" + } + }, + "type": "object" + }, + "storage.RedisRunConfig": { + "description": "RedisConfig is the Redis-specific configuration when Type is \"redis\".", + "properties": { + "acl_user_config": { + "$ref": "#/components/schemas/storage.ACLUserRunConfig" + }, + "addr": { + "description": "Addr is the Redis server address (host:port). Required for standalone and cluster modes.\nMutually exclusive with SentinelConfig.", + "type": "string" + }, + "auth_type": { + "description": "AuthType must be \"aclUser\" - only ACL user authentication is supported.", + "type": "string" + }, + "cluster_mode": { + "description": "ClusterMode enables the Redis Cluster protocol. Requires Addr to be set.", + "type": "boolean" + }, + "dial_timeout": { + "description": "DialTimeout is the timeout for establishing connections (e.g., \"5s\").", + "type": "string" + }, + "key_prefix": { + "description": "KeyPrefix for multi-tenancy, typically \"thv:auth:{ns}:{name}:\".", + "type": "string" + }, + "read_timeout": { + "description": "ReadTimeout is the timeout for read operations (e.g., \"3s\").", + "type": "string" + }, + "sentinel_config": { + "$ref": "#/components/schemas/storage.SentinelRunConfig" + }, + "sentinel_tls": { + "$ref": "#/components/schemas/storage.RedisTLSRunConfig" + }, + "tls": { + "$ref": "#/components/schemas/storage.RedisTLSRunConfig" + }, + "token_encryption": { + "$ref": "#/components/schemas/storage.TokenEncryptionRunConfig" + }, + "write_timeout": { + "description": "WriteTimeout is the timeout for write operations (e.g., \"3s\").", + "type": "string" + } + }, + "type": "object" + }, + "storage.RedisTLSRunConfig": { + "description": "SentinelTLS configures TLS for Sentinel connections. Only applies when SentinelConfig is set.", + "properties": { + "ca_cert_file": { + "description": "CACertFile is the path to a PEM-encoded CA certificate file.", + "type": "string" + }, + "insecure_skip_verify": { + "description": "InsecureSkipVerify skips certificate verification.", + "type": "boolean" + } + }, + "type": "object" + }, + "storage.RunConfig": { + "description": "Storage configures the storage backend for the auth server.\nIf nil, defaults to in-memory storage.", + "properties": { + "redis_config": { + "$ref": "#/components/schemas/storage.RedisRunConfig" + }, + "type": { + "description": "Type specifies the storage backend type. Defaults to \"memory\".", + "type": "string" + } + }, + "type": "object" + }, + "storage.SentinelRunConfig": { + "description": "SentinelConfig contains Sentinel-specific configuration.\nMutually exclusive with Addr.", + "properties": { + "db": { + "description": "DB is the Redis database number (default: 0).", + "type": "integer" + }, + "master_name": { + "description": "MasterName is the name of the Redis Sentinel master.", + "type": "string" + }, + "sentinel_addrs": { + "description": "SentinelAddrs is the list of Sentinel addresses (host:port).", + "items": { + "type": "string" + }, + "type": "array", + "uniqueItems": false + } + }, + "type": "object" + }, + "storage.TokenEncryptionRunConfig": { + "description": "TokenEncryption configures envelope encryption for upstream OAuth tokens\nat rest. Presence enables encryption; absence preserves the current\nplaintext behavior.", + "properties": { + "active_key_id": { + "description": "ActiveKeyID identifies the KEK used to encrypt new writes. Other IDs in\nKeys remain decrypt-only (retired), supporting lazy read-old/write-new\nrotation.", + "type": "string" + }, + "keys": { + "additionalProperties": { + "type": "string" + }, + "description": "Keys maps key ID → name of the environment variable holding the\nbase64-encoded 32-byte key-encryption key.", + "type": "object" + } + }, + "type": "object" + }, "v0.ServerJSON": { "properties": { "$schema": { diff --git a/docs/server/swagger.yaml b/docs/server/swagger.yaml index 3ba6e5b194..24256ff973 100644 --- a/docs/server/swagger.yaml +++ b/docs/server/swagger.yaml @@ -350,6 +350,15 @@ components: When set along with EmbeddedAuthServerConfig, this middleware exchanges ToolHive JWTs for upstream IdP tokens before forwarding requests to the MCP server. properties: + authorize_url: + description: |- + AuthorizeURL is the ToolHive authorization server's authorize-endpoint URL + ({issuer}/oauth/authorize). When set, the 401 consent response includes it + so clients can direct the user to consent. It is only an endpoint: the + client merges its own client_id, redirect_uri, and PKCE parameters. It + must never be derived from the request Host header (attacker-controlled). + Optional; when empty the 401 body omits authorize_url. + type: string custom_header_name: description: CustomHeaderName is the header name when HeaderStrategy is "custom". @@ -708,7 +717,7 @@ components: signing_key_config: $ref: '#/components/schemas/github_com_stacklok_toolhive_pkg_authserver.SigningKeyRunConfig' storage: - $ref: '#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RunConfig' + $ref: '#/components/schemas/storage.RunConfig' token_lifespans: $ref: '#/components/schemas/github_com_stacklok_toolhive_pkg_authserver.TokenLifespanRunConfig' upstreams: @@ -874,96 +883,6 @@ components: If not specified, defaults to GET. type: string type: object - github_com_stacklok_toolhive_pkg_authserver_storage.ACLUserRunConfig: - description: ACLUserConfig contains ACL user authentication configuration. - properties: - password_env_var: - description: PasswordEnvVar is the environment variable containing the Redis - password. - type: string - username_env_var: - description: UsernameEnvVar is the environment variable containing the Redis - username. - type: string - type: object - github_com_stacklok_toolhive_pkg_authserver_storage.RedisRunConfig: - description: RedisConfig is the Redis-specific configuration when Type is "redis". - properties: - acl_user_config: - $ref: '#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.ACLUserRunConfig' - addr: - description: |- - Addr is the Redis server address (host:port). Required for standalone and cluster modes. - Mutually exclusive with SentinelConfig. - type: string - auth_type: - description: AuthType must be "aclUser" - only ACL user authentication is - supported. - type: string - cluster_mode: - description: ClusterMode enables the Redis Cluster protocol. Requires Addr - to be set. - type: boolean - dial_timeout: - description: DialTimeout is the timeout for establishing connections (e.g., - "5s"). - type: string - key_prefix: - description: KeyPrefix for multi-tenancy, typically "thv:auth:{ns}:{name}:". - type: string - read_timeout: - description: ReadTimeout is the timeout for read operations (e.g., "3s"). - type: string - sentinel_config: - $ref: '#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.SentinelRunConfig' - sentinel_tls: - $ref: '#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RedisTLSRunConfig' - tls: - $ref: '#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RedisTLSRunConfig' - write_timeout: - description: WriteTimeout is the timeout for write operations (e.g., "3s"). - type: string - type: object - github_com_stacklok_toolhive_pkg_authserver_storage.RedisTLSRunConfig: - description: SentinelTLS configures TLS for Sentinel connections. Only applies - when SentinelConfig is set. - properties: - ca_cert_file: - description: CACertFile is the path to a PEM-encoded CA certificate file. - type: string - insecure_skip_verify: - description: InsecureSkipVerify skips certificate verification. - type: boolean - type: object - github_com_stacklok_toolhive_pkg_authserver_storage.RunConfig: - description: |- - Storage configures the storage backend for the auth server. - If nil, defaults to in-memory storage. - properties: - redis_config: - $ref: '#/components/schemas/github_com_stacklok_toolhive_pkg_authserver_storage.RedisRunConfig' - type: - description: Type specifies the storage backend type. Defaults to "memory". - type: string - type: object - github_com_stacklok_toolhive_pkg_authserver_storage.SentinelRunConfig: - description: |- - SentinelConfig contains Sentinel-specific configuration. - Mutually exclusive with Addr. - properties: - db: - description: 'DB is the Redis database number (default: 0).' - type: integer - master_name: - description: MasterName is the name of the Redis Sentinel master. - type: string - sentinel_addrs: - description: SentinelAddrs is the list of Sentinel addresses (host:port). - items: - type: string - type: array - uniqueItems: false - type: object github_com_stacklok_toolhive_pkg_authz.Config: description: |- DEPRECATED: Middleware configuration. @@ -3853,6 +3772,118 @@ components: predicate_type: type: string type: object + storage.ACLUserRunConfig: + description: ACLUserConfig contains ACL user authentication configuration. + properties: + password_env_var: + description: PasswordEnvVar is the environment variable containing the Redis + password. + type: string + username_env_var: + description: UsernameEnvVar is the environment variable containing the Redis + username. + type: string + type: object + storage.RedisRunConfig: + description: RedisConfig is the Redis-specific configuration when Type is "redis". + properties: + acl_user_config: + $ref: '#/components/schemas/storage.ACLUserRunConfig' + addr: + description: |- + Addr is the Redis server address (host:port). Required for standalone and cluster modes. + Mutually exclusive with SentinelConfig. + type: string + auth_type: + description: AuthType must be "aclUser" - only ACL user authentication is + supported. + type: string + cluster_mode: + description: ClusterMode enables the Redis Cluster protocol. Requires Addr + to be set. + type: boolean + dial_timeout: + description: DialTimeout is the timeout for establishing connections (e.g., + "5s"). + type: string + key_prefix: + description: KeyPrefix for multi-tenancy, typically "thv:auth:{ns}:{name}:". + type: string + read_timeout: + description: ReadTimeout is the timeout for read operations (e.g., "3s"). + type: string + sentinel_config: + $ref: '#/components/schemas/storage.SentinelRunConfig' + sentinel_tls: + $ref: '#/components/schemas/storage.RedisTLSRunConfig' + tls: + $ref: '#/components/schemas/storage.RedisTLSRunConfig' + token_encryption: + $ref: '#/components/schemas/storage.TokenEncryptionRunConfig' + write_timeout: + description: WriteTimeout is the timeout for write operations (e.g., "3s"). + type: string + type: object + storage.RedisTLSRunConfig: + description: SentinelTLS configures TLS for Sentinel connections. Only applies + when SentinelConfig is set. + properties: + ca_cert_file: + description: CACertFile is the path to a PEM-encoded CA certificate file. + type: string + insecure_skip_verify: + description: InsecureSkipVerify skips certificate verification. + type: boolean + type: object + storage.RunConfig: + description: |- + Storage configures the storage backend for the auth server. + If nil, defaults to in-memory storage. + properties: + redis_config: + $ref: '#/components/schemas/storage.RedisRunConfig' + type: + description: Type specifies the storage backend type. Defaults to "memory". + type: string + type: object + storage.SentinelRunConfig: + description: |- + SentinelConfig contains Sentinel-specific configuration. + Mutually exclusive with Addr. + properties: + db: + description: 'DB is the Redis database number (default: 0).' + type: integer + master_name: + description: MasterName is the name of the Redis Sentinel master. + type: string + sentinel_addrs: + description: SentinelAddrs is the list of Sentinel addresses (host:port). + items: + type: string + type: array + uniqueItems: false + type: object + storage.TokenEncryptionRunConfig: + description: |- + TokenEncryption configures envelope encryption for upstream OAuth tokens + at rest. Presence enables encryption; absence preserves the current + plaintext behavior. + properties: + active_key_id: + description: |- + ActiveKeyID identifies the KEK used to encrypt new writes. Other IDs in + Keys remain decrypt-only (retired), supporting lazy read-old/write-new + rotation. + type: string + keys: + additionalProperties: + type: string + description: |- + Keys maps key ID → name of the environment variable holding the + base64-encoded 32-byte key-encryption key. + type: object + type: object v0.ServerJSON: properties: $schema: diff --git a/pkg/egressbroker/audit.go b/pkg/egressbroker/audit.go new file mode 100644 index 0000000000..8c3b1f1e72 --- /dev/null +++ b/pkg/egressbroker/audit.go @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "log/slog" + "os" + "time" +) + +// Audit events for the credential broker (ADR-0001 D11). The audit trail is +// the primary detection surface for authority-abuse-within-scope (ADR §9.5): +// every credential injection, every denial, and every suppressed response-side +// leak emits exactly one structured slog line to stdout → cluster log pipeline +// (same sink as every other component). +// +// Cardinality rule (D11): the user sub appears ONLY here — hashed +// (SHA-256[:16hex], correlation without raw PII in stdout) — and never in +// metric labels. +// +// NEVER logged: token values, request/response bodies, KEKs, CA material. + +// DenyReason is the fixed vocabulary of injection-denial causes. Reasons must +// carry no token data and no request body content. +type DenyReason string + +const ( + // DenyReasonNoPolicy means no provider allowlists the destination host. + DenyReasonNoPolicy DenyReason = "no-policy" + // DenyReasonMethodNotAllowed means the method is outside the provider's policy. + DenyReasonMethodNotAllowed DenyReason = "method-not-allowed" + // DenyReasonPathNotAllowed means the path is outside the provider's policy. + DenyReasonPathNotAllowed DenyReason = "path-not-allowed" + // DenyReasonCredentialUnavailable means no (valid) credential exists for + // the provider; re-consent is required. + DenyReasonCredentialUnavailable DenyReason = "credential-unavailable" //nolint:gosec // G101: a reason label, not a credential + // DenyReasonBindingMismatch means the stored credential's owner binding + // does not match the pod's identity (or the row cannot prove its owner in + // Strict mode). + DenyReasonBindingMismatch DenyReason = "binding-mismatch" + // DenyReasonStoreError means the credential store itself failed. + DenyReasonStoreError DenyReason = "store-error" + // DenyReasonMalformed means the ext_authz request carried no parseable + // destination (fail-closed parse refusal). + DenyReasonMalformed DenyReason = "malformed-request" +) + +// InjectEvent is emitted once per successful credential injection. +type InjectEvent struct { + MCPServer string + Pod string + // Subject is the raw OIDC sub; it is hashed before it ever reaches the + // log line (see SubjectHash). + Subject string + Provider string + Host string + Method string + Path string +} + +// DenyEvent is emitted once per injection denial. +type DenyEvent struct { + InjectEvent + Reason DenyReason +} + +// LeakEvent is emitted when the response scanner suppresses a response that +// echoed the injected credential back (D6c). RequestID is the Envoy +// x-request-id correlator. +type LeakEvent struct { + MCPServer string + Provider string + Host string + RequestID string + Where LeakLocation +} + +// AuditLogger is the broker's audit sink. Implementations must be safe for +// concurrent use and must never emit credential material. +type AuditLogger interface { + Inject(ctx context.Context, e InjectEvent) + Deny(ctx context.Context, e DenyEvent) + Leak(ctx context.Context, e LeakEvent) +} + +// SubjectHash returns the correlation-safe form of a raw OIDC sub: the first +// 16 hex chars of its SHA-256. Raw subs never reach stdout. +func SubjectHash(subject string) string { + sum := sha256.Sum256([]byte(subject)) + return hex.EncodeToString(sum[:])[:16] +} + +// slogAuditLogger emits one structured slog line per event to the given +// handler's sink (stdout in production). +type slogAuditLogger struct { + logger *slog.Logger +} + +// NewAuditLogger returns the stdout slog-backed audit sink (production). +func NewAuditLogger() AuditLogger { + return NewAuditLoggerWith(slog.NewJSONHandler(os.Stdout, nil)) +} + +// NewAuditLoggerWith returns an audit sink on an arbitrary slog handler +// (tests capture with an in-memory handler). +func NewAuditLoggerWith(handler slog.Handler) AuditLogger { + if handler == nil { + // Fail loudly: a nil handler would silently drop the audit trail of a + // credential broker. + panic("egressbroker: audit slog handler must not be nil") + } + return &slogAuditLogger{logger: slog.New(handler)} +} + +// Inject implements AuditLogger. +func (a *slogAuditLogger) Inject(ctx context.Context, e InjectEvent) { + a.logger.LogAttrs(ctx, slog.LevelInfo, "egressbroker credential injected", auditAttrs(e)...) +} + +// Deny implements AuditLogger. +func (a *slogAuditLogger) Deny(ctx context.Context, e DenyEvent) { + attrs := append(auditAttrs(e.InjectEvent), slog.String("reason", string(e.Reason))) + a.logger.LogAttrs(ctx, slog.LevelInfo, "egressbroker injection denied", attrs...) +} + +// Leak implements AuditLogger. +func (a *slogAuditLogger) Leak(ctx context.Context, e LeakEvent) { + a.logger.WarnContext(ctx, "egressbroker response suppressed: credential echo detected", + slog.String("mcpserver", e.MCPServer), + slog.String("provider", e.Provider), + slog.String("host", e.Host), + slog.String("request_id", e.RequestID), + slog.String("where", string(e.Where)), + ) +} + +// auditAttrs renders the shared Inject/Deny fields. The sub is hashed here — +// the only place the raw value is consumed. +func auditAttrs(e InjectEvent) []slog.Attr { + return []slog.Attr{ + slog.Time("ts", time.Now()), + slog.String("mcpserver", e.MCPServer), + slog.String("pod", e.Pod), + slog.String("sub_hash", SubjectHash(e.Subject)), + slog.String("provider", e.Provider), + slog.String("host", e.Host), + slog.String("method", e.Method), + slog.String("path", e.Path), + } +} + +// AuditEvent builds an InjectEvent for one decision from the pod identity and +// destination. podName is optional context (downward-API pod name; empty is +// tolerated — the event still carries the correlation keys). +func AuditEvent(identity PodIdentity, podName string, dest Destination, provider string) InjectEvent { + return InjectEvent{ + MCPServer: identity.MCPServer, + Pod: podName, + Subject: identity.Subject, + Provider: provider, + Host: dest.Host, + Method: dest.Method, + Path: dest.Path, + } +} + +// validateDenyReason guards the enum at the Deny call sites (fail loudly on a +// reason outside the fixed vocabulary — a typo'd reason would break +// alerting dashboards keyed on it). +func validateDenyReason(r DenyReason) { + switch r { + case DenyReasonNoPolicy, DenyReasonMethodNotAllowed, DenyReasonPathNotAllowed, + DenyReasonCredentialUnavailable, DenyReasonBindingMismatch, DenyReasonStoreError, + DenyReasonMalformed: + default: + panic(fmt.Sprintf("egressbroker: unknown deny reason %q", r)) + } +} diff --git a/pkg/egressbroker/audit_test.go b/pkg/egressbroker/audit_test.go new file mode 100644 index 0000000000..ce81d1237f --- /dev/null +++ b/pkg/egressbroker/audit_test.go @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +var _ = time.Second + +// §2.3 #1: injection success → exactly one Inject line with all fields and +// no token substring anywhere. +func TestAudit_InjectEvent(t *testing.T) { + t.Parallel() + h, audit := newCaptureAudit() + + audit.Inject(context.Background(), egressbroker.InjectEvent{ + MCPServer: "github-mcp", + Pod: "pod-abc", + Subject: "user-123", + Provider: "github", + Host: "api.github.com", + Method: "GET", + Path: "/repos/foo", + }) + + lines := h.byMsg("egressbroker credential injected") + require.Len(t, lines, 1, "exactly one Inject line") + rec := lines[0] + assert.Equal(t, "github-mcp", rec.attrs["mcpserver"]) + assert.Equal(t, "pod-abc", rec.attrs["pod"]) + assert.Equal(t, "github", rec.attrs["provider"]) + assert.Equal(t, "api.github.com", rec.attrs["host"]) + assert.Equal(t, "GET", rec.attrs["method"]) + assert.Equal(t, "/repos/foo", rec.attrs["path"]) + // Sub appears hashed only: 16 hex chars, never the raw value. + assert.Regexp(t, `^[0-9a-f]{16}$`, rec.attrs["sub_hash"]) + assert.NotContains(t, rec.rawLog, "user-123", "raw sub must never reach the log line") + // No token material ever (the event struct cannot even carry it). + assert.NotContains(t, rec.rawLog, testToken) + // Timestamp present (the ts attr carries a parsed time). + assert.NotEmpty(t, rec.attrs["ts"]) +} + +// §2.3 #2: each deny reason → one Deny line with the correct reason enum. +func TestAudit_DenyEvents(t *testing.T) { + t.Parallel() + reasons := []egressbroker.DenyReason{ + egressbroker.DenyReasonNoPolicy, + egressbroker.DenyReasonMethodNotAllowed, + egressbroker.DenyReasonPathNotAllowed, + egressbroker.DenyReasonCredentialUnavailable, + egressbroker.DenyReasonBindingMismatch, + egressbroker.DenyReasonStoreError, + egressbroker.DenyReasonMalformed, + } + for _, reason := range reasons { + t.Run(string(reason), func(t *testing.T) { + t.Parallel() + h, audit := newCaptureAudit() + audit.Deny(context.Background(), egressbroker.DenyEvent{ + InjectEvent: egressbroker.InjectEvent{ + MCPServer: "github-mcp", + Pod: "pod-abc", + Subject: "user-123", + Provider: "github", + Host: "api.github.com", + Method: "GET", + Path: "/", + }, + Reason: reason, + }) + lines := h.byMsg("egressbroker injection denied") + require.Len(t, lines, 1) + assert.Equal(t, string(reason), lines[0].attrs["reason"]) + assert.Equal(t, "github-mcp", lines[0].attrs["mcpserver"]) + assert.NotContains(t, lines[0].rawLog, "user-123") + assert.NotContains(t, lines[0].rawLog, testToken) + }) + } +} + +// §2.3 #3: leak block → one Leak line, no token substring. +func TestAudit_LeakEvent(t *testing.T) { + t.Parallel() + h, audit := newCaptureAudit() + + audit.Leak(context.Background(), egressbroker.LeakEvent{ + MCPServer: "github-mcp", + Provider: "github", + Host: "api.github.com", + RequestID: "req-77", + Where: egressbroker.LeakLocationBody, + }) + + lines := h.byMsg("egressbroker response suppressed: credential echo detected") + require.Len(t, lines, 1, "exactly one Leak line") + rec := lines[0] + assert.Equal(t, "github-mcp", rec.attrs["mcpserver"]) + assert.Equal(t, "github", rec.attrs["provider"]) + assert.Equal(t, "api.github.com", rec.attrs["host"]) + assert.Equal(t, "req-77", rec.attrs["request_id"]) + assert.Equal(t, "body", rec.attrs["where"]) + // The event has no token fields at all; belt-and-suspenders substring + // guard against future field additions. + assert.NotContains(t, rec.rawLog, testToken) + assert.NotContains(t, rec.rawLog, "user-123") +} + +// §2.3 #4: concurrent emissions → no interleave/corruption, no race. +func TestAudit_ConcurrentEmissions(t *testing.T) { + t.Parallel() + h, audit := newCaptureAudit() + + const emitters = 32 + const perEmitter = 25 + var wg sync.WaitGroup + wg.Add(emitters) + for i := 0; i < emitters; i++ { + go func(i int) { + defer wg.Done() + for j := 0; j < perEmitter; j++ { + audit.Inject(context.Background(), egressbroker.InjectEvent{ + MCPServer: "github-mcp", + Subject: fmt.Sprintf("user-%d", i), + Provider: "github", + Host: "api.github.com", + Method: "GET", + Path: "/", + }) + } + }(i) + } + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for concurrent audit emissions") + } + + lines := h.byMsg("egressbroker credential injected") + assert.Len(t, lines, emitters*perEmitter, "every emission lands exactly once") + for _, rec := range lines { + assert.Regexp(t, `^[0-9a-f]{16}$`, rec.attrs["sub_hash"], "no torn/corrupted records") + } +} + +// TestAudit_SubjectHashIsCorrelationStable proves the hash form is stable +// (same sub → same hash) and distinguishes subs. +func TestAudit_SubjectHashIsCorrelationStable(t *testing.T) { + t.Parallel() + a := egressbroker.SubjectHash("user-123") + b := egressbroker.SubjectHash("user-123") + c := egressbroker.SubjectHash("user-456") + assert.Equal(t, a, b) + assert.NotEqual(t, a, c) + assert.Len(t, a, 16) +} + +// TestAudit_NilHandlerFailsLoudly guards the constructor-validation rule. +func TestAudit_NilHandlerFailsLoudly(t *testing.T) { + t.Parallel() + assert.Panics(t, func() { egressbroker.NewAuditLoggerWith(nil) }) +} + +// TestAuditEvent_BuilderFields covers the shared event builder used by the +// injector (pod name optional; sub carried raw only until the log call). +func TestAuditEvent_BuilderFields(t *testing.T) { + t.Parallel() + e := egressbroker.AuditEvent(testIdentity, "pod-abc", + egressbroker.Destination{Host: "api.github.com", Method: "GET", Path: "/x"}, "github") + assert.Equal(t, "github-mcp", e.MCPServer) + assert.Equal(t, "pod-abc", e.Pod) + assert.Equal(t, "user-123", e.Subject) + assert.Equal(t, "github", e.Provider) + assert.Equal(t, "api.github.com", e.Host) +} diff --git a/pkg/egressbroker/config.go b/pkg/egressbroker/config.go index c61d86e49d..bb00212fc3 100644 --- a/pkg/egressbroker/config.go +++ b/pkg/egressbroker/config.go @@ -8,6 +8,7 @@ import ( "os" "strconv" "strings" + "time" ) // Environment-driven configuration (the sidecar is configured exclusively @@ -27,15 +28,32 @@ const ( // EnvDialAllowlist is a comma-separated CIDR/IP list for the D7 per-dial // resolved-IP validation. EnvDialAllowlist = "THV_EGRESSBROKER_DIAL_ALLOWLIST" + // EnvScanFailClosed flips the response scanner (D6c) off its DOCUMENTED + // fail-open default: when "true", a scanner-internal error suppresses the + // response (502) instead of passing it. Governs scanner errors only — a + // detected credential echo ALWAYS blocks in both modes. + EnvScanFailClosed = "THV_EGRESSBROKER_SCAN_FAIL_CLOSED" + // EnvScanMaxBodyBytes overrides the buffered-body scan cap (bytes). + EnvScanMaxBodyBytes = "THV_EGRESSBROKER_SCAN_MAX_BODY_BYTES" ) -// Defaults for the gRPC listener and the Envoy explicit-proxy port. +// Defaults for the gRPC listener, the Envoy explicit-proxy port, and the +// response scanner (D6c). const ( defaultListenAddress = "127.0.0.1" defaultListenPort = 9001 // DefaultProxyPort is the Envoy explicit-proxy port the backend's // HTTP(S)_PROXY env points at (must match the clone-time wiring). DefaultProxyPort = 15001 + + // DefaultScanMaxBodyBytes is the buffered-body scan cap (1 MiB). The + // rendered Envoy ext_proc config buffers at most this much per response. + DefaultScanMaxBodyBytes = 1 << 20 + // ScanCorrelationTTL bounds how long an injection's scan record lives + // (request/response round trip plus slack). + ScanCorrelationTTL = 60 * time.Second + // ScanCorrelationMaxEntries bounds the request-id → token map (10k). + ScanCorrelationMaxEntries = 10000 ) // Config is the validated runtime configuration of the broker process. @@ -52,6 +70,11 @@ type Config struct { ListenPort int // DialAllowlist is the D7 per-dial IP allowlist (required, non-empty). DialAllowlist []string + // ScanFailClosed suppresses responses on scanner-internal errors + // (default false — the documented fail-open posture, ADR D6c). + ScanFailClosed bool + // ScanMaxBodyBytes is the buffered-body scan cap (default 1 MiB). + ScanMaxBodyBytes int64 } // LoadConfig reads and validates the process environment. Fails loudly on @@ -71,13 +94,12 @@ func LoadConfig(getenv func(string) string) (*Config, error) { if cfg.ListenAddress == "" { cfg.ListenAddress = defaultListenAddress } - cfg.ListenPort = defaultListenPort - if raw := strings.TrimSpace(getenv(EnvListenPort)); raw != "" { - port, err := strconv.Atoi(raw) - if err != nil || port <= 0 || port > 65535 { - return nil, fmt.Errorf("egressbroker: %s value %q is not a valid port", EnvListenPort, raw) - } - cfg.ListenPort = port + var err error + if cfg.ListenPort, err = parsePort(getenv(EnvListenPort)); err != nil { + return nil, err + } + if err := loadScanConfig(cfg, getenv); err != nil { + return nil, err } if cfg.PolicyFile == "" { return nil, fmt.Errorf("egressbroker: %s must be set (policy ConfigMap mount)", EnvPolicyFile) @@ -138,6 +160,40 @@ func (c *Config) ReadBumpCA() (*BumpCA, error) { return ParseBumpCA(certPEM, keyPEM) } +// parsePort reads an optional port env value (default when empty). +func parsePort(raw string) (int, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return defaultListenPort, nil + } + port, err := strconv.Atoi(raw) + if err != nil || port <= 0 || port > 65535 { + return 0, fmt.Errorf("egressbroker: %s value %q is not a valid port", EnvListenPort, raw) + } + return port, nil +} + +// loadScanConfig reads the D6c scanner tunables (fail-closed opt-in, body +// cap) with their documented defaults. +func loadScanConfig(cfg *Config, getenv func(string) string) error { + if raw := strings.TrimSpace(getenv(EnvScanFailClosed)); raw != "" { + failClosed, err := strconv.ParseBool(raw) + if err != nil { + return fmt.Errorf("egressbroker: %s value %q is not a boolean", EnvScanFailClosed, raw) + } + cfg.ScanFailClosed = failClosed + } + cfg.ScanMaxBodyBytes = DefaultScanMaxBodyBytes + if raw := strings.TrimSpace(getenv(EnvScanMaxBodyBytes)); raw != "" { + maxBytes, err := strconv.ParseInt(raw, 10, 64) + if err != nil || maxBytes <= 0 { + return fmt.Errorf("egressbroker: %s value %q is not a positive byte count", EnvScanMaxBodyBytes, raw) + } + cfg.ScanMaxBodyBytes = maxBytes + } + return nil +} + func splitTrim(s string) []string { if strings.TrimSpace(s) == "" { return nil diff --git a/pkg/egressbroker/envoyconfig.go b/pkg/egressbroker/envoyconfig.go index a613f8d249..f69933d32d 100644 --- a/pkg/egressbroker/envoyconfig.go +++ b/pkg/egressbroker/envoyconfig.go @@ -25,6 +25,19 @@ type EnvoyConfigParams struct { // SDS server will mint bump certs for — anything else fails the // downstream handshake. AllowedHosts []string + // ScanFailOpen renders the ext_proc response scanner's failure_mode_allow + // (ADR D6c): true = pass responses when the scanner is down/errored + // (documented default); false = fail closed (502) for high-security + // tenants. ext_authz stays failure_mode_allow: false either way. + ScanFailOpen bool + // ScanMaxBodyBytes is the body cap the broker's scanner enforces in-band + // (must match the broker's scanner config; default 1 MiB). It does NOT + // appear in the rendered bootstrap: ext_proc has no per-filter byte cap, + // so Envoy buffers the whole response body and the broker refuses to scan + // past the cap (recorded as a skip, headers still scanned). The cap is a + // cost bound, not a security boundary — the allowlist + destination + // binding are the boundary (ADR-0001 §5). + ScanMaxBodyBytes int64 } // Validate fails loudly on an unrenderable bootstrap (constructor @@ -42,6 +55,9 @@ func (p EnvoyConfigParams) Validate() error { if len(p.AllowedHosts) == 0 { return fmt.Errorf("egressbroker: allowed hosts must not be empty") } + if p.ScanMaxBodyBytes <= 0 { + return fmt.Errorf("egressbroker: scan body cap must be positive") + } return nil } @@ -66,8 +82,29 @@ func (p EnvoyConfigParams) Validate() error { // Security invariants baked into the template (do not relax): // - ext_authz failure_mode_allow: false — a dead injector denies, never // passes unauthenticated traffic. +// - A Lua HTTP filter runs BEFORE ext_authz/ext_proc and copies +// x-request-id and :authority into dynamic metadata namespace +// io.toolhive.egress. This is the ONLY working D6c correlation path: +// route filter_metadata values are stored literally (Envoy performs no +// formatter substitution there), and the response-path ext_proc cannot +// see request headers. ext_proc forwards the namespace per its +// metadata_options.forwarding_namespaces. +// - ext_proc (D6c response scanner) runs response headers+body only +// (request phase is pass-through; injection stays in ext_authz), with +// message_timeout 2s and failure_mode_allow wired from broker config +// (documented fail-open default; fail-closed for high-security tenants). +// Body mode is BUFFERED: ext_proc has no per-filter byte cap, so Envoy +// buffers the whole body; the byte cap lives in the broker's in-band +// scanner (over-cap bodies pass with a skip metric, headers still +// scanned). The cap is a cost bound, not a security boundary — the +// allowlist + destination binding are the boundary (ADR-0001 §5). // - The route table matches only allowlisted hosts; anything else has no // route → connection refused before ext_authz is even consulted. +// - preserve_external_request_id is false (the Envoy default, pinned by +// test): the untrusted server controls its own outbound x-request-id, so +// client-supplied ids are always discarded and Envoy generates one — +// collision poisoning and premature-consumption evasion against the D6c +// correlation map are impossible. // - No redirect-following filter exists: Envoy as a forward proxy passes // 3xx responses through untouched (D6a). // - Listeners and the broker cluster bind loopback only; the SDS secret @@ -104,11 +141,29 @@ const envoyBootstrapTemplate = `static_resources: upgrade_configs: - upgrade_type: CONNECT enabled: true + request_id_extension: + typed_config: + "@type": type.googleapis.com/envoy.extensions.request_id.uuid.v3.UuidRequestIdConfig route_config: name: egress_routes virtual_hosts: {{VHOSTS}} http_filters: + - name: envoy.filters.http.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + default_source_code: + inline_string: | + -- Copies the D6c correlation values into dynamic metadata so + -- the response-path ext_proc scanner can find the injection's + -- scan record (it cannot see request headers, and route + -- filter_metadata values are stored literally — no formatter + -- substitution happens there). + function envoy_on_request(request_handle) + local metadata = request_handle:metadata() + metadata:set("io.toolhive.egress", "request_id", request_handle:headers():get("x-request-id") or "") + metadata:set("io.toolhive.egress", "host", request_handle:headers():get(":authority") or "") + end - name: envoy.filters.http.ext_authz typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz @@ -119,6 +174,32 @@ const envoyBootstrapTemplate = `static_resources: envoy_grpc: cluster_name: egress_broker timeout: 5s + - name: envoy.filters.http.ext_proc + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + failure_mode_allow: {{SCAN_FAIL_OPEN}} + message_timeout: 2s + # NOTE: ext_proc has no per-filter body byte cap (no max_bytes or + # allow_partial_message field exists on this filter in the pinned + # Envoy v1.36 / go-control-plane v1.37): Envoy buffers the whole + # response body per the BUFFERED mode below. The byte cap is + # enforced in-band by the broker's scanner (over-cap bodies pass + # with a skip metric; headers are always scanned). + grpc_service: + envoy_grpc: + cluster_name: egress_broker + timeout: 2s + processing_mode: + request_header_mode: SKIP + request_body_mode: NONE + request_trailer_mode: SKIP + response_header_mode: SEND + response_body_mode: BUFFERED + response_trailer_mode: SKIP + metadata_options: + forwarding_namespaces: + typed: + - io.toolhive.egress - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -178,7 +259,10 @@ const envoyBootstrapTemplate = `static_resources: // forward proxy; every request — the CONNECT itself and the decrypted tunneled // requests — passes ext_authz (no per-route disable: the credential injection // on tunneled traffic IS the feature). Non-CONNECT routes (plain absolute-form -// HTTP through the proxy) share the same upstream. +// HTTP through the proxy) share the same upstream. The D6c correlation +// metadata is NOT set here: route filter_metadata values are stored literally +// (no %REQ()% formatter substitution) — the Lua HTTP filter upstream in the +// chain sets them instead. const vhostTemplate = ` - name: {{HOST_NAME}} domains: ["{{HOST}}"] routes: @@ -217,6 +301,7 @@ func RenderEnvoyBootstrap(params EnvoyConfigParams) ([]byte, error) { out := strings.ReplaceAll(envoyBootstrapTemplate, "{{PROXY_PORT}}", fmt.Sprintf("%d", params.ProxyPort)) out = strings.ReplaceAll(out, "{{EXT_AUTHZ_ADDRESS}}", params.ExtAuthzAddress) out = strings.ReplaceAll(out, "{{EXT_AUTHZ_PORT}}", fmt.Sprintf("%d", params.ExtAuthzPort)) + out = strings.ReplaceAll(out, "{{SCAN_FAIL_OPEN}}", fmt.Sprintf("%t", params.ScanFailOpen)) out = strings.ReplaceAll(out, "{{VHOSTS}}", vhosts.String()) return []byte(out), nil } diff --git a/pkg/egressbroker/envoyconfig_test.go b/pkg/egressbroker/envoyconfig_test.go index 114d54caf3..415500f491 100644 --- a/pkg/egressbroker/envoyconfig_test.go +++ b/pkg/egressbroker/envoyconfig_test.go @@ -20,10 +20,12 @@ import ( func testParams() egressbroker.EnvoyConfigParams { return egressbroker.EnvoyConfigParams{ - ExtAuthzAddress: "127.0.0.1", - ExtAuthzPort: 9001, - ProxyPort: 15001, - AllowedHosts: []string{"api.github.com", "*.example.com"}, + ExtAuthzAddress: "127.0.0.1", + ExtAuthzPort: 9001, + ProxyPort: 15001, + AllowedHosts: []string{"api.github.com", "*.example.com"}, + ScanFailOpen: true, + ScanMaxBodyBytes: egressbroker.DefaultScanMaxBodyBytes, } } @@ -131,13 +133,126 @@ func TestRenderEnvoyBootstrap(t *testing.T) { chain := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any)[0] hcm := dig(t, chain, "filters").([]any)[1].(map[string]any) httpFilters := dig(t, hcm, "typed_config", "http_filters").([]any) - require.Len(t, httpFilters, 2) + require.Len(t, httpFilters, 4) - extAuthz := dig(t, httpFilters[0], "typed_config").(map[string]any) + assert.Equal(t, "envoy.filters.http.lua", httpFilters[0].(map[string]any)["name"], + "the Lua correlation filter runs BEFORE ext_authz/ext_proc") + extAuthz := dig(t, httpFilters[1], "typed_config").(map[string]any) assert.Equal(t, false, extAuthz["failure_mode_allow"], "ext_authz must deny when the broker is down") cluster := dig(t, extAuthz, "grpc_service", "envoy_grpc").(map[string]any)["cluster_name"] assert.Equal(t, "egress_broker", cluster) - assert.Equal(t, "envoy.filters.http.router", httpFilters[1].(map[string]any)["name"]) + assert.Equal(t, "envoy.filters.http.ext_proc", httpFilters[2].(map[string]any)["name"], + "ext_proc runs after ext_authz (D6c)") + assert.Equal(t, "envoy.filters.http.router", httpFilters[3].(map[string]any)["name"]) + }) + + t.Run("lua filter renders the D6c correlation metadata copy", func(t *testing.T) { + t.Parallel() + doc := render(t, testParams()) + chain := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any)[0] + hcm := dig(t, chain, "filters").([]any)[1].(map[string]any) + httpFilters := dig(t, hcm, "typed_config", "http_filters").([]any) + lua := dig(t, httpFilters[0], "typed_config").(map[string]any) + assert.Equal(t, "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua", lua["@type"]) + code := dig(t, lua, "default_source_code").(map[string]any)["inline_string"].(string) + assert.Contains(t, code, `metadata:set("io.toolhive.egress", "request_id"`) + assert.Contains(t, code, `headers():get("x-request-id")`) + assert.Contains(t, code, `metadata:set("io.toolhive.egress", "host"`) + assert.Contains(t, code, `headers():get(":authority")`) + }) + + t.Run("ext_proc scans response headers+body only, fail-open default, 2s timeout (D6c)", func(t *testing.T) { + t.Parallel() + doc := render(t, testParams()) + chain := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any)[0] + hcm := dig(t, chain, "filters").([]any)[1].(map[string]any) + httpFilters := dig(t, hcm, "typed_config", "http_filters").([]any) + extProc := dig(t, httpFilters[2], "typed_config").(map[string]any) + + assert.Equal(t, true, extProc["failure_mode_allow"], + "the documented fail-open default passes responses when the scanner is down") + assert.Equal(t, "2s", extProc["message_timeout"]) + cluster := dig(t, extProc, "grpc_service", "envoy_grpc").(map[string]any)["cluster_name"] + assert.Equal(t, "egress_broker", cluster) + + mode := extProc["processing_mode"].(map[string]any) + assert.Equal(t, "SKIP", mode["request_header_mode"], "request phase is a pass-through") + assert.Equal(t, "NONE", mode["request_body_mode"]) + assert.Equal(t, "SKIP", mode["request_trailer_mode"]) + assert.Equal(t, "SEND", mode["response_header_mode"]) + assert.Equal(t, "BUFFERED", mode["response_body_mode"], "response body buffered for scanning") + assert.Equal(t, "SKIP", mode["response_trailer_mode"]) + + // The D6c correlation metadata namespace is forwarded to ext_proc. + namespaces := dig(t, extProc, "metadata_options", "forwarding_namespaces", "typed").([]any) + assert.Contains(t, namespaces, "io.toolhive.egress") + }) + + t.Run("fail-closed scan config flips ext_proc failure_mode_allow to false", func(t *testing.T) { + t.Parallel() + params := testParams() + params.ScanFailOpen = false + doc := render(t, params) + chain := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any)[0] + hcm := dig(t, chain, "filters").([]any)[1].(map[string]any) + extProc := dig(t, hcm, "typed_config", "http_filters").([]any)[2].(map[string]any)["typed_config"].(map[string]any) + assert.Equal(t, false, extProc["failure_mode_allow"], + "fail-closed tenants get 502s on scanner unavailability") + // ext_authz posture never changes. + extAuthz := dig(t, hcm, "typed_config", "http_filters").([]any)[1].(map[string]any)["typed_config"].(map[string]any) + assert.Equal(t, false, extAuthz["failure_mode_allow"]) + }) + + t.Run("request-id extension renders so x-request-id exists at ext_authz time", func(t *testing.T) { + t.Parallel() + doc := render(t, testParams()) + chain := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any)[0] + hcm := dig(t, chain, "filters").([]any)[1].(map[string]any)["typed_config"].(map[string]any) + ridExt := dig(t, hcm, "request_id_extension", "typed_config").(map[string]any) + assert.Equal(t, + "type.googleapis.com/envoy.extensions.request_id.uuid.v3.UuidRequestIdConfig", ridExt["@type"]) + }) + + t.Run("client-supplied x-request-id is never trusted (no accept_from_client; default pinned)", func(t *testing.T) { + t.Parallel() + // The pinned Envoy version (v1.36 / go-control-plane v1.37) has NO + // request_id_config/accept_from_client field on the HCM — the trust + // knob is preserve_external_request_id, whose default (false) discards + // any downstream-supplied x-request-id and generates a fresh one. This + // test pins BOTH that the field is absent and that the template does + // not opt back into preserving client ids, so a future config edit or + // Envoy upgrade that re-enables client-supplied ids fails loudly here. + data, err := egressbroker.RenderEnvoyBootstrap(testParams()) + require.NoError(t, err) + assert.NotContains(t, string(data), "preserve_external_request_id", + "never opt into keeping client-supplied request ids (collision poisoning / "+ + "premature-consumption evasion against the D6c correlation map)") + doc := render(t, testParams()) + chain := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any)[0] + hcm := dig(t, chain, "filters").([]any)[1].(map[string]any)["typed_config"].(map[string]any) + assert.NotContains(t, hcm, "preserve_external_request_id") + assert.NotContains(t, hcm, "request_id_config", + "no accept_from_client exists on the pinned HCM; if this fails after an Envoy bump, "+ + "set request_id_config.accept_from_client: false explicitly") + }) + + t.Run("routes carry no D6c filter_metadata (Lua sets it; route values are stored literally)", func(t *testing.T) { + t.Parallel() + data, err := egressbroker.RenderEnvoyBootstrap(testParams()) + require.NoError(t, err) + assert.NotContains(t, string(data), "%REQ(", + "route filter_metadata performs no formatter substitution — %REQ% values would be "+ + "stored literally and every scanner lookup would miss") + doc := render(t, testParams()) + chain := dig(t, doc, "static_resources", "listeners").([]any)[0].(map[string]any)["filter_chains"].([]any)[0] + vhosts := dig(t, chain, "filters").([]any)[1].(map[string]any) + vl := dig(t, vhosts, "typed_config", "route_config", "virtual_hosts").([]any) + for _, v := range vl { + routes := v.(map[string]any)["routes"].([]any) + require.Len(t, routes, 2) + assert.NotContains(t, routes[1], "metadata", + "the D6c correlation metadata comes from the Lua filter, not route filter_metadata") + } }) t.Run("routes cover exactly the allowlisted hosts; CONNECT terminates with upgrade config", func(t *testing.T) { diff --git a/pkg/egressbroker/extproc.go b/pkg/egressbroker/extproc.go new file mode 100644 index 0000000000..8f4725afda --- /dev/null +++ b/pkg/egressbroker/extproc.go @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + typev3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +// ExternalProcessorServer is the Envoy ext_proc gRPC endpoint implementing +// response-side credential scanning (ADR-0001 D6c). The ext_authz Check that +// injects a credential records (x-request-id → scan record); this service +// looks the request-id up on each response and scans the headers + buffered +// body for the injected credential value (exact and base64-encoded). +// +// Correlation model: Envoy forwards x-request-id in the ext_proc dynamic +// metadata (namespace "io.toolhive.egress", key "request_id") because the +// response-path ext_proc cannot see request headers. The rendered bootstrap +// sets that metadata with a Lua HTTP filter running BEFORE ext_authz (route +// filter_metadata values are stored literally — no formatter substitution — +// so they cannot carry it). +// +// Failure posture: the DOCUMENTED fail-open default lives in the RENDERED +// ENVOY CONFIG (failure_mode_allow: a dead/unreachable/slow scanner passes +// responses); this service additionally applies the configured posture +// (failOpen) to in-band decisions — an unknown request-id (deny path, +// scanner restart, TTL eviction, direct hit) and an over-cap body pass with +// a metric when failOpen, and suppress the response (502) when !failOpen. +// Detection of an actual leak ALWAYS blocks — the failure mode governs +// scanner unavailability only, never matches. +type ExternalProcessorServer struct { + extprocv3.UnimplementedExternalProcessorServer + tokens *TokenMap + bounds ScannerBounds + failOpen bool + identity PodIdentity + metrics *BrokerMetrics + audit AuditLogger +} + +// NewExternalProcessorServer wires the scanner. Fails loudly on nil/invalid +// dependencies (constructor validation: a misconfigured scanner must not +// start). +func NewExternalProcessorServer( + tokens *TokenMap, + bounds ScannerBounds, + failOpen bool, + identity PodIdentity, + metrics *BrokerMetrics, + audit AuditLogger, +) (*ExternalProcessorServer, error) { + if tokens == nil { + return nil, fmt.Errorf("egressbroker: token map must not be nil") + } + if err := validateScannerBounds(bounds); err != nil { + return nil, err + } + if audit == nil { + return nil, fmt.Errorf("egressbroker: audit logger must not be nil") + } + if identity.MCPServer == "" { + return nil, fmt.Errorf("egressbroker: pod identity is incomplete; refusing to build response scanner") + } + return &ExternalProcessorServer{ + tokens: tokens, + bounds: bounds, + failOpen: failOpen, + identity: identity, + metrics: metrics, + audit: audit, + }, nil +} + +// Process implements envoy.service.ext_proc.v3.ExternalProcessor. One stream +// corresponds to one HTTP request/response exchange; messages arrive in +// request-then-response order (request phase is a pass-through — injection +// stays in ext_authz). +func (s *ExternalProcessorServer) Process(stream extprocv3.ExternalProcessor_ProcessServer) error { + state := &StreamState{} + for { + req, err := stream.Recv() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + // Transport-level failure: nothing to decide; the filter's own + // failure_mode_allow governs the in-flight response. + return err + } + resp := s.Handle(stream.Context(), state, req) + if resp == nil { + continue + } + if err := stream.Send(resp); err != nil { + return err + } + } +} + +// StreamState carries the correlation across the phases of one stream. +type StreamState struct { + requestID string + record ScanRecord + known bool + host string +} + +// Handle computes the response (nil = no response needed) for one processing +// request, carrying per-stream correlation state. It never returns an error +// to the stream: every internal failure is decided locally per the +// fail-open/fail-closed posture so a scanner bug can never wedge every +// upstream call of the workload. Exported so tests can drive the scanner +// without a gRPC stream; production callers go through Process. +func (s *ExternalProcessorServer) Handle(ctx context.Context, st *StreamState, req *extprocv3.ProcessingRequest, +) *extprocv3.ProcessingResponse { + switch r := req.GetRequest().(type) { + case *extprocv3.ProcessingRequest_ResponseHeaders: + // The request-id arrives via the Lua-set dynamic metadata (the + // response-side ext_proc cannot see request headers; the rendered + // bootstrap copies x-request-id into io.toolhive.egress upstream of + // ext_authz). + st.requestID = requestIDFromMetadata(req.GetMetadataContext()) + st.host = requestHostFromMetadata(req.GetMetadataContext()) + record, ok := s.tokens.Lookup(st.requestID) + if !ok { + // Unknown request-id: deny path (no injection), scanner restart, + // TTL eviction, or a direct hit. Fail-open: pass + metric; + // fail-closed: suppress (502) — nothing proves this response does + // not echo a credential. + s.metrics.RecordScan(ctx, s.identity.MCPServer, "", metricResultUnknown, "") + if !s.failOpen { + slog.WarnContext(ctx, "egressbroker: unknown request-id; suppressing response (fail-closed)", + "mcpserver", s.identity.MCPServer, "host", st.host) + return s.suppress() + } + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ResponseHeaders{}, + } + } + st.record = record + st.known = true + if scanHeaders(r.ResponseHeaders.GetHeaders(), record.Needles) { + return s.block(ctx, st, LeakLocationHeader) + } + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ResponseHeaders{}, + } + + case *extprocv3.ProcessingRequest_ResponseBody: + if !st.known { + // Unknown request-id was already counted at the header phase. + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ResponseBody{}, + } + } + result := scanBody(r.ResponseBody.GetBody(), st.record.Needles, s.bounds.MaxBodyBytes) + switch result { + case scanLeak: + return s.block(ctx, st, LeakLocationBody) + case scanOversize: + // Body not scanned (the cap is a cost bound, not a security + // boundary — the allowlist + destination binding are). Record ONLY + // the skip: a result=ok datapoint here would double-count the body + // phase as scanned when it was not. + s.metrics.RecordScanSkipped(ctx, s.identity.MCPServer, st.record.Provider) + slog.WarnContext(ctx, "egressbroker: response body over scan cap; body not scanned", + "mcpserver", s.identity.MCPServer, "provider", st.record.Provider, + "cap_bytes", s.bounds.MaxBodyBytes) + if !s.failOpen { + return s.suppress() + } + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ResponseBody{}, + } + case scanClean: + // fall through to the pass-through below + } + s.metrics.RecordScan(ctx, s.identity.MCPServer, st.record.Provider, metricResultOK, "") + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ResponseBody{}, + } + + default: + // Request bodies/trailers and response trailers are not configured for + // delivery; answer with the matching pass-through variant if one ever + // arrives so the filter never stalls. + return passThroughFor(req) + } +} + +// block suppresses a leaking response: 502 + generic body, audit + metric. +// The matched value NEVER appears in any log, metric, or response. +func (s *ExternalProcessorServer) block( + ctx context.Context, st *StreamState, where LeakLocation, +) *extprocv3.ProcessingResponse { + s.metrics.RecordScan(ctx, s.identity.MCPServer, st.record.Provider, metricResultLeak, string(where)) + s.audit.Leak(ctx, LeakEvent{ + MCPServer: s.identity.MCPServer, + Provider: st.record.Provider, + Host: st.host, + RequestID: st.requestID, + Where: where, + }) + slog.WarnContext(ctx, "egressbroker: response suppressed (injected credential echoed back)", + "mcpserver", s.identity.MCPServer, "provider", st.record.Provider, "where", string(where)) + return s.suppress() +} + +// suppress is the 502 + generic-body immediate response (no credential +// material, no reason detail that would help an attacker tune the evasion). +func (*ExternalProcessorServer) suppress() *extprocv3.ProcessingResponse { + return &extprocv3.ProcessingResponse{ + Response: &extprocv3.ProcessingResponse_ImmediateResponse{ + ImmediateResponse: &extprocv3.ImmediateResponse{ + Status: &typev3.HttpStatus{Code: typev3.StatusCode_BadGateway}, + Body: []byte("response suppressed by egress policy"), + Details: "egress_broker_response_leak", + }, + }, + } +} + +// passThroughFor returns the empty response variant matching the incoming +// request variant so unconfigured phases never stall the filter. +func passThroughFor(req *extprocv3.ProcessingRequest) *extprocv3.ProcessingResponse { + switch req.GetRequest().(type) { + case *extprocv3.ProcessingRequest_RequestBody: + return &extprocv3.ProcessingResponse{Response: &extprocv3.ProcessingResponse_RequestBody{}} + case *extprocv3.ProcessingRequest_RequestTrailers: + return &extprocv3.ProcessingResponse{Response: &extprocv3.ProcessingResponse_RequestTrailers{}} + case *extprocv3.ProcessingRequest_ResponseTrailers: + return &extprocv3.ProcessingResponse{Response: &extprocv3.ProcessingResponse_ResponseTrailers{}} + default: + return nil + } +} diff --git a/pkg/egressbroker/extproc_test.go b/pkg/egressbroker/extproc_test.go new file mode 100644 index 0000000000..cfa33fb09b --- /dev/null +++ b/pkg/egressbroker/extproc_test.go @@ -0,0 +1,923 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker_test + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + "log/slog" + "net" + "sync" + "testing" + "time" + + envoycore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/noop" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken" + "github.com/stacklok/toolhive/pkg/auth/upstreamtoken/mocks" + "github.com/stacklok/toolhive/pkg/egressbroker" +) + +// --- shared test plumbing ------------------------------------------------- + +func newTestMeterProvider(t *testing.T) (metric.MeterProvider, *sdkmetric.ManualReader) { + t.Helper() + reader := sdkmetric.NewManualReader() + return sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)), reader +} + +func collectMetrics(t *testing.T, reader *sdkmetric.ManualReader) map[string]metricdata.Metrics { + t.Helper() + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + out := map[string]metricdata.Metrics{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + out[m.Name] = m + } + } + return out +} + +// metricPoints flattens a counter's datapoints into attr-set → value. +func metricPoints(t *testing.T, m metricdata.Metrics) []map[string]string { + t.Helper() + sum, ok := m.Data.(metricdata.Sum[int64]) + require.True(t, ok, "expected a Sum counter, got %T", m.Data) + var out []map[string]string + for _, dp := range sum.DataPoints { + attrs := map[string]string{} + for _, kv := range dp.Attributes.ToSlice() { + attrs[string(kv.Key)] = kv.Value.AsString() + } + for i := int64(0); i < dp.Value; i++ { + out = append(out, attrs) + } + } + return out +} + +// captureHandler records slog records for audit assertions. +type captureHandler struct { + mu sync.Mutex + records []slogRecord +} + +type slogRecord struct { + msg string + attrs map[string]string + rawLog string // every attribute rendered (for token-substring guards) +} + +func (*captureHandler) Enabled(_ context.Context, _ slog.Level) bool { return true } + +func (h *captureHandler) Handle(_ context.Context, r slog.Record) error { + h.mu.Lock() + defer h.mu.Unlock() + rec := slogRecord{msg: r.Message, attrs: map[string]string{}} + r.Attrs(func(a slog.Attr) bool { + v := a.Value.String() + rec.attrs[a.Key] = v + rec.rawLog += a.Key + "=" + v + " " + return true + }) + h.records = append(h.records, rec) + return nil +} + +func (h *captureHandler) WithAttrs(_ []slog.Attr) slog.Handler { return h } +func (h *captureHandler) WithGroup(_ string) slog.Handler { return h } + +func (h *captureHandler) byMsg(msg string) []slogRecord { + h.mu.Lock() + defer h.mu.Unlock() + var out []slogRecord + for _, r := range h.records { + if r.msg == msg { + out = append(out, r) + } + } + return out +} + +func newCaptureAudit() (*captureHandler, egressbroker.AuditLogger) { + h := &captureHandler{} + return h, egressbroker.NewAuditLoggerWith(h) +} + +const ( + testHost = "api.github.com" + testRequestID = "req-scan-1" + testToken = "gho_s3cr3t-t0ken" + testHeaderVal = "Bearer " + testToken +) + +// scanFixture wires map + scanner + metrics + audit with one recorded +// injection for testRequestID (mirrors what the injector does on allow). +type scanFixture struct { + tokens *egressbroker.TokenMap + scanner *egressbroker.ExternalProcessorServer + metrics *egressbroker.BrokerMetrics + reader *sdkmetric.ManualReader + audit *captureHandler + state *egressbroker.StreamState +} + +func newScanFixture(t *testing.T) *scanFixture { + t.Helper() + return newScanFixtureWithFailOpen(t, true) +} + +func newScanFixtureWithFailOpen(t *testing.T, failOpen bool) *scanFixture { + t.Helper() + tokens, err := egressbroker.NewTokenMap(egressbroker.ScanCorrelationTTL, egressbroker.ScanCorrelationMaxEntries) + require.NoError(t, err) + mp, reader := newTestMeterProvider(t) + metrics, err := egressbroker.NewBrokerMetrics(mp) + require.NoError(t, err) + auditHandler, auditLog := newCaptureAudit() + scanner, err := egressbroker.NewExternalProcessorServer( + tokens, + egressbroker.ScannerBounds{MaxBodyBytes: egressbroker.DefaultScanMaxBodyBytes}, + failOpen, + testIdentity, + metrics, + auditLog, + ) + require.NoError(t, err) + return &scanFixture{tokens: tokens, scanner: scanner, metrics: metrics, reader: reader, audit: auditHandler} +} + +// record mirrors the injector's injection-time hook (same map, same API). +func (f *scanFixture) record() { + f.tokens.Record(testRequestID, testHeaderVal, "github") +} + +// handleHeaders / handleBody drive one response exchange with shared stream +// state (as the gRPC Process loop would). +func (f *scanFixture) handleHeaders(req *extprocv3.ProcessingRequest) *extprocv3.ProcessingResponse { + f.state = &egressbroker.StreamState{} + return f.scanner.Handle(context.Background(), f.state, req) +} + +func (f *scanFixture) handleBody(req *extprocv3.ProcessingRequest) *extprocv3.ProcessingResponse { + return f.scanner.Handle(context.Background(), f.state, req) +} + +// respHeaders builds the response-headers processing request with the D6c +// correlation metadata the rendered route would set. +func respHeaders(requestID string, headers ...*envoycore.HeaderValue) *extprocv3.ProcessingRequest { + ns, err := structpb.NewStruct(map[string]any{"request_id": requestID, "host": testHost}) + requireNotErr(err) + return &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_ResponseHeaders{ + ResponseHeaders: &extprocv3.HttpHeaders{ + Headers: &envoycore.HeaderMap{Headers: headers}, + }, + }, + MetadataContext: &envoycore.Metadata{ + FilterMetadata: map[string]*structpb.Struct{"io.toolhive.egress": ns}, + }, + } +} + +func respBody(body []byte, requestID string) *extprocv3.ProcessingRequest { + ns, err := structpb.NewStruct(map[string]any{"request_id": requestID, "host": testHost}) + requireNotErr(err) + return &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_ResponseBody{ + ResponseBody: &extprocv3.HttpBody{Body: body, EndOfStream: true}, + }, + MetadataContext: &envoycore.Metadata{ + FilterMetadata: map[string]*structpb.Struct{"io.toolhive.egress": ns}, + }, + } +} + +func requireNotErr(err error) { + if err != nil { + panic(err) + } +} + +func isImmediate(resp *extprocv3.ProcessingResponse) *extprocv3.ImmediateResponse { + imm, ok := resp.GetResponse().(*extprocv3.ProcessingResponse_ImmediateResponse) + if !ok { + return nil + } + return imm.ImmediateResponse +} + +// --- §1.3 test matrix ------------------------------------------------------ + +// §1.3 #1: response body contains the injected token → 502 + generic body; +// the token appears nowhere; metric + audit emitted. +func TestExtProc_LeakInBodyBlocked(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + + resp := f.handleHeaders(respHeaders(testRequestID)) + assert.Nil(t, isImmediate(resp), "clean headers must pass") + + body := []byte(`{"debug":"` + testHeaderVal + `"}`) + resp = f.handleBody(respBody(body, testRequestID)) + imm := isImmediate(resp) + require.NotNil(t, imm, "echoed credential in body must suppress the response") + assert.Equal(t, int32(502), int32(imm.GetStatus().GetCode())) + assert.Equal(t, "response suppressed by egress policy", string(imm.GetBody())) + assert.NotContains(t, string(imm.GetBody()), testToken, "generic body must never carry the matched value") + assert.NotContains(t, imm.GetDetails(), testToken) + + // Metric: one leak scan with mcpserver+provider+where labels. + scans := metricPoints(t, collectMetrics(t, f.reader)["egress_broker_response_scan_total"]) + require.Len(t, scans, 1) + assert.Equal(t, map[string]string{ + "mcpserver": "github-mcp", "provider": "github", "result": "leak", "where": "body", + }, scans[0]) + + // Audit: exactly one Leak line, no token substring anywhere. + leaks := f.audit.byMsg("egressbroker response suppressed: credential echo detected") + require.Len(t, leaks, 1) + assert.Equal(t, "github-mcp", leaks[0].attrs["mcpserver"]) + assert.Equal(t, "github", leaks[0].attrs["provider"]) + assert.Equal(t, "api.github.com", leaks[0].attrs["host"]) + assert.Equal(t, testRequestID, leaks[0].attrs["request_id"]) + assert.Equal(t, "body", leaks[0].attrs["where"]) + assert.NotContains(t, leaks[0].rawLog, testToken) + assert.NotContains(t, leaks[0].rawLog, testHeaderVal) +} + +// §1.3 #2: response headers contain the token → same suppression. +func TestExtProc_LeakInHeadersBlocked(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + + echoed := &envoycore.HeaderValue{Key: "x-echo-authorization", Value: testHeaderVal} + resp := f.handleHeaders(respHeaders(testRequestID, echoed)) + imm := isImmediate(resp) + require.NotNil(t, imm, "echoed credential in a response header must suppress the response") + assert.Equal(t, int32(502), int32(imm.GetStatus().GetCode())) + assert.NotContains(t, string(imm.GetBody()), testToken) + + scans := metricPoints(t, collectMetrics(t, f.reader)["egress_broker_response_scan_total"]) + require.Len(t, scans, 1) + assert.Equal(t, "header", scans[0]["where"]) + leaks := f.audit.byMsg("egressbroker response suppressed: credential echo detected") + require.Len(t, leaks, 1) + assert.Equal(t, "header", leaks[0].attrs["where"]) + assert.NotContains(t, leaks[0].rawLog, testToken) +} + +// §1.3 #3: clean response passes untouched. +func TestExtProc_CleanResponsePasses(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + + resp := f.handleHeaders(respHeaders(testRequestID, + &envoycore.HeaderValue{Key: "content-type", Value: "application/json"})) + assert.Nil(t, isImmediate(resp)) + assert.IsType(t, &extprocv3.ProcessingResponse_ResponseHeaders{}, resp.GetResponse(), + "clean headers must get a pass-through response") + + resp = f.handleBody(respBody([]byte(`{"ok":true}`), testRequestID)) + assert.Nil(t, isImmediate(resp)) + assert.IsType(t, &extprocv3.ProcessingResponse_ResponseBody{}, resp.GetResponse(), + "clean body must get a pass-through response") + + scans := metricPoints(t, collectMetrics(t, f.reader)["egress_broker_response_scan_total"]) + require.Len(t, scans, 1) + assert.Equal(t, "ok", scans[0]["result"]) + assert.Empty(t, f.audit.byMsg("egressbroker response suppressed: credential echo detected")) +} + +// §1.3 #4: token present base64-encoded → blocked. +func TestExtProc_Base64TokenBlocked(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + + resp := f.handleHeaders(respHeaders(testRequestID)) + assert.Nil(t, isImmediate(resp)) + encoded := base64.StdEncoding.EncodeToString([]byte(testHeaderVal)) + resp = f.handleBody(respBody([]byte("data: "+encoded), testRequestID)) + imm := isImmediate(resp) + require.NotNil(t, imm, "base64-encoded echo must suppress the response") + assert.NotContains(t, string(imm.GetBody()), testToken) +} + +// §1.3 #5: body over cap → body not scanned (header scan still applies) + +// scan_skipped_total metric. Q1 metric semantics: the skip must NOT also +// record a result=ok datapoint for the unscanned body phase. +func TestExtProc_OversizeBodySkipped(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + + // Exactly at the cap is scanned: token within the cap must block... + bodyCap := egressbroker.DefaultScanMaxBodyBytes + atCap := append(bytes.Repeat([]byte("a"), bodyCap-len(testHeaderVal)), testHeaderVal...) + require.Len(t, atCap, bodyCap) + _ = f.handleHeaders(respHeaders(testRequestID)) + resp := f.handleBody(respBody(atCap, testRequestID)) + assert.NotNil(t, isImmediate(resp), "token at [cap-len, cap) is scanned and blocks") + + // ...and over the cap is NOT: the same body one byte longer passes with a + // skip (never scan past the cap). + over := append(atCap, 'b') + f.record() // the previous lookup consumed the entry + resp = f.handleHeaders(respHeaders(testRequestID)) + assert.Nil(t, isImmediate(resp)) + resp = f.handleBody(respBody(over, testRequestID)) + assert.Nil(t, isImmediate(resp), "over-cap body passes (fail-open default for the body)") + + got := collectMetrics(t, f.reader) + skips := metricPoints(t, got["egress_broker_scan_skipped_total"]) + require.Len(t, skips, 1) + assert.Equal(t, map[string]string{"mcpserver": "github-mcp", "provider": "github"}, skips[0]) + + // Q1: the skipped body phase must not double-count as result=ok — the + // only scan datapoint is the earlier leak. + scans := metricPoints(t, got["egress_broker_response_scan_total"]) + require.Len(t, scans, 1, "skip-then-ok double counting: an unscanned body must not record result=ok") + assert.Equal(t, "leak", scans[0]["result"]) +} + +// §1.3 #5b: fail-closed + over-cap body → 502 (the configured posture +// governs in-band decisions, not just Envoy-side unavailability). +func TestExtProc_FailClosedOversizeBodyBlocked(t *testing.T) { + t.Parallel() + f := newScanFixtureWithFailOpen(t, false) + f.record() + + over := bytes.Repeat([]byte("a"), egressbroker.DefaultScanMaxBodyBytes+1) + resp := f.handleHeaders(respHeaders(testRequestID)) + assert.Nil(t, isImmediate(resp), "clean headers pass even fail-closed") + resp = f.handleBody(respBody(over, testRequestID)) + imm := isImmediate(resp) + require.NotNil(t, imm, "fail-closed: an unscannable (over-cap) body must be suppressed") + assert.Equal(t, int32(502), int32(imm.GetStatus().GetCode())) + + got := collectMetrics(t, f.reader) + skips := metricPoints(t, got["egress_broker_scan_skipped_total"]) + require.Len(t, skips, 1, "the skip is recorded before the fail-closed suppression") + // An unscanned body records no scan-result datapoint (no skip-then-ok + // double count): the only body outcome was the skip above. + if m, ok := got["egress_broker_response_scan_total"]; ok { + assert.Empty(t, metricPoints(t, m)) + } +} + +// §1.3 #6b: fail-closed + unknown request-id → 502 (nothing proves the +// response does not echo a credential). +func TestExtProc_FailClosedUnknownRequestIDBlocked(t *testing.T) { + t.Parallel() + f := newScanFixtureWithFailOpen(t, false) + // Nothing recorded. + + resp := f.handleHeaders(respHeaders("never-injected")) + imm := isImmediate(resp) + require.NotNil(t, imm, "fail-closed: unknown request-id must be suppressed") + assert.Equal(t, int32(502), int32(imm.GetStatus().GetCode())) + assert.NotContains(t, string(imm.GetBody()), testToken) + + scans := metricPoints(t, collectMetrics(t, f.reader)["egress_broker_response_scan_total"]) + require.Len(t, scans, 1, "unknown is still counted exactly once") + assert.Equal(t, "unknown_request", scans[0]["result"]) +} + +// §1.3 #6: unknown request-id → pass + unknown metric. +func TestExtProc_UnknownRequestIDPasses(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + // Nothing recorded. + + resp := f.handleHeaders(respHeaders("never-injected")) + assert.Nil(t, isImmediate(resp)) + resp = f.handleBody(respBody([]byte("anything"), "never-injected")) + assert.Nil(t, isImmediate(resp)) + + scans := metricPoints(t, collectMetrics(t, f.reader)["egress_broker_response_scan_total"]) + require.Len(t, scans, 1, "unknown counted once (header phase)") + assert.Equal(t, "unknown_request", scans[0]["result"]) +} + +// Expired entries read as unknown (scanner restart / TTL eviction equivalent). +// Uses the injected fake clock — no time.Sleep against the TTL. +func TestTokenMap_ExpiredEntryReadsAsUnknown(t *testing.T) { + t.Parallel() + now := time.Now() + clock := &fakeClock{now: now} + short, err := egressbroker.NewTokenMapWithClock(time.Minute, 10, clock.Now) + require.NoError(t, err) + short.Record("req-expired", testHeaderVal, "github") + + clock.Advance(2 * time.Minute) + + _, ok := short.Lookup("req-expired") + assert.False(t, ok, "expired entry must read as unknown") +} + +// §1.3 #8: fail-closed config renders failure_mode_allow: false (the 502 on +// scanner error is then Envoy's). Asserted end-to-end in envoyconfig_test; +// here the constructor rejects a nonsensical scan cap (fail loudly). +func TestExtProc_ConstructorValidation(t *testing.T) { + t.Parallel() + tokens, err := egressbroker.NewTokenMap(time.Minute, 10) + require.NoError(t, err) + _, auditLog := newCaptureAudit() + + _, err = egressbroker.NewExternalProcessorServer(nil, + egressbroker.ScannerBounds{MaxBodyBytes: 1}, true, testIdentity, nil, auditLog) + assert.Error(t, err, "nil token map") + _, err = egressbroker.NewExternalProcessorServer(tokens, + egressbroker.ScannerBounds{MaxBodyBytes: 0}, true, testIdentity, nil, auditLog) + assert.Error(t, err, "zero body cap must fail loudly") + _, err = egressbroker.NewExternalProcessorServer(tokens, + egressbroker.ScannerBounds{MaxBodyBytes: 1}, true, testIdentity, nil, nil) + assert.Error(t, err, "nil audit logger") + _, err = egressbroker.NewExternalProcessorServer(tokens, + egressbroker.ScannerBounds{MaxBodyBytes: 1}, true, egressbroker.PodIdentity{}, nil, auditLog) + assert.Error(t, err, "incomplete identity") + + // nil metrics is allowed (uninstrumented path), noop provider too. + m, err := egressbroker.NewBrokerMetrics(noop.NewMeterProvider()) + require.NoError(t, err) + _, err = egressbroker.NewExternalProcessorServer(tokens, + egressbroker.ScannerBounds{MaxBodyBytes: 1}, false, testIdentity, m, auditLog) + assert.NoError(t, err) +} + +// §1.3 #9: TTL-map eviction under load → no panic, oldest evicted, scan miss +// counted as unknown. +func TestTokenMap_EvictionUnderLoad(t *testing.T) { + t.Parallel() + m, err := egressbroker.NewTokenMap(time.Minute, 100) + require.NoError(t, err) + + // Oldest ("req-0") must be evicted when entry 101 lands. + for i := 0; i < 99; i++ { + m.Record(fmt.Sprintf("req-%d", i), testHeaderVal, "github") + } + m.Record("req-99", testHeaderVal, "github") + m.Record("req-100", testHeaderVal, "github") + assert.Equal(t, 100, m.Len()) + _, ok := m.Lookup("req-0") + assert.False(t, ok, "oldest entry must be evicted (scan miss)") + _, ok = m.Lookup("req-100") + assert.True(t, ok, "newest entry survives") + + // Lookup consumes (one injection ↔ one scanned response). + _, ok = m.Lookup("req-100") + assert.False(t, ok, "consumed entry reports unknown") + + // Re-recording the same request-id refreshes in place. + m.Record("req-1", testHeaderVal, "github") + m.Record("req-1", testHeaderVal, "github") + assert.Equal(t, 99, m.Len(), "two consumed (one evicted, one looked up) + one refreshed") +} + +// §1.3 #10: deny path (no injection) → no map entry; response passes. +// Covered at the scanner boundary by TestExtProc_UnknownRequestIDPasses; +// here the injector proves the deny path never records. +func TestInjector_DenyPathRecordsNothing(t *testing.T) { + t.Parallel() + tokens, err := egressbroker.NewTokenMap(time.Minute, 10) + require.NoError(t, err) + inj := mustInjector(t, newDenyAllReader(t)) + inj.WithScanCorrelation(tokens) + + res := inj.Evaluate(context.Background(), + egressbroker.Destination{Host: "evil.example.com", Method: "GET", Path: "/"}, "req-denied") + assert.False(t, res.Allow) + assert.Equal(t, 0, tokens.Len(), "a denied request must leave no scan-correlation entry") + _, ok := tokens.Lookup("req-denied") + assert.False(t, ok) +} + +// --- token map bounds ------------------------------------------------------ + +func TestTokenMap_ConstructorValidation(t *testing.T) { + t.Parallel() + _, err := egressbroker.NewTokenMap(0, 10) + assert.Error(t, err, "zero TTL must fail loudly") + _, err = egressbroker.NewTokenMap(time.Minute, 0) + assert.Error(t, err, "zero bound must fail loudly") +} + +// TestScanCorrelationRecordedOnInject proves the injector records +// (request-id → token) exactly when it allows, keyed by the x-request-id. +func TestScanCorrelationRecordedOnInject(t *testing.T) { + t.Parallel() + tokens, err := egressbroker.NewTokenMap(time.Minute, 10) + require.NoError(t, err) + inj := mustInjector(t, newGithubTokenReader(t, testToken)) + inj.WithScanCorrelation(tokens) + + res := inj.Evaluate(context.Background(), + egressbroker.Destination{Host: "api.github.com", Method: "GET", Path: "/repos/foo"}, testRequestID) + require.True(t, res.Allow) + rec, ok := tokens.Lookup(testRequestID) + require.True(t, ok, "allow must record the scan correlation") + assert.Equal(t, "github", rec.Provider) + // The hash is of the exact injected header value. + assert.Equal(t, sha256Of(testHeaderVal), rec.TokenHash) +} + +// --- low-cardinality guard (ADR D11) --------------------------------------- + +// TestMetricLabelsAreLowCardinality asserts no datapoint on any broker +// instrument carries a user/pod/request-id label. +func TestMetricLabelsAreLowCardinality(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + f.handleHeaders(respHeaders(testRequestID, + &envoycore.HeaderValue{Key: "x-echo", Value: testHeaderVal})) + f.handleHeaders(respHeaders("unknown-req")) + + allowed := map[string]map[string]bool{ + "egress_broker_response_scan_total": {"mcpserver": true, "provider": true, "result": true, "where": true}, + "egress_broker_scan_skipped_total": {"mcpserver": true, "provider": true}, + "egress_broker_injections_total": {"mcpserver": true, "provider": true}, + "egress_broker_denials_total": {"mcpserver": true, "provider": true, "result": true}, + } + for name, m := range collectMetrics(t, f.reader) { + keys, ok := allowed[name] + require.True(t, ok, "unexpected instrument %q", name) + sum, isSum := m.Data.(metricdata.Sum[int64]) + require.True(t, isSum, "%s must be a counter", name) + for _, dp := range sum.DataPoints { + for _, attr := range dp.Attributes.ToSlice() { + assert.True(t, keys[string(attr.Key)], + "%s carries forbidden label %q (D11: no user/pod/request-id labels)", name, attr.Key) + assert.NotContains(t, attr.Value.AsString(), "user-123") + assert.NotContains(t, attr.Value.AsString(), testRequestID) + } + } + } +} + +// newDenyAllReader returns a TokenReader mock with no expectations: any +// credential load fails the test (deny paths must never load). +func newDenyAllReader(t *testing.T) *mocks.MockTokenReader { + t.Helper() + return mocks.NewMockTokenReader(gomock.NewController(t)) +} + +// newGithubTokenReader returns a TokenReader mock serving one github +// credential for the test session. +func newGithubTokenReader(t *testing.T, token string) *mocks.MockTokenReader { + t.Helper() + reader := mocks.NewMockTokenReader(gomock.NewController(t)) + reader.EXPECT(). + GetAllUpstreamCredentials(gomock.Any(), testIdentity.SessionID, gomock.Any()). + Return(map[string]upstreamtoken.UpstreamCredential{ + "github": {AccessToken: token}, + }, nil, nil) + return reader +} + +func sha256Of(v string) [32]byte { return sha256.Sum256([]byte(v)) } + +var _ slog.Handler = (*captureHandler)(nil) + +// --- TokenMap concurrency + eviction --------------------------------------- + +// fakeClock is the injected now-seam for TokenMap tests (no time.Sleep). +type fakeClock struct { + mu sync.Mutex + now time.Time +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +// Concurrency flood (run with -race): concurrent Record/Lookup against +// distinct ids ≫ maxEntries must keep Len ≤ maxEntries, never panic, and +// evict oldest-first. +func TestTokenMap_ConcurrencyFlood(t *testing.T) { + t.Parallel() + const maxEntries = 64 + m, err := egressbroker.NewTokenMap(time.Minute, maxEntries) + require.NoError(t, err) + + var wg sync.WaitGroup + ids := make([]string, 0, 8*maxEntries) + for i := 0; i < 8*maxEntries; i++ { + ids = append(ids, fmt.Sprintf("flood-%d", i)) + } + for _, id := range ids { + wg.Add(2) + go func(id string) { + defer wg.Done() + m.Record(id, testHeaderVal, "github") + }(id) + go func(id string) { + defer wg.Done() + _, _ = m.Lookup(id) // racing a record of the same id is fine + }(id) + } + waitWithTimeout(t, &wg, "record/lookup flood") + + assert.LessOrEqual(t, m.Len(), maxEntries, "the map must stay bounded under flood") + // The oldest ids must have been evicted; the newest survive. + for _, id := range ids[:maxEntries] { + _, ok := m.Lookup(id) + assert.False(t, ok, "oldest entry %s must be evicted", id) + } + survivors := 0 + for _, id := range ids[len(ids)-maxEntries:] { + if _, ok := m.Lookup(id); ok { + survivors++ + } + } + assert.Positive(t, survivors, "recently recorded entries must survive") +} + +// Two streams racing one request-id: exactly one gets the scan record; the +// other reads unknown (one injection ↔ one scanned response) — asserted at +// the scanner level so the metric semantics are pinned too. +func TestExtProc_ConcurrentSameRequestIDExactlyOneScan(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + + respHeadersForRace := func() *extprocv3.ProcessingRequest { + return respHeaders(testRequestID) + } + const winner = 0 + results := make([]*extprocv3.ProcessingResponse, 2) + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + results[i] = f.scanner.Handle(context.Background(), &egressbroker.StreamState{}, respHeadersForRace()) + }(i) + } + waitWithTimeout(t, &wg, "two streams racing one request-id") + + // Both pass (fail-open), but exactly one was the known scan. + scans := metricPoints(t, collectMetrics(t, f.reader)["egress_broker_response_scan_total"]) + unknowns := 0 + for _, s := range scans { + if s["result"] == "unknown_request" { + unknowns++ + } + } + assert.Equal(t, 1, unknowns, "exactly one racer reads unknown_request") + assert.Nil(t, isImmediate(results[winner])) +} + +// TokenMap-level race guard: the consuming lookup gives exactly one winner. +func TestTokenMap_ConcurrentLookupSameRequestID(t *testing.T) { + t.Parallel() + m, err := egressbroker.NewTokenMap(time.Minute, 10) + require.NoError(t, err) + m.Record(testRequestID, testHeaderVal, "github") + + const racers = 16 + found := make(chan bool, racers) + var wg sync.WaitGroup + for i := 0; i < racers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, ok := m.Lookup(testRequestID) + found <- ok + }() + } + waitWithTimeout(t, &wg, "concurrent same-id lookup") + close(found) + wins := 0 + for ok := range found { + if ok { + wins++ + } + } + assert.Equal(t, 1, wins, "exactly one stream gets the scan record; the rest read unknown_request") +} + +// RunEvictionLoop exits on ctx cancel (no goroutine leak) and sweeps expired +// entries on tick. +func TestTokenMap_RunEvictionLoop(t *testing.T) { + t.Parallel() + clock := &fakeClock{now: time.Now()} + m, err := egressbroker.NewTokenMapWithClock(time.Minute, 100, clock.Now) + require.NoError(t, err) + m.Record("stale", testHeaderVal, "github") + m.Record("fresh", testHeaderVal, "github") + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + m.RunEvictionLoop(ctx, time.Millisecond) + close(done) + }() + + // Only "stale" is past TTL: the sweep drops it and keeps "fresh". + clock.Advance(2 * time.Minute) + m.Record("fresh", testHeaderVal, "github") // re-record with the advanced clock so only "stale" is expired + require.Eventually(t, func() bool { return m.Len() == 1 }, + 5*time.Second, time.Millisecond, "the sweep must evict exactly the expired entry") + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("RunEvictionLoop did not exit on ctx cancel (goroutine leak)") + } +} + +// waitWithTimeout fails fast instead of hanging the suite on a deadlock. +func waitWithTimeout(t *testing.T, wg *sync.WaitGroup, what string) { + t.Helper() + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("timeout waiting for %s", what) + } +} + +// hungStreamServer wedges every ext_proc stream until told otherwise: this +// is the pathological stream that would block grpc.GracefulStop forever. +// entered is closed when the first stream handler starts, so the test wedges +// shutdown only after a handler is genuinely mid-stream. +type hungStreamServer struct { + extprocv3.UnimplementedExternalProcessorServer + release chan struct{} + entered chan struct{} + once sync.Once +} + +func (h *hungStreamServer) Process(extprocv3.ExternalProcessor_ProcessServer) error { + h.once.Do(func() { close(h.entered) }) + <-h.release + return nil +} + +// GracefulStop wedge (R4): with a stream that never returns, canceling Run's +// ctx must still return — GracefulStop times out and Stop() breaks the wedge. +// This test takes gracefulStopTimeout to run (the wedge IS the budget). +func TestServerRunHungStreamBoundedShutdown(t *testing.T) { + t.Parallel() + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + hung := &hungStreamServer{release: make(chan struct{}), entered: make(chan struct{})} + grpcServer := grpc.NewServer() + extprocv3.RegisterExternalProcessorServer(grpcServer, hung) + server := egressbroker.NewTestServer(grpcServer, listener) + + ctx, cancel := context.WithCancel(context.Background()) + runDone := make(chan error, 1) + go func() { runDone <- server.Run(ctx) }() + + // Open one stream and leave it hanging (no messages, never closed). The + // client stream uses its own ctx so it outlives Run's cancellation. + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + client := extprocv3.NewExternalProcessorClient(conn) + stream, err := client.Process(context.Background()) + require.NoError(t, err) + + // Wait until the handler is genuinely mid-stream, then cancel: Run must + // return within gracefulStopTimeout + slack even though GracefulStop + // blocks on the hung stream. + select { + case <-hung.entered: + case <-time.After(5 * time.Second): + t.Fatal("hung stream handler never started") + } + cancel() + select { + case err := <-runDone: + assert.NoError(t, err) + case <-time.After(egressbroker.GracefulStopTimeoutForTest() + 10*time.Second): + t.Fatal("Run wedged on a hung stream: GracefulStop timeout path broken") + } + close(hung.release) + _ = stream.CloseSend() + _ = conn.Close() +} + +// --- documented scanner gaps (ADR-0001 §5) --------------------------------- + +// ACCEPTED GAP (ADR-0001 §5 "does NOT defeat" #3): a server that transforms +// the echo evades the byte-exact scan. A gzipped body containing the token +// passes unscanned. The allowlist + destination binding are the real +// boundary; the scan is a tripwire. This test documents the gap so it can +// never be mistaken for a regression. +func TestExtProc_GzipEvasionIsAnAcceptedGap(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + _, err := gz.Write([]byte(`{"debug":"` + testHeaderVal + `"}`)) + require.NoError(t, err) + require.NoError(t, gz.Close()) + + resp := f.handleHeaders(respHeaders(testRequestID, + &envoycore.HeaderValue{Key: "content-encoding", Value: "gzip"})) + assert.Nil(t, isImmediate(resp)) + resp = f.handleBody(respBody(buf.Bytes(), testRequestID)) + assert.Nil(t, isImmediate(resp), + "ACCEPTED GAP (ADR §5): gzip-transformed echoes evade the byte-exact scan; "+ + "the allowlist + destination binding are the boundary") +} + +// ACCEPTED GAP (ADR-0001 §5): response trailers are never delivered to the +// scanner (the rendered config sets response_trailer_mode: SKIP). If one +// ever arrives, the broker answers with the matching pass-through variant so +// the filter never stalls — and asserts nothing about its content. +func TestExtProc_ResponseTrailersAreUnscanned(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + + // A trailer carrying the credential would pass: the phase is not scanned + // (SKIP mode asserted in the envoyconfig tests). + req := &extprocv3.ProcessingRequest{ + Request: &extprocv3.ProcessingRequest_ResponseTrailers{ + ResponseTrailers: &extprocv3.HttpTrailers{ + Trailers: &envoycore.HeaderMap{Headers: []*envoycore.HeaderValue{ + {Key: "x-echo", Value: testHeaderVal}, + }}, + }, + }, + } + resp := f.scanner.Handle(context.Background(), &egressbroker.StreamState{}, req) + require.NotNil(t, resp, "trailer messages must be answered so the filter never stalls") + assert.IsType(t, &extprocv3.ProcessingResponse_ResponseTrailers{}, resp.GetResponse()) + assert.Nil(t, isImmediate(resp), + "ACCEPTED GAP (ADR §5): response trailers are unscanned (SKIP mode)") +} + +// base64 of the injected header value leaked into a response header must +// block: the needle set is {exact header value, base64(header value)}, and +// the header scan matches both (§1.3 #4 covers the body path). +func TestExtProc_Base64HeaderLeak(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + + // base64 of the full injected header value IS a needle and must block. + encoded := base64.StdEncoding.EncodeToString([]byte(testHeaderVal)) + resp := f.handleHeaders(respHeaders(testRequestID, + &envoycore.HeaderValue{Key: "x-data", Value: "payload=" + encoded})) + imm := isImmediate(resp) + require.NotNil(t, imm, "base64 of the injected header value in a response header must block") + assert.Equal(t, int32(502), int32(imm.GetStatus().GetCode())) + + scans := metricPoints(t, collectMetrics(t, f.reader)["egress_broker_response_scan_total"]) + require.Len(t, scans, 1) + assert.Equal(t, "header", scans[0]["where"]) +} + +// containsRawValue: a non-UTF8 header value (delivered via raw_value, the +// value field empty) carrying the credential must block. +func TestExtProc_NonUTF8RawValueHeaderBlocked(t *testing.T) { + t.Parallel() + f := newScanFixture(t) + f.record() + + // 0xff makes the value invalid UTF-8 → Envoy would deliver it as + // raw_value with an empty value string. + raw := append([]byte{0xff}, []byte(testHeaderVal)...) + resp := f.handleHeaders(respHeaders(testRequestID, + &envoycore.HeaderValue{Key: "x-bin", RawValue: raw})) + imm := isImmediate(resp) + require.NotNil(t, imm, "a non-UTF8 raw_value echo must be scanned and blocked") + assert.Equal(t, int32(502), int32(imm.GetStatus().GetCode())) +} diff --git a/pkg/egressbroker/identity.go b/pkg/egressbroker/identity.go index 48e1631d7d..b64323f1d4 100644 --- a/pkg/egressbroker/identity.go +++ b/pkg/egressbroker/identity.go @@ -49,6 +49,10 @@ const ( EnvSessionID = "THV_UNTRUSTED_SESSION" // EnvMCPServer mirrors the pod's AnnotationMCPServerName annotation. EnvMCPServer = "THV_UNTRUSTED_MCPSERVER" + // EnvPodName carries the pod's own name (metadata.name fieldRef) for audit + // context. Not part of the annotation contract (the name needs no + // annotation) and not identity-load-bearing: optional, empty tolerated. + EnvPodName = "THV_UNTRUSTED_POD_NAME" ) // EnvToAnnotation is the complete downward-API contract: each env var name to @@ -218,3 +222,9 @@ func NewPodIdentityResolver(getenv func(string) string) (*PodIdentityResolver, e func (r *PodIdentityResolver) PodIdentity() PodIdentity { return r.identity } + +// PodName returns the optional pod-name env (audit context only; "" when the +// deployment does not set it — never identity-load-bearing). +func (*PodIdentityResolver) PodName(getenv func(string) string) string { + return strings.TrimSpace(getenv(EnvPodName)) +} diff --git a/pkg/egressbroker/injector.go b/pkg/egressbroker/injector.go index 6629b90241..33f8eccef0 100644 --- a/pkg/egressbroker/injector.go +++ b/pkg/egressbroker/injector.go @@ -5,6 +5,7 @@ package egressbroker import ( "context" + "errors" "fmt" "log/slog" "slices" @@ -24,11 +25,15 @@ type Destination struct { Path string } -// Decision is the injector's verdict. On Deny, HeaderName/HeaderValue are -// guaranteed empty: no deny path may ever carry header material. -type Decision struct { +// Result is the injector's verdict plus the structured context the caller +// (ext_authz server) needs for metrics/audit/scan-correlation. On Deny, +// HeaderName/HeaderValue are guaranteed empty: no deny path may ever carry +// header material. +type Result struct { Allow bool - DenyReason string + Reason DenyReason + DenyDetail string // coarse operator-facing text; never token data + Provider string // the provider whose policy matched ("" on no-policy deny) HeaderName string HeaderValue string //nolint:gosec // G117: field legitimately holds sensitive data } @@ -47,9 +52,13 @@ type Decision struct { // a header. The token reader must be wrapped in upstreamtoken.NewStrictTokenReader // by the caller (the constructor does it here so the wrap cannot be forgotten). type CredentialInjector struct { - identity PodIdentity - policy *EgressPolicy - tokenReader upstreamtoken.TokenReader + identity PodIdentity + podName string + policy *EgressPolicy + reader upstreamtoken.TokenReader + tokens *TokenMap // nil: no response scanning (scanner not wired) + metrics *BrokerMetrics + audit AuditLogger // nil: audit disabled (never in production) } // NewCredentialInjector wires the injector. tokenReader is wrapped with @@ -70,38 +79,66 @@ func NewCredentialInjector( return nil, fmt.Errorf("egressbroker: pod identity is incomplete; refusing to build injector") } return &CredentialInjector{ - identity: identity, - policy: policy, - tokenReader: upstreamtoken.NewStrictTokenReader(tokenReader), + identity: identity, + policy: policy, + reader: upstreamtoken.NewStrictTokenReader(tokenReader), }, nil } +// WithScanCorrelation attaches the D6c response-scan correlation map: every +// successful injection records (request-id → SHA-256(header value) + scan +// needles) so the ext_proc scanner can detect the credential echoed back. +// tokens must come from the same process's scanner (in-memory only). +func (i *CredentialInjector) WithScanCorrelation(tokens *TokenMap) *CredentialInjector { + i.tokens = tokens + return i +} + +// WithObservability attaches metrics and audit sinks. Both may be nil (tests); +// production wires both (cmd/thv-egressbroker). +func (i *CredentialInjector) WithObservability(metrics *BrokerMetrics, audit AuditLogger, podName string) *CredentialInjector { + i.metrics = metrics + i.audit = audit + i.podName = podName + return i +} + // Evaluate decides whether dest may carry the pod-owner's credential and, if // so, returns the Authorization header mutation. It never logs credential -// material; deny reasons carry no token data. -func (i *CredentialInjector) Evaluate(ctx context.Context, dest Destination) Decision { +// material; deny reasons carry no token data. On allow, the scan-correlation +// record (keyed by requestID, the Envoy x-request-id) is written BEFORE the +// caller emits the header mutation, and the audit/metric events fire. +func (i *CredentialInjector) Evaluate(ctx context.Context, dest Destination, requestID string) Result { // D5 step 1: provider lookup — before any credential load. provider, ok := i.policy.ProviderFor(dest.Host) if !ok { - return deny("no provider allowlists this destination") + return i.deny(ctx, dest, "", DenyReasonNoPolicy, "no provider allowlists this destination") } // D5 step 2: method + path check — still before any credential load. - if !i.policy.Allows(provider, dest.Method, dest.Path) { - return deny("method/path outside provider policy") + if !i.policy.AllowsMethod(provider, dest.Method) { + return i.deny(ctx, dest, provider, DenyReasonMethodNotAllowed, "method outside provider policy") + } + if !i.policy.AllowsPath(provider, dest.Path) { + return i.deny(ctx, dest, provider, DenyReasonPathNotAllowed, "path outside provider policy") } // Step 3: credential load, bound to the pod's own identity. The expected // binding's UserID is the pod-owner's raw sub from the downward-API // registry; Strict (forced by NewStrictTokenReader) fails closed on rows // that cannot prove their owner. - creds, failed, err := i.tokenReader.GetAllUpstreamCredentials(ctx, i.identity.SessionID, + creds, failed, err := i.reader.GetAllUpstreamCredentials(ctx, i.identity.SessionID, &storage.ExpectedBinding{UserID: i.identity.Subject, Strict: true}) if err != nil { // Fail closed on storage errors (Redis down, binding violations): - // never passthrough. + // never passthrough. A binding violation is reported by the store as an + // error; distinguish it for the audit reason vocabulary. + reason := DenyReasonStoreError + if errors.Is(err, storage.ErrInvalidBinding) { + reason = DenyReasonBindingMismatch + } slog.DebugContext(ctx, "egressbroker: credential load failed; denying", "provider", provider, "error", err) - return deny("upstream credential load failed") + return i.deny(ctx, dest, provider, reason, "upstream credential load failed") } cred, ok := creds[provider] if !ok || slices.Contains(failed, provider) { @@ -112,21 +149,47 @@ func (i *CredentialInjector) Evaluate(ctx context.Context, dest Destination) Dec // path, never from here: the denial surfaces on the server's own // outbound HTTPS call, not on the vMCP channel, so no consent URL can // or should be threaded through this response. - return deny("upstream credential unavailable; re-consent required") + return i.deny(ctx, dest, provider, DenyReasonCredentialUnavailable, + "upstream credential unavailable; re-consent required") } if cred.AccessToken == "" { // Defensive: a successful load with an empty token must never produce // an "Authorization: Bearer " header. - return deny("upstream credential unavailable; re-consent required") + return i.deny(ctx, dest, provider, DenyReasonCredentialUnavailable, + "upstream credential unavailable; re-consent required") } - return Decision{ + // Step 4: the header value is in scope. Record the D6c scan correlation + // BEFORE returning (the caller emits the mutation after this returns) — + // the response can race ahead of any later hook. + headerValue := "Bearer " + cred.AccessToken + if i.tokens != nil { + i.tokens.Record(requestID, headerValue, provider) + } + if i.audit != nil { + i.audit.Inject(ctx, AuditEvent(i.identity, i.podName, dest, provider)) + } + i.metrics.RecordInjection(ctx, i.identity.MCPServer, provider) + + return Result{ Allow: true, + Provider: provider, HeaderName: AuthorizationHeader, - HeaderValue: "Bearer " + cred.AccessToken, + HeaderValue: headerValue, } } -func deny(reason string) Decision { - return Decision{Allow: false, DenyReason: reason} +// deny fires the deny audit + metric and returns the negative verdict. +func (i *CredentialInjector) deny( + ctx context.Context, dest Destination, provider string, reason DenyReason, detail string, +) Result { + validateDenyReason(reason) + if i.audit != nil { + i.audit.Deny(ctx, DenyEvent{ + InjectEvent: AuditEvent(i.identity, i.podName, dest, provider), + Reason: reason, + }) + } + i.metrics.RecordDenial(ctx, i.identity.MCPServer, provider, reason) + return Result{Allow: false, Reason: reason, Provider: provider, DenyDetail: detail} } diff --git a/pkg/egressbroker/injector_test.go b/pkg/egressbroker/injector_test.go index 1c88d2c1b6..9df553059f 100644 --- a/pkg/egressbroker/injector_test.go +++ b/pkg/egressbroker/injector_test.go @@ -55,33 +55,51 @@ func TestCredentialInjector(t *testing.T) { }, nil, nil }) - decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + decision := mustInjector(t, reader).Evaluate(ctx, githubDest, "req-1") assert.True(t, decision.Allow) assert.Equal(t, egressbroker.AuthorizationHeader, decision.HeaderName) assert.Equal(t, "Bearer gho_secret", decision.HeaderValue) - assert.Empty(t, decision.DenyReason) + assert.Empty(t, decision.Reason) }) t.Run("D5 ordering: credential load never happens on policy deny", func(t *testing.T) { t.Parallel() - denyDests := map[string]egressbroker.Destination{ - "non-allowlisted host": {Host: "evil.example.com", Method: "GET", Path: "/"}, - "superdomain of allowed": {Host: "github.com", Method: "GET", Path: "/"}, - "disallowed method": {Host: "api.github.com", Method: "DELETE", Path: "/repos/foo"}, - "disallowed path": {Host: "api.github.com", Method: "GET", Path: "/admin"}, - "method default is read-only": {Host: "slack.com", Method: "POST", Path: "/api/chat"}, + denyDests := map[string]struct { + dest egressbroker.Destination + reason egressbroker.DenyReason + }{ + "non-allowlisted host": { + egressbroker.Destination{Host: "evil.example.com", Method: "GET", Path: "/"}, + egressbroker.DenyReasonNoPolicy, + }, + "superdomain of allowed": { + egressbroker.Destination{Host: "github.com", Method: "GET", Path: "/"}, + egressbroker.DenyReasonNoPolicy, + }, + "disallowed method": { + egressbroker.Destination{Host: "api.github.com", Method: "DELETE", Path: "/repos/foo"}, + egressbroker.DenyReasonMethodNotAllowed, + }, + "disallowed path": { + egressbroker.Destination{Host: "api.github.com", Method: "GET", Path: "/admin"}, + egressbroker.DenyReasonPathNotAllowed, + }, + "method default is read-only": { + egressbroker.Destination{Host: "slack.com", Method: "POST", Path: "/api/chat"}, + egressbroker.DenyReasonMethodNotAllowed, + }, } - for name, dest := range denyDests { + for name, tc := range denyDests { t.Run(name, func(t *testing.T) { t.Parallel() ctrl := gomock.NewController(t) reader := mocks.NewMockTokenReader(ctrl) // No EXPECT: any call to the token reader fails the test. // This proves D5 is evaluated before any credential load. - decision := mustInjector(t, reader).Evaluate(ctx, dest) + decision := mustInjector(t, reader).Evaluate(ctx, tc.dest, "req-1") assert.False(t, decision.Allow) - assert.NotEmpty(t, decision.DenyReason) + assert.Equal(t, tc.reason, decision.Reason, "the deny reason names the failing policy dimension") assert.Empty(t, decision.HeaderName, "deny path must never carry header material") assert.Empty(t, decision.HeaderValue, "deny path must never carry header material") }) @@ -95,7 +113,7 @@ func TestCredentialInjector(t *testing.T) { // The reader would return a valid credential for ANY provider — but the // destination binding must deny before it is ever consulted. decision := mustInjector(t, reader).Evaluate(ctx, - egressbroker.Destination{Host: "attacker-controlled.example.com", Method: "GET", Path: "/"}) + egressbroker.Destination{Host: "attacker-controlled.example.com", Method: "GET", Path: "/"}, "req-1") assert.False(t, decision.Allow) assert.Empty(t, decision.HeaderValue) }) @@ -108,9 +126,9 @@ func TestCredentialInjector(t *testing.T) { GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{}, nil, nil) - decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + decision := mustInjector(t, reader).Evaluate(ctx, githubDest, "req-1") assert.False(t, decision.Allow) - assert.Contains(t, decision.DenyReason, "re-consent") + assert.Contains(t, decision.DenyDetail, "re-consent") assert.Empty(t, decision.HeaderValue) }) @@ -122,7 +140,7 @@ func TestCredentialInjector(t *testing.T) { GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). Return(map[string]upstreamtoken.UpstreamCredential{}, []string{"github"}, nil) - decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + decision := mustInjector(t, reader).Evaluate(ctx, githubDest, "req-1") assert.False(t, decision.Allow) assert.Empty(t, decision.HeaderValue) }) @@ -137,7 +155,7 @@ func TestCredentialInjector(t *testing.T) { "github": {AccessToken: ""}, }, nil, nil) - decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + decision := mustInjector(t, reader).Evaluate(ctx, githubDest, "req-1") assert.False(t, decision.Allow) assert.Empty(t, decision.HeaderValue) }) @@ -150,11 +168,11 @@ func TestCredentialInjector(t *testing.T) { GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). Return(nil, nil, fmt.Errorf("row has no owner: %w", storage.ErrInvalidBinding)) - decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + decision := mustInjector(t, reader).Evaluate(ctx, githubDest, "req-1") assert.False(t, decision.Allow) assert.Empty(t, decision.HeaderValue) // The deny reason must not leak storage internals. - assert.NotContains(t, decision.DenyReason, "row") + assert.NotContains(t, decision.DenyDetail, "row") }) t.Run("store down (Redis error) → deny, never passthrough", func(t *testing.T) { @@ -165,7 +183,7 @@ func TestCredentialInjector(t *testing.T) { GetAllUpstreamCredentials(gomock.Any(), "session-abc", gomock.Any()). Return(nil, nil, errors.New("connection refused")) - decision := mustInjector(t, reader).Evaluate(ctx, githubDest) + decision := mustInjector(t, reader).Evaluate(ctx, githubDest, "req-1") assert.False(t, decision.Allow) assert.Empty(t, decision.HeaderValue) }) @@ -183,7 +201,7 @@ func TestCredentialInjector(t *testing.T) { // Wildcard github host: github credential, never the slack one. decision := mustInjector(t, reader).Evaluate(ctx, - egressbroker.Destination{Host: "raw.githubusercontent.com", Method: "GET", Path: "/repos/x"}) + egressbroker.Destination{Host: "raw.githubusercontent.com", Method: "GET", Path: "/repos/x"}, "req-1") assert.True(t, decision.Allow) assert.Equal(t, "Bearer gho_secret", decision.HeaderValue) assert.NotContains(t, decision.HeaderValue, "xoxb") diff --git a/pkg/egressbroker/metrics.go b/pkg/egressbroker/metrics.go new file mode 100644 index 0000000000..d54f61d581 --- /dev/null +++ b/pkg/egressbroker/metrics.go @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// instrumentationName matches the vMCP-wide OTel instrumentation scope used by +// the other broker-adjacent meters (pkg/vmcp/session/untrusted). +const instrumentationName = "github.com/stacklok/toolhive/pkg/vmcp" + +// Metric label vocabularies (ADR-0001 D11 — low cardinality: workload, +// provider, and a small fixed result/location/where vocabulary only; NEVER a +// user identifier, pod name, request-id, or token material). +const ( + metricAttrMCPServer = "mcpserver" + metricAttrProvider = "provider" + metricAttrResult = "result" + metricAttrWhere = "where" + + // metricResultOK: response passed the scanner untouched. + metricResultOK = "ok" + // metricResultLeak: response suppressed — injected credential echoed back. + metricResultLeak = "leak" + // metricResultUnknown: response carried no known request-id (direct hit, + // scanner restart, TTL eviction) and passed under the fail-open default. + metricResultUnknown = "unknown_request" +) + +// BrokerMetrics holds the egress-broker instruments. All instruments are +// optional: a nil *brokerMetrics (or nil individual instruments) is a silent +// no-op so the broker runs uninstrumented when no MeterProvider is wired. +type BrokerMetrics struct { + scans metric.Int64Counter + skips metric.Int64Counter + inject metric.Int64Counter + denies metric.Int64Counter +} + +// NewBrokerMetrics registers the broker instruments. Registration failures +// are startup-fatal (fail loudly): a silently uninstrumented credential broker +// hides the leak detector — the one signal paged on (ADR D11). +func NewBrokerMetrics(meterProvider metric.MeterProvider) (*BrokerMetrics, error) { + if meterProvider == nil { + return nil, fmt.Errorf("egressbroker: meter provider must not be nil") + } + meter := meterProvider.Meter(instrumentationName) + + scans, err := meter.Int64Counter( + "egress_broker_response_scan_total", + metric.WithDescription("Response-side credential scans by outcome (ADR D6c)"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create egress_broker_response_scan_total counter: %w", err) + } + skips, err := meter.Int64Counter( + "egress_broker_scan_skipped_total", + metric.WithDescription("Response scans skipped (body over cap; headers still scanned)"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create egress_broker_scan_skipped_total counter: %w", err) + } + inject, err := meter.Int64Counter( + "egress_broker_injections_total", + metric.WithDescription("Successful credential injections"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create egress_broker_injections_total counter: %w", err) + } + denies, err := meter.Int64Counter( + "egress_broker_denials_total", + metric.WithDescription("Injection denials by reason"), + ) + if err != nil { + return nil, fmt.Errorf("failed to create egress_broker_denials_total counter: %w", err) + } + return &BrokerMetrics{scans: scans, skips: skips, inject: inject, denies: denies}, nil +} + +// RecordScan records one scan outcome. result is one of the metricResult* +// constants; where is only meaningful for leaks (empty otherwise). +func (m *BrokerMetrics) RecordScan(ctx context.Context, mcpserver, provider, result, where string) { + if m == nil || m.scans == nil { + return + } + attrs := []attribute.KeyValue{ + attribute.String(metricAttrMCPServer, mcpserver), + attribute.String(metricAttrProvider, provider), + attribute.String(metricAttrResult, result), + } + if where != "" { + attrs = append(attrs, attribute.String(metricAttrWhere, where)) + } + m.scans.Add(ctx, 1, metric.WithAttributes(attrs...)) +} + +// RecordScanSkipped records a body-over-cap skip (headers were still scanned). +func (m *BrokerMetrics) RecordScanSkipped(ctx context.Context, mcpserver, provider string) { + if m == nil || m.skips == nil { + return + } + m.skips.Add(ctx, 1, metric.WithAttributes( + attribute.String(metricAttrMCPServer, mcpserver), + attribute.String(metricAttrProvider, provider), + )) +} + +// RecordInjection records one successful injection. +func (m *BrokerMetrics) RecordInjection(ctx context.Context, mcpserver, provider string) { + if m == nil || m.inject == nil { + return + } + m.inject.Add(ctx, 1, metric.WithAttributes( + attribute.String(metricAttrMCPServer, mcpserver), + attribute.String(metricAttrProvider, provider), + )) +} + +// RecordDenial records one injection denial. reason must be a DenyReason +// vocabulary value (validated at the call site). +func (m *BrokerMetrics) RecordDenial(ctx context.Context, mcpserver, provider string, reason DenyReason) { + if m == nil || m.denies == nil { + return + } + attrs := []attribute.KeyValue{attribute.String(metricAttrResult, string(reason))} + if mcpserver != "" { + attrs = append(attrs, attribute.String(metricAttrMCPServer, mcpserver)) + } + if provider != "" { + attrs = append(attrs, attribute.String(metricAttrProvider, provider)) + } + m.denies.Add(ctx, 1, metric.WithAttributes(attrs...)) +} diff --git a/pkg/egressbroker/policy.go b/pkg/egressbroker/policy.go index 5e968279d7..4364fdaee0 100644 --- a/pkg/egressbroker/policy.go +++ b/pkg/egressbroker/policy.go @@ -124,15 +124,26 @@ func (e *EgressPolicy) ProviderFor(host string) (string, bool) { // second half of D5, evaluated before any credential load or header write). // provider must be a name previously returned by ProviderFor. func (e *EgressPolicy) Allows(provider, method, path string) bool { + return e.AllowsMethod(provider, method) && e.AllowsPath(provider, path) +} + +// AllowsMethod is the method half of Allows (used by the injector to name +// the deny-reason dimension). +func (e *EgressPolicy) AllowsMethod(provider, method string) bool { for i := range e.providers { - if e.providers[i].Provider != provider { - continue + if e.providers[i].Provider == provider { + return methodAllowed(e.providers[i], method) } - p := e.providers[i] - if !methodAllowed(p, method) { - return false + } + return false +} + +// AllowsPath is the path half of Allows. +func (e *EgressPolicy) AllowsPath(provider, path string) bool { + for i := range e.providers { + if e.providers[i].Provider == provider { + return pathAllowed(e.providers[i], path) } - return pathAllowed(p, path) } return false } diff --git a/pkg/egressbroker/requestid.go b/pkg/egressbroker/requestid.go new file mode 100644 index 0000000000..1032dea12f --- /dev/null +++ b/pkg/egressbroker/requestid.go @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + envoycore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +// Correlation plumbing between ext_authz (injection) and ext_proc (response +// scan), ADR D6c: the Envoy-generated x-request-id is the shared key. It is +// present at ext_authz time (request headers); on the response path the +// allowlisted routes set dynamic metadata (namespace metadataNamespace, key +// metadataKeyRequestID) from %REQ(X-REQUEST-ID)%, which ext_proc forwards per +// its metadata_options.forwarding_namespaces config (the response-side +// ext_proc cannot see request headers). +const ( + // requestIDHeader is the Envoy request-id header name (canonical + // lowercase, as HTTP/2 header maps are). + requestIDHeader = "x-request-id" + + // metadataNamespace is the dynamic-metadata namespace carrying the + // correlation values from the route to ext_proc. + metadataNamespace = "io.toolhive.egress" + // metadataKeyRequestID is the metadata key holding the x-request-id copy. + metadataKeyRequestID = "request_id" + // metadataKeyHost is the metadata key holding the request host (for audit). + metadataKeyHost = "host" +) + +// metadataString extracts a string value from ext_proc-forwarded dynamic +// metadata: metadata.filter_metadata[metadataNamespace].fields[key]. +func metadataString(meta *envoycore.Metadata, key string) string { + if meta == nil { + return "" + } + ns, ok := meta.GetFilterMetadata()[metadataNamespace] + if !ok { + return "" + } + return ns.GetFields()[key].GetStringValue() +} + +// requestIDFromMetadata returns the route-forwarded x-request-id copy. +func requestIDFromMetadata(meta *envoycore.Metadata) string { + return metadataString(meta, metadataKeyRequestID) +} + +// requestHostFromMetadata returns the route-forwarded request host (audit +// context only — never a routing decision). +func requestHostFromMetadata(meta *envoycore.Metadata) string { + return metadataString(meta, metadataKeyHost) +} diff --git a/pkg/egressbroker/scanner.go b/pkg/egressbroker/scanner.go new file mode 100644 index 0000000000..302b218b19 --- /dev/null +++ b/pkg/egressbroker/scanner.go @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "fmt" + + envoycore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +// LeakLocation identifies where the echoed credential was found. +type LeakLocation string + +const ( + // LeakLocationHeader means a response header carried the injected credential. + LeakLocationHeader LeakLocation = "header" + // LeakLocationBody means the response body carried the injected credential. + LeakLocationBody LeakLocation = "body" +) + +// ScannerBounds are the response-scanner limits. +type ScannerBounds struct { + // MaxBodyBytes caps the buffered body actually scanned (default 1 MiB). + // The cap is enforced HERE, in-band: Envoy's ext_proc filter has no + // per-filter byte cap (no max_bytes / allow_partial_message field exists), + // so Envoy buffers the whole response body and this bound refuses to scan + // past the cap (a cost bound) and records a skip — headers are always + // scanned. The cap is NOT a security boundary: the allowlist + destination + // binding are (ADR-0001 §5). + MaxBodyBytes int64 +} + +// scanResult is the scanner's verdict for one response part. +type scanResult int + +const ( + // scanClean: no credential material found; the response passes untouched. + scanClean scanResult = iota + // scanLeak: the injected credential (exact or base64-encoded) was found; + // the response must be suppressed. + scanLeak + // scanOversize: the body exceeded the scan cap and was NOT scanned + // (headers still were). Not a leak verdict. + scanOversize +) + +// buildNeedles derives the byte sequences scanned for from a recorded token +// hash. The scan compares exact + base64 encodings of the INJECTED credential +// header value; the raw value is recoverable here only in the sense that the +// broker itself injected it (the map retains hashes, so the scanner needs the +// header value back — see recordNeedles: the hash map entry carries the +// needles, not the token). +// +// IMPORTANT: needles are computed at record time (injection), when the +// plaintext header value is legitimately in scope, and retained alongside the +// hash — the scanner never re-derives them from a credential store. +func buildNeedles(headerValue string) [][]byte { + b64 := base64.StdEncoding.EncodeToString([]byte(headerValue)) + return [][]byte{[]byte(headerValue), []byte(b64)} +} + +// scanHeaders looks for any needle in any response header key or value. The +// matched needle is never reported, logged, or returned. +func scanHeaders(headers *envoycore.HeaderMap, needles [][]byte) bool { + if headers == nil { + return false + } + for _, hv := range headers.GetHeaders() { + for _, n := range needles { + if bytes.Contains([]byte(hv.GetKey()), n) || + bytes.Contains([]byte(hv.GetValue()), n) || + containsRawValue(hv, n) { + return true + } + } + } + return false +} + +// containsRawValue checks the envoy HeaderValue raw form (present when the +// value is not valid UTF-8). +func containsRawValue(hv *envoycore.HeaderValue, needle []byte) bool { + raw := hv.GetRawValue() + return len(raw) > 0 && bytes.Contains(raw, needle) +} + +// scanBody looks for any needle in a buffered response body. Bodies larger +// than maxBytes are refused (scanOversize): Envoy buffers the whole body +// (ext_proc has no byte cap), so the cap is enforced here — never scan past +// it (cost bound). +func scanBody(body []byte, needles [][]byte, maxBytes int64) scanResult { + if maxBytes <= 0 { + return scanOversize + } + if int64(len(body)) > maxBytes { + return scanOversize + } + for _, n := range needles { + if bytes.Contains(body, n) { + return scanLeak + } + } + return scanClean +} + +// hashOf is the SHA-256 the token map stores (kept here so the scanner and +// the injector share exactly one hash definition). +func hashOf(headerValue string) [32]byte { + return sha256.Sum256([]byte(headerValue)) +} + +// validateScannerBounds fails loudly on a nonsensical scan cap (a zero/negative +// cap would silently disable body scanning entirely). +func validateScannerBounds(b ScannerBounds) error { + if b.MaxBodyBytes <= 0 { + return fmt.Errorf("egressbroker: scanner body cap must be positive") + } + return nil +} diff --git a/pkg/egressbroker/server.go b/pkg/egressbroker/server.go index dd045280cb..a2d0585018 100644 --- a/pkg/egressbroker/server.go +++ b/pkg/egressbroker/server.go @@ -16,6 +16,7 @@ import ( envoytls "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" envoyauth "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3" envoydiscovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" envoysecret "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3" "google.golang.org/genproto/googleapis/rpc/code" "google.golang.org/genproto/googleapis/rpc/status" @@ -32,6 +33,10 @@ import ( type AuthorizationServer struct { envoyauth.UnimplementedAuthorizationServer injector *CredentialInjector + audit AuditLogger // nil: deny events from unparseable requests are not audited + metrics *BrokerMetrics + identity PodIdentity + podName string } // NewAuthorizationServer builds the ext_authz endpoint on an injector. @@ -42,22 +47,50 @@ func NewAuthorizationServer(injector *CredentialInjector) (*AuthorizationServer, return &AuthorizationServer{injector: injector}, nil } +// WithObservability attaches the audit/metrics sinks used for Check-level +// denials that never reach the injector (unparseable destination). +func (s *AuthorizationServer) WithObservability( + audit AuditLogger, metrics *BrokerMetrics, identity PodIdentity, podName string, +) *AuthorizationServer { + s.audit = audit + s.metrics = metrics + s.identity = identity + s.podName = podName + return s +} + // Check implements envoy.service.auth.v3.Authorization. func (s *AuthorizationServer) Check(ctx context.Context, req *envoyauth.CheckRequest) (*envoyauth.CheckResponse, error) { dest, err := destinationFromCheckRequest(req) if err != nil { slog.DebugContext(ctx, "egressbroker: denying request with unparseable destination", "error", err) + if s.audit != nil { + s.audit.Deny(ctx, DenyEvent{ + InjectEvent: AuditEvent(s.identity, s.podName, Destination{}, ""), + Reason: DenyReasonMalformed, + }) + } + s.metrics.RecordDenial(ctx, s.identity.MCPServer, "", DenyReasonMalformed) return deniedResponse(codes.InvalidArgument, "unparseable destination"), nil } - decision := s.injector.Evaluate(ctx, dest) + requestID := requestIDFromCheckRequest(req) + decision := s.injector.Evaluate(ctx, dest, requestID) if !decision.Allow { slog.DebugContext(ctx, "egressbroker: denied egress", - "host", dest.Host, "method", dest.Method, "reason", decision.DenyReason) - return deniedResponse(codes.PermissionDenied, decision.DenyReason), nil + "host", dest.Host, "method", dest.Method, "reason", decision.Reason) + return deniedResponse(codes.PermissionDenied, decision.DenyDetail), nil } return okResponse(decision.HeaderName, decision.HeaderValue), nil } +// requestIDFromCheckRequest returns the Envoy x-request-id header (present: +// the rendered bootstrap enables the request-id extension, so Envoy generates +// one when the downstream did not supply it). Empty when the header map is +// absent; the injector skips scan-correlation for an empty id. +func requestIDFromCheckRequest(req *envoyauth.CheckRequest) string { + return req.GetAttributes().GetRequest().GetHttp().GetHeaders()[requestIDHeader] +} + // destinationFromCheckRequest extracts host/method/path from the attribute // context. Host is port-stripped and lowercased; an empty host or method // fails closed. @@ -106,6 +139,10 @@ func okResponse(headerName, headerValue string) *envoyauth.CheckResponse { OkResponse: &envoyauth.OkHttpResponse{ Headers: []*envoycore.HeaderValueOption{{ Header: &envoycore.HeaderValue{Key: headerName, Value: headerValue}, + // The injected credential REPLACES anything the workload + // sent — explicit so a go-control-plane default change can + // never silently turn this into an append. + AppendAction: envoycore.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD, }}, // Never echo the credential anywhere but the upstream request // header: response headers and dynamic metadata stay empty. @@ -233,15 +270,17 @@ type Server struct { } // NewServer creates the gRPC server on listenAddress:port, registering the -// ext_authz and SDS services. +// ext_authz, SDS, and ext_proc (response scanner, D6c) services on one +// loopback socket. func NewServer( listenAddress string, port int, authz *AuthorizationServer, sds *SecretDiscoveryServer, + extproc *ExternalProcessorServer, ) (*Server, error) { - if authz == nil || sds == nil { - return nil, fmt.Errorf("egressbroker: authz and sds servers must not be nil") + if authz == nil || sds == nil || extproc == nil { + return nil, fmt.Errorf("egressbroker: authz, sds, and ext_proc servers must not be nil") } listener, err := net.Listen("tcp", net.JoinHostPort(listenAddress, fmt.Sprintf("%d", port))) if err != nil { @@ -250,17 +289,73 @@ func NewServer( grpcServer := grpc.NewServer() envoyauth.RegisterAuthorizationServer(grpcServer, authz) envoysecret.RegisterSecretDiscoveryServiceServer(grpcServer, sds) + extprocv3.RegisterExternalProcessorServer(grpcServer, extproc) return &Server{grpcServer: grpcServer, listener: listener}, nil } -// Run serves until ctx is canceled, then gracefully stops. +// gracefulStopTimeout bounds the drain on shutdown: a hung ext_proc stream +// must not wedge the pod past its termination grace period. After the budget +// the server is stopped forcefully (in-flight RPCs error out; Envoy's +// failure_mode_allow governs those responses). +const gracefulStopTimeout = 5 * time.Second + +// NewTestServer wraps an arbitrary grpc.Server + listener in a Server so the +// shutdown path (GracefulStop + bounded Stop) is testable without the +// production service trio. Test-only constructor. +func NewTestServer(grpcServer *grpc.Server, listener net.Listener) *Server { + return &Server{grpcServer: grpcServer, listener: listener} +} + +// GracefulStopTimeoutForTest exposes the drain budget so the wedge test can +// bound its own wait (kept out of the constant's scope so tests cannot +// mutate it). +func GracefulStopTimeoutForTest() time.Duration { return gracefulStopTimeout } + +// Run serves until ctx is canceled, then stops with a bounded drain. On +// cancellation GracefulStop closes the listener and drains in-flight RPCs; +// a hung ext_proc stream would block that drain forever, so after +// gracefulStopTimeout the server is force-stopped — wedged streams error +// out and Envoy's failure_mode_allow governs those responses. +// +// Run does NOT wait for Serve to return on the forced path: grpc-go's Serve +// blocks until the graceful stop fully completes (it waits out hung +// handlers), so Run itself bounds the wait at the drain budget and returns +// while the wedge unwinds in the background. func (s *Server) Run(ctx context.Context) error { go func() { <-ctx.Done() s.grpcServer.GracefulStop() }() - if err := s.grpcServer.Serve(s.listener); err != nil { - return fmt.Errorf("egressbroker: gRPC server failed: %w", err) + go func() { + <-ctx.Done() + timer := time.NewTimer(gracefulStopTimeout) + defer timer.Stop() + <-timer.C + slog.Warn("egressbroker: graceful gRPC stop exceeded the drain budget; forcing stop", + "timeout", gracefulStopTimeout) + s.grpcServer.Stop() + }() + serveDone := make(chan error, 1) + go func() { serveDone <- s.grpcServer.Serve(s.listener) }() + select { + case err := <-serveDone: + if err != nil { + return fmt.Errorf("egressbroker: gRPC server failed: %w", err) + } + return nil + case <-ctx.Done(): + // Shutdown in progress: give the graceful drain its budget, then + // return — the forced Stop above unwinds any wedge asynchronously. + timer := time.NewTimer(gracefulStopTimeout) + defer timer.Stop() + select { + case err := <-serveDone: + if err != nil { + return fmt.Errorf("egressbroker: gRPC server failed: %w", err) + } + return nil + case <-timer.C: + return nil + } } - return nil } diff --git a/pkg/egressbroker/tokenmap.go b/pkg/egressbroker/tokenmap.go new file mode 100644 index 0000000000..2c2ab30302 --- /dev/null +++ b/pkg/egressbroker/tokenmap.go @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package egressbroker + +import ( + "container/list" + "context" + "fmt" + "sync" + "time" +) + +// TokenMap is the request-id → injected-token correlation the response +// scanner (ADR D6c) needs: ext_authz records (x-request-id → scan record) at +// injection time; ext_proc looks it up on the response. +// +// Retention discipline: no token plaintext is RETAINED at rest in this map. +// The entry carries the SHA-256 of the injected header value (the correlation +// proof) plus the derived scan needles (exact + base64 of the header value), +// computed at injection time — the only moment the plaintext is legitimately +// in scope. The needles live in process memory only, are never logged, never +// leave the process, and are dropped on lookup (one injection ↔ one scanned +// response) or TTL eviction. +// +// Bounds: entries expire after a TTL (the request/response round trip plus +// slack) and the whole map is capped; under load the oldest entries are +// evicted (a response whose entry was evicted scans as unknown-request and +// passes under the fail-open default — recorded as a scan miss). +// +// Concurrency: a single mutex guards map + eviction list (one synchronization +// primitive per data structure). +type TokenMap struct { + mu sync.Mutex + entries map[string]*list.Element // requestID → list element + lru *list.List // front = most recently recorded + ttl time.Duration + maxEntries int + now func() time.Time +} + +// ScanRecord is one entry's payload. +type ScanRecord struct { + TokenHash [32]byte + Needles [][]byte + Provider string +} + +type tokenMapEntry struct { + requestID string + record ScanRecord + expiresAt time.Time +} + +// NewTokenMap builds the bounded TTL map. ttl must be positive; maxEntries must be +// >= 1 (fail loudly on nonsensical bounds — a zero bound would silently +// disable the scanner's correlation and every response would pass as +// unknown-request). +func NewTokenMap(ttl time.Duration, maxEntries int) (*TokenMap, error) { + return NewTokenMapWithClock(ttl, maxEntries, time.Now) +} + +// NewTokenMapWithClock is NewTokenMap with an injectable clock (tests use a +// fake clock instead of time.Sleep against the TTL). +func NewTokenMapWithClock(ttl time.Duration, maxEntries int, now func() time.Time) (*TokenMap, error) { + if ttl <= 0 { + return nil, fmt.Errorf("egressbroker: token map TTL must be positive") + } + if maxEntries < 1 { + return nil, fmt.Errorf("egressbroker: token map bound must be >= 1") + } + if now == nil { + return nil, fmt.Errorf("egressbroker: clock must not be nil") + } + return &TokenMap{ + entries: map[string]*list.Element{}, + lru: list.New(), + ttl: ttl, + maxEntries: maxEntries, + now: now, + }, nil +} + +// Record stores the scan record for an injected credential header value under +// requestID. provider travels with the entry for scan-time metrics/audit +// labeling (low-cardinality policy names only). A re-recorded requestID +// refreshes in place. +func (m *TokenMap) Record(requestID, headerValue, provider string) { + if requestID == "" || headerValue == "" { + return + } + entry := &tokenMapEntry{ + requestID: requestID, + record: ScanRecord{ + TokenHash: hashOf(headerValue), + Needles: buildNeedles(headerValue), + Provider: provider, + }, + expiresAt: m.now().Add(m.ttl), + } + m.mu.Lock() + defer m.mu.Unlock() + if el, ok := m.entries[requestID]; ok { + el.Value = entry + m.lru.MoveToFront(el) + return + } + m.entries[requestID] = m.lru.PushFront(entry) + for m.lru.Len() > m.maxEntries { + oldest := m.lru.Back() + m.lru.Remove(oldest) + delete(m.entries, oldest.Value.(*tokenMapEntry).requestID) + } +} + +// Lookup returns the scan record for requestID, consuming the entry (one +// injection corresponds to one scanned response; a second lookup reports +// unknown). Expired entries count as unknown. +func (m *TokenMap) Lookup(requestID string) (ScanRecord, bool) { + if requestID == "" { + return ScanRecord{}, false + } + m.mu.Lock() + defer m.mu.Unlock() + el, found := m.entries[requestID] + if !found { + return ScanRecord{}, false + } + delete(m.entries, requestID) + m.lru.Remove(el) + entry := el.Value.(*tokenMapEntry) + if m.now().After(entry.expiresAt) { + return ScanRecord{}, false + } + return entry.record, true +} + +// evictExpired drops all expired entries (background hygiene; lookup +// already treats expired entries as unknown, so this only bounds memory +// under a flood of distinct request-ids within one TTL window). +func (m *TokenMap) evictExpired() { + now := m.now() + m.mu.Lock() + defer m.mu.Unlock() + for el := m.lru.Back(); el != nil; { + prev := el.Prev() + entry := el.Value.(*tokenMapEntry) + if now.After(entry.expiresAt) { + m.lru.Remove(el) + delete(m.entries, entry.requestID) + } + el = prev + } +} + +// Len returns the live entry count. +func (m *TokenMap) Len() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.lru.Len() +} + +// RunEvictionLoop sweeps expired entries every interval until ctx is done +// (goroutine owned by the caller; exits cleanly on cancellation — no leak). +func (m *TokenMap) RunEvictionLoop(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + m.evictExpired() + } + } +} diff --git a/pkg/vmcp/cli/untrusted.go b/pkg/vmcp/cli/untrusted.go index 956a2e5287..f18766ec99 100644 --- a/pkg/vmcp/cli/untrusted.go +++ b/pkg/vmcp/cli/untrusted.go @@ -46,6 +46,17 @@ const ( untrustedTokenStoreKEKIDsEnvVar = "THV_UNTRUSTED_TOKEN_STORE_KEK_IDS" // #nosec G101 -- env var name, not a credential ) +// untrustedTokenStorePassword*EnvVars mirror the operator-side env vars +// carrying the auth-server Redis ACL password Secret COORDINATES (name + key, +// never the value — see buildUntrustedTokenStoreEnvVars). The sidecar reads +// the password itself from THV_SESSION_REDIS_PASSWORD, rendered as a +// SecretKeyRef env at clone time. +const ( + // #nosec G101 -- env var names, not credentials. + untrustedTokenStorePasswordSecretEnvVar = "THV_UNTRUSTED_TOKEN_STORE_PASSWORD_SECRET" + untrustedTokenStorePasswordKeyEnvVar = "THV_UNTRUSTED_TOKEN_STORE_PASSWORD_KEY" // #nosec G101 +) + // Platform-operator tunables (Wave-5 spec §3.1/§3.2/§4), resolved ONCE here at // the composition root — never hot-reloaded. Every one fails startup on an // unparseable/zero/negative value (fail loud). @@ -231,6 +242,23 @@ func resolveTokenStoreConfig(namespace, vmcpName string) *untrusted.TokenStoreCo RedisAddr: addr, KeyPrefix: authstorage.DeriveKeyPrefix(namespace, vmcpName), } + // The token-store Redis password: Secret coordinates only (KEK pattern — + // the value never transits an env literal). Missing coordinates mean the + // sidecar cannot AUTH and every injection denies: warn loudly so the + // operator sees the misconfiguration before the first credentialed call. + passwordSecret := os.Getenv(untrustedTokenStorePasswordSecretEnvVar) + passwordKey := os.Getenv(untrustedTokenStorePasswordKeyEnvVar) + if passwordSecret != "" && passwordKey != "" { + ts.RedisPasswordSecret = passwordSecret + ts.RedisPasswordKey = passwordKey + } else { + slog.Warn("untrusted token-store Redis password coordinates are not configured; "+ + "egress-broker sidecars cannot authenticate against the token store "+ + "(every upstream credential injection will deny)", + "secret_env_var", untrustedTokenStorePasswordSecretEnvVar, + "key_env_var", untrustedTokenStorePasswordKeyEnvVar, + "password_env_var", config.RedisPasswordEnvVar) + } secretName := os.Getenv(untrustedTokenStoreKEKSecretEnvVar) activeID := os.Getenv(untrustedTokenStoreKEKKeyEnvVar) idsRaw := os.Getenv(untrustedTokenStoreKEKIDsEnvVar) diff --git a/pkg/vmcp/cli/untrusted_test.go b/pkg/vmcp/cli/untrusted_test.go index c7cc638335..a291654e00 100644 --- a/pkg/vmcp/cli/untrusted_test.go +++ b/pkg/vmcp/cli/untrusted_test.go @@ -59,6 +59,24 @@ func TestResolveTokenStoreConfig(t *testing.T) { assert.Empty(t, ts.KEKIDs) }) + t.Run("Redis password coordinates populate the Secret reference the sidecar clones", func(t *testing.T) { + t.Setenv(untrustedTokenStoreAddrEnvVar, "redis.auth:6379") + t.Setenv(untrustedTokenStorePasswordSecretEnvVar, "redis-creds") + t.Setenv(untrustedTokenStorePasswordKeyEnvVar, "password") + ts := resolveTokenStoreConfig("toolhive", "my-vmcp") + require.NotNil(t, ts) + assert.Equal(t, "redis-creds", ts.RedisPasswordSecret) + assert.Equal(t, "password", ts.RedisPasswordKey) + }) + + t.Run("absent password coordinates render no password reference (broker fails loud)", func(t *testing.T) { + t.Setenv(untrustedTokenStoreAddrEnvVar, "redis.auth:6379") + ts := resolveTokenStoreConfig("toolhive", "my-vmcp") + require.NotNil(t, ts) + assert.Empty(t, ts.RedisPasswordSecret) + assert.Empty(t, ts.RedisPasswordKey) + }) + t.Run("KEK coordinates populate the multi-key set the sidecar clones", func(t *testing.T) { t.Setenv(untrustedTokenStoreAddrEnvVar, "redis.auth:6379") t.Setenv(untrustedTokenStoreKEKSecretEnvVar, "my-vmcp-kek") diff --git a/pkg/vmcp/session/untrusted/egress.go b/pkg/vmcp/session/untrusted/egress.go index 29099252fc..e26c9e5fdd 100644 --- a/pkg/vmcp/session/untrusted/egress.go +++ b/pkg/vmcp/session/untrusted/egress.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "github.com/stacklok/toolhive/pkg/egressbroker" + vmcpconfig "github.com/stacklok/toolhive/pkg/vmcp/config" ) // Wave-3 egress-broker wiring constants. Data-plane resource names come from @@ -332,6 +333,14 @@ func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { {Name: "THV_EGRESSBROKER_LISTEN_ADDRESS", Value: "127.0.0.1"}, {Name: "THV_EGRESSBROKER_LISTEN_PORT", Value: fmt.Sprintf("%d", brokerListenPort)}, {Name: "THV_EGRESSBROKER_ENVOY_BOOTSTRAP_OUT", Value: envoyConfigPath + "/envoy.yaml"}, + // Pod name for the D11 audit trail (metadata context only — the + // identity contract above stays annotation-mirrored). + { + Name: egressbroker.EnvPodName, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }, } for _, mapping := range sidecarEnv { brokerEnv = append(brokerEnv, corev1.EnvVar{ @@ -354,6 +363,20 @@ func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { corev1.EnvVar{Name: "THV_EGRESSBROKER_REDIS_ADDR", Value: req.TokenStore.RedisAddr}, corev1.EnvVar{Name: "THV_EGRESSBROKER_REDIS_KEY_PREFIX", Value: req.TokenStore.KeyPrefix}, ) + // The token store's Redis password: SecretKeyRef only, never a literal + // (KEK pattern). The broker reads it via vmcpconfig.RedisPasswordEnvVar + // and fails loud at startup when it is absent. + if req.TokenStore.RedisPasswordSecret != "" { + brokerEnv = append(brokerEnv, corev1.EnvVar{ + Name: vmcpconfig.RedisPasswordEnvVar, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: req.TokenStore.RedisPasswordSecret}, + Key: req.TokenStore.RedisPasswordKey, + }, + }, + }) + } if req.TokenStore.KEKSecret != "" { brokerEnv = append(brokerEnv, corev1.EnvVar{Name: "THV_EGRESSBROKER_KEK_ID", Value: req.TokenStore.KEKActiveID}, diff --git a/pkg/vmcp/session/untrusted/egress_test.go b/pkg/vmcp/session/untrusted/egress_test.go index c32ea8f7f0..1d269ed6aa 100644 --- a/pkg/vmcp/session/untrusted/egress_test.go +++ b/pkg/vmcp/session/untrusted/egress_test.go @@ -205,11 +205,13 @@ func TestApplyEgressBrokerSidecar(t *testing.T) { t.Parallel() req := testRequest() req.TokenStore = &TokenStoreConfig{ - RedisAddr: "redis.auth:6379", - KeyPrefix: "thv:auth:{toolhive:my-vmcp}:", - KEKSecret: "my-vmcp-token-kek", - KEKActiveID: "kek-2", - KEKIDs: []string{"kek-1", "kek-2"}, + RedisAddr: "redis.auth:6379", + KeyPrefix: "thv:auth:{toolhive:my-vmcp}:", + RedisPasswordSecret: "redis-creds", + RedisPasswordKey: "password", + KEKSecret: "my-vmcp-token-kek", + KEKActiveID: "kek-2", + KEKIDs: []string{"kek-1", "kek-2"}, } pod, err := clonePodFromTemplate(validTemplate(), "backend-app", req, "vmcp-1") require.NoError(t, err) @@ -222,6 +224,15 @@ func TestApplyEgressBrokerSidecar(t *testing.T) { assert.Equal(t, "redis.auth:6379", env["THV_EGRESSBROKER_REDIS_ADDR"].Value) assert.Equal(t, "thv:auth:{toolhive:my-vmcp}:", env["THV_EGRESSBROKER_REDIS_KEY_PREFIX"].Value) + // The Redis password is a SecretKeyRef env (KEK pattern — never a + // literal); the broker reads it via vmcpconfig.RedisPasswordEnvVar. + pw := env["THV_SESSION_REDIS_PASSWORD"] + require.NotNil(t, pw.ValueFrom, "the Redis password must come from a Secret env reference") + require.NotNil(t, pw.ValueFrom.SecretKeyRef) + assert.Equal(t, "redis-creds", pw.ValueFrom.SecretKeyRef.Name) + assert.Equal(t, "password", pw.ValueFrom.SecretKeyRef.Key) + assert.Empty(t, pw.Value, "the Redis password must never be a pod-spec literal") + // The active key ID is a non-secret literal. assert.Equal(t, "kek-2", env["THV_EGRESSBROKER_KEK_ID"].Value) @@ -277,6 +288,10 @@ func TestApplyEgressBrokerSidecar(t *testing.T) { RedisAddr: "r:6379", KeyPrefix: "thv:auth:{a:b}:", KEKSecret: "s", KEKActiveID: "kek.1", KEKIDs: []string{"kek.1"}, }}, + {"password coordinates partial", &TokenStoreConfig{ + RedisAddr: "r:6379", KeyPrefix: "thv:auth:{a:b}:", + RedisPasswordSecret: "redis-creds", + }}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() diff --git a/pkg/vmcp/session/untrusted/tokenstore.go b/pkg/vmcp/session/untrusted/tokenstore.go index 889a109a5c..870591854d 100644 --- a/pkg/vmcp/session/untrusted/tokenstore.go +++ b/pkg/vmcp/session/untrusted/tokenstore.go @@ -30,6 +30,13 @@ type TokenStoreConfig struct { // KeyPrefix is the auth-server per-tenant key prefix (e.g. // "thv:auth:{ns:name}:"). Required; must end with ':'. KeyPrefix string + // RedisPasswordSecret and RedisPasswordKey are the Secret COORDINATES of + // the token store's Redis ACL password — never the value (the KEK + // pattern). They render as a SecretKeyRef env (THV_SESSION_REDIS_PASSWORD) + // on the sidecar. All-or-nothing; empty on both means no password is + // wired and the broker fails loud at startup naming the env var. + RedisPasswordSecret string + RedisPasswordKey string // KEKSecret, when non-empty, names the Secret holding the base64 // token-encryption KEKs (one data entry per key ID, 32 bytes decoded). // Every ID in KEKIDs is mounted as a per-ID SecretKeyRef env on the @@ -64,6 +71,12 @@ func (c *TokenStoreConfig) validate() error { if c.KeyPrefix == "" || c.KeyPrefix[len(c.KeyPrefix)-1] != ':' { return fmt.Errorf("untrusted token store: KeyPrefix must be non-empty and end with ':'") } + // Redis password coordinates are all-or-nothing (a half-reference would + // render a SecretKeyRef pointing nowhere). + if (c.RedisPasswordSecret == "") != (c.RedisPasswordKey == "") { + return fmt.Errorf("untrusted token store: RedisPasswordSecret and RedisPasswordKey " + + "must be set together (Secret coordinates of the Redis ACL password)") + } // KEK coordinates are all-or-nothing: secret name + active ID + at least // one key ID, with the active ID a member of the set. if c.KEKSecret == "" && c.KEKActiveID == "" && len(c.KEKIDs) == 0 { From 8ec8b408affbe473a9a3729e8f14a1fcec1c67c5 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 23 Jul 2026 09:02:18 +0300 Subject: [PATCH 06/12] Document untrusted mode architecture and operations --- docs/arch/05-runconfig-and-permissions.md | 20 +- docs/arch/09-operator-architecture.md | 63 +++ docs/arch/13-vmcp-scalability.md | 39 ++ docs/arch/16-untrusted-mode.md | 415 ++++++++++++++++++ docs/arch/README.md | 16 + .../adr/0001-untrusted-mcp-egress-broker.md | 46 +- docs/operator/untrusted-mode.md | 162 +++++++ 7 files changed, 759 insertions(+), 2 deletions(-) create mode 100644 docs/arch/16-untrusted-mode.md create mode 100644 docs/operator/untrusted-mode.md diff --git a/docs/arch/05-runconfig-and-permissions.md b/docs/arch/05-runconfig-and-permissions.md index 0d3ab48578..2ea013b676 100644 --- a/docs/arch/05-runconfig-and-permissions.md +++ b/docs/arch/05-runconfig-and-permissions.md @@ -742,9 +742,26 @@ Custom permission profiles can be defined in JSON files for reusable security po **Pod security context:** - RunConfig permission profile → Security context -- Network policies generated from profile - Volume mounts → PersistentVolumeClaims or HostPath +**Network policies (untrusted mode):** + +NetworkPolicy generation is real — but scoped to **untrusted mode**, not +general permission profiles. For MCPServers with `spec.untrusted: true`, the +operator renders an egress-only NetworkPolicy from `spec.egressPolicy`: session +pods may reach only loopback (the in-pod Envoy/broker sidecar), cluster DNS +(kube-system / `k8s-app=kube-dns` pods only), and the CIDRs the policy's +`allowedHosts` resolve to. In that mode +`permissionProfile.network.outbound` is **ignored** for the backend pod — the +two are intentionally separate dialects: `permissionProfile` remains the +Docker/Squid dialect (local mode, above), `egressPolicy` is the K8s +untrusted-mode dialect. General profile → NetworkPolicy conversion for trusted +workloads remains unimplemented; the load-bearing untrusted-mode controls are +the broker's destination binding and per-dial IP validation, with the +NetworkPolicy as defense-in-depth. See +[Untrusted Mode](16-untrusted-mode.md) and +[ADR-0001](adr/0001-untrusted-mcp-egress-broker.md) (D7). + **Pod template patches:** ```json { @@ -771,4 +788,5 @@ Custom permission profiles can be defined in JSON files for reusable security po - [Deployment Modes](01-deployment-modes.md) - RunConfig portability - [Transport Architecture](03-transport-architecture.md) - Transport configuration - [Operator Architecture](09-operator-architecture.md) - K8s-specific configuration +- [Untrusted Mode](16-untrusted-mode.md) - K8s egressPolicy dialect and NetworkPolicy generation - [Registry System](06-registry-system.md) - Registry metadata that seeds RunConfig for a server diff --git a/docs/arch/09-operator-architecture.md b/docs/arch/09-operator-architecture.md index 525c41e457..7e29c31361 100644 --- a/docs/arch/09-operator-architecture.md +++ b/docs/arch/09-operator-architecture.md @@ -179,6 +179,8 @@ MCPServer supports referencing shared configuration CRDs: **Status fields** include phase (Ready, Pending, Failed, Terminating), the accessible URL, and config hashes (`oidcConfigHash`, `telemetryConfigHash`, `authServerConfigHash`) for change detection on referenced CRDs. +**Untrusted mode**: `spec.untrusted: true` + `spec.egressPolicy` mark the server as untrusted third-party code running single-tenant behind the egress credential broker. CEL rules then require `groupRef` + `sessionAffinity: ClientIP` and forbid `secrets`, `podTemplateSpec`, and `backendReplicas`. See [Untrusted Mode](16-untrusted-mode.md) and the resources below under "Untrusted-mode resources". + For examples, see: - [`examples/operator/mcp-servers/mcpserver_github.yaml`](../../examples/operator/mcp-servers/mcpserver_github.yaml) - Basic GitHub MCP server - [`examples/operator/mcp-servers/mcpserver_with_oidcconfig_ref.yaml`](../../examples/operator/mcp-servers/mcpserver_with_oidcconfig_ref.yaml) - With shared MCPOIDCConfig reference @@ -507,6 +509,66 @@ The operator binary's `main` package delegates to `cmd/thv-operator/app/app.go`. - For RBAC permissions - Pod identity +### Untrusted-mode resources + +**For each MCPServer with `spec.untrusted: true`, the operator additionally creates** +(`cmd/thv-operator/controllers/mcpserver_untrusted_resources.go`, ADR-0001): + +1. **Bump CA Secret** (per CA generation, `-bump-ca-`) + - Self-signed ECDSA P-256 per-tenant CA (90-day validity, rotation due at 50%) + - Cert + key of ONE generation in ONE Secret, so a pod cloned mid-rotation + always mounts a consistent pair + - Generation-named, never updated in place; N and N-1 generations retained, + older ones GC'd each reconcile + +2. **Bump CA bundle ConfigMap** (per generation, `-bump-ca-bundle-`) + - Public cert only; mounted by the trusted CA-seed init container in session pods + +3. **Egress-policy ConfigMap** (`-egress-policy`) + - The broker-consumed policy document rendered from `spec.egressPolicy` plus the + operator-resolved `dialAllowlist` (destination CIDRs — the same list the + NetworkPolicy carries, so the sidecar's per-dial check and the pod-level + policy enforce one destination set) + - Policy-shape problems are terminal (`UntrustedEgressPolicyInvalid` status + condition); DNS lookup failures retry with backoff + +4. **NetworkPolicy** (`-egress`, egress-only) + - Selects untrusted session pods (`toolhive.stacklok.dev/untrusted=true` + + `mcpserver-uid` label) + - Permits: loopback (the in-pod sidecar), cluster DNS (kube-system / + `k8s-app=kube-dns` pods only — a documented limitation on clusters whose + DNS lives elsewhere), and the policy-resolved destination CIDRs + - Requires a CNI that enforces NetworkPolicy (hard dependency) + +5. **CA generation stamp** on the backend StatefulSet's pod template + (`toolhive.stacklok.dev/bump-ca-generation` annotation) — the vMCP session + lifecycle clones pods mounting exactly that generation's Secret/bundle. + +All untrusted resources are owner-referenced to the MCPServer (GC on deletion) +and carry the `toolhive.stacklok.dev/untrusted-resource=true` label; the +untrusted→trusted flip deletes them explicitly. Session pods themselves are +created by vMCP, not the operator — see +[Untrusted Mode](16-untrusted-mode.md). + +**RBAC additions for untrusted mode** (on the MCPServer controller): +`networkpolicies` (create/delete/get/list/patch/update/watch) and `secrets` +(get/list/watch) — the operator mints and GCs bump-CA Secrets. The +VirtualMCPServer controller additionally holds `secrets` create (KEK coordinate +resolution for the untrusted token-store wiring). + +The **terminal/terminal-DNS split**: policy-shape errors (invalid hosts, a host +resolving to nothing, empty resolved allowlist) are terminal spec errors — +retrying cannot fix the spec — while DNS resolution failures are transient +ordinary errors so controller-runtime retries with backoff. + +**Token encryption wiring**: when `spec.authServerConfig.storage.tokenEncryption` +is set on a VirtualMCPServer, the vMCP Deployment gains one SecretKeyRef env per +KEK (`TOOLHIVE_AUTHSERVER_TOKEN_ENCRYPTION_KEK_`) and the untrusted +token-store coordinate env (`THV_UNTRUSTED_TOKEN_STORE_*`) forwarded to cloned +sidecars — KEK values never appear in pod specs or ConfigMaps. Sentinel-managed +Redis is unsupported for untrusted egress (Warning event +`TokenEncryptionNotSupportedForUntrusted`). + ## Deployment Pattern ```mermaid @@ -729,5 +791,6 @@ spec: - [Core Concepts](02-core-concepts.md) - Operator concepts - [Registry System](06-registry-system.md) - MCPRegistry CRD - [Virtual MCP Server Architecture](10-virtual-mcp-architecture.md) - VirtualMCPServer details +- [Untrusted Mode](16-untrusted-mode.md) - untrusted MCPServer egress broker resources - [Transport Architecture](03-transport-architecture.md) - Transport and proxy setup the proxy-runner performs in-cluster - Operator Design: `cmd/thv-operator/DESIGN.md` diff --git a/docs/arch/13-vmcp-scalability.md b/docs/arch/13-vmcp-scalability.md index 8acc86dc19..b2d67c2585 100644 --- a/docs/arch/13-vmcp-scalability.md +++ b/docs/arch/13-vmcp-scalability.md @@ -190,6 +190,39 @@ session to a single pod. For production stateful workloads, prefer vertical scaling over horizontal scaling. See `docs/arch/10-virtual-mcp-architecture.md` for session affinity design details. +## Single-tenant untrusted backend fan-out + +MCPServers marked `spec.untrusted: true` (ADR-0001) do **not** share the +replica-based backend StatefulSet: each active `(user, MCPServer)` pair gets +its own cloned backend pod carrying an Envoy + credential-broker sidecar. This +is the deliberate cost of per-user egress attribution — pod identity *is* user +identity — and it changes the capacity math above: + +- **Fan-out** ≈ active users × untrusted servers each user touches. 100 users × + 3 untrusted servers = 300 extra pods, each with ~75m CPU / ~96Mi RAM of + sidecar overhead on top of the backend container. +- **Global cap**: total untrusted pods is bounded at `0.8 × session cache + capacity` (0.8 × 1,000 = 800 by default), so untrusted fan-out can never + crowd out the 1,000-session LRU entirely. Tunable via + `THV_UNTRUSTED_GLOBAL_CAP_RATIO`. +- **DoS admission controls** (checked before any pod write, all fail closed on + Redis outage): per-user concurrent quota (10, `THV_UNTRUSTED_PER_USER_QUOTA`), + per-user create rate (5/min), per-MCPServer cap (200, + `THV_UNTRUSTED_PER_SERVER_CAP`). Denials are soft — the backend is excluded + from the session under partial-init semantics, not a session failure. +- **Cold start**: session restore blocks up to the readiness budget (120s + default, `THV_UNTRUSTED_READINESS_TIMEOUT`) while the pod is cloned and goes + Ready; pods that never go Ready are reaped. +- **Teardown**: idle pods are deleted after the idle TTL (30m default, + `THV_UNTRUSTED_IDLE_TTL`); a reaper (60s tick, every vMCP replica) owns GC — + readiness-timeout, idle, and zombie (owning vMCP heartbeat gone) rules — + and rebuilds admission counters from the authoritative pod LIST each tick. + +Sizing implication: with untrusted backends, plan node capacity for the pod +fan-out in addition to the per-vMCP-pod fd/memory limits in this document, and +watch the `untrusted_backend_pods` gauge per MCPServer against the 200 cap. +Full lifecycle details: [Untrusted Mode](16-untrusted-mode.md). + ## Hardcoded limits summary | Limit | Value | Source | Tunable? | @@ -201,6 +234,11 @@ for session affinity design details. | Redis read timeout | 3 s | `toolhive-core/redis/config.go:DefaultReadTimeout` | Via `tcredis.Config.ReadTimeout` | | Redis write timeout | 3 s | `toolhive-core/redis/config.go:DefaultWriteTimeout` | Via `tcredis.Config.WriteTimeout` | | forEach max iterations | 1,000 | `vmcp/config/composite_validation.go:MaxForEachIterations` | Via `WorkflowStepConfig.MaxIterations` (capped at 1,000) | +| Untrusted pods per user | 10 | `vmcp/session/untrusted/admission.go:defaultPerUserPodQuota` | `THV_UNTRUSTED_PER_USER_QUOTA` | +| Untrusted pods per MCPServer | 200 | `vmcp/session/untrusted/admission.go:defaultPerMCPServerCap` | `THV_UNTRUSTED_PER_SERVER_CAP` | +| Untrusted pods global | 0.8 × cache capacity (800) | `vmcp/session/untrusted/admission.go:defaultGlobalCapFactor` | `THV_UNTRUSTED_GLOBAL_CAP_RATIO` | +| Untrusted pod idle TTL | 30 minutes | `vmcp/session/untrusted/reaper.go:defaultIdleTTL` | `THV_UNTRUSTED_IDLE_TTL` | +| Untrusted pod readiness budget | 120 s | `vmcp/session/untrusted/lifecycle.go:DefaultReadyBudget` | `THV_UNTRUSTED_READINESS_TIMEOUT` | ## Related @@ -210,3 +248,4 @@ for session affinity design details. - `github.com/stacklok/toolhive-core/redis/config.go` — Redis client config and timeout defaults - `docs/arch/10-virtual-mcp-architecture.md` — overall vMCP architecture - `docs/arch/11-auth-server-storage.md` — Redis Sentinel for auth server sessions +- `docs/arch/16-untrusted-mode.md` — single-tenant untrusted backend lifecycle and caps diff --git a/docs/arch/16-untrusted-mode.md b/docs/arch/16-untrusted-mode.md new file mode 100644 index 0000000000..0d32d1cd3a --- /dev/null +++ b/docs/arch/16-untrusted-mode.md @@ -0,0 +1,415 @@ +# Untrusted Mode: Egress Credential Broker + +> **Audience**: operators enabling `spec.untrusted` on an MCPServer, and MCP +> server authors whose servers will run untrusted. For the design rationale and +> rejected alternatives, see [ADR-0001](adr/0001-untrusted-mcp-egress-broker.md). + +Untrusted mode runs a third-party MCP server — arbitrary community code the +platform operator does not trust — while still letting that server call upstream +APIs (GitHub, Google, …) **as the human driving the agent**, without the server +ever possessing a real credential. Kubernetes only; the flag is inert in +CLI/Docker mode. + +## 1. What untrusted mode is + +One paragraph version: the untrusted server runs **single-tenant** — one backend +pod per `(user, MCPServer)` pair, cloned on first use by vMCP — and every +credentialed upstream call it makes is transparently intercepted by an **Envoy +sidecar inside its own pod** that terminates TLS, asks a co-located **Go +credential broker** which token applies, and injects `Authorization` at the +boundary. The server's env contains only a sentinel literal; the real token +lives in the auth server's Redis store and is touched only by the broker. + +```mermaid +graph LR + Agent["Agent / MCP client"] --> VMCP["VirtualMCPServer
(trusted, identity-aware front)"] + VMCP --> PR["proxy-runner"] + PR --> Pod + + subgraph Pod["Session pod — one per (user, MCPServer)"] + MCP["mcp container
(untrusted server)
token env = sentinel
HTTP(S)_PROXY=127.0.0.1:15001"] + Envoy["thv-envoy
TLS-bump forward proxy
ext_authz + ext_proc"] + Broker["thv-egressbroker
ext_authz injector + SDS
+ response scanner
+ /healthz :15083"] + Init["thv-ca-seed (init)
seeds bump CA into emptyDir"] + Init --> MCP + MCP -->|CONNECT api.github.com| Envoy + Envoy <-->|"gRPC (loopback)"| Broker + end + + Broker -->|"reads upstream token (refresh-on-read)"| Redis[("Auth-server Redis
upstream token store
(AES-256-GCM at rest)")] + Envoy -->|"re-encrypted TLS to real destination"| Upstream["api.github.com"] + + style Pod fill:#fff3e0 + style MCP fill:#ffb74d + style Envoy fill:#90caf9 + style Broker fill:#90caf9 + style VMCP fill:#a5d6a7 +``` + +**Implementation**: broker `pkg/egressbroker/` + `cmd/thv-egressbroker/`; pod +lifecycle `pkg/vmcp/session/untrusted/`; vMCP wiring `pkg/vmcp/cli/untrusted.go`; +operator resources `cmd/thv-operator/controllers/mcpserver_untrusted_resources.go`. + +## 2. Threat model + +The untrusted server is assumed to be **fully malicious or compromised**: + +- It can read its own environment, filesystem, and process memory. +- It can initiate arbitrary outbound network connections and shape every byte + of them (host, path, method, headers, body). +- It can return anything it wants in MCP tool responses to the agent. +- It cannot be relied on to forward any value faithfully (no per-call dispatch + IDs — ADR D1), so nothing the security boundary depends on ever transits it. +- It shares a pod with the trusted sidecar but has no K8s API access beyond what + the pod's ServiceAccount grants (none, by default). + +What the design assumes is *not* compromised: vMCP, the operator, the auth +server and its Redis, the broker sidecar image, and the cluster CNI's +NetworkPolicy enforcement (a hard dependency — see §8). + +## 3. THE SECURITY BOUNDARY — read before enabling + +> **Untrusted mode defeats credential theft, replay, cross-user use, and +> post-hoc use.** The server never holds a token it can carry off, replay +> later, or use as another user. The pod↔user binding plus destination binding +> plus response scanning enforce this at points the server cannot reach. + +> **It does NOT defeat:** +> +> 1. **Authority abuse within scope for the current call.** The server shapes +> every outbound request, so it directs the current user's credential within +> granted scopes for the current session. A compromised GitHub MCP server can +> still call `DELETE /repos/{user}/{repo}` as the user. The actual mitigations +> are: least-privilege upstream consent scopes; per-provider +> method+path-prefix ACLs in `egressPolicy`; and the audit trail of every +> injected call as the primary detection surface (§8). +> 2. **Data exfiltration via MCP responses.** Anything the upstream API returns +> to the server can be embedded in tool responses to the agent. The credential +> never leaks; the data it fetches can. +> 3. **Transformed echoes of the credential.** The response scanner (§4) matches +> the injected header value **byte-exactly, plus its standard base64 form**. +> It is a tripwire, not a proof: a server that transforms the echo — gzip, +> hex, base64-of-base64, splitting across chunks or fields, or smuggling in +> response trailers (unscanned, SKIP mode) — evades the match. The real +> boundary remains the allowlist + destination binding: the credential can +> only travel to consented hosts in the first place, and every injection is +> audited. + +"Untrusted" means **credential-safe**, not **harmless**. Do not read it as a +sandbox that makes a malicious server safe to point at production APIs with +broad scopes. + +## 4. How credentials flow + +1. **Consent.** The first tool call that needs an upstream provider fails with a + consent-required signal (§9): the user opens the ToolHive authorize endpoint, + completes the upstream OAuth flow (PKCE) against the vMCP's embedded auth + server, and retries the call. +2. **tsid.** On successful auth, the auth server mints a session ID (`tsid`) + into the user's ToolHive JWT and stores the upstream tokens under + `{prefix}upstream:{sessionID}:{provider}` in Redis — AES-256-GCM envelope + encrypted when `tokenEncryption` is configured (§7). +3. **Single-tenant pod.** vMCP clones a backend pod for the `(user, MCPServer)` + pair and stamps pod annotations: OIDC `iss`, raw `sub`, session ID, MCPServer + name (`pkg/egressbroker/identity.go`). The broker container receives them via + downward-API env (`THV_UNTRUSTED_ISS` / `_SUB_RAW` / `_SESSION` / + `_MCPSERVER`); missing identity is a startup error — the pod is the registry. +4. **Injection (ext_authz).** The server's HTTPS egress is CONNECT-tunneled + through the loopback Envoy sidecar (TLS-bump with a per-tenant CA). For each + tunneled request, Envoy calls the broker's `ext_authz`, which: resolves + pod→identity locally (no lookup of anything the request carries), loads the + session's upstream token from Redis with refresh-on-read, asserts the stored + row's owner binding matches the pod's `(iss, sub)` in Strict mode, and — + **only if the destination host/method/path is allowlisted for that provider + (D5)** — returns a header mutation injecting `Authorization`. The broker + records the injected header value in a bounded TTL map keyed by + `x-request-id` (60s TTL, 10k entries, no token plaintext retained). +5. **Destination binding (D5, D7).** D5 is evaluated *before* any header is + written: provider P's credential is emitted only to hosts in P's + `allowedHosts`, only on `allowedMethods`, only under `allowedPathPrefixes`. + Additionally the broker validates the *resolved destination IP* per dial + against the operator-resolved CIDR allowlist (D7, DNS-rebinding protection), + and a NetworkPolicy confines pod egress to loopback + cluster DNS + those + same CIDRs. +6. **Response scanning (D6c).** Envoy's `ext_proc` filter (response headers + + body only, `message_timeout: 2s`) calls the broker, which scans for the + injected header value byte-exactly and in standard base64. A Lua filter + copies `x-request-id` and `:authority` into dynamic metadata on the request + path — the only working correlation path, since response-path ext_proc cannot + see request headers. On a match the response is **replaced with a generic + 502** (`"response suppressed by egress policy"`), a Leak audit event and the + `egress_broker_response_scan_total{result="leak"}` metric are emitted, and + the matched value never appears in any log or event. Bodies over the scan cap + (default 1 MiB, `THV_EGRESSBROKER_SCAN_MAX_BODY_BYTES`) are not body-scanned + (headers still are); this is a cost bound, not a security boundary. Redirects + are never followed: Envoy passes 3xx through untouched (D6a), and + `preserve_external_request_id` stays false so a hostile server cannot poison + the correlation map with its own request IDs. + +## 5. Single-tenant lifecycle + +vMCP owns the lifecycle (`pkg/vmcp/session/untrusted/lifecycle.go`, +`reaper.go`), cloning bare pods from the operator-built backend StatefulSet's +pod template. + +**Fan-out math.** Total untrusted pods ≈ active users × untrusted servers each +user touches. 100 users each driving 3 untrusted servers = 300 pods. This is +the deliberate cost of attribution (ADR D2): pod identity *is* user identity. + +**Caps** (admission gate, all fail closed when Redis is unreachable): + +| Control | Default | Tunable | +|---|---|---| +| Per-user concurrent pods (across servers) | 10 | `THV_UNTRUSTED_PER_USER_QUOTA` | +| Per-user create rate | 5/min | (internal) | +| Per-MCPServer concurrent pods | 200 | `THV_UNTRUSTED_PER_SERVER_CAP` | +| Global pods | 0.8 × session cache capacity (1000) = 800 | `THV_UNTRUSTED_GLOBAL_CAP_RATIO` | + +Admission denials are soft: the backend is excluded from the session +(partial-init semantics), never a hard session failure. The per-user counter is +authoritatively enforced inside the pod-create transaction; admission is a +read-only pre-flight; the reaper rebuilds counters from the pod LIST each tick. + +**Cold start.** The resolver waits up to the readiness budget (default 120s, +`THV_UNTRUSTED_READINESS_TIMEOUT`) for the pod to get an IP and a Ready +condition — including image pull, CA seeding, and Envoy/broker start. A pod +older than the readiness timeout and not Ready is deleted by the reaper. + +**Idle teardown.** The reaper (60s tick) deletes a pod when its `podttl` lease +lapses — default idle TTL 30m (`THV_UNTRUSTED_IDLE_TTL`), renewed while the +owning session's metadata exists (sliding window). A zombie rule deletes pods +whose vMCP heartbeat has been absent for the grace period (owning vMCP died); +session termination deletes the pod immediately. + +**Cluster sizing.** Each session pod carries the backend container plus three +sidecars: Envoy (50m/64Mi req, 500m/256Mi limit), broker (25m/32Mi req, +250m/128Mi limit), and a one-shot init container. Budget node capacity for +`expected_concurrent_pods × (backend + ~75m CPU / ~96Mi RAM sidecar overhead)`. + +## 6. Failure modes + +| Component | Posture | Behavior | +|---|---|---| +| Broker startup | **fail closed** | Missing policy file, CA files, identity env, or dial allowlist → process exits non-zero; pod never Ready; reaper deletes it after the readiness timeout. A nil token-store wiring likewise refuses to start. | +| ext_authz (injection) | **fail closed** | `failure_mode_allow: false` — a dead injector denies; nothing ever passes uncredentialed-by-mistake. Denial reasons are a fixed vocabulary (`no-policy`, `method-not-allowed`, `path-not-allowed`, `credential-unavailable`, `binding-mismatch`, `store-error`, `malformed-request`). | +| ext_proc (response scanner) | **fail open (default)** | `failure_mode_allow: true` — a scanner outage passes responses + alert metric, rather than hard-downing every untrusted workload. Set `THV_EGRESSBROKER_SCAN_FAIL_CLOSED=true` to flip (scanner errors → 502). A *detected* echo always blocks in both modes. | +| Unknown request-id (direct hit, scanner restart, TTL eviction) | **fail open (default)** | Response passes + `egress_broker_response_scan_total{result="unknown_request"}`. | +| Admission (vMCP) | **fail closed** | Redis unreachable → every admission check denies (untrusted mode already requires Redis-backed session storage; startup refuses otherwise). | +| Reaper | **fail safe** | Redis down at tick → tick skipped entirely; no deletions without Redis evidence. | +| Encrypted token row, no KEK wired | **fail closed** | Open rejects non-legacy values; every injection denies with `store-error`/`credential-unavailable`. | +| DNS resolution at operator reconcile | **retry with backoff** | NXDOMAIN blips must not poison the workload; a permanently unresolvable host is a terminal spec error. | + +## 7. Configuration reference + +### MCPServer CRD fields + +```yaml +spec: + untrusted: true # marks the server as untrusted code (K8s-only) + egressPolicy: # required when untrusted; ≥1 provider + providers: + - provider: github # must match the auth-server upstream name + allowedHosts: ["api.github.com"] # exact hosts or one-label wildcards + allowedMethods: [GET, POST] # empty = GET/HEAD/OPTIONS only (read-only) + allowedPathPrefixes: ["/repos/"] # empty = all paths on allowed hosts + credentialEnvName: GITHUB_TOKEN # backend env that gets the sentinel +``` + +When `untrusted: true`, `permissionProfile.network.outbound` is **ignored** for +the backend pod — the NetworkPolicy derives solely from `egressPolicy` (+ DNS + +loopback). The two network dialects are intentionally separate: +`permissionProfile` remains the Docker/Squid dialect; `egressPolicy` is the K8s +untrusted-mode dialect. + +### CEL validation rules (MCPServerSpec, R1–R6) + +| # | Rule | Message (abbreviated) | +|---|---|---| +| R1 | `untrusted ⇒ egressPolicy with ≥1 provider` | "egressPolicy … is required when untrusted is true" | +| R2 | `untrusted ⇒ groupRef set` | "must belong to an MCPGroup fronted by a VirtualMCPServer" | +| R3 | `untrusted ⇒ spec.secrets empty` | "declare providers in egressPolicy and use sentinel env" | +| R4 | `untrusted ⇒ no podTemplateSpec` | the session lifecycle owns the pod shape | +| R5 | `untrusted ⇒ sessionAffinity: ClientIP` | per-user routing requires sticky sessions | +| R6 | `untrusted ⇒ no backendReplicas` | managed by the untrusted-mode session lifecycle | + +The runtime env gate additionally rejects Secret/ConfigMap-sourced env and +Secret/ConfigMap volumes in the backend template (Wave-0 hardening), and +sentinel-forgery validation reserves the `thv-untrusted-sentinel:` prefix for +operator-injected values only. Violations surface as terminal status conditions +(`GroupRefNotVMCPFronted`, `SecretEnvRejected`, `UntrustedEgressPolicyInvalid`). + +### Platform tunables (vMCP env vars, resolved once at startup — no hot reload) + +| Env var | Default | Meaning | +|---|---|---| +| `THV_UNTRUSTED_IDLE_TTL` | `30m` | pod liveness lease the reaper enforces | +| `THV_UNTRUSTED_PER_USER_QUOTA` | `10` | concurrent untrusted pods per user | +| `THV_UNTRUSTED_PER_SERVER_CAP` | `200` | concurrent untrusted pods per MCPServer | +| `THV_UNTRUSTED_GLOBAL_CAP_RATIO` | `0.8` | fraction of the session cache capacity bounding total untrusted pods | +| `THV_UNTRUSTED_READINESS_TIMEOUT` | `120s` | cold-start budget (resolver wait + reaper sweep rule) | +| `THV_UNTRUSTED_ENVOY_IMAGE` | `envoyproxy/envoy:v1.36.2@sha256:4972…80ba` (tag+digest pinned) | Envoy data-plane image override (air-gapped mirrors) | +| `THV_UNTRUSTED_BROKER_IMAGE` | `ghcr.io/stacklok/toolhive/egressbroker:v0.17.0` | broker sidecar image override | +| `THV_UNTRUSTED_SIDECAR_CPU` / `THV_UNTRUSTED_SIDECAR_MEM` | `1.0` | multiplier on envoy/broker sidecar requests+limits (must be in (0, 100]) | + +Every tunable fails startup on an unparseable/zero/negative value. Image +overrides tagged `:latest` are rejected outright; non-digest-pinned overrides +are honored with a loud warning. + +### Broker-side env (set by the clone wiring — informational) + +`THV_EGRESSBROKER_POLICY_FILE`, `THV_EGRESSBROKER_CA_FILE`, +`THV_EGRESSBROKER_CA_KEY_FILE`, `THV_EGRESSBROKER_LISTEN_ADDRESS`/`_PORT` +(default 127.0.0.1:9001), `THV_EGRESSBROKER_DIAL_ALLOWLIST` (override; normally +carried in the policy document), `THV_EGRESSBROKER_SCAN_FAIL_CLOSED`, +`THV_EGRESSBROKER_SCAN_MAX_BODY_BYTES` (default 1 MiB), +`THV_EGRESSBROKER_REDIS_ADDR`, `THV_EGRESSBROKER_REDIS_KEY_PREFIX`, +`THV_SESSION_REDIS_PASSWORD` (SecretKeyRef), `THV_EGRESSBROKER_KEK_ID` + +`THV_EGRESSBROKER_KEK_` per key ID (SecretKeyRef, active + retired), +`THV_EGRESSBROKER_ENVOY_BOOTSTRAP_OUT`. + +### Token encryption at rest (KEK) + +`VirtualMCPServer.spec.authServerConfig.storage.tokenEncryption` (CEL: +requires storage `type: redis`): + +```yaml +spec: + authServerConfig: + storage: + type: redis + redis: { addr: "redis:6379", aclUserConfig: { passwordSecretRef: { name: redis-acl, key: password } } } + tokenEncryption: + activeKeyId: kek-2026-07 # key used for new writes + keySecretRef: { name: thv-token-keks } # Secret: one base64 32-byte KEK per data key +``` + +The operator renders one SecretKeyRef env per Secret data key onto the vMCP +container (`TOOLHIVE_AUTHSERVER_TOKEN_ENCRYPTION_KEK_`, max 16 keys) and +forwards the KEK *coordinates* (never values) to cloned sidecars via +`THV_UNTRUSTED_TOKEN_STORE_KEK_SECRET` / `_KEK_KEY` / `_KEK_IDS`. Sidecar +keyrings receive the full key-ID set so rotation never orphans ciphertext +sealed under a retired ID. Key IDs must match `[A-Za-z0-9_-]+` (they become +env-var suffixes). Sentinel-backed Redis cannot be dialed by sidecars — +`tokenEncryption` with Sentinel storage emits a +`TokenEncryptionNotSupportedForUntrusted` Warning event and no sidecar wiring. + +## 8. Operations + +### Metrics (OTel, low-cardinality per ADR D11 — never a user identifier) + +| Metric | Labels | Meaning | +|---|---|---| +| `untrusted_backend_pods` (gauge) | `mcpserver` | live untrusted pods, refreshed from the pod LIST each reaper tick | +| `untrusted_pod_admissions_total` | `result` (`admitted`/`quota_exceeded`/`error`) | admission decisions | +| `egress_broker_injections_total` | `mcpserver`, `provider` | successful injections | +| `egress_broker_denials_total` | `result` (deny-reason vocabulary), `mcpserver`, `provider` | injection denials | +| `egress_broker_response_scan_total` | `mcpserver`, `provider`, `result` (`ok`/`leak`/`unknown_request`), `where` (`header`/`body`, leaks only) | D6c scan outcomes | +| `egress_broker_scan_skipped_total` | `mcpserver`, `provider` | body-over-cap skips (headers still scanned) | + +**Audit**: every injection, denial, and leak emits one structured slog JSON line +to stdout (Inject/Deny/Leak events; `pkg/egressbroker/audit.go`). The user sub +appears only here, as `SHA-256(sub)[:16hex]` — never in metric labels. Token +values, bodies, KEKs, and CA material are never logged. + +### Recommended alerts + +- `egress_broker_response_scan_total{result="leak"} > 0` → **page**: an + upstream endpoint echoed the injected credential back. +- `rate(untrusted_pod_admissions_total{result="quota_exceeded"}[10m]) > 0` → + possible session-creation DoS. +- `untrusted_backend_pods` per mcpserver approaching 200 → capacity. +- `egress_broker_denials_total{result="credential-unavailable"}` spike → + consent-flow breakage. +- Bump CA `notAfter` within 14 days → run the rotation check below (rotation is + automatic; alert only if generations stop advancing). + +### CA rotation runbook (generation-named, N/N-1) + +The per-tenant bump CA is a self-signed ECDSA P-256 CA valid 90 days; rotation +becomes due at 50% of validity. Rotation is fully automatic and +generation-named, never in-place: + +1. The operator mints a **new** CA into a new Secret `-bump-ca-` + (generation = `sha256(cert)[:16hex]`) plus a matching public bundle + ConfigMap, and stamps the new generation on the backend StatefulSet's + `toolhive.stacklok.dev/bump-ca-generation` pod-template annotation. +2. New session pods clone the new template and mount exactly one consistent + cert/key pair; existing pods keep the CA their emptyDir was seeded with + (no cross-pod TLS). +3. GC retains the current and previous generations (N and N-1) and deletes + everything older. + +Manual verification: list `secrets -l toolhive.stacklok.dev/untrusted-resource=true` +and confirm at most two `-bump-ca-*` generations exist per MCPServer. To force a +rotation early, delete the current generation Secret; the next reconcile mints +a fresh CA (existing pods are unaffected until their idle TTL lapses). + +### Upgrade story + +Sidecar images are part of the cloned pod template. New sessions clone the new +template; existing pods run to idle-TTL (30m default). CA generations are +backward-compatible (N and N-1 retained). **No in-place pod mutation ever +happens** — that invariant is what makes upgrades trivially safe. + +### Known limitations + +- **kube-dns selector assumption**: the NetworkPolicy permits DNS only to pods + labeled `k8s-app: kube-dns` in namespace `kube-system`. On clusters whose DNS + lives elsewhere, backend DNS lookups fail loudly until the policy is adjusted + — an explicit failure, never a silent port-53-to-anywhere hole. +- **Sentinel-managed Redis unsupported for the token store**: sidecars need a + standalone/cluster Redis address. `tokenEncryption` with Sentinel storage + emits a Warning event and sidecars cannot decrypt. +- **stdio proxy-env caveat (D10)**: a stdio server that ignores + `HTTP_PROXY`/`HTTPS_PROXY` makes direct egress attempts → NetworkPolicy blocks + them → the server cannot call upstreams. Compatibility limit, not a security + gap. +- **No non-HTTP upstreams**: databases, gRPC, SSH-git are denied by default; + only HTTP(S) through the forward proxy is credentialed. +- **Cert-pinning servers fail closed**: pinning against a public CA bundle + breaks the TLS-bump handshake (D9) — see §9. +- **Byte-exact scanner limits**: see §3 item 3 — the scanner is a tripwire, not + a proof. +- **Container-level confinement**: NetworkPolicy is pod-level, not + container-level; the backend container's confinement rests on the proxy-env + routing and the fact that no credential ever resides in it. The load-bearing + controls are D5 destination binding and D7 per-dial IP validation. + +## 9. For MCP server authors + +Your server runs unmodified, but the environment it sees is shaped by untrusted +mode: + +- **Sentinel semantics.** The env var that would normally carry your upstream + token (declared as `credentialEnvName` in the egress policy) contains the + literal `thv-untrusted-sentinel:` — **not** a token. It exists so + servers that refuse to start tokenless still boot. Never send it anywhere. +- **Make the call anyway.** Issue the upstream HTTPS request exactly as you + would with a real token (any placeholder `Authorization` header is fine — it + is replaced). The broker injects the real credential at the sidecar, only for + policy-allowlisted hosts/methods/paths. Calls outside the policy are denied + (403 from the proxy) or have no route (connection refused). +- **Honor proxy env.** Your HTTP client must respect `HTTP_PROXY`/`HTTPS_PROXY` + (most stdlib clients do by default). Direct egress is blocked by + NetworkPolicy — there is no fallback path. +- **Cert pinning fails closed.** If you pin against public CA bundles or + specific leaf certs, the TLS-bump handshake fails. Your pod already trusts + the per-tenant bump CA via `SSL_CERT_FILE`, `NODE_EXTRA_CA_CERTS`, + `REQUESTS_CA_BUNDLE`, and `CURL_CA_BUNDLE`; use the default trust store. +- **Request-driven only.** Background polling cannot work: pods are per-session + and torn down after the idle TTL (default 30m), and every upstream call needs + a live session's consent. Do work only in response to tool calls. +- **Consent UX.** When the user has not consented the provider, your tool call + chain surfaces an error result whose text starts with + `UPSTREAM_CONSENT_REQUIRED` followed by a JSON payload + `{"provider":"","authorize_url":""}` — pass it through untouched; + MCP clients render it as an actionable consent prompt. + +## Related documentation + +- [ADR-0001: Untrusted MCP Server Egress Credential Broker](adr/0001-untrusted-mcp-egress-broker.md) — decisions D1–D11, rejected alternatives +- [Virtual MCP Server Architecture](10-virtual-mcp-architecture.md) — the trusted front +- [vMCP Scalability Limits](13-vmcp-scalability.md) — session cache cap the global pod cap derives from +- [Auth Server Storage](11-auth-server-storage.md) — upstream token store, key prefixes +- [RunConfig and Permissions](05-runconfig-and-permissions.md) — the Docker/Squid network dialect +- Operator how-to: [docs/operator/untrusted-mode.md](../operator/untrusted-mode.md) diff --git a/docs/arch/README.md b/docs/arch/README.md index a39a67a97c..112381171a 100644 --- a/docs/arch/README.md +++ b/docs/arch/README.md @@ -127,6 +127,14 @@ Welcome to the ToolHive architecture documentation. This directory contains comp - Dynamic forward proxy with per-request DNS and native HTTPS CONNECT tunnelling - Squid-vs-Envoy comparison and known limitations (`AllowPort` gap, V4_ONLY DNS, deny-all on empty profile) +16. **[Untrusted Mode: Egress Credential Broker](16-untrusted-mode.md)** + - Single-tenant backends (one pod per (user, MCPServer)) + Envoy/broker sidecar + - THE SECURITY BOUNDARY stated loudly (theft/replay defeated; authority abuse + exfil not) + - Credential flow: consent → tsid → encrypted store → sidecar injection → D5 binding → D6c scanning + - Failure modes, configuration reference (CEL R1–R6, THV_UNTRUSTED_* tunables, KEK) + - Operations: alerts, generation-named CA rotation, upgrade story, known limitations + - Design decisions: [ADR-0001](adr/0001-untrusted-mcp-egress-broker.md) + ### Existing Documentation For middleware architecture, see: **[docs/middleware.md](../middleware.md)** @@ -177,6 +185,10 @@ graph TB Plugins[14: Plugins System
MaterializationAdapter] end + subgraph "Security Boundaries" + Untrusted[15: Untrusted Mode
Egress credential broker] + end + %% Navigation paths Overview --> Concepts Overview --> Deployment @@ -207,6 +219,8 @@ graph TB Workloads --> Operator vMCP --> Operator AuthStorage --> Operator + Untrusted --> vMCP + Untrusted --> Operator %% Styling style Overview fill:#e1f5fe,stroke:#01579b,stroke-width:3px @@ -223,6 +237,7 @@ graph TB style vMCP fill:#e0f2f1,stroke:#004d40,stroke-width:2px style AuthStorage fill:#e0f2f1,stroke:#004d40,stroke-width:2px style Skills fill:#e8eaf6,stroke:#283593,stroke-width:2px + style Untrusted fill:#ffebee,stroke:#b71c1c,stroke-width:2px ``` **Color Legend:** @@ -280,6 +295,7 @@ Review all documents in order (00 → 01 → 02 → 03 → ...) - [Groups](07-groups.md) - [Workloads Lifecycle](08-workloads-lifecycle.md) - [Kubernetes Operator](09-operator-architecture.md) +- [Untrusted Mode](16-untrusted-mode.md) ## Architecture Principles diff --git a/docs/arch/adr/0001-untrusted-mcp-egress-broker.md b/docs/arch/adr/0001-untrusted-mcp-egress-broker.md index e88ee06595..8ae9cb4680 100644 --- a/docs/arch/adr/0001-untrusted-mcp-egress-broker.md +++ b/docs/arch/adr/0001-untrusted-mcp-egress-broker.md @@ -1,6 +1,6 @@ # ADR-0001: Untrusted MCP Server Egress Credential Broker -- **Status**: Accepted +- **Status**: Implemented (Waves 0–7) - **Date**: 2026-07-21 - **Deciders**: ToolHive core team - **Source**: Final consolidated plan, `.claude/scratch/untrusted-mcp-egress-broker-plan.md` §9 @@ -336,3 +336,47 @@ is not over-trusted. 2. Per-tenant vs cluster bump CA — default per-tenant where feasible (D9). 3. Idle-TTL default and cold-start SLO for session-scoped pods — Wave 2. 4. Non-HTTP upstreams (DBs, gRPC, SSH-git): deny by default; revisit per demand. + +--- + +## 8. As-built deviations (recorded at closeout, Waves 0–7) + +The shipped system matches D1–D11 with these recorded deviations from the +original text: + +- **Sidecar topology confirmed** (D3): Envoy per session pod + Go broker serving + `ext_authz` (injection), on-demand SDS (bump certs), **and** `ext_proc` + (response scanning, D6c) on one loopback socket. The Wave-3 deferral of + ext_proc was resolved in Wave 5 — both filters now ship, with a Lua filter + copying `x-request-id`/`:authority` into dynamic metadata as the only working + request→response correlation path (route `filter_metadata` is literal; + response-path ext_proc cannot see request headers). +- **D6c scanner default is fail-open** (`failure_mode_allow: true`, + `THV_EGRESSBROKER_SCAN_FAIL_CLOSED=true` to flip): a scanner outage must not + hard-down every untrusted workload. A detected echo always blocks in both + modes. The byte-exact (+base64) match limit is recorded in §5 item 3. +- **Audit is slog, not `pkg/audit`** (D11): `pkg/audit` is HTTP-middleware-shaped; + the broker is gRPC. One structured slog line per Inject/Deny/Leak event to + stdout; sub appears hashed only there, never in metric labels. +- **Tunables are env vars, not CRD fields** (`THV_UNTRUSTED_*`, resolved once at + the vMCP composition root): platform-operator knobs, not per-workload API. +- **KEK surface added**: `tokenEncryption` on the auth-server storage config + (CEL: requires `type: redis`); KEKs reach the vMCP and the cloned sidecars as + SecretKeyRef env only — coordinates, never values. Sentinel-managed Redis is + unsupported for the untrusted token store (Warning event, fail closed). +- **Bump CA is generation-named** (D9 implementation): cert+key of one + generation in one Secret (`-bump-ca-`), N/N-1 retained, + StatefulSet pod-template annotation publishes the current generation — no + in-place rotation race. +- **Images pinned**: Envoy by tag+digest, broker by release tag; per-install + overrides via `THV_UNTRUSTED_ENVOY_IMAGE` / `THV_UNTRUSTED_BROKER_IMAGE` + (`:latest` rejected). +- **Sidecar resources set** (envoy 50m/64Mi–500m/256Mi, broker 25m/32Mi–250m/128Mi, + init 10m/16Mi–50m/32Mi) and broker `/healthz` on :15083 (Redis + policy + CA) + wired to readiness/liveness probes. +- **Data-plane resource naming is shared** across the operator/vMCP module + boundary via `pkg/egressbroker` (`ResourceNamesFor`, `CAGeneration`, + identity annotation↔env contract), pinned by tests on both sides. + +Full architecture and operations documentation: `docs/arch/16-untrusted-mode.md`; +operator how-to: `docs/operator/untrusted-mode.md`. diff --git a/docs/operator/untrusted-mode.md b/docs/operator/untrusted-mode.md new file mode 100644 index 0000000000..5e8ba398f7 --- /dev/null +++ b/docs/operator/untrusted-mode.md @@ -0,0 +1,162 @@ +# Untrusted Mode Operator Guide + +How to run a third-party MCP server whose code you do not trust while still +letting it call upstream APIs as the consenting user. Architecture, threat +model, and the full security boundary: +[Untrusted Mode architecture](../arch/16-untrusted-mode.md) and +[ADR-0001](../arch/adr/0001-untrusted-mcp-egress-broker.md). + +> **Read the security boundary first.** Untrusted mode stops the server from +> *stealing* credentials. It does not stop the server from *using* the current +> user's credential within granted scopes, and it does not stop data fetched +> with that credential from leaving through MCP tool responses. + +## Prerequisites + +- A `VirtualMCPServer` fronting an `MCPGroup` (untrusted workloads must be + group members — enforced by CEL). +- The vMCP's embedded auth server with the upstream provider(s) configured and + **Redis storage** (standalone/cluster — Sentinel-managed Redis is not + supported for untrusted egress). +- A CNI that enforces `NetworkPolicy` (hard dependency). +- Session affinity `ClientIP` on the MCPServer (required by CEL). + +## Marking a server untrusted + +Set `spec.untrusted: true` and declare an `egressPolicy`: + +```yaml +apiVersion: toolhive.stacklok.dev/v1beta1 +kind: MCPServer +metadata: + name: github-community + namespace: default +spec: + image: ghcr.io/example/github-mcp:latest + transport: streamable-http + port: 8080 + groupRef: research-group # required: an MCPGroup fronted by a VirtualMCPServer + sessionAffinity: ClientIP # required + untrusted: true + egressPolicy: + providers: + - provider: github # must match the auth-server upstream provider name + allowedHosts: + - api.github.com + - "*.githubusercontent.com" # one-label wildcards allowed + allowedMethods: [GET, POST] # omit for read-only (GET/HEAD/OPTIONS only) + allowedPathPrefixes: ["/repos/", "/user"] # omit for all paths on the allowed hosts + credentialEnvName: GITHUB_TOKEN # the env var the server reads its "token" from +``` + +CEL validation enforces, when `untrusted: true`: + +- `egressPolicy` with at least one provider (required); +- `groupRef` set (the server must be fronted by a vMCP); +- `spec.secrets` forbidden — declare providers in `egressPolicy` and let the + broker inject; no Secret/ConfigMap-sourced env or volumes on the backend + either (runtime gate, terminal `SecretEnvRejected` condition); +- `podTemplateSpec` and `backendReplicas` forbidden — the untrusted session + lifecycle owns the pod shape and the replica count (one pod per + (user, MCPServer) pair). + +The same provider name must appear in the vMCP's embedded auth server upstream +chain; the broker looks the user's token up by it. + +## How the pieces fit at runtime + +1. The operator reconciles the untrusted MCPServer into: a per-server + **bump CA** (generation-named Secret + public bundle ConfigMap), an + **egress-policy ConfigMap** the sidecar compiles, and a + **NetworkPolicy** confining session-pod egress to loopback, cluster DNS, + and the CIDRs your `allowedHosts` resolve to. +2. On a user's first call, vMCP clones a backend pod from the server template + and adds three containers: a CA-seed init container, an Envoy sidecar, and + the credential broker. The pod is stamped with the user's identity. +3. The server's egress goes through the sidecar; the broker injects the user's + consented token only for policy-allowlisted destinations and scans responses + for credential echoes. + +Idle pods are torn down after 30 minutes (`THV_UNTRUSTED_IDLE_TTL`); cold start +is bounded at 120s (`THV_UNTRUSTED_READINESS_TIMEOUT`). Capacity caps default to +10 pods/user, 200 pods/server, 0.8× session-cache globally — see the +[configuration reference](../arch/16-untrusted-mode.md#7-configuration-reference) +for the `THV_UNTRUSTED_*` tunables and image overrides. + +## Sentinel semantics + +The env var named by `credentialEnvName` contains the literal +`thv-untrusted-sentinel:` — a boot-compatibility shim, **not** a +token. The server must make its upstream call anyway; the broker replaces the +placeholder at the sidecar. Servers that ignore `HTTP_PROXY`/`HTTPS_PROXY`, pin +certificates, or poll in the background will not work (see +[limitations](../arch/16-untrusted-mode.md#known-limitations)). + +## kube-dns limitation + +The generated NetworkPolicy permits DNS only to pods labeled `k8s-app: +kube-dns` in namespace `kube-system` (kubeadm/kind/GKE/EKS defaults). If your +cluster DNS lives elsewhere, backend name resolution fails loudly — patch the +`-egress` NetworkPolicy's DNS peer to match your DNS pods. + +## Consent UX for end users + +When a user who has not consented the upstream provider calls a tool that needs +it, the tool result is an error whose text is: + +``` +UPSTREAM_CONSENT_REQUIRED {"provider":"github","authorize_url":"https://thv.example.com/oauth/authorize"} +``` + +The `UPSTREAM_CONSENT_REQUIRED` marker is machine-detectable; MCP clients +surface it as a consent prompt. The `authorize_url` is only an endpoint — the +client merges its own `client_id`, `redirect_uri`, and PKCE parameters, drives +the OAuth flow in a browser, and retries the tool call. The URL comes from the +vMCP's auth-server issuer configuration; when unset, the payload carries just +the provider name and the user must be directed to consent out of band. + +## Token encryption at rest (recommended) + +Encrypt upstream tokens in Redis with AES-256-GCM envelope encryption: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: thv-token-keks +stringData: + kek-2026-07: # data keys are key IDs +--- +apiVersion: toolhive.stacklok.dev/v1beta1 +kind: VirtualMCPServer +spec: + authServerConfig: + storage: + type: redis + redis: + addr: redis:6379 + aclUserConfig: + passwordSecretRef: { name: redis-acl, key: password } + tokenEncryption: + activeKeyId: kek-2026-07 + keySecretRef: { name: thv-token-keks } +``` + +Rotation: add a new key to the Secret, flip `activeKeyId`, keep the old key +until all rows are re-sealed or expired — both the auth server and the broker +sidecars receive the full key set, so retired IDs keep decrypting. Key IDs must +match `[A-Za-z0-9_-]+`. Requires standalone/cluster Redis; Sentinel storage +emits a `TokenEncryptionNotSupportedForUntrusted` Warning and sidecars cannot +decrypt. + +## Troubleshooting + +| Symptom | Likely cause | +|---|---| +| Server boots, upstream calls fail with 403 | destination/method/path not in `egressPolicy` for that provider | +| Connection refused to a host | host not in `allowedHosts` (no route at the proxy) | +| Tool result says `UPSTREAM_CONSENT_REQUIRED` | user has not consented; drive the authorize URL flow | +| Denials with `credential-unavailable` | consent expired/revoked, or token-store Redis unreachable from sidecars | +| Pods created then deleted ~2 min later | cold start exceeded the readiness timeout (image pull, broker health) — check the broker `/healthz` (Redis reachable, policy loaded, CA valid) | +| All egress broken, DNS timeouts | cluster DNS not in `kube-system`/`k8s-app=kube-dns` — see kube-dns limitation | +| Warning event `TokenEncryptionNotSupportedForUntrusted` | `tokenEncryption` set with Sentinel Redis storage | From 2c57e02364fa4c363a1db5bc10e7de8e64e0d9c3 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Wed, 22 Jul 2026 17:39:15 +0300 Subject: [PATCH 07/12] Harden untrusted mode after duplication and reuse review Consolidate the private-IP classifier: the egress broker's dial allowlist now uses networking.IsPrivateIP instead of a second local table, gaining NAT64 embedded-IPv4 decoding (64:ff9b::/96, 64:ff9b:1::/48) so a crafted DNS answer can no longer smuggle the cloud metadata endpoint past D7 on NAT64 networks. The shared classifier absorbs the two benchmarking prefixes the egress table had, making the merged set a strict superset. Render the Envoy bootstrap as data: all config-derived values (vhosts, routes, ext_authz address, scan posture) go through yaml.Marshal instead of string substitution, matching the operator side and removing the reliance on cross-package validation alone. TokenMap rebuilt on hashicorp/golang-lru/v2 (already a dependency), keeping consume-on-read semantics via Peek+Remove under the map mutex; the hand-rolled container/list machinery is gone. Converge tokenenc on pkg/secrets/aes: new EncryptWithAAD/ DecryptWithAAD helpers carry the nonce|ct|tag convention and size bound; tokenenc deletes its duplicated GCM boilerplate and calls them, keeping the redis-key AAD cut-and-paste defense. Also: slices.Contains/Sort/ContainsFunc in the policy hot path, and cross-referenced drift-guard comments between the operator and vMCP secret-material gate test suites. --- .../untrusted_env_validation_test.go | 9 + pkg/authserver/tokenenc/tokenenc.go | 46 +-- pkg/egressbroker/envoyconfig.go | 317 +++++++++++++++--- pkg/egressbroker/policy.go | 22 +- pkg/egressbroker/resolve.go | 43 +-- pkg/egressbroker/resolve_test.go | 31 ++ pkg/egressbroker/tokenmap.go | 69 ++-- pkg/networking/utilities.go | 2 + pkg/secrets/aes/aes.go | 18 +- .../session/untrusted/template_clone_test.go | 10 + 10 files changed, 388 insertions(+), 179 deletions(-) diff --git a/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation_test.go b/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation_test.go index 42ab821779..db663eb275 100644 --- a/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation_test.go +++ b/cmd/thv-operator/pkg/controllerutil/untrusted_env_validation_test.go @@ -39,6 +39,15 @@ func configMapEnvVar(name, configMapName, key string) corev1.EnvVar { } } +// TestValidateNoSecretEnvForUntrusted pins the operator admission-side +// untrusted pod gate. CHANNEL-COVERAGE MIRROR: the vmcp runtime-side gate's +// test — pkg/vmcp/session/untrusted/template_clone_test.go +// (TestClonePodFromTemplate_FailClosed / TestReverifyAllowsFieldRefEnv) — +// covers the same secret-injection channels (secretKeyRef / configMapKeyRef +// env, envFrom secret/configmap refs, secret / configmap / projected +// volumes). When a new injection channel is added or allowed on one side, +// update the other side's suite too — drift between the two is a silent gap +// in the untrusted isolation boundary. func TestValidateNoSecretEnvForUntrusted(t *testing.T) { t.Parallel() diff --git a/pkg/authserver/tokenenc/tokenenc.go b/pkg/authserver/tokenenc/tokenenc.go index f8fb8d3d4c..33df572e7a 100644 --- a/pkg/authserver/tokenenc/tokenenc.go +++ b/pkg/authserver/tokenenc/tokenenc.go @@ -29,14 +29,14 @@ package tokenenc import ( - "crypto/aes" - "crypto/cipher" "crypto/rand" "encoding/base64" "encoding/json" "errors" "fmt" "io" + + "github.com/stacklok/toolhive/pkg/secrets/aes" ) const ( @@ -144,12 +144,12 @@ func Seal(kr Keyring, redisKey string, plaintext []byte) ([]byte, error) { activeID, kek := kr.Active() - edek, err := gcmSeal(kek, dek, nil) + edek, err := aes.Encrypt(dek, kek) if err != nil { return nil, fmt.Errorf("token encryption: failed to wrap data key: %w", err) } - ct, err := gcmSeal(dek, plaintext, []byte(redisKey)) + ct, err := aes.EncryptWithAAD(plaintext, dek, []byte(redisKey)) if err != nil { return nil, fmt.Errorf("token encryption: failed to encrypt record: %w", err) } @@ -201,7 +201,7 @@ func Open(kr Keyring, redisKey string, value []byte) (plaintext []byte, legacy b if err != nil { return nil, false, fmt.Errorf("token encryption: malformed wrapped data key: %w", err) } - dek, err := gcmOpen(kek, edek, nil) + dek, err := aes.Decrypt(edek, kek) if err != nil { return nil, false, fmt.Errorf("token encryption: failed to unwrap data key: %w", err) } @@ -210,7 +210,7 @@ func Open(kr Keyring, redisKey string, value []byte) (plaintext []byte, legacy b if err != nil { return nil, false, fmt.Errorf("token encryption: malformed ciphertext: %w", err) } - plaintext, err = gcmOpen(dek, ct, []byte(redisKey)) + plaintext, err = aes.DecryptWithAAD(ct, dek, []byte(redisKey)) if err != nil { return nil, false, fmt.Errorf("token encryption: failed to decrypt record: %w", err) } @@ -246,37 +246,3 @@ func NeedsRotation(kr Keyring, value []byte) bool { activeID, _ := kr.Active() return env.KID != activeID } - -// gcmSeal encrypts plaintext with AES-256-GCM under key, returning -// nonce|ciphertext|tag. aad is additional authenticated data (may be nil). -func gcmSeal(key, plaintext, aad []byte) ([]byte, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, err - } - nonce := make([]byte, gcm.NonceSize()) - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { - return nil, err - } - return gcm.Seal(nonce, nonce, plaintext, aad), nil -} - -// gcmOpen reverses gcmSeal. -func gcmOpen(key, ciphertext, aad []byte) ([]byte, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, err - } - if len(ciphertext) < gcm.NonceSize() { - return nil, errors.New("malformed ciphertext") - } - return gcm.Open(nil, ciphertext[:gcm.NonceSize()], ciphertext[gcm.NonceSize():], aad) -} diff --git a/pkg/egressbroker/envoyconfig.go b/pkg/egressbroker/envoyconfig.go index f69933d32d..17fac050da 100644 --- a/pkg/egressbroker/envoyconfig.go +++ b/pkg/egressbroker/envoyconfig.go @@ -6,8 +6,10 @@ package egressbroker import ( "fmt" "os" - "sort" + "slices" "strings" + + "gopkg.in/yaml.v3" ) // EnvoyConfigParams carries the rendered Envoy bootstrap's variable parts. @@ -118,7 +120,7 @@ const envoyBootstrapTemplate = `static_resources: address: socket_address: address: 127.0.0.1 - port_value: {{PROXY_PORT}} + port_value: 0 # set from params at render time listener_filters: - name: envoy.filters.listener.tls_inspector typed_config: @@ -146,8 +148,7 @@ const envoyBootstrapTemplate = `static_resources: "@type": type.googleapis.com/envoy.extensions.request_id.uuid.v3.UuidRequestIdConfig route_config: name: egress_routes - virtual_hosts: -{{VHOSTS}} + virtual_hosts: [] # one vhost per allowlisted host, built as data at render time http_filters: - name: envoy.filters.http.lua typed_config: @@ -177,7 +178,7 @@ const envoyBootstrapTemplate = `static_resources: - name: envoy.filters.http.ext_proc typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor - failure_mode_allow: {{SCAN_FAIL_OPEN}} + failure_mode_allow: true # wired from broker scan config at render time message_timeout: 2s # NOTE: ext_proc has no per-filter body byte cap (no max_bytes or # allow_partial_message field exists on this filter in the pinned @@ -232,8 +233,8 @@ const envoyBootstrapTemplate = `static_resources: - endpoint: address: socket_address: - address: {{EXT_AUTHZ_ADDRESS}} - port_value: {{EXT_AUTHZ_PORT}} + address: 127.0.0.1 # set from params at render time + port_value: 0 # set from params at render time - name: dynamic_forward_proxy_cluster lb_policy: CLUSTER_PROVIDED cluster_type: @@ -253,57 +254,279 @@ const envoyBootstrapTemplate = `static_resources: filename: /etc/ssl/certs/ca-certificates.crt ` -// vhostTemplate is one allowlisted-host route entry. The CONNECT upgrade -// terminates the tunnel downstream via the filter chain's on-demand SDS cert -// (resource "host:") and re-originates upstream through the dynamic -// forward proxy; every request — the CONNECT itself and the decrypted tunneled -// requests — passes ext_authz (no per-route disable: the credential injection -// on tunneled traffic IS the feature). Non-CONNECT routes (plain absolute-form -// HTTP through the proxy) share the same upstream. The D6c correlation -// metadata is NOT set here: route filter_metadata values are stored literally -// (no %REQ()% formatter substitution) — the Lua HTTP filter upstream in the -// chain sets them instead. -const vhostTemplate = ` - name: {{HOST_NAME}} - domains: ["{{HOST}}"] - routes: - - match: - connect_matcher: {} - route: - cluster: dynamic_forward_proxy_cluster - upgrade_configs: - - upgrade_type: CONNECT - connect_config: - terminate_connect: true - - match: - prefix: / - route: - cluster: dynamic_forward_proxy_cluster -` +// bootstrapDoc is the typed mirror of the rendered bootstrap: only the +// config-derived values are typed fields; everything else round-trips through +// raw yaml.Nodes (gopkg.in/yaml.v3 cannot decode into *yaml.Node sequence +// elements, so every raw list is a single wrapping node). +type bootstrapDoc struct { + StaticResources struct { + Listeners []struct { + Name string `yaml:"name"` + Address struct { + SocketAddress struct { + Address string `yaml:"address"` + PortValue int `yaml:"port_value"` + } `yaml:"socket_address"` + } `yaml:"address"` + ListenerFilters yaml.Node `yaml:"listener_filters"` + FilterChains []struct { + Filters yaml.Node `yaml:"filters"` + TransportSocket yaml.Node `yaml:"transport_socket"` + } `yaml:"filter_chains"` + } `yaml:"listeners"` + Clusters []clusterDoc `yaml:"clusters"` + } `yaml:"static_resources"` +} + +// clusterDoc mirrors one static cluster; the broker cluster's endpoint address +// is config-derived (navigated into LoadAssignment), the rest is static. +type clusterDoc struct { + Name string `yaml:"name"` + Type string `yaml:"type,omitempty"` + TypedExtensionProtocolOptions yaml.Node `yaml:"typed_extension_protocol_options,omitempty"` + LoadAssignment yaml.Node `yaml:"load_assignment,omitempty"` + LBPolicy string `yaml:"lb_policy,omitempty"` + ClusterType yaml.Node `yaml:"cluster_type,omitempty"` + TransportSocket yaml.Node `yaml:"transport_socket,omitempty"` +} // RenderEnvoyBootstrap renders the Envoy bootstrap YAML for params. The // result contains no secrets — the bump-CA material is delivered to Envoy // via SDS at runtime, not through the bootstrap file. AllowedHosts is sorted // so the rendered document is deterministic. +// +// Hybrid render (mirrors the operator's renderEgressPolicyYAML approach): the +// static listener/filter skeleton stays a YAML template; EVERY config-derived +// value (ports, addresses, scan fail-open, the allowlisted-host route table) +// is built as typed data and re-emitted through yaml.Marshal — no +// string substitution ever touches a value an operator could influence. func RenderEnvoyBootstrap(params EnvoyConfigParams) ([]byte, error) { if err := params.Validate(); err != nil { return nil, err } - hosts := make([]string, len(params.AllowedHosts)) - copy(hosts, params.AllowedHosts) - sort.Strings(hosts) + var doc bootstrapDoc + if err := yaml.Unmarshal([]byte(envoyBootstrapTemplate), &doc); err != nil { + return nil, fmt.Errorf("egressbroker: failed to parse envoy bootstrap template: %w", err) + } + + doc.StaticResources.Listeners[0].Address.SocketAddress.PortValue = params.ProxyPort + + filters := doc.StaticResources.Listeners[0].FilterChains[0].Filters + hcm, err := nodeByName(filters.Content, "envoy.filters.network.http_connection_manager") + if err != nil { + return nil, err + } + if err := setNodeMapping(hcm, "typed_config.route_config.virtual_hosts", + buildVirtualHosts(params.AllowedHosts)); err != nil { + return nil, err + } + httpFilters, err := nodeByPath(hcm, "typed_config", "http_filters") + if err != nil { + return nil, err + } + if httpFilters.Kind != yaml.SequenceNode { + return nil, fmt.Errorf("egressbroker: bootstrap template: http_filters is not a sequence") + } + extProc, err := nodeByName(httpFilters.Content, "envoy.filters.http.ext_proc") + if err != nil { + return nil, err + } + if err := setNodeMapping(extProc, "typed_config.failure_mode_allow", + boolNode(params.ScanFailOpen)); err != nil { + return nil, err + } + + broker, err := clusterByName(doc.StaticResources.Clusters, "egress_broker") + if err != nil { + return nil, err + } + socketAddr, err := nodeByPath(&broker.LoadAssignment, "endpoints") + if err != nil { + return nil, err + } + socketAddr, err = nodeByIndex(socketAddr, 0, "lb_endpoints", 0, "endpoint", "address", "socket_address") + if err != nil { + return nil, err + } + if err := setNodeMapping(socketAddr, "address", scalarNode(params.ExtAuthzAddress)); err != nil { + return nil, err + } + if err := setNodeMapping(socketAddr, "port_value", intNode(params.ExtAuthzPort)); err != nil { + return nil, err + } - var vhosts strings.Builder + out, err := yaml.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to marshal envoy bootstrap: %w", err) + } + return out, nil +} + +// buildVirtualHosts builds one allowlisted-host route entry per host as data. +// The CONNECT upgrade terminates the tunnel downstream via the filter chain's +// on-demand SDS cert (resource "host:") and re-originates upstream +// through the dynamic forward proxy; every request — the CONNECT itself and +// the decrypted tunneled requests — passes ext_authz (no per-route disable: +// the credential injection on tunneled traffic IS the feature). Non-CONNECT +// routes (plain absolute-form HTTP through the proxy) share the same +// upstream. The D6c correlation metadata is NOT set here: route +// filter_metadata values are stored literally (no %REQ()% formatter +// substitution) — the Lua HTTP filter upstream in the chain sets them +// instead. Host patterns are sorted so the render is deterministic. +func buildVirtualHosts(allowedHosts []string) yaml.Node { + hosts := slices.Clone(allowedHosts) + slices.Sort(hosts) + vhosts := make([]any, 0, len(hosts)) for i, host := range hosts { - entry := strings.ReplaceAll(vhostTemplate, "{{HOST_NAME}}", sanitizeRouteName(host, i)) - entry = strings.ReplaceAll(entry, "{{HOST}}", host) - vhosts.WriteString(entry) - } - out := strings.ReplaceAll(envoyBootstrapTemplate, "{{PROXY_PORT}}", fmt.Sprintf("%d", params.ProxyPort)) - out = strings.ReplaceAll(out, "{{EXT_AUTHZ_ADDRESS}}", params.ExtAuthzAddress) - out = strings.ReplaceAll(out, "{{EXT_AUTHZ_PORT}}", fmt.Sprintf("%d", params.ExtAuthzPort)) - out = strings.ReplaceAll(out, "{{SCAN_FAIL_OPEN}}", fmt.Sprintf("%t", params.ScanFailOpen)) - out = strings.ReplaceAll(out, "{{VHOSTS}}", vhosts.String()) - return []byte(out), nil + vhosts = append(vhosts, map[string]any{ + "name": sanitizeRouteName(host, i), + "domains": []string{host}, + "routes": []any{ + map[string]any{ + "match": map[string]any{"connect_matcher": map[string]any{}}, + "route": map[string]any{ + "cluster": "dynamic_forward_proxy_cluster", + "upgrade_configs": []any{ + map[string]any{ + "upgrade_type": "CONNECT", + "connect_config": map[string]any{"terminate_connect": true}, + }, + }, + }, + }, + map[string]any{ + "match": map[string]any{"prefix": "/"}, + "route": map[string]any{"cluster": "dynamic_forward_proxy_cluster"}, + }, + }, + }) + } + var node yaml.Node + // The value tree is fully static-typed above; a marshal failure is + // unreachable. + _ = node.Encode(vhosts) + return node +} + +// nodeByName finds the entry of a YAML sequence of {"name": ..., ...} maps +// whose name matches (the elements are the same *yaml.Node pointers the +// caller's struct holds, so mutations through the result are visible on +// re-marshal). +func nodeByName(seq []*yaml.Node, name string) (*yaml.Node, error) { + for _, entry := range seq { + if entry == nil || entry.Kind != yaml.MappingNode { + continue + } + for i := 0; i+1 < len(entry.Content); i += 2 { + if entry.Content[i].Value == "name" && entry.Content[i+1].Value == name { + return entry, nil + } + } + } + return nil, fmt.Errorf("egressbroker: bootstrap template: entry %q not found", name) +} + +// nodeByPath walks mapping keys from node. +func nodeByPath(node *yaml.Node, path ...string) (*yaml.Node, error) { + cur := node + for _, key := range path { + next, err := mappingValue(cur, key) + if err != nil { + return nil, err + } + cur = next + } + return cur, nil +} + +// nodeByIndex walks sequence indexes (interleaved with mapping keys). +func nodeByIndex(node *yaml.Node, index int, path ...any) (*yaml.Node, error) { + cur := node + if cur.Kind != yaml.SequenceNode || index >= len(cur.Content) { + return nil, fmt.Errorf("egressbroker: bootstrap template: missing sequence index %d", index) + } + cur = cur.Content[index] + for _, step := range path { + switch s := step.(type) { + case string: + next, err := mappingValue(cur, s) + if err != nil { + return nil, err + } + cur = next + case int: + if cur.Kind != yaml.SequenceNode || s >= len(cur.Content) { + return nil, fmt.Errorf("egressbroker: bootstrap template: missing sequence index %d", s) + } + cur = cur.Content[s] + } + } + return cur, nil +} + +// mappingValue returns the value node for key in a mapping node. +func mappingValue(node *yaml.Node, key string) (*yaml.Node, error) { + if node.Kind != yaml.MappingNode { + return nil, fmt.Errorf("egressbroker: bootstrap template: expected mapping at %q, got kind %d", key, node.Kind) + } + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return node.Content[i+1], nil + } + } + return nil, fmt.Errorf("egressbroker: bootstrap template: key %q not found", key) +} + +// setNodeMapping replaces the value at the end of a mapping-key path. +func setNodeMapping(node *yaml.Node, path string, value yaml.Node) error { + parent := node + keys := strings.Split(path, ".") + for _, key := range keys[:len(keys)-1] { + next, err := mappingValue(parent, key) + if err != nil { + return err + } + parent = next + } + if parent.Kind != yaml.MappingNode { + return fmt.Errorf("egressbroker: bootstrap template: expected mapping at %q", path) + } + for i := 0; i+1 < len(parent.Content); i += 2 { + if parent.Content[i].Value == keys[len(keys)-1] { + v := value + parent.Content[i+1] = &v + return nil + } + } + return fmt.Errorf("egressbroker: bootstrap template: key %q not found", path) +} + +// clusterByName finds a static cluster by name. +func clusterByName(clusters []clusterDoc, name string) (*clusterDoc, error) { + for i := range clusters { + if clusters[i].Name == name { + return &clusters[i], nil + } + } + return nil, fmt.Errorf("egressbroker: bootstrap template: cluster %q not found", name) +} + +func boolNode(b bool) yaml.Node { + var node yaml.Node + _ = node.Encode(b) // bool encode cannot fail + return node +} + +func intNode(i int) yaml.Node { + var node yaml.Node + _ = node.Encode(i) // int encode cannot fail + return node +} + +func scalarNode(s string) yaml.Node { + var node yaml.Node + _ = node.Encode(s) // string encode cannot fail + return node } // WriteEnvoyBootstrap renders and writes the bootstrap to path with diff --git a/pkg/egressbroker/policy.go b/pkg/egressbroker/policy.go index 4364fdaee0..2cc3ea6691 100644 --- a/pkg/egressbroker/policy.go +++ b/pkg/egressbroker/policy.go @@ -13,7 +13,7 @@ package egressbroker import ( "fmt" - "sort" + "slices" "strings" "gopkg.in/yaml.v3" @@ -97,7 +97,7 @@ func (e *EgressPolicy) ProviderNames() []string { for _, p := range e.providers { names = append(names, p.Provider) } - sort.Strings(names) + slices.Sort(names) return names } @@ -162,7 +162,7 @@ func (e *EgressPolicy) HostAllowlist() []string { } } } - sort.Strings(out) + slices.Sort(out) return out } @@ -251,12 +251,7 @@ func methodAllowed(p ProviderPolicy, method string) bool { } return false } - for _, m := range p.AllowedMethods { - if m == method { - return true - } - } - return false + return slices.Contains(p.AllowedMethods, method) } // pathAllowed: empty AllowedPathPrefixes means all paths ("/" prefix). @@ -267,12 +262,9 @@ func pathAllowed(p ProviderPolicy, path string) bool { if len(p.AllowedPathPrefixes) == 0 { return true } - for _, prefix := range p.AllowedPathPrefixes { - if strings.HasPrefix(path, prefix) { - return true - } - } - return false + return slices.ContainsFunc(p.AllowedPathPrefixes, func(prefix string) bool { + return strings.HasPrefix(path, prefix) + }) } func knownMethod(m string) bool { diff --git a/pkg/egressbroker/resolve.go b/pkg/egressbroker/resolve.go index c3efc223b7..c4d2afb4a6 100644 --- a/pkg/egressbroker/resolve.go +++ b/pkg/egressbroker/resolve.go @@ -10,6 +10,8 @@ import ( "net/netip" "slices" "strings" + + "github.com/stacklok/toolhive/pkg/networking" ) // ErrDNSResolution marks DNS lookup failures during dial-allowlist resolution. @@ -60,7 +62,7 @@ func ResolveDialAllowlist(policy *EgressPolicy, lookupHost func(string) ([]net.I continue } addr = addr.Unmap() - if isNonPublic(addr) { + if networking.IsPrivateIP(addr.AsSlice()) { continue } prefix := netip.PrefixFrom(addr, addr.BitLen()).String() @@ -77,36 +79,9 @@ func ResolveDialAllowlist(policy *EgressPolicy, lookupHost func(string) ([]net.I return out, nil } -// nonPublicPrefixes are never allowlisted from DNS answers (OWASP SSRF set: -// loopback, RFC 1918, link-local, CGNAT, IPv6 ULA/link-local). A name that -// resolves into these is a rebinding suspect and must not widen the dial -// allowlist. -var nonPublicPrefixes = []netip.Prefix{ - netip.MustParsePrefix("0.0.0.0/8"), - netip.MustParsePrefix("10.0.0.0/8"), - netip.MustParsePrefix("100.64.0.0/10"), - netip.MustParsePrefix("127.0.0.0/8"), - netip.MustParsePrefix("169.254.0.0/16"), - netip.MustParsePrefix("172.16.0.0/12"), - netip.MustParsePrefix("192.0.0.0/24"), - netip.MustParsePrefix("192.0.2.0/24"), - netip.MustParsePrefix("192.168.0.0/16"), - netip.MustParsePrefix("198.18.0.0/15"), - netip.MustParsePrefix("198.51.100.0/24"), - netip.MustParsePrefix("203.0.113.0/24"), - netip.MustParsePrefix("224.0.0.0/4"), - netip.MustParsePrefix("240.0.0.0/4"), - netip.MustParsePrefix("::1/128"), - netip.MustParsePrefix("fc00::/7"), - netip.MustParsePrefix("fe80::/10"), - netip.MustParsePrefix("ff00::/8"), -} - -func isNonPublic(addr netip.Addr) bool { - for _, prefix := range nonPublicPrefixes { - if prefix.Contains(addr) { - return true - } - } - return false -} +// The non-public address classifier is networking.IsPrivateIP (single shared +// implementation — do not reintroduce a local table): it rejects the OWASP +// SSRF set (loopback, RFC 1918, link-local, CGNAT, IPv6 ULA/link-local, +// multicast/reserved, documentation ranges, 192.0.0.0/24, 198.18.0.0/15) AND +// decodes NAT64 64:ff9b::/96 + 64:ff9b:1::/48 embeddings, so a rebinding +// suspect hidden behind a NAT64 address never widens the dial allowlist. diff --git a/pkg/egressbroker/resolve_test.go b/pkg/egressbroker/resolve_test.go index 9d7c8d45c3..a4499357eb 100644 --- a/pkg/egressbroker/resolve_test.go +++ b/pkg/egressbroker/resolve_test.go @@ -105,6 +105,37 @@ providers: assert.Contains(t, err.Error(), "empty dial allowlist") }) + t.Run("NAT64-embedded private answers never widen the allowlist", func(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com"] +`) + _, err := egressbroker.ResolveDialAllowlist(p, lookup(map[string][]net.IP{ + // 64:ff9b::/96 (RFC 6052) and 64:ff9b:1::/96 (RFC 8215 local-use) + // embeddings of 169.254.169.254 (cloud metadata endpoint). + "api.github.com": {net.ParseIP("64:ff9b::a9fe:a9fe"), net.ParseIP("64:ff9b:1::a9fe:a9fe")}, + })) + require.Error(t, err, "a host resolving only to NAT64-embedded private IPs must fail, not allowlist them") + assert.Contains(t, err.Error(), "empty dial allowlist") + }) + + t.Run("NAT64-embedded global answers stay allowlisted", func(t *testing.T) { + t.Parallel() + p := mustParse(t, ` +providers: +- provider: github + allowedHosts: ["api.github.com"] +`) + out, err := egressbroker.ResolveDialAllowlist(p, lookup(map[string][]net.IP{ + // 64:ff9b::/96 embedding of the public 140.82.114.26. + "api.github.com": {net.ParseIP("64:ff9b::8c52:721a")}, + })) + require.NoError(t, err) + assert.Equal(t, []string{"64:ff9b::8c52:721a/128"}, out) + }) + t.Run("mixed global + private answers keep only the global ones", func(t *testing.T) { t.Parallel() p := mustParse(t, ` diff --git a/pkg/egressbroker/tokenmap.go b/pkg/egressbroker/tokenmap.go index 2c2ab30302..418ed4793f 100644 --- a/pkg/egressbroker/tokenmap.go +++ b/pkg/egressbroker/tokenmap.go @@ -4,11 +4,12 @@ package egressbroker import ( - "container/list" "context" "fmt" "sync" "time" + + lru "github.com/hashicorp/golang-lru/v2" ) // TokenMap is the request-id → injected-token correlation the response @@ -28,15 +29,16 @@ import ( // evicted (a response whose entry was evicted scans as unknown-request and // passes under the fail-open default — recorded as a scan miss). // -// Concurrency: a single mutex guards map + eviction list (one synchronization +// Concurrency: the backing store is hashicorp/golang-lru/v2 (the same LRU +// pkg/cache wraps for the session manager). A single mutex serializes the +// map's operations so the consume-on-read Lookup (get + remove) is atomic +// with respect to Record and the eviction sweep (one synchronization // primitive per data structure). type TokenMap struct { - mu sync.Mutex - entries map[string]*list.Element // requestID → list element - lru *list.List // front = most recently recorded - ttl time.Duration - maxEntries int - now func() time.Time + mu sync.Mutex + lru *lru.Cache[string, *tokenMapEntry] + ttl time.Duration + now func() time.Time } // ScanRecord is one entry's payload. @@ -47,7 +49,6 @@ type ScanRecord struct { } type tokenMapEntry struct { - requestID string record ScanRecord expiresAt time.Time } @@ -72,25 +73,23 @@ func NewTokenMapWithClock(ttl time.Duration, maxEntries int, now func() time.Tim if now == nil { return nil, fmt.Errorf("egressbroker: clock must not be nil") } - return &TokenMap{ - entries: map[string]*list.Element{}, - lru: list.New(), - ttl: ttl, - maxEntries: maxEntries, - now: now, - }, nil + cache, err := lru.New[string, *tokenMapEntry](maxEntries) + if err != nil { + return nil, fmt.Errorf("egressbroker: failed to build token map: %w", err) + } + return &TokenMap{lru: cache, ttl: ttl, now: now}, nil } // Record stores the scan record for an injected credential header value under // requestID. provider travels with the entry for scan-time metrics/audit // labeling (low-cardinality policy names only). A re-recorded requestID -// refreshes in place. +// refreshes in place; past the cap the least-recently-used entry is evicted +// by the backing LRU. func (m *TokenMap) Record(requestID, headerValue, provider string) { if requestID == "" || headerValue == "" { return } entry := &tokenMapEntry{ - requestID: requestID, record: ScanRecord{ TokenHash: hashOf(headerValue), Needles: buildNeedles(headerValue), @@ -100,35 +99,27 @@ func (m *TokenMap) Record(requestID, headerValue, provider string) { } m.mu.Lock() defer m.mu.Unlock() - if el, ok := m.entries[requestID]; ok { - el.Value = entry - m.lru.MoveToFront(el) - return - } - m.entries[requestID] = m.lru.PushFront(entry) - for m.lru.Len() > m.maxEntries { - oldest := m.lru.Back() - m.lru.Remove(oldest) - delete(m.entries, oldest.Value.(*tokenMapEntry).requestID) - } + m.lru.Add(requestID, entry) } // Lookup returns the scan record for requestID, consuming the entry (one // injection corresponds to one scanned response; a second lookup reports -// unknown). Expired entries count as unknown. +// unknown). Expired entries count as unknown. The peek+remove pair runs under +// the map's mutex so no concurrent Record/Lookup can interleave between them. +// Peek (not Get) keeps the recency order strictly record-ordered: a lookup is +// consumption, not reuse, so it must not shield an entry from being the next +// eviction victim — the oldest un-consumed injection is always evicted first. func (m *TokenMap) Lookup(requestID string) (ScanRecord, bool) { if requestID == "" { return ScanRecord{}, false } m.mu.Lock() defer m.mu.Unlock() - el, found := m.entries[requestID] + entry, found := m.lru.Peek(requestID) if !found { return ScanRecord{}, false } - delete(m.entries, requestID) - m.lru.Remove(el) - entry := el.Value.(*tokenMapEntry) + m.lru.Remove(requestID) if m.now().After(entry.expiresAt) { return ScanRecord{}, false } @@ -142,14 +133,10 @@ func (m *TokenMap) evictExpired() { now := m.now() m.mu.Lock() defer m.mu.Unlock() - for el := m.lru.Back(); el != nil; { - prev := el.Prev() - entry := el.Value.(*tokenMapEntry) - if now.After(entry.expiresAt) { - m.lru.Remove(el) - delete(m.entries, entry.requestID) + for _, key := range m.lru.Keys() { + if entry, ok := m.lru.Peek(key); ok && now.After(entry.expiresAt) { + m.lru.Remove(key) } - el = prev } } diff --git a/pkg/networking/utilities.go b/pkg/networking/utilities.go index bf2c7aeebc..f13703838b 100644 --- a/pkg/networking/utilities.go +++ b/pkg/networking/utilities.go @@ -99,9 +99,11 @@ func init() { "fe80::/10", // IPv6 link-local "fc00::/7", // IPv6 unique local addr "100.64.0.0/10", // RFC6598 shared address space (CGN) + "192.0.0.0/24", // RFC6890 IETF protocol assignments "192.0.2.0/24", // RFC5737 documentation (TEST-NET-1) "198.51.100.0/24", // RFC5737 documentation (TEST-NET-2) "203.0.113.0/24", // RFC5737 documentation (TEST-NET-3) + "198.18.0.0/15", // RFC2544 benchmarking (device interconnect tests) "224.0.0.0/4", // IPv4 multicast "240.0.0.0/4", // RFC1112 reserved (Class E), incl. 255.255.255.255 broadcast "ff00::/8", // IPv6 multicast diff --git a/pkg/secrets/aes/aes.go b/pkg/secrets/aes/aes.go index f686e68f8d..9b28cd4772 100644 --- a/pkg/secrets/aes/aes.go +++ b/pkg/secrets/aes/aes.go @@ -21,6 +21,14 @@ var ErrExceedsMaxSize = errors.New("plaintext is too large, limited to 32MiB") // the data and provides a check that it hasn't been altered. Output takes the // form nonce|ciphertext|tag where '|' indicates concatenation. func Encrypt(plaintext []byte, key []byte) ([]byte, error) { + return EncryptWithAAD(plaintext, key, nil) +} + +// EncryptWithAAD is Encrypt with additional authenticated data (may be nil). +// The AAD is authenticated but not encrypted: decryption with different AAD +// fails, which cryptographically binds the ciphertext to its context (e.g. +// the storage key a value is written under). +func EncryptWithAAD(plaintext, key, aad []byte) ([]byte, error) { if len(plaintext) > maxPlaintextSize { return nil, ErrExceedsMaxSize } @@ -41,13 +49,19 @@ func Encrypt(plaintext []byte, key []byte) ([]byte, error) { return nil, err } - return gcm.Seal(nonce, nonce, plaintext, nil), nil + return gcm.Seal(nonce, nonce, plaintext, aad), nil } // Decrypt decrypts data using 256-bit AES-GCM. This both hides the content of // the data and provides a check that it hasn't been altered. Expects input // form nonce|ciphertext|tag where '|' indicates concatenation. func Decrypt(ciphertext []byte, key []byte) ([]byte, error) { + return DecryptWithAAD(ciphertext, key, nil) +} + +// DecryptWithAAD is Decrypt with additional authenticated data; it must match +// the AAD passed to EncryptWithAAD or decryption fails. +func DecryptWithAAD(ciphertext, key, aad []byte) ([]byte, error) { block, err := aes.NewCipher(key[:]) if err != nil { return nil, err @@ -65,6 +79,6 @@ func Decrypt(ciphertext []byte, key []byte) ([]byte, error) { return gcm.Open(nil, ciphertext[:gcm.NonceSize()], ciphertext[gcm.NonceSize():], - nil, + aad, ) } diff --git a/pkg/vmcp/session/untrusted/template_clone_test.go b/pkg/vmcp/session/untrusted/template_clone_test.go index 7cbdadd72a..2aa6dd7981 100644 --- a/pkg/vmcp/session/untrusted/template_clone_test.go +++ b/pkg/vmcp/session/untrusted/template_clone_test.go @@ -148,6 +148,16 @@ func TestClonePodFromTemplate(t *testing.T) { }) } +// TestClonePodFromTemplate_FailClosed pins the vmcp runtime-side untrusted +// pod gate (the template clone refuses secret-injecting pod specs). +// CHANNEL-COVERAGE MIRROR: the operator admission-side gate's test — +// cmd/thv-operator/pkg/controllerutil/untrusted_env_validation_test.go +// (TestValidateNoSecretEnvForUntrusted) — covers the same secret-injection +// channels (secretKeyRef / configMapKeyRef env, envFrom secret/configmap +// refs, secret / configmap / projected volumes). When a new injection +// channel is added or allowed on one side, update the other side's suite too +// — drift between the two is a silent gap in the untrusted isolation +// boundary. func TestClonePodFromTemplate_FailClosed(t *testing.T) { t.Parallel() From 7e26c8eda010eeb2cbd7648813a2c5bc9fc3daac Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 23 Jul 2026 00:15:59 +0300 Subject: [PATCH 08/12] Fix untrusted-mode data plane after live-cluster validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End-to-end testing against a real cluster surfaced a chain of defects in the untrusted pod data plane that unit tests could not reach. All were reproduced, fixed, and re-verified live: the pod now comes up 3/3 and a malicious-server attack playbook is fully blocked (sentinel env, direct-egress 403, proxy-to-evil denied, echo attack suppressed, DNS exfil refused). Health/probe contract: the broker health listener bound 127.0.0.1 but kubelet probes reach the pod IP, so every probe failed and liveness killed the broker at the threshold before it could initialize. Bind all interfaces, and split /livez (dependency-free, served from process start) from /healthz (readiness gated on CA/policy/Redis) so a slow Redis/CA init can no longer kill a healthy broker mid-startup. Envoy bootstrap: three render bugs — the typed bootstrapDoc dropped the SDS-required node id/cluster on re-marshal, terminate_connect is a v1.37 field the pinned Envoy v1.36 rejects, and with_request_body max_request_bytes 0 fails proto validation. Also removed the Envoy/ broker startup race by making Envoy wait for the rendered bootstrap. Token store reachability: NetworkPolicy egress is evaluated against the post-DNAT pod IP, not the Service ClusterIP, so the generated policy never matched the in-cluster token store and the broker could not reach Redis. Resolve the Redis Service's EndpointSlice pod IPs into the egress allowlist (external Redis falls back to DNS), add the discovery.k8s.io/endpointslices RBAC, and route the broker's token-store dial around the D7 destination guard (it is broker infrastructure, not credential egress). Also: ca-seed init no longer needs a shell (the image is distroless); the broker gains a seed-ca mode. Operator ClusterRole gains pods create/delete so the untrusted lifecycle can be granted to vMCP. Deterministic fix for the pre-existing TestTokenMap_ConcurrencyFlood flake (racy oldest-eviction assertion under a concurrent flood). --- cmd/thv-egressbroker/health_test.go | 4 +- cmd/thv-egressbroker/main.go | 116 ++++++++--- cmd/thv-egressbroker/main_test.go | 24 +-- .../controllers/mcpserver_controller.go | 1 + .../mcpserver_untrusted_resources.go | 191 +++++++++++++++--- .../virtualmcpserver_controller.go | 1 + cmd/thv-operator/internal/testutil/scheme.go | 2 + .../operator/templates/clusterrole/role.yaml | 10 + pkg/egressbroker/envoyconfig.go | 15 +- pkg/egressbroker/envoyconfig_test.go | 6 +- pkg/egressbroker/extproc_test.go | 34 +++- pkg/vmcp/session/untrusted/egress.go | 21 +- pkg/vmcp/session/untrusted/egress_test.go | 6 +- 13 files changed, 338 insertions(+), 93 deletions(-) diff --git a/cmd/thv-egressbroker/health_test.go b/cmd/thv-egressbroker/health_test.go index a6640a472e..405dfa0f8f 100644 --- a/cmd/thv-egressbroker/health_test.go +++ b/cmd/thv-egressbroker/health_test.go @@ -87,7 +87,7 @@ func TestHealthz(t *testing.T) { t.Parallel() h, _ := healthFixture(t, nil) var loadedFlag atomic.Bool - srv := newHealthServer(h.ca, &loadedFlag, h.ping) - assert.Equal(t, "127.0.0.1:15083", srv.Addr) + srv := newHealthServerHandler(&healthServer{ca: h.ca, policyLoaded: &loadedFlag, ping: h.ping}) + assert.Equal(t, ":15083", srv.Addr) }) } diff --git a/cmd/thv-egressbroker/main.go b/cmd/thv-egressbroker/main.go index 717f2f0371..09037efe9e 100644 --- a/cmd/thv-egressbroker/main.go +++ b/cmd/thv-egressbroker/main.go @@ -14,7 +14,6 @@ import ( "errors" "fmt" "log/slog" - "net" "net/http" "os" "os/signal" @@ -67,6 +66,17 @@ const ( ) func main() { + // Seed mode: the untrusted pod's ca-seed init container runs the broker + // binary with `seed-ca` because the production image is distroless (no + // shell) — a plain `cp` in an init container is impossible without it. + // Copies the operator's CA bundle into the shared emptyDir and exits. + if len(os.Args) > 1 && os.Args[1] == "seed-ca" { + if err := seedCA(os.Args[2:]); err != nil { + slog.Error("seed-ca failed", "error", err) + os.Exit(1) + } + return + } if err := run(); err != nil { // Startup failures are loud and carry no credential material. slog.Error("egressbroker failed to start", "error", err) @@ -74,6 +84,22 @@ func main() { } } +// seedCA copies to read-only (0444). Usage: seed-ca . +func seedCA(args []string) error { + if len(args) != 2 { + return fmt.Errorf("seed-ca requires , got %d args", len(args)) + } + data, err := os.ReadFile(args[0]) + if err != nil { + return fmt.Errorf("read bundle: %w", err) + } + // #nosec G306 -- the CA public cert is not secret; 0444 is the intended mode + if err := os.WriteFile(args[1], data, 0o444); err != nil { + return fmt.Errorf("write CA: %w", err) + } + return nil +} + func run() error { cfg, err := egressbroker.LoadConfig(os.Getenv) if err != nil { @@ -95,8 +121,10 @@ func run() error { if err != nil { return err } - dialGuard, err := egressbroker.ParseIPAllowlist(dialCIDRs) - if err != nil { + // The dial allowlist is enforced at the Envoy data plane (D7), not by the + // broker's own dials. Parse it here only to fail fast on a corrupt policy + // before the pod goes further. + if _, err := egressbroker.ParseIPAllowlist(dialCIDRs); err != nil { return err } if err := renderEnvoyBootstrap(cfg, policy); err != nil { @@ -106,7 +134,25 @@ func run() error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - tokens, err := buildTokenReader(ctx, dialGuard) + // Start the health listener BEFORE the blocking dependency init so /livez + // answers immediately (liveness = process alive). /healthz (readiness) + // flips to 200 only once policyLoaded is set and Redis pings. A wedged + // listener wedges the pod — the untrusted reaper deletes it once it is + // older than the readiness timeout — so log a listener failure at Error. + var policyLoaded atomic.Bool + health := &healthServer{ca: ca, policyLoaded: &policyLoaded, ping: func(context.Context) error { + // Before dependencies load, readiness reports not-ready without + // touching Redis (liveness is the separate, always-200 /livez). + return errDependenciesNotReady + }} + healthSrv := newHealthServerHandler(health) + go func() { + if err := runHealthServer(ctx, healthSrv); err != nil { + slog.Error("egressbroker: health listener stopped; probes will now fail", "error", err) + } + }() + + tokens, err := buildTokenReader(ctx) if err != nil { return err } @@ -116,23 +162,9 @@ func run() error { return err } - // Health listener (readiness + liveness probes): answers 200 only when - // the broker can actually serve — Redis reachable AND policy loaded AND - // the bump CA not past rotation-due. Everything past this point is - // already loaded, so policyLoaded flips exactly once at startup. - var policyLoaded atomic.Bool + // Dependencies are up: readiness may now answer 200 once Redis pings. + health.ping = tokens.ping policyLoaded.Store(true) - healthSrv := newHealthServer(ca, &policyLoaded, tokens.ping) - go func() { - if err := runHealthServer(ctx, healthSrv); err != nil { - // A wedged health listener wedges the pod: probes start failing, - // the pod never goes Ready, and the untrusted reaper deletes it - // once it is older than the readiness timeout. Log at Error so the - // failure is visible before that teardown. - slog.Error("egressbroker: health listener stopped; readiness/liveness probes will now fail", - "error", err) - } - }() return server.Run(ctx) } @@ -196,6 +228,8 @@ type redisPinger func(ctx context.Context) error // rotation-due AND the policy is loaded AND Redis answers PING within // healthPingTimeout; otherwise 503 with a coarse reason (never credential // material). +var errDependenciesNotReady = fmt.Errorf("dependencies not ready") + type healthServer struct { ca *egressbroker.BumpCA policyLoaded *atomic.Bool @@ -205,12 +239,24 @@ type healthServer struct { // newHealthServer builds the loopback-only HTTP server bound to the health // port. The handler is exported through the returned server's Handler for // tests. -func newHealthServer(ca *egressbroker.BumpCA, policyLoaded *atomic.Bool, ping redisPinger) *http.Server { - h := &healthServer{ca: ca, policyLoaded: policyLoaded, ping: ping} +func newHealthServerHandler(h *healthServer) *http.Server { mux := http.NewServeMux() mux.HandleFunc("/healthz", h.handle) + // /livez is dependency-free: 200 whenever the process is alive. Served + // from process start so the liveness probe never kills a broker that is + // still initialising (slow Redis/CA). Readiness stays on /healthz. + mux.HandleFunc("/livez", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) + }) return &http.Server{ - Addr: net.JoinHostPort("127.0.0.1", fmt.Sprintf("%d", untrusted.BrokerHealthPort)), + // Bind all interfaces, not 127.0.0.1: kubelet httpGet probes are + // delivered to the pod IP (from the node network), not loopback, so a + // loopback-only bind makes every probe fail and the container is killed + // at the liveness threshold before it can initialize. The port is not + // otherwise reachable: the pod's NetworkPolicy confines ingress, and + // /healthz discloses only coarse subsystem names (no addresses/dates). + Addr: fmt.Sprintf(":%d", untrusted.BrokerHealthPort), Handler: mux, ReadHeaderTimeout: 5 * time.Second, } @@ -288,17 +334,23 @@ type tokenStore struct { } // buildTokenReader constructs the upstream-token reader against the vMCP's -// session-storage Redis. The Redis dials go through the D7 guard: the Redis -// address is operator-injected config, and the guard keeps the broker from -// ever dialing outside the policy's destination set. No refresh is wired (the -// broker holds no OAuth client material): expired rows surface on the failed -// list and deny with "re-consent required" (Wave 4 consent UX). +// session-storage Redis. The Redis address is operator-injected config. The +// dial does NOT pass through the D7 destination allowlist: that allowlist is +// derived from the credential-egress allowedHosts and deliberately excludes +// private IPs, which would brick any in-cluster token store (the common +// case). Instead the broker dials exactly the configured Redis address and +// nothing else — the connection target is fixed by operator config, not by +// anything the untrusted workload can influence, so the SSRF concern the D7 +// guard addresses for upstream egress does not apply here. No refresh is +// wired (the broker holds no OAuth client material): expired rows surface on +// the failed list and deny with "re-consent required" (Wave 4 consent UX). // // The Redis ACL password arrives via vmcpconfig.RedisPasswordEnvVar, // rendered at clone time as a SecretKeyRef env (never a literal). It is // REQUIRED: without it every Redis call fails AUTH and every credential +// lookup denies. // injection denies — fail loud at startup instead of crash-looping on 403s. -func buildTokenReader(_ context.Context, dialGuard *egressbroker.IPAllowlist) (*tokenStore, error) { +func buildTokenReader(_ context.Context) (*tokenStore, error) { addr := strings.TrimSpace(os.Getenv(envRedisAddr)) keyPrefix := strings.TrimSpace(os.Getenv(envRedisKeyPrefix)) if addr == "" || keyPrefix == "" { @@ -317,10 +369,14 @@ func buildTokenReader(_ context.Context, dialGuard *egressbroker.IPAllowlist) (* return nil, err } + // The token-store connection is broker infrastructure, not credential + // egress: it must NOT pass through the D7 destination allowlist (which is + // derived from allowedHosts and excludes private IPs like the Redis + // ClusterIP). Guarding it would brick the broker whenever the token store + // is in-cluster. The D7 guard applies to upstream egress only. client := goredis.NewClient(&goredis.Options{ Addr: addr, Password: password, - Dialer: dialGuard.DialContext, }) stor := storage.NewRedisStorageWithClient(client, keyPrefix, opts...) return &tokenStore{ diff --git a/cmd/thv-egressbroker/main_test.go b/cmd/thv-egressbroker/main_test.go index 6d832d5dd6..cb5074c1e8 100644 --- a/cmd/thv-egressbroker/main_test.go +++ b/cmd/thv-egressbroker/main_test.go @@ -34,18 +34,19 @@ func TestBrokerPortContract(t *testing.T) { "the Envoy proxy port must match the backend HTTP(S)_PROXY env the clone wiring renders") } -// TestHealthListenerLoopbackBind performs a real loopback bind on the pinned -// health port: newHealthServer must bind 127.0.0.1 (never a pod-external -// interface — the handler answers unauthenticated) and serve /healthz. -// The bind is the assertion; port conflicts skip rather than fail (another -// listener on 15083 says nothing about where this server would have bound). -func TestHealthListenerLoopbackBind(t *testing.T) { +// TestHealthListenerBind performs a real bind on the pinned health port. The +// listener must bind all interfaces (the pod IP): kubelet httpGet probes are +// delivered to the pod IP from the node network, so a loopback-only bind makes +// every probe fail and the container is killed at the liveness threshold +// before it can initialize. The bind is the assertion; port conflicts skip +// rather than fail (another listener on 15083 says nothing about the bind). +func TestHealthListenerBind(t *testing.T) { t.Parallel() h, _ := healthFixture(t, nil) var loadedFlag atomic.Bool loadedFlag.Store(true) - srv := newHealthServer(h.ca, &loadedFlag, h.ping) + srv := newHealthServerHandler(&healthServer{ca: h.ca, policyLoaded: &loadedFlag, ping: h.ping}) ln, err := net.Listen("tcp", srv.Addr) if err != nil { @@ -53,12 +54,11 @@ func TestHealthListenerLoopbackBind(t *testing.T) { } defer ln.Close() - // The bound address must be loopback-only. - host, _, err := net.SplitHostPort(ln.Addr().String()) + // The bound address must be the wildcard (all interfaces) so kubelet + // probes delivered to the pod IP reach it. + _, port, err := net.SplitHostPort(ln.Addr().String()) require.NoError(t, err) - ip := net.ParseIP(host) - require.NotNil(t, ip, "bound host %q is not an IP", host) - assert.True(t, ip.IsLoopback(), "health listener must bind loopback only, got %s", host) + assert.Equal(t, "15083", port) // Serve one real request through the bound listener. go func() { _ = srv.Serve(ln) }() diff --git a/cmd/thv-operator/controllers/mcpserver_controller.go b/cmd/thv-operator/controllers/mcpserver_controller.go index 69ca38a23e..bdc92e6f39 100644 --- a/cmd/thv-operator/controllers/mcpserver_controller.go +++ b/cmd/thv-operator/controllers/mcpserver_controller.go @@ -168,6 +168,7 @@ func (r *MCPServerReconciler) detectPlatform(ctx context.Context) (kubernetes.Pl // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=pods/attach,verbs=create;get // +kubebuilder:rbac:groups="",resources=pods/log,verbs=get diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go index 8af5408f53..a1828708f8 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go @@ -9,6 +9,7 @@ import ( "fmt" "net" "sort" + "strconv" "strings" "sync" "time" @@ -16,6 +17,7 @@ import ( "gopkg.in/yaml.v3" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" "github.com/stacklok/toolhive/pkg/egressbroker" @@ -163,7 +166,15 @@ func (r *MCPServerReconciler) ensureUntrustedResources(ctx context.Context, m *m if err := r.ensureEgressPolicyConfigMap(ctx, m, policyDoc); err != nil { return fmt.Errorf("failed to ensure egress policy ConfigMap: %w", err) } - if err := r.ensureUntrustedNetworkPolicy(ctx, m, dialAllowlist); err != nil { + // The token store is broker infrastructure, not a credential destination: + // the pod's NetworkPolicy must let the sidecar reach it or every + // credential lookup fails (the pod is otherwise locked to loopback + + // kube-dns + allowedHosts). Resolve the fronting vMCP's Redis address and + // add it to the egress allowlist. Absent/unresolvable = fail closed (the + // broker already denies without a token store, and a pod that can reach + // nothing but its sidecar is the safe posture). + tokenStoreCIDRs, tokenStorePort := r.resolveUntrustedTokenStoreEgress(ctx, m) + if err := r.ensureUntrustedNetworkPolicy(ctx, m, dialAllowlist, tokenStoreCIDRs, tokenStorePort); err != nil { return fmt.Errorf("failed to ensure egress NetworkPolicy: %w", err) } if err := r.stampUntrustedCAGeneration(ctx, m, generation); err != nil { @@ -568,9 +579,9 @@ func (r *MCPServerReconciler) ensureEgressPolicyConfigMap( // load-bearing controls are the sidecar's destination binding (D5) and // per-dial IP validation (D7). func (r *MCPServerReconciler) ensureUntrustedNetworkPolicy( - ctx context.Context, m *mcpv1beta1.MCPServer, dialAllowlist []string, + ctx context.Context, m *mcpv1beta1.MCPServer, dialAllowlist, tokenStoreCIDRs []string, tokenStorePort *intstr.IntOrString, ) error { - desired := desiredUntrustedNetworkPolicy(m, dialAllowlist) + desired := desiredUntrustedNetworkPolicy(m, dialAllowlist, tokenStoreCIDRs, tokenStorePort) if err := controllerutil.SetControllerReference(m, desired, r.Scheme); err != nil { return fmt.Errorf("failed to set owner reference on egress NetworkPolicy: %w", err) } @@ -609,9 +620,111 @@ const ( dnsPodSelectorName = "kube-dns" ) +// resolveUntrustedTokenStoreEgress finds the VirtualMCPServer fronting this +// workload's group and resolves its auth-server Redis address to CIDRs for +// the NetworkPolicy egress allowlist. Returns (nil, nil) when no vMCP fronts +// the group or its Redis is not standalone/cluster-addressable — in both +// cases the broker cannot function anyway, so the pod correctly fails closed +// with no token-store rule. The port is returned separately so the NP rule +// can be scoped to the Redis port only. +func (r *MCPServerReconciler) resolveUntrustedTokenStoreEgress( + ctx context.Context, m *mcpv1beta1.MCPServer, +) (cidrs []string, port *intstr.IntOrString) { + if m.Spec.GroupRef == nil { + return nil, nil + } + vmcps := &mcpv1beta1.VirtualMCPServerList{} + if err := r.List(ctx, vmcps, client.InNamespace(m.Namespace)); err != nil { + log.FromContext(ctx).Error(err, "Failed to list VirtualMCPServers for untrusted token-store egress") + return nil, nil + } + groupName := m.Spec.GroupRef.Name + for i := range vmcps.Items { + vmcp := &vmcps.Items[i] + if vmcp.Spec.GroupRef.GetName() != groupName { + continue + } + as := vmcp.Spec.AuthServerConfig + if as == nil || as.Storage == nil || + as.Storage.Type != mcpv1beta1.AuthServerStorageTypeRedis || + as.Storage.Redis == nil || as.Storage.Redis.Addr == "" { + return nil, nil + } + host, portStr, err := net.SplitHostPort(as.Storage.Redis.Addr) + if err != nil { + log.FromContext(ctx).Error(err, "untrusted token-store addr is not host:port", + "addr", as.Storage.Redis.Addr) + return nil, nil + } + portNum, err := strconv.Atoi(portStr) + if err != nil { + return nil, nil + } + p := intstr.FromInt32(int32(portNum)) // #nosec G109 G115 -- portNum is a validated 0-65535 TCP port, fits int32 + + // NetworkPolicy egress is evaluated against the packet's destination + // AFTER kube-proxy DNAT — i.e. the backend POD IP, not the Service + // ClusterIP. So for an in-cluster Redis (a Service DNS name) we must + // allow the Service's *endpoint* IPs, or the rule silently never + // matches and the broker cannot reach the token store. For an external + // Redis (a routable host/IP) fall back to plain DNS resolution. + if svcCIDRs := r.resolveServiceEndpointCIDRs(ctx, m.Namespace, host); len(svcCIDRs) > 0 { + return svcCIDRs, &p + } + ips, err := resolveEgressHost(host) + if err != nil { + log.FromContext(ctx).Error(err, "untrusted token-store host does not resolve", + "host", host) + return nil, nil + } + for _, ip := range ips { + cidrs = append(cidrs, ip.String()+"/32") + } + return cidrs, &p + } + return nil, nil +} + +// resolveServiceEndpointCIDRs returns the endpoint pod IPs (as /32 CIDRs) of +// the Kubernetes Service named by host, when host is a cluster Service name +// (either bare "redis" or "redis.namespace[.svc.cluster.local]"). Returns nil +// when host is not a Service name this operator can resolve (e.g. an external +// host), so callers fall back to DNS. +func (r *MCPServerReconciler) resolveServiceEndpointCIDRs( + ctx context.Context, defaultNS, host string, +) []string { + name, ns := host, defaultNS + if i := strings.Index(host, "."); i >= 0 { + name = host[:i] + if parts := strings.Split(host, "."); len(parts) > 1 && parts[1] != "svc" { + ns = parts[1] + } + } + slices := &discoveryv1.EndpointSliceList{} + if err := r.List(ctx, slices, + client.InNamespace(ns), + client.MatchingLabels{discoveryv1.LabelServiceName: name}); err != nil { + return nil + } + seen := map[string]struct{}{} + var out []string + for _, es := range slices.Items { + for _, ep := range es.Endpoints { + for _, addr := range ep.Addresses { + if _, dup := seen[addr]; !dup { + seen[addr] = struct{}{} + out = append(out, addr+"/32") + } + } + } + } + sort.Strings(out) + return out +} + // desiredUntrustedNetworkPolicy renders the egress lockdown. func desiredUntrustedNetworkPolicy( - m *mcpv1beta1.MCPServer, dialAllowlist []string, + m *mcpv1beta1.MCPServer, dialAllowlist, tokenStoreCIDRs []string, tokenStorePort *intstr.IntOrString, ) *networkingv1.NetworkPolicy { udp := corev1.ProtocolUDP tcp := corev1.ProtocolTCP @@ -626,6 +739,49 @@ func desiredUntrustedNetworkPolicy( networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}}) } + egressRules := []networkingv1.NetworkPolicyEgressRule{ + // Loopback: the in-pod sidecar (Envoy + broker). + {To: []networkingv1.NetworkPolicyPeer{ + {IPBlock: &networkingv1.IPBlock{CIDR: "127.0.0.1/32"}}, + {IPBlock: &networkingv1.IPBlock{CIDR: "::1/128"}}, + }}, + // DNS: restricted to the cluster DNS pods (kube-system + + // k8s-app=kube-dns) — never 0.0.0.0/0:53, which would let the + // backend run or reach an arbitrary DNS server to smuggle data + // or resolve attacker-controlled names off-policy. + { + To: []networkingv1.NetworkPolicyPeer{{ + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": dnsNamespaceSelectorName, + }}, + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "k8s-app": dnsPodSelectorName, + }}, + }}, + Ports: []networkingv1.NetworkPolicyPort{ + {Protocol: &udp, Port: &dnsPort}, + {Protocol: &tcp, Port: &dnsPort}, + }, + }, + destinationRule, + } + + // Token store (broker infrastructure): the sidecar must reach the + // auth-server Redis or every credential lookup fails. Scoped to the Redis + // port only. Added only when resolvable (fail-closed otherwise — the pod + // already cannot inject, and confining it is the safe posture). + if len(tokenStoreCIDRs) > 0 && tokenStorePort != nil { + tcp2 := corev1.ProtocolTCP + rule := networkingv1.NetworkPolicyEgressRule{ + Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp2, Port: tokenStorePort}}, + } + for _, cidr := range tokenStoreCIDRs { + rule.To = append(rule.To, + networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}}) + } + egressRules = append(egressRules, rule) + } + return &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: m.Name + untrustedNetworkPolicySuffix, @@ -638,32 +794,7 @@ func desiredUntrustedNetworkPolicy( "toolhive.stacklok.dev/mcpserver-uid": string(m.UID), }}, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, - Egress: []networkingv1.NetworkPolicyEgressRule{ - // Loopback: the in-pod sidecar (Envoy + broker). - {To: []networkingv1.NetworkPolicyPeer{ - {IPBlock: &networkingv1.IPBlock{CIDR: "127.0.0.1/32"}}, - {IPBlock: &networkingv1.IPBlock{CIDR: "::1/128"}}, - }}, - // DNS: restricted to the cluster DNS pods (kube-system + - // k8s-app=kube-dns) — never 0.0.0.0/0:53, which would let the - // backend run or reach an arbitrary DNS server to smuggle data - // or resolve attacker-controlled names off-policy. - { - To: []networkingv1.NetworkPolicyPeer{{ - NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ - "kubernetes.io/metadata.name": dnsNamespaceSelectorName, - }}, - PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ - "k8s-app": dnsPodSelectorName, - }}, - }}, - Ports: []networkingv1.NetworkPolicyPort{ - {Protocol: &udp, Port: &dnsPort}, - {Protocol: &tcp, Port: &dnsPort}, - }, - }, - destinationRule, - }, + Egress: egressRules, }, } } diff --git a/cmd/thv-operator/controllers/virtualmcpserver_controller.go b/cmd/thv-operator/controllers/virtualmcpserver_controller.go index 2879067fd1..2589edfc94 100644 --- a/cmd/thv-operator/controllers/virtualmcpserver_controller.go +++ b/cmd/thv-operator/controllers/virtualmcpserver_controller.go @@ -166,6 +166,7 @@ type VirtualMCPServerReconciler struct { // +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=embeddingservers,verbs=get;list;watch // +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=embeddingservers/status,verbs=get // +kubebuilder:rbac:groups=toolhive.stacklok.dev,resources=mcptelemetryconfigs,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/cmd/thv-operator/internal/testutil/scheme.go b/cmd/thv-operator/internal/testutil/scheme.go index 84dc816d44..ee01ae69ff 100644 --- a/cmd/thv-operator/internal/testutil/scheme.go +++ b/cmd/thv-operator/internal/testutil/scheme.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/require" + discoveryv1 "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -35,6 +36,7 @@ func NewScheme(t *testing.T, extra ...func(*runtime.Scheme) error) *runtime.Sche scheme := runtime.NewScheme() adders := append([]func(*runtime.Scheme) error{ clientgoscheme.AddToScheme, + discoveryv1.AddToScheme, mcpv1beta1.AddToScheme, mcpv1alpha1.AddToScheme, }, extra...) diff --git a/deploy/charts/operator/templates/clusterrole/role.yaml b/deploy/charts/operator/templates/clusterrole/role.yaml index 8172b420b5..77d15dc2bc 100644 --- a/deploy/charts/operator/templates/clusterrole/role.yaml +++ b/deploy/charts/operator/templates/clusterrole/role.yaml @@ -25,6 +25,8 @@ rules: resources: - pods verbs: + - create + - delete - get - list - watch @@ -81,6 +83,14 @@ rules: - patch - update - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch - apiGroups: - events.k8s.io resources: diff --git a/pkg/egressbroker/envoyconfig.go b/pkg/egressbroker/envoyconfig.go index 17fac050da..212c7d1c15 100644 --- a/pkg/egressbroker/envoyconfig.go +++ b/pkg/egressbroker/envoyconfig.go @@ -114,7 +114,10 @@ func (p EnvoyConfigParams) Validate() error { // - upstream_validation_context pins trust to the system CA bundle — the // bump CA is only ever presented to the backend container, never used to // validate real upstreams. -const envoyBootstrapTemplate = `static_resources: +const envoyBootstrapTemplate = `node: + id: thv-egress-sidecar + cluster: thv-untrusted-egress +static_resources: listeners: - name: https_proxy address: @@ -169,8 +172,6 @@ const envoyBootstrapTemplate = `static_resources: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz failure_mode_allow: false - with_request_body: - max_request_bytes: 0 grpc_service: envoy_grpc: cluster_name: egress_broker @@ -259,6 +260,12 @@ const envoyBootstrapTemplate = `static_resources: // raw yaml.Nodes (gopkg.in/yaml.v3 cannot decode into *yaml.Node sequence // elements, so every raw list is a single wrapping node). type bootstrapDoc struct { + // Node carries the Envoy node id/cluster required by the SDS API. Kept as + // a yaml.Node (not typed fields) so it round-trips verbatim through the + // unmarshal/mutate/marshal render path — a typed struct that omits it + // silently drops the section and Envoy rejects the bootstrap with + // "node 'id' and 'cluster' are required". + Node yaml.Node `yaml:"node"` StaticResources struct { Listeners []struct { Name string `yaml:"name"` @@ -389,7 +396,7 @@ func buildVirtualHosts(allowedHosts []string) yaml.Node { "upgrade_configs": []any{ map[string]any{ "upgrade_type": "CONNECT", - "connect_config": map[string]any{"terminate_connect": true}, + "connect_config": map[string]any{}, }, }, }, diff --git a/pkg/egressbroker/envoyconfig_test.go b/pkg/egressbroker/envoyconfig_test.go index 415500f491..04e67fd630 100644 --- a/pkg/egressbroker/envoyconfig_test.go +++ b/pkg/egressbroker/envoyconfig_test.go @@ -275,7 +275,10 @@ func TestRenderEnvoyBootstrap(t *testing.T) { connect := routes[0].(map[string]any) upgrade := dig(t, connect, "route", "upgrade_configs").([]any)[0].(map[string]any) assert.Equal(t, "CONNECT", upgrade["upgrade_type"]) - assert.Equal(t, true, dig(t, upgrade, "connect_config").(map[string]any)["terminate_connect"]) + // terminate_connect was removed: it is a v1.37 field the pinned + // Envoy v1.36 rejects (bootstrap invalid). Presence of connect_config + // (empty) is the bump marker. + assert.Contains(t, upgrade, "connect_config") assert.Equal(t, "dynamic_forward_proxy_cluster", dig(t, connect, "route").(map[string]any)["cluster"]) } @@ -408,7 +411,6 @@ func TestBootstrapMentionsSDSFetchWiring(t *testing.T) { "tls_certificate_sds_secret_configs", "tls_inspector", "sni_dynamic_forward_proxy", - "terminate_connect: true", "failure_mode_allow: false", } { assert.Contains(t, string(data), want) diff --git a/pkg/egressbroker/extproc_test.go b/pkg/egressbroker/extproc_test.go index cfa33fb09b..80b5c50df5 100644 --- a/pkg/egressbroker/extproc_test.go +++ b/pkg/egressbroker/extproc_test.go @@ -634,6 +634,22 @@ func TestTokenMap_ConcurrencyFlood(t *testing.T) { for i := 0; i < 8*maxEntries; i++ { ids = append(ids, fmt.Sprintf("flood-%d", i)) } + // Phase 1: concurrent Record flood (no racing Lookups — a racing Lookup can + // legitimately consume an "oldest" entry, freeing capacity so it is NOT + // evicted, which made the old assertion racy by design). Bounded-ness and + // oldest-first eviction are only deterministic once the flood settles. + for _, id := range ids { + wg.Add(1) + go func(id string) { + defer wg.Done() + m.Record(id, testHeaderVal, "github") + }(id) + } + waitWithTimeout(t, &wg, "record flood") + + assert.LessOrEqual(t, m.Len(), maxEntries, "the map must stay bounded under flood") + // Phase 2: concurrent Record/Lookup mix must never panic and must keep the + // map bounded — the only invariants that hold while Lookups consume. for _, id := range ids { wg.Add(2) go func(id string) { @@ -642,20 +658,26 @@ func TestTokenMap_ConcurrencyFlood(t *testing.T) { }(id) go func(id string) { defer wg.Done() - _, _ = m.Lookup(id) // racing a record of the same id is fine + _, _ = m.Lookup(id) }(id) } waitWithTimeout(t, &wg, "record/lookup flood") + assert.LessOrEqual(t, m.Len(), maxEntries, "the map must stay bounded under a record/lookup mix") - assert.LessOrEqual(t, m.Len(), maxEntries, "the map must stay bounded under flood") - // The oldest ids must have been evicted; the newest survive. + // Phase 3: a SEQUENTIAL record flood evicts oldest-first — this is the only + // ordering that is deterministic (concurrent records acquire the map lock + // in arbitrary order, so "the oldest N" is not well-defined under a flood). + // Drive it from this goroutine, not spawned ones. + for _, id := range ids { + m.Record(id+"-fresh", testHeaderVal, "github") + } for _, id := range ids[:maxEntries] { - _, ok := m.Lookup(id) - assert.False(t, ok, "oldest entry %s must be evicted", id) + _, ok := m.Lookup(id + "-fresh") + assert.False(t, ok, "oldest entry %s-fresh must be evicted", id) } survivors := 0 for _, id := range ids[len(ids)-maxEntries:] { - if _, ok := m.Lookup(id); ok { + if _, ok := m.Lookup(id + "-fresh"); ok { survivors++ } } diff --git a/pkg/vmcp/session/untrusted/egress.go b/pkg/vmcp/session/untrusted/egress.go index e26c9e5fdd..9dfbb2aae6 100644 --- a/pkg/vmcp/session/untrusted/egress.go +++ b/pkg/vmcp/session/untrusted/egress.go @@ -289,9 +289,10 @@ func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{ Name: caSeedInitContainerName, Image: images.EgressBroker, - Command: []string{"sh", "-c"}, + Command: []string{"/ko-app/thv-egressbroker", "seed-ca"}, Args: []string{ - fmt.Sprintf("cp /bundle/%s /ca/%s && chmod 0444 /ca/%s", caKeyCert, caFileName, caFileName), + fmt.Sprintf("/bundle/%s", caKeyCert), + fmt.Sprintf("/ca/%s", caFileName), }, Resources: caSeedResources, VolumeMounts: []corev1.VolumeMount{ @@ -311,9 +312,14 @@ func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { // Envoy data plane: consumes the broker-rendered bootstrap. pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ - Name: envoyContainerName, - Image: images.EnvoyProxy, - Command: []string{"envoy", "-c", envoyConfigPath + "/envoy.yaml"}, + Name: envoyContainerName, + Image: images.EnvoyProxy, + // Wait for the broker to render the bootstrap before starting; the + // broker writes it at startup and Envoy crashes on a missing file. + Command: []string{"sh", "-c"}, + Args: []string{ + fmt.Sprintf("until [ -f %s/envoy.yaml ]; do sleep 0.2; done; exec envoy -c %s/envoy.yaml", envoyConfigPath, envoyConfigPath), + }, Resources: scaledResources(defaultEnvoyResources, res.CPUMultiplier, res.MemoryMultiplier), Ports: []corev1.ContainerPort{ {Name: "egress-proxy", ContainerPort: proxyPort}, @@ -418,7 +424,10 @@ func applyEgressBrokerSidecar(pod *corev1.Pod, req EnsurePodRequest) error { LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/healthz", + // /livez is dependency-free (process alive), served from + // process start — a slow Redis/CA init must not let + // liveness kill a healthy broker mid-startup. + Path: "/livez", Port: intstr.FromInt32(BrokerHealthPort), }, }, diff --git a/pkg/vmcp/session/untrusted/egress_test.go b/pkg/vmcp/session/untrusted/egress_test.go index 1d269ed6aa..112bd5fd52 100644 --- a/pkg/vmcp/session/untrusted/egress_test.go +++ b/pkg/vmcp/session/untrusted/egress_test.go @@ -411,16 +411,20 @@ func TestApplyEgressBrokerSidecar(t *testing.T) { pod := newPod() broker := findContainer(t, pod, brokerContainerName) + // Liveness targets /livez (dependency-free, served from process start) + // so a slow Redis/CA init cannot kill a healthy broker; readiness + // targets /healthz (gated on CA/policy/Redis). for name, probe := range map[string]*corev1.Probe{ "readiness": broker.ReadinessProbe, "liveness": broker.LivenessProbe, } { require.NotNil(t, probe, "%s probe must be set", name) require.NotNil(t, probe.HTTPGet, "%s probe must be httpGet", name) - assert.Equal(t, "/healthz", probe.HTTPGet.Path) assert.Equal(t, int32(BrokerHealthPort), probe.HTTPGet.Port.IntVal) assert.Equal(t, int32(5), probe.PeriodSeconds, "%s probe period", name) } + assert.Equal(t, "/healthz", broker.ReadinessProbe.HTTPGet.Path) + assert.Equal(t, "/livez", broker.LivenessProbe.HTTPGet.Path) assert.Equal(t, int32(3), broker.LivenessProbe.FailureThreshold) // Envoy and the backend container get no probes (the broker's health From c6d451318cadb989d4a714f0bf5ab72743125e0a Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 23 Jul 2026 08:48:29 +0300 Subject: [PATCH 09/12] Fix codespell findings and the DNS stub test race Codespell: redeclared (not re-declared) and unparsable (not unparseable) across the untrusted-mode packages and docs; add userA to the codespell ignore list (false positive on a test identifier). The untrusted egress tests were flaky under parallel runs: reconcileOnce installed the package-level DNS stub per call via untrustedDNSTestLock, whose re-entry check treats any installed stub (including a parallel sibling's) as "held" and skips the mutex. A multi-reconcile test could then overwrite another test's stub mid-flight, and a cleared stub sent the next reconcile to real DNS (api.github.com fails offline). Install the fixture stub exactly once per test, holding the mutex for the test's whole lifetime; parallel siblings block until cleanup. Green at -count=8 (previously failed at -count=3). --- .codespellrc | 2 +- .../mcpserver_untrusted_env_test.go | 39 +++++++++++++++++-- .../mcpserver_untrusted_resources.go | 4 +- .../mcpserver_untrusted_resources_test.go | 2 +- docs/arch/16-untrusted-mode.md | 2 +- pkg/egressbroker/dialcontrol.go | 2 +- pkg/egressbroker/dialcontrol_test.go | 2 +- pkg/egressbroker/server.go | 8 ++-- pkg/vmcp/cli/untrusted.go | 4 +- pkg/vmcp/cli/untrusted_test.go | 8 ++-- pkg/vmcp/session/untrusted/addressing.go | 2 +- 11 files changed, 54 insertions(+), 21 deletions(-) diff --git a/.codespellrc b/.codespellrc index b1d5725306..c618b6e64b 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,3 @@ [codespell] -ignore-words-list = NotIn,notin,AfterAll,ND,aks,deriver,te,clientA,AtMost,atmost,convertIn +ignore-words-list = NotIn,notin,AfterAll,ND,aks,deriver,te,clientA,AtMost,atmost,convertIn,userA skip = *.svg,*.mod,*.sum diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go index b9bfbc5cd5..bc18680a9f 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "net" "strings" + "sync" "testing" "time" @@ -107,9 +108,13 @@ func setupUntrustedReconciler(t *testing.T, objs ...client.Object) (*MCPServerRe func reconcileOnce(t *testing.T, r *MCPServerReconciler, mcpServer *mcpv1beta1.MCPServer) ctrl.Result { t.Helper() - untrustedDNSTestLock(t, func(string) ([]net.IP, error) { - return []net.IP{net.ParseIP("140.82.114.26")}, nil - }) + // Install the fixture stub ONCE per test, held for its whole lifetime. + // untrustedDNSTestLock treats any installed stub (including a parallel + // sibling's) as "held" and skips locking, so calling it per reconcile + // lets a multi-reconcile test overwrite another test's stub mid-flight. + // stubOnce guarantees one lock+install per test no matter how many + // reconcileOnce calls follow; t.Cleanup releases it. + stubUntrustedDNSOnce(t) ctx := log.IntoContext(t.Context(), log.Log) req := ctrl.Request{NamespacedName: types.NamespacedName{ @@ -121,6 +126,34 @@ func reconcileOnce(t *testing.T, r *MCPServerReconciler, mcpServer *mcpv1beta1.M return result } +// stubUntrustedDNSOnce installs the fixture DNS stub exactly once per test, +// holding untrustedDNSTestMu for the test's whole lifetime. Reconciles after +// the first reuse the already-installed stub; a parallel sibling blocks on +// the mutex until t.Cleanup releases it. This removes the re-entry race in +// untrustedDNSTestLock that let a multi-reconcile test clobber a sibling's +// stub without holding the lock. +var untrustedDNSStubOnce sync.Map // *testing.T -> struct{}{} + +func stubUntrustedDNSOnce(t *testing.T) { + t.Helper() + if _, done := untrustedDNSStubOnce.LoadOrStore(t, struct{}{}); done { + return + } + untrustedDNSTestMu.Lock() + untrustedDNSLookupMu.Lock() + untrustedDNSLookupStub = func(string) ([]net.IP, error) { + return []net.IP{net.ParseIP("140.82.114.26")}, nil + } + untrustedDNSLookupMu.Unlock() + t.Cleanup(func() { + untrustedDNSLookupMu.Lock() + untrustedDNSLookupStub = nil + untrustedDNSLookupMu.Unlock() + untrustedDNSTestMu.Unlock() + untrustedDNSStubOnce.Delete(t) + }) +} + func TestMCPServerReconciler_UntrustedSecretEnvRejected(t *testing.T) { t.Parallel() diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go index a1828708f8..a75b34112c 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go @@ -32,7 +32,7 @@ import ( ) // Untrusted-mode wiring constants (ADR-0001 Wave 3). Resource naming is NOT -// re-declared here: it comes from egressbroker.ResourceNamesFor / +// redeclared here: it comes from egressbroker.ResourceNamesFor / // BaseCASecretName / TrimGeneration, the shared naming contract both sides of // the module boundary import (the vMCP side resolves the same names in // pkg/vmcp/session/untrusted/egress.go). Data keys come from @@ -327,7 +327,7 @@ func (r *MCPServerReconciler) ensureUntrustedCA(ctx context.Context, m *mcpv1bet // currentCAMaterial selects the current CA generation from the listed // generation Secrets and returns its cert PEM (plus key PEM only when a fresh // CA was minted this pass — no Secret, or rotation due). The current -// generation is the parseable Secret with the latest notAfter; unparseable +// generation is the parseable Secret with the latest notAfter; unparsable // Secrets are ignored here (the GC pass deletes them — they match no // published generation). func currentCAMaterial(mcpserverName string, secrets []corev1.Secret) (certPEM, keyPEM []byte, err error) { diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go index 2a89c24004..4c5222cc0b 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go @@ -265,7 +265,7 @@ func TestEnsureUntrustedResources(t *testing.T) { }) //nolint:paralleltest // Swaps the package-level DNS lookup stub (restored after each ensure). - t.Run("unparseable CA Secret is garbage-collected and a fresh generation minted", func(t *testing.T) { + t.Run("unparsable CA Secret is garbage-collected and a fresh generation minted", func(t *testing.T) { m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) stubEgressDNS(t, githubDNS) r, _ := setupUntrustedReconciler(t, m) diff --git a/docs/arch/16-untrusted-mode.md b/docs/arch/16-untrusted-mode.md index 0d32d1cd3a..39bb673850 100644 --- a/docs/arch/16-untrusted-mode.md +++ b/docs/arch/16-untrusted-mode.md @@ -251,7 +251,7 @@ operator-injected values only. Violations surface as terminal status conditions | `THV_UNTRUSTED_BROKER_IMAGE` | `ghcr.io/stacklok/toolhive/egressbroker:v0.17.0` | broker sidecar image override | | `THV_UNTRUSTED_SIDECAR_CPU` / `THV_UNTRUSTED_SIDECAR_MEM` | `1.0` | multiplier on envoy/broker sidecar requests+limits (must be in (0, 100]) | -Every tunable fails startup on an unparseable/zero/negative value. Image +Every tunable fails startup on an unparsable/zero/negative value. Image overrides tagged `:latest` are rejected outright; non-digest-pinned overrides are honored with a loud warning. diff --git a/pkg/egressbroker/dialcontrol.go b/pkg/egressbroker/dialcontrol.go index 308fff99ce..f046c9475d 100644 --- a/pkg/egressbroker/dialcontrol.go +++ b/pkg/egressbroker/dialcontrol.go @@ -58,7 +58,7 @@ func ParseIPAllowlist(entries []string) (*IPAllowlist, error) { } // Allows reports whether addr (a host:port or bare IP dial target) resolves -// inside the allowlist. Fails closed: unparseable targets, and IPs outside +// inside the allowlist. Fails closed: unparsable targets, and IPs outside // every prefix, are refused. func (a *IPAllowlist) Allows(addr string) bool { host, _, err := net.SplitHostPort(addr) diff --git a/pkg/egressbroker/dialcontrol_test.go b/pkg/egressbroker/dialcontrol_test.go index 73f209c7cc..4233806cbd 100644 --- a/pkg/egressbroker/dialcontrol_test.go +++ b/pkg/egressbroker/dialcontrol_test.go @@ -72,7 +72,7 @@ func TestIPAllowlist(t *testing.T) { require.Error(t, allow.DialControl("tcp", "[::ffff:10.0.0.5]:443", nil)) }) - t.Run("unparseable target fails closed", func(t *testing.T) { + t.Run("unparsable target fails closed", func(t *testing.T) { t.Parallel() require.Error(t, allow.DialControl("tcp", "not-an-ip-literal:443", nil)) }) diff --git a/pkg/egressbroker/server.go b/pkg/egressbroker/server.go index a2d0585018..164e542de5 100644 --- a/pkg/egressbroker/server.go +++ b/pkg/egressbroker/server.go @@ -33,7 +33,7 @@ import ( type AuthorizationServer struct { envoyauth.UnimplementedAuthorizationServer injector *CredentialInjector - audit AuditLogger // nil: deny events from unparseable requests are not audited + audit AuditLogger // nil: deny events from unparsable requests are not audited metrics *BrokerMetrics identity PodIdentity podName string @@ -48,7 +48,7 @@ func NewAuthorizationServer(injector *CredentialInjector) (*AuthorizationServer, } // WithObservability attaches the audit/metrics sinks used for Check-level -// denials that never reach the injector (unparseable destination). +// denials that never reach the injector (unparsable destination). func (s *AuthorizationServer) WithObservability( audit AuditLogger, metrics *BrokerMetrics, identity PodIdentity, podName string, ) *AuthorizationServer { @@ -63,7 +63,7 @@ func (s *AuthorizationServer) WithObservability( func (s *AuthorizationServer) Check(ctx context.Context, req *envoyauth.CheckRequest) (*envoyauth.CheckResponse, error) { dest, err := destinationFromCheckRequest(req) if err != nil { - slog.DebugContext(ctx, "egressbroker: denying request with unparseable destination", "error", err) + slog.DebugContext(ctx, "egressbroker: denying request with unparsable destination", "error", err) if s.audit != nil { s.audit.Deny(ctx, DenyEvent{ InjectEvent: AuditEvent(s.identity, s.podName, Destination{}, ""), @@ -71,7 +71,7 @@ func (s *AuthorizationServer) Check(ctx context.Context, req *envoyauth.CheckReq }) } s.metrics.RecordDenial(ctx, s.identity.MCPServer, "", DenyReasonMalformed) - return deniedResponse(codes.InvalidArgument, "unparseable destination"), nil + return deniedResponse(codes.InvalidArgument, "unparsable destination"), nil } requestID := requestIDFromCheckRequest(req) decision := s.injector.Evaluate(ctx, dest, requestID) diff --git a/pkg/vmcp/cli/untrusted.go b/pkg/vmcp/cli/untrusted.go index f18766ec99..14482280af 100644 --- a/pkg/vmcp/cli/untrusted.go +++ b/pkg/vmcp/cli/untrusted.go @@ -59,7 +59,7 @@ const ( // Platform-operator tunables (Wave-5 spec §3.1/§3.2/§4), resolved ONCE here at // the composition root — never hot-reloaded. Every one fails startup on an -// unparseable/zero/negative value (fail loud). +// unparsable/zero/negative value (fail loud). const ( // envUntrustedEnvoyImage overrides the pinned Envoy data-plane image // (air-gapped mirrors). @@ -311,7 +311,7 @@ type untrustedTunables struct { } // resolveUntrustedTunables reads the THV_UNTRUSTED_* env vars once. Absent -// vars take defaults; an unparseable or non-positive value is a startup-fatal +// vars take defaults; an unparsable or non-positive value is a startup-fatal // error (fail loud — a silently-defaulted quota or timeout is a misconfig the // operator cannot see). func resolveUntrustedTunables() (*untrustedTunables, error) { diff --git a/pkg/vmcp/cli/untrusted_test.go b/pkg/vmcp/cli/untrusted_test.go index a291654e00..6855bb799c 100644 --- a/pkg/vmcp/cli/untrusted_test.go +++ b/pkg/vmcp/cli/untrusted_test.go @@ -113,7 +113,7 @@ func TestResolveTokenStoreConfig(t *testing.T) { } // TestResolveUntrustedTunables pins the Wave-5 platform knobs: defaults when -// unset, overrides honored, and startup-fatal on unparseable/zero/negative +// unset, overrides honored, and startup-fatal on unparsable/zero/negative // values. t.Setenv serializes subtests (no t.Parallel). // //nolint:paralleltest // t.Setenv modifies the process environment. @@ -177,17 +177,17 @@ func TestResolveUntrustedTunables(t *testing.T) { name string env map[string]string }{ - {"unparseable idle TTL", map[string]string{envUntrustedIdleTTL: "not-a-duration"}}, + {"unparsable idle TTL", map[string]string{envUntrustedIdleTTL: "not-a-duration"}}, {"zero idle TTL", map[string]string{envUntrustedIdleTTL: "0s"}}, {"negative readiness timeout", map[string]string{envUntrustedReadinessTimeout: "-5s"}}, {"zero per-user quota", map[string]string{envUntrustedPerUserQuota: "0"}}, {"negative per-server cap", map[string]string{envUntrustedPerServerCap: "-3"}}, - {"unparseable per-server cap", map[string]string{envUntrustedPerServerCap: "abc"}}, + {"unparsable per-server cap", map[string]string{envUntrustedPerServerCap: "abc"}}, {"zero global cap ratio", map[string]string{envUntrustedGlobalCapRatio: "0"}}, {"negative global cap ratio", map[string]string{envUntrustedGlobalCapRatio: "-0.5"}}, {"NaN global cap ratio", map[string]string{envUntrustedGlobalCapRatio: "NaN"}}, {"negative sidecar cpu", map[string]string{envUntrustedSidecarCPU: "-1"}}, - {"unparseable sidecar mem", map[string]string{envUntrustedSidecarMem: "lots"}}, + {"unparsable sidecar mem", map[string]string{envUntrustedSidecarMem: "lots"}}, {"NaN sidecar cpu", map[string]string{envUntrustedSidecarCPU: "NaN"}}, {"+Inf sidecar mem", map[string]string{envUntrustedSidecarMem: "+Inf"}}, {"-Inf sidecar cpu", map[string]string{envUntrustedSidecarCPU: "-Inf"}}, diff --git a/pkg/vmcp/session/untrusted/addressing.go b/pkg/vmcp/session/untrusted/addressing.go index 57ad3d2ff1..ee4393c608 100644 --- a/pkg/vmcp/session/untrusted/addressing.go +++ b/pkg/vmcp/session/untrusted/addressing.go @@ -146,7 +146,7 @@ func (r *podAddressResolver) resolveOne( port, suffix, err := BackendPortAndSuffix(b.BaseURL) if err != nil { - return nil, fmt.Errorf("untrusted backend %q has unparseable BaseURL: %w", b.Name, err) + return nil, fmt.Errorf("untrusted backend %q has unparsable BaseURL: %w", b.Name, err) } pod, err := r.lifecycle.EnsurePod(ctx, EnsurePodRequest{ From 09aacf11e0c8e64a727f5324785b495a6a700761 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 23 Jul 2026 12:03:21 +0300 Subject: [PATCH 10/12] Honor permissionProfile egress on Kubernetes via NetworkPolicy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the 4-year-old pkg/container/kubernetes/client.go TODO: trusted K8s workloads with spec.permissionProfile.network.outbound.allowHost or allowPort set now get a NetworkPolicy, making the profile real on K8s for the first time instead of silently discarded. Opt-in semantics: InsecureAllowAll or an empty allowHost/allowPort set generates no policy (today's behavior, never auto-default-deny); the policy is deleted when the profile is removed or turns permissive. Builtin profiles map to the shared toolhive-core builtins; configmap profiles load the referenced document. allowHost resolves to CIDRs (transient DNS failures retry, never a terminal spec error); allowPort produces port rules. The NetworkPolicy rendering primitive is now shared between the trusted and untrusted paths (loopback, cluster-DNS, destination-CIDR helpers and a common render core), with two render functions on top: renderUntrustedEgressNetworkPolicy (unchanged behavior) and the new renderTrustedEgressNetworkPolicy (podSelector from labelsForMCPServer, no sidecar/broker/token-store wiring). Security invariant documented in code and docs: a trusted NetworkPolicy without the untrusted sidecar is blast-radius reduction ONLY, never a credential boundary — trusted workloads hold their own credentials, so the credential guarantee exists only in untrusted mode. The two egress dialects stay separate (permissionProfile = trusted firewall, egressPolicy = untrusted credential binding); only the render machinery is shared. Also fixes the parallel DNS-stub test race at the root: stubEgressDNS/ stubEgressDNSStrict used untrustedDNSTestLock, which treated any installed stub as "held" and skipped the serialization mutex, letting parallel tests clobber each other's stub or hit real DNS. Stub install is now once-per-test-chain, holding the mutex for the test's lifetime. Green at -count=8 with -race. --- .../api/v1beta1/mcpserver_types.go | 13 +- .../controllers/mcpserver_controller.go | 36 ++ .../mcpserver_untrusted_env_test.go | 36 +- .../mcpserver_untrusted_resources.go | 426 +++++++++++++++--- .../mcpserver_untrusted_resources_test.go | 248 +++++++++- .../toolhive.stacklok.dev_mcpservers.yaml | 12 +- .../toolhive.stacklok.dev_mcpservers.yaml | 12 +- docs/arch/05-runconfig-and-permissions.md | 41 +- pkg/container/kubernetes/client.go | 5 +- 9 files changed, 701 insertions(+), 128 deletions(-) diff --git a/cmd/thv-operator/api/v1beta1/mcpserver_types.go b/cmd/thv-operator/api/v1beta1/mcpserver_types.go index 244c03fa86..864aa69927 100644 --- a/cmd/thv-operator/api/v1beta1/mcpserver_types.go +++ b/cmd/thv-operator/api/v1beta1/mcpserver_types.go @@ -65,6 +65,13 @@ const ( // ConditionReasonNotReady so operators can tell "fix the spec" apart from // "wait for convergence". ConditionReasonUntrustedPolicyInvalid = "UntrustedEgressPolicyInvalid" + + // ConditionReasonPermissionProfileInvalid indicates a terminal rejection of + // a trusted workload's spec.permissionProfile for egress NetworkPolicy + // rendering (unknown builtin name, missing ConfigMap/key, malformed profile + // JSON). Distinct from ConditionReasonNotReady so operators can tell "fix + // the spec" apart from "wait for convergence". + ConditionReasonPermissionProfileInvalid = "PermissionProfileInvalid" ) // Condition type for CA bundle validation @@ -416,7 +423,11 @@ type MCPServerSpec struct { // When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend // pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar // + vMCP). Do not merge the two vocabularies: OutboundNetworkPermissions is the - // Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + // Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. Only the + // NetworkPolicy rendering machinery is shared between them. SECURITY INVARIANT: the + // trusted-mode NetworkPolicy rendered from PermissionProfile is blast-radius reduction + // only, never a credential boundary — the credential guarantee comes solely from this + // field's broker + single-tenant pods in untrusted mode. // +optional EgressPolicy *EgressPolicy `json:"egressPolicy,omitempty"` diff --git a/cmd/thv-operator/controllers/mcpserver_controller.go b/cmd/thv-operator/controllers/mcpserver_controller.go index bdc92e6f39..35e1648f03 100644 --- a/cmd/thv-operator/controllers/mcpserver_controller.go +++ b/cmd/thv-operator/controllers/mcpserver_controller.go @@ -408,6 +408,42 @@ func (r *MCPServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, err } + // Ensure the TRUSTED-mode egress NetworkPolicy from spec.permissionProfile + // (opt-in: only when network.outbound pins allowHost/allowPort and is NOT + // InsecureAllowAll; deleted otherwise). Terminal spec errors (unknown + // builtin, missing ConfigMap/key, malformed profile) surface as a condition + // and stop without requeue, per the gate pattern. SECURITY INVARIANT: this + // policy is blast-radius reduction only, never a credential boundary — the + // credential guarantee exists only in untrusted mode (broker + single- + // tenancy), which trusted workloads do not have. + if err := r.ensureTrustedEgressNetworkPolicy(ctx, mcpServer); err != nil { + var specErr *SpecValidationError + if stderrors.As(err, &specErr) { + ctxLogger.Error(err, "Permission profile rejected for egress NetworkPolicy") + mcpServer.Status.Phase = mcpv1beta1.MCPServerPhaseFailed + mcpServer.Status.Message = fmt.Sprintf("Invalid permission profile: %s", specErr.Error()) + meta.SetStatusCondition(&mcpServer.Status.Conditions, metav1.Condition{ + Type: mcpv1beta1.ConditionTypeReady, + Status: metav1.ConditionFalse, + Reason: mcpv1beta1.ConditionReasonPermissionProfileInvalid, + Message: mcpServer.Status.Message, + ObservedGeneration: mcpServer.Generation, + }) + if statusErr := r.Status().Update(ctx, mcpServer); statusErr != nil { + ctxLogger.Error(statusErr, "Failed to update MCPServer status after permission profile validation failure") + } + return ctrl.Result{}, nil + } + ctxLogger.Error(err, "Failed to ensure trusted egress NetworkPolicy") + mcpServer.Status.Phase = mcpv1beta1.MCPServerPhaseFailed + mcpServer.Status.Message = fmt.Sprintf("Failed to ensure trusted egress NetworkPolicy: %s", err.Error()) + setReadyCondition(mcpServer, metav1.ConditionFalse, mcpv1beta1.ConditionReasonNotReady, mcpServer.Status.Message) + if statusErr := r.Status().Update(ctx, mcpServer); statusErr != nil { + ctxLogger.Error(statusErr, "Failed to update MCPServer status after trusted egress NetworkPolicy error") + } + return ctrl.Result{}, err + } + // Ensure RunConfig ConfigMap exists and is up to date if err := r.ensureRunConfigConfigMap(ctx, mcpServer); err != nil { ctxLogger.Error(err, "Failed to ensure RunConfig ConfigMap") diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go index bc18680a9f..2b88db8603 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go @@ -7,7 +7,6 @@ import ( "encoding/json" "net" "strings" - "sync" "testing" "time" @@ -109,11 +108,9 @@ func reconcileOnce(t *testing.T, r *MCPServerReconciler, mcpServer *mcpv1beta1.M t.Helper() // Install the fixture stub ONCE per test, held for its whole lifetime. - // untrustedDNSTestLock treats any installed stub (including a parallel - // sibling's) as "held" and skips locking, so calling it per reconcile - // lets a multi-reconcile test overwrite another test's stub mid-flight. - // stubOnce guarantees one lock+install per test no matter how many - // reconcileOnce calls follow; t.Cleanup releases it. + // installEgressDNSStubOnce guarantees one lock+install per test no matter + // how many reconcileOnce calls follow; a parallel sibling stubber blocks + // on the mutex until t.Cleanup releases it. stubUntrustedDNSOnce(t) ctx := log.IntoContext(t.Context(), log.Log) @@ -126,31 +123,14 @@ func reconcileOnce(t *testing.T, r *MCPServerReconciler, mcpServer *mcpv1beta1.M return result } -// stubUntrustedDNSOnce installs the fixture DNS stub exactly once per test, -// holding untrustedDNSTestMu for the test's whole lifetime. Reconciles after -// the first reuse the already-installed stub; a parallel sibling blocks on -// the mutex until t.Cleanup releases it. This removes the re-entry race in -// untrustedDNSTestLock that let a multi-reconcile test clobber a sibling's -// stub without holding the lock. -var untrustedDNSStubOnce sync.Map // *testing.T -> struct{}{} - +// stubUntrustedDNSOnce installs the fixture DNS stub (every host resolves to +// the shared fixture IP) exactly once per test via the shared once-per-test +// install. Reconciles after the first reuse the already-installed stub; a +// parallel sibling blocks on the mutex until t.Cleanup releases it. func stubUntrustedDNSOnce(t *testing.T) { t.Helper() - if _, done := untrustedDNSStubOnce.LoadOrStore(t, struct{}{}); done { - return - } - untrustedDNSTestMu.Lock() - untrustedDNSLookupMu.Lock() - untrustedDNSLookupStub = func(string) ([]net.IP, error) { + installEgressDNSStubOnce(t, func(string) ([]net.IP, error) { return []net.IP{net.ParseIP("140.82.114.26")}, nil - } - untrustedDNSLookupMu.Unlock() - t.Cleanup(func() { - untrustedDNSLookupMu.Lock() - untrustedDNSLookupStub = nil - untrustedDNSLookupMu.Unlock() - untrustedDNSTestMu.Unlock() - untrustedDNSStubOnce.Delete(t) }) } diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go index a75b34112c..ea6c466384 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go @@ -5,6 +5,7 @@ package controllers import ( "context" + "encoding/json" stderrors "errors" "fmt" "net" @@ -27,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + corepermissions "github.com/stacklok/toolhive-core/permissions" mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" "github.com/stacklok/toolhive/pkg/egressbroker" ) @@ -58,8 +60,8 @@ const ( // untrustedDNSLookup is the resolver for egress-policy destination hosts. // Production leaves the stub nil (net.LookupIP). Tests install a stub via -// stubUntrustedDNSLookup, which SERIALIZES the whole calling test (parent or -// subtest): the installer holds untrustedDNSTestMu until the t.Cleanup +// installEgressDNSStubOnce, which SERIALIZES the whole calling test (parent +// or subtest): the installer holds untrustedDNSTestMu until the t.Cleanup // restore runs. Tests that exercise an untrusted ensure must stub (via // stubEgressDNS or reconcileOnce) — real round-robin DNS answers vary // between calls and would churn the rendered allowlist if a parallel @@ -71,34 +73,47 @@ var ( untrustedDNSLookupStub func(string) ([]net.IP, error) ) -// untrustedDNSTestLock serializes a test (or subtest) that stubs the egress -// DNS lookup against every other such test and installs the stub for the -// caller's whole lifetime (released via the registered t.Cleanup). A -// same-chain re-entry (parent already holds the lock) installs the stub -// without re-locking, so serial subtests compose with a stubbing parent -// without deadlock. -func untrustedDNSTestLock(t interface { +// egressDNSStubInstalls records the tests (or subtests) that have already +// installed an egress DNS stub, so repeat installs within one test chain +// compose without re-locking untrustedDNSTestMu (which they already hold). +var egressDNSStubInstalls sync.Map // *testing.T -> struct{}{} + +// installEgressDNSStubOnce installs swap as the egress DNS stub exactly once +// per test chain, holding untrustedDNSTestMu for the chain's whole lifetime. +// The first caller in a chain (parent or first serial subtest) blocks on the +// mutex until any parallel sibling stubber's t.Cleanup has restored the +// stub; later callers from subtests of an already-stubbed parent install +// their own stub value (serial subtests run after the parent body returns) +// and register their own cleanup. Parent cleanup then restores the parent's +// stub, leaving the lock-holder's stub installed until the lock itself is +// released — a sibling never runs under a cleared stub or real DNS. +// +// This once-per-test pattern replaces the previous per-call lock, whose +// "any installed stub means the chain holds the lock" re-entry check let a +// test overwrite a PARALLEL sibling's stub without holding the mutex, and +// let a cleanup clear the stub while a sibling was mid-flight. +func installEgressDNSStubOnce(t interface { Helper() Cleanup(func()) }, swap func(string) ([]net.IP, error)) { t.Helper() - untrustedDNSLookupMu.RLock() - held := untrustedDNSLookupStub != nil - untrustedDNSLookupMu.RUnlock() - if !held { - untrustedDNSTestMu.Lock() + // *testing.T (and *testing.B) is the identity used across a test chain: + // a subtest's t differs from its parent's, so parent+serial-subtests each + // install exactly once and never re-acquire a lock the parent holds. + if _, done := egressDNSStubInstalls.LoadOrStore(t, struct{}{}); done { + return } + untrustedDNSTestMu.Lock() untrustedDNSLookupMu.Lock() untrustedDNSLookupStub = swap untrustedDNSLookupMu.Unlock() - if !held { - t.Cleanup(func() { - untrustedDNSLookupMu.Lock() - untrustedDNSLookupStub = nil - untrustedDNSLookupMu.Unlock() - untrustedDNSTestMu.Unlock() - }) - } + t.Cleanup(func() { + untrustedDNSLookupMu.Lock() + untrustedDNSLookupStub = nil + untrustedDNSLookupMu.Unlock() + untrustedDNSTestMu.Unlock() + egressDNSStubInstalls.Delete(t) + }) } // resolveEgressHost resolves one policy destination host through the @@ -581,7 +596,7 @@ func (r *MCPServerReconciler) ensureEgressPolicyConfigMap( func (r *MCPServerReconciler) ensureUntrustedNetworkPolicy( ctx context.Context, m *mcpv1beta1.MCPServer, dialAllowlist, tokenStoreCIDRs []string, tokenStorePort *intstr.IntOrString, ) error { - desired := desiredUntrustedNetworkPolicy(m, dialAllowlist, tokenStoreCIDRs, tokenStorePort) + desired := renderUntrustedEgressNetworkPolicy(m, dialAllowlist, tokenStoreCIDRs, tokenStorePort) if err := controllerutil.SetControllerReference(m, desired, r.Scheme); err != nil { return fmt.Errorf("failed to set owner reference on egress NetworkPolicy: %w", err) } @@ -722,48 +737,87 @@ func (r *MCPServerReconciler) resolveServiceEndpointCIDRs( return out } -// desiredUntrustedNetworkPolicy renders the egress lockdown. -func desiredUntrustedNetworkPolicy( - m *mcpv1beta1.MCPServer, dialAllowlist, tokenStoreCIDRs []string, tokenStorePort *intstr.IntOrString, -) *networkingv1.NetworkPolicy { +// loopbackEgressRule permits egress to the pod's own loopback addresses. +// Untrusted mode: the in-pod sidecar (Envoy + broker). Trusted mode: loopback +// stays reachable when an egress policy otherwise exists. +func loopbackEgressRule() networkingv1.NetworkPolicyEgressRule { + return networkingv1.NetworkPolicyEgressRule{To: []networkingv1.NetworkPolicyPeer{ + {IPBlock: &networkingv1.IPBlock{CIDR: "127.0.0.1/32"}}, + {IPBlock: &networkingv1.IPBlock{CIDR: "::1/128"}}, + }} +} + +// clusterDNSEgressRule permits DNS to the cluster DNS pods only +// (kube-system + k8s-app=kube-dns) — never 0.0.0.0/0:53, which would let the +// backend run or reach an arbitrary DNS server to smuggle data or resolve +// attacker-controlled names off-policy. +func clusterDNSEgressRule() networkingv1.NetworkPolicyEgressRule { udp := corev1.ProtocolUDP tcp := corev1.ProtocolTCP dnsPort := intstr.FromInt32(53) + return networkingv1.NetworkPolicyEgressRule{ + To: []networkingv1.NetworkPolicyPeer{{ + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": dnsNamespaceSelectorName, + }}, + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "k8s-app": dnsPodSelectorName, + }}, + }}, + Ports: []networkingv1.NetworkPolicyPort{ + {Protocol: &udp, Port: &dnsPort}, + {Protocol: &tcp, Port: &dnsPort}, + }, + } +} - // Destination allowlist (operator-resolved AllowedHosts; best-effort - // defense-in-depth — the sidecar re-validates per dial, D7). The list is - // sorted by the resolver, so rule order is deterministic (no churn). - destinationRule := networkingv1.NetworkPolicyEgressRule{} - for _, cidr := range dialAllowlist { - destinationRule.To = append(destinationRule.To, +// destinationCIDREgressRule renders one egress rule permitting the given +// CIDRs. An empty list renders an empty rule (deny-all — no To means no +// destination matches). CIDRs are sorted upstream so rule order is +// deterministic (no churn). +func destinationCIDREgressRule(cidrs []string) networkingv1.NetworkPolicyEgressRule { + rule := networkingv1.NetworkPolicyEgressRule{} + for _, cidr := range cidrs { + rule.To = append(rule.To, networkingv1.NetworkPolicyPeer{IPBlock: &networkingv1.IPBlock{CIDR: cidr}}) } + return rule +} - egressRules := []networkingv1.NetworkPolicyEgressRule{ - // Loopback: the in-pod sidecar (Envoy + broker). - {To: []networkingv1.NetworkPolicyPeer{ - {IPBlock: &networkingv1.IPBlock{CIDR: "127.0.0.1/32"}}, - {IPBlock: &networkingv1.IPBlock{CIDR: "::1/128"}}, - }}, - // DNS: restricted to the cluster DNS pods (kube-system + - // k8s-app=kube-dns) — never 0.0.0.0/0:53, which would let the - // backend run or reach an arbitrary DNS server to smuggle data - // or resolve attacker-controlled names off-policy. - { - To: []networkingv1.NetworkPolicyPeer{{ - NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ - "kubernetes.io/metadata.name": dnsNamespaceSelectorName, - }}, - PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ - "k8s-app": dnsPodSelectorName, - }}, - }}, - Ports: []networkingv1.NetworkPolicyPort{ - {Protocol: &udp, Port: &dnsPort}, - {Protocol: &tcp, Port: &dnsPort}, - }, +// renderEgressNetworkPolicy is the shared NetworkPolicy rendering core for +// both egress dialects (trusted permissionProfile and untrusted egressPolicy). +// The two dialects stay separate CRD vocabularies; only this rendering +// machinery is shared. +func renderEgressNetworkPolicy( + name, namespace string, labels map[string]string, + podSelector metav1.LabelSelector, egressRules ...networkingv1.NetworkPolicyEgressRule, +) *networkingv1.NetworkPolicy { + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, }, - destinationRule, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: podSelector, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} + +// renderUntrustedEgressNetworkPolicy renders the egress lockdown for +// untrusted session pods: loopback (the sidecar) + cluster DNS + the +// policy-resolved destination CIDRs + (when resolvable) the token-store rule. +func renderUntrustedEgressNetworkPolicy( + m *mcpv1beta1.MCPServer, dialAllowlist, tokenStoreCIDRs []string, tokenStorePort *intstr.IntOrString, +) *networkingv1.NetworkPolicy { + egressRules := []networkingv1.NetworkPolicyEgressRule{ + loopbackEgressRule(), + clusterDNSEgressRule(), + // Destination allowlist (operator-resolved AllowedHosts; best-effort + // defense-in-depth — the sidecar re-validates per dial, D7). + destinationCIDREgressRule(dialAllowlist), } // Token store (broker infrastructure): the sidecar must reach the @@ -771,9 +825,9 @@ func desiredUntrustedNetworkPolicy( // port only. Added only when resolvable (fail-closed otherwise — the pod // already cannot inject, and confining it is the safe posture). if len(tokenStoreCIDRs) > 0 && tokenStorePort != nil { - tcp2 := corev1.ProtocolTCP + tcp := corev1.ProtocolTCP rule := networkingv1.NetworkPolicyEgressRule{ - Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp2, Port: tokenStorePort}}, + Ports: []networkingv1.NetworkPolicyPort{{Protocol: &tcp, Port: tokenStorePort}}, } for _, cidr := range tokenStoreCIDRs { rule.To = append(rule.To, @@ -782,21 +836,247 @@ func desiredUntrustedNetworkPolicy( egressRules = append(egressRules, rule) } - return &networkingv1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: m.Name + untrustedNetworkPolicySuffix, - Namespace: m.Namespace, - Labels: untrustedResourceLabels(m), - }, - Spec: networkingv1.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{ - "toolhive.stacklok.dev/untrusted": "true", - "toolhive.stacklok.dev/mcpserver-uid": string(m.UID), - }}, - PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, - Egress: egressRules, - }, + return renderEgressNetworkPolicy( + m.Name+untrustedNetworkPolicySuffix, m.Namespace, untrustedResourceLabels(m), + metav1.LabelSelector{MatchLabels: map[string]string{ + "toolhive.stacklok.dev/untrusted": "true", + "toolhive.stacklok.dev/mcpserver-uid": string(m.UID), + }}, + egressRules..., + ) +} + +// renderTrustedEgressNetworkPolicy renders the blast-radius-reduction +// NetworkPolicy for a TRUSTED workload whose permission profile pins outbound +// hosts/ports: loopback + cluster DNS + the profile-resolved destination +// CIDRs, plus the allowPort port rule. +// +// SECURITY INVARIANT: a trusted NetworkPolicy without the untrusted sidecar +// is blast-radius reduction ONLY, never a credential boundary. Trusted mode +// has no broker and no single-tenancy: the workload holds its own +// credentials, so a compromise confined by this policy can still exfiltrate +// them to any ALLOWED destination. The credential boundary guarantee exists +// only in untrusted mode (broker + single-tenant pods); never present the +// trusted policy as an equivalent control. Deliberately absent vs. the +// untrusted render: no token-store rule, no untrusted podSelector labels. +func renderTrustedEgressNetworkPolicy( + m *mcpv1beta1.MCPServer, destinationCIDRs []string, ports []int32, +) *networkingv1.NetworkPolicy { + egressRules := []networkingv1.NetworkPolicyEgressRule{ + loopbackEgressRule(), + clusterDNSEgressRule(), + destinationCIDREgressRule(destinationCIDRs), + } + if len(ports) > 0 { + sorted := make([]int32, len(ports)) + copy(sorted, ports) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + rule := networkingv1.NetworkPolicyEgressRule{} + for _, p := range sorted { + tcp := corev1.ProtocolTCP + port := intstr.FromInt32(p) + rule.Ports = append(rule.Ports, networkingv1.NetworkPolicyPort{Protocol: &tcp, Port: &port}) + } + egressRules = append(egressRules, rule) + } + return renderEgressNetworkPolicy( + m.Name+untrustedNetworkPolicySuffix, m.Namespace, labelsForMCPServer(m.Name), + metav1.LabelSelector{MatchLabels: labelsForMCPServer(m.Name)}, + egressRules..., + ) +} + +// ensureTrustedEgressNetworkPolicy reconciles the TRUSTED-mode egress +// NetworkPolicy from spec.permissionProfile (this closes the +// pkg/container/kubernetes/client.go TODO for the operator path). Opt-in +// semantics: InsecureAllowAll or an empty allowHost/allowPort set means NO +// NetworkPolicy (today's behavior — never auto-default-deny); any existing +// trusted policy is deleted when the profile is removed or turns permissive. +// +// SECURITY INVARIANT: this policy is blast-radius reduction only, never a +// credential boundary (see renderTrustedEgressNetworkPolicy). +func (r *MCPServerReconciler) ensureTrustedEgressNetworkPolicy(ctx context.Context, m *mcpv1beta1.MCPServer) error { + if isUntrusted(m) || m.Spec.PermissionProfile == nil { + return r.deleteTrustedEgressNetworkPolicy(ctx, m) + } + profile, err := r.resolvePermissionProfile(ctx, m) + if err != nil { + return err + } + outbound := profile.Network.Outbound + if outbound == nil || outbound.InsecureAllowAll || + (len(outbound.AllowHost) == 0 && len(outbound.AllowPort) == 0) { + return r.deleteTrustedEgressNetworkPolicy(ctx, m) + } + + cidrs, err := resolveTrustedAllowHosts(ctx, outbound.AllowHost) + if err != nil { + return err + } + ports, err := validateTrustedAllowPorts(outbound.AllowPort) + if err != nil { + return err } + desired := renderTrustedEgressNetworkPolicy(m, cidrs, ports) + return r.applyTrustedEgressNetworkPolicy(ctx, m, desired) +} + +// applyTrustedEgressNetworkPolicy converges the rendered trusted policy: +// create when absent, rewrite spec+labels only when they drifted (no-op +// reconciles perform no write). The untrusted render shares this upsert via +// its own ensure — see ensureUntrustedNetworkPolicy. +func (r *MCPServerReconciler) applyTrustedEgressNetworkPolicy( + ctx context.Context, m *mcpv1beta1.MCPServer, desired *networkingv1.NetworkPolicy, +) error { + if err := controllerutil.SetControllerReference(m, desired, r.Scheme); err != nil { + return fmt.Errorf("failed to set owner reference on egress NetworkPolicy: %w", err) + } + + existing := &networkingv1.NetworkPolicy{} + err := r.Get(ctx, types.NamespacedName{Name: desired.Name, Namespace: desired.Namespace}, existing) + if errors.IsNotFound(err) { + if err := r.Create(ctx, desired); err != nil { + return fmt.Errorf("failed to create egress NetworkPolicy: %w", err) + } + return nil + } + if err != nil { + return fmt.Errorf("failed to get egress NetworkPolicy: %w", err) + } + if networkPolicyNeedsUpdate(existing, desired) { + existing.Spec = desired.Spec + existing.Labels = desired.Labels + if err := r.Update(ctx, existing); err != nil { + return fmt.Errorf("failed to update egress NetworkPolicy: %w", err) + } + } + return nil +} + +// deleteTrustedEgressNetworkPolicy removes the trusted-mode egress +// NetworkPolicy (profile removed, permissive, or the untrusted flip — the +// untrusted ensure renders its own policy under the same name). +func (r *MCPServerReconciler) deleteTrustedEgressNetworkPolicy(ctx context.Context, m *mcpv1beta1.MCPServer) error { + return r.deleteIfExists(ctx, &networkingv1.NetworkPolicy{}, + m.Name+untrustedNetworkPolicySuffix, m.Namespace, "NetworkPolicy") +} + +// resolvePermissionProfile resolves spec.permissionProfile to a toolhive-core +// permissions.Profile for NetworkPolicy rendering: builtin names map to the +// shared builtins (network = InsecureAllowAll → no policy; none = empty → no +// policy); configmap loads the profile JSON from the referenced ConfigMap key +// (the same document the runner mounts at /etc/toolhive/profiles/). +// Failures are terminal spec problems (SpecValidationError): retrying cannot +// fix a missing ConfigMap/key or malformed profile. +func (r *MCPServerReconciler) resolvePermissionProfile( + ctx context.Context, m *mcpv1beta1.MCPServer, +) (*corepermissions.Profile, error) { + ref := m.Spec.PermissionProfile + switch ref.Type { + case mcpv1beta1.PermissionProfileTypeBuiltin: + switch ref.Name { + case corepermissions.ProfileNone: + return corepermissions.BuiltinNoneProfile(), nil + case corepermissions.ProfileNetwork: + return corepermissions.BuiltinNetworkProfile(), nil + default: + return nil, &SpecValidationError{Message: fmt.Sprintf( + "unknown builtin permission profile %q (must be %q or %q)", + ref.Name, corepermissions.ProfileNone, corepermissions.ProfileNetwork)} + } + case mcpv1beta1.PermissionProfileTypeConfigMap: + cm := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Name: ref.Name, Namespace: m.Namespace}, cm); err != nil { + return nil, &SpecValidationError{Message: fmt.Sprintf( + "failed to get permission profile ConfigMap %q: %v", ref.Name, err)} + } + data, ok := cm.Data[ref.Key] + if !ok { + return nil, &SpecValidationError{Message: fmt.Sprintf( + "permission profile ConfigMap %q has no key %q", ref.Name, ref.Key)} + } + var profile corepermissions.Profile + if err := json.Unmarshal([]byte(data), &profile); err != nil { + return nil, &SpecValidationError{Message: fmt.Sprintf( + "failed to parse permission profile %q/%q: %v", ref.Name, ref.Key, err)} + } + return &profile, nil + default: + return nil, &SpecValidationError{Message: fmt.Sprintf( + "unknown permission profile type %q", ref.Type)} + } +} + +// validateTrustedAllowPorts converts profile allowPort entries to int32 for +// the NetworkPolicy render. Invalid ports are terminal spec problems: the +// profile schema accepts any int, so range validation happens here. +func validateTrustedAllowPorts(allowPorts []int) ([]int32, error) { + ports := make([]int32, 0, len(allowPorts)) + for _, p := range allowPorts { + if p <= 0 || p > 65535 { + return nil, &SpecValidationError{Message: fmt.Sprintf( + "permission profile allowPort %d is not a valid TCP port (1-65535)", p)} + } + ports = append(ports, int32(p)) // #nosec G109 G115 -- p is validated 1-65535 above, fits int32 + } + return ports, nil +} + +// resolveTrustedAllowHosts resolves each allowHost entry to CIDRs for the +// trusted NetworkPolicy. Unlike the untrusted path (egressbroker +// ResolveDialAllowlist, which must fail closed to protect a credential +// boundary) this policy is blast-radius reduction only, so hosts that are +// unresolvable or resolve to non-public addresses are skipped with a warning +// rather than failing the reconcile. The result is sorted for churn-free +// idempotent renders. +func resolveTrustedAllowHosts(ctx context.Context, hosts []string) ([]string, error) { + seen := map[string]struct{}{} + var out []string + for _, host := range hosts { + if ip := net.ParseIP(host); ip != nil { + bits := 32 + if ip.To4() == nil { + bits = 128 + } + cidr := fmt.Sprintf("%s/%d", host, bits) + if _, dup := seen[cidr]; !dup { + seen[cidr] = struct{}{} + out = append(out, cidr) + } + continue + } + if _, ipNet, err := net.ParseCIDR(host); err == nil { + cidr := ipNet.String() + if _, dup := seen[cidr]; !dup { + seen[cidr] = struct{}{} + out = append(out, cidr) + } + continue + } + ips, err := resolveEgressHost(host) + if err != nil { + // Transient: DNS is an external dependency; retry with backoff. + return nil, fmt.Errorf("%w: permission profile host %q: %w", egressbroker.ErrDNSResolution, host, err) + } + if len(ips) == 0 { + log.FromContext(ctx).Info("permission profile allowHost resolved to no addresses; skipping", + "host", host) + continue + } + for _, ip := range ips { + bits := 32 + if ip.To4() == nil { + bits = 128 + } + cidr := fmt.Sprintf("%s/%d", ip.String(), bits) + if _, dup := seen[cidr]; !dup { + seen[cidr] = struct{}{} + out = append(out, cidr) + } + } + } + sort.Strings(out) + return out, nil } // upsertConfigMapData creates the ConfigMap or updates only its Data (and diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go index 4c5222cc0b..4889c65f19 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go @@ -40,26 +40,29 @@ func withEgressPolicy() v1beta1test.MCPServerOption { // Unit fixtures use hostname allowedHosts (the CRD grammar forbids IP // literals) plus a stubbed DNS lookup — no real resolution in unit tests. The -// stub spans the test's whole lifetime (t.Cleanup restores it): a mid-test -// restore would let a parallel sibling's Reconcile hit real DNS or a stale -// stub. Tests calling this must NOT run in parallel (paralleltest nolint). +// stub is installed once per test chain and spans the chain's whole lifetime +// (t.Cleanup restores it and releases the serialization mutex): a parallel +// sibling stubber blocks until this chain's cleanup runs, so it never hits +// real DNS or this chain's stub. Subtests calling this must NOT run in +// parallel (paralleltest nolint) — they share the parent's lock and run +// serially after the parent body returns. func stubEgressDNS(t *testing.T, ips map[string][]net.IP) { t.Helper() - untrustedDNSTestLock(t, stubDNSFor(ips, false)) + installEgressDNSStubOnce(t, stubDNSFor(ips, false)) } // stubEgressDNSStrict stubs DNS with exact map semantics (unknown hosts -// fail) for tests asserting resolution failure. +// fail) for tests asserting resolution failure. A strict-stubbing test +// serializes against every other stubber, so no sibling's permissive stub +// can leak into it (and vice versa). func stubEgressDNSStrict(t *testing.T, ips map[string][]net.IP) { t.Helper() - untrustedDNSTestLock(t, stubDNSFor(ips, true)) + installEgressDNSStubOnce(t, stubDNSFor(ips, true)) } // stubDNSFor builds a DNS stub over ips. Non-strict stubs resolve unknown -// hosts to the shared fixture IP so any test's stub is a valid stand-in for -// any other's (a parallel sibling may run under this stub after this test's -// cleanup already ran). Strict stubs fail unknown hosts (resolution-failure -// assertions). +// hosts to the shared fixture IP; strict stubs fail unknown hosts +// (resolution-failure assertions). func stubDNSFor(ips map[string][]net.IP, strict bool) func(string) ([]net.IP, error) { return func(host string) ([]net.IP, error) { if resolved, ok := ips[host]; ok { @@ -580,3 +583,228 @@ func TestRenderEgressPolicyYAML(t *testing.T) { assert.False(t, policy.Allows("github", "DELETE", "/repos/x")) assert.False(t, policy.Allows("github", "GET", "/admin")) } + +// trustedProfileFixture is the ConfigMap-backed permission profile used by the +// trusted egress NetworkPolicy tests (the same JSON document the runner mounts +// at /etc/toolhive/profiles/). +const trustedProfileFixture = `{ + "name": "custom-egress", + "network": { + "outbound": { + "insecure_allow_all": false, + "allow_host": ["api.github.com"], + "allow_port": [443] + } + } +}` + +func withTrustedConfigMapProfile() v1beta1test.MCPServerOption { + return v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.PermissionProfile = &mcpv1beta1.PermissionProfileRef{ + Type: mcpv1beta1.PermissionProfileTypeConfigMap, + Name: "my-profile", + Key: "profile.json", + } + }) +} + +// trustedProfileConfigMapWith returns the profile ConfigMap carrying a profile +// whose allowHost is the given list. DNS-failure tests use a self-describing +// fixture (a host absent from the strict stub's map) so the failure assertion +// is pinned to the fixture, not to which stub happens to be installed. +func trustedProfileConfigMapWith(allowHost string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "my-profile", Namespace: "default"}, + Data: map[string]string{"profile.json": fmt.Sprintf(`{ + "name": "custom-egress", + "network": {"outbound": {"insecure_allow_all": false, "allow_host": [%q], "allow_port": [443]}} +}`, allowHost)}, + } +} + +func trustedProfileConfigMap() *corev1.ConfigMap { + return trustedProfileConfigMapWith("api.github.com") +} + +//nolint:tparallel // Subtests swap the package-level DNS lookup stub; they must run serially. +func TestEnsureTrustedEgressNetworkPolicy(t *testing.T) { + t.Parallel() + + //nolint:paralleltest // Swaps the package-level DNS lookup stub (held for the test lifetime). + t.Run("configmap profile with allowHost+allowPort renders loopback+DNS+destination+ports", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withTrustedConfigMapProfile()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m, trustedProfileConfigMap()) + ctx := t.Context() + + require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) + + np := &networkingv1.NetworkPolicy{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, np)) + assertOwnerRef(t, np.OwnerReferences, m) + assert.Contains(t, np.Spec.PolicyTypes, networkingv1.PolicyTypeEgress) + + // The podSelector is the trusted workload label set — NOT the untrusted + // UID labels. + assert.Equal(t, labelsForMCPServer(m.Name), np.Spec.PodSelector.MatchLabels) + assert.NotContains(t, np.Spec.PodSelector.MatchLabels, "toolhive.stacklok.dev/mcpserver-uid") + assert.NotContains(t, np.Labels, "toolhive.stacklok.dev/untrusted-resource") + + var cidrs []string + for _, rule := range np.Spec.Egress { + for _, peer := range rule.To { + if peer.IPBlock != nil { + cidrs = append(cidrs, peer.IPBlock.CIDR) + } + } + } + assert.Contains(t, cidrs, "127.0.0.1/32", "loopback must be permitted") + assert.Contains(t, cidrs, "140.82.114.26/32", "resolved allowHost must be permitted") + assertDNSRuleRestricted(t, np) + + // allowPort produces a ports-only rule on TCP/443. + var foundPort bool + for _, rule := range np.Spec.Egress { + for _, p := range rule.Ports { + if p.Port != nil && p.Port.IntVal == 443 { + foundPort = true + assert.Empty(t, rule.To, "the allowPort rule must not be destination-scoped") + } + } + } + assert.True(t, foundPort, "allowPort 443 must produce a port rule") + + // Idempotency: a second ensure performs no write. + before := np.ResourceVersion + require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) + after := &networkingv1.NetworkPolicy{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, after)) + assert.Equal(t, before, after.ResourceVersion, "no-op reconcile must not rewrite the NetworkPolicy") + }) + + //nolint:paralleltest // Swaps the package-level DNS lookup stub (held for the test lifetime). + t.Run("builtin network profile (InsecureAllowAll) renders no NetworkPolicy", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.PermissionProfile = &mcpv1beta1.PermissionProfileRef{ + Type: mcpv1beta1.PermissionProfileTypeBuiltin, Name: "network", + } + })) + r, _ := setupUntrustedReconciler(t, m) + + require.NoError(t, r.ensureTrustedEgressNetworkPolicy(t.Context(), m)) + np := &networkingv1.NetworkPolicy{} + err := r.Get(t.Context(), types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, np) + assert.True(t, apierrors.IsNotFound(err), "InsecureAllowAll must never render a policy") + }) + + //nolint:paralleltest // Serial with sibling subtests that swap the DNS lookup stub. + t.Run("builtin none profile (empty allowHost) renders no NetworkPolicy", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.PermissionProfile = &mcpv1beta1.PermissionProfileRef{ + Type: mcpv1beta1.PermissionProfileTypeBuiltin, Name: "none", + } + })) + r, _ := setupUntrustedReconciler(t, m) + + require.NoError(t, r.ensureTrustedEgressNetworkPolicy(t.Context(), m)) + np := &networkingv1.NetworkPolicy{} + err := r.Get(t.Context(), types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, np) + assert.True(t, apierrors.IsNotFound(err), "empty allowHost/allowPort must never render a policy") + }) + + //nolint:paralleltest // Swaps the package-level DNS lookup stub (held for the test lifetime). + t.Run("policy is deleted when the profile is removed or becomes permissive", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withTrustedConfigMapProfile()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m, trustedProfileConfigMap()) + ctx := t.Context() + + require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) + np := &networkingv1.NetworkPolicy{} + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, np)) + + // Profile turns permissive → policy deleted. + cm := trustedProfileConfigMap() + cm.Data["profile.json"] = `{"name":"p","network":{"outbound":{"insecure_allow_all":true}}}` + require.NoError(t, r.Update(ctx, cm)) + require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) + err := r.Get(ctx, types.NamespacedName{Name: "github-mcp-egress", Namespace: "default"}, np) + assert.True(t, apierrors.IsNotFound(err), "a permissive profile must delete the policy") + + // Restrictive again, then the profile is removed → policy deleted. + cm.Data["profile.json"] = trustedProfileFixture + require.NoError(t, r.Update(ctx, cm)) + require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) + require.NoError(t, r.Get(ctx, types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, np)) + m.Spec.PermissionProfile = nil + require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) + err = r.Get(ctx, types.NamespacedName{Name: "github-mcp-egress", Namespace: "default"}, np) + assert.True(t, apierrors.IsNotFound(err), "removing the profile must delete the policy") + }) + + //nolint:paralleltest // Serial with sibling subtests that swap the DNS lookup stub. + t.Run("missing ConfigMap or malformed profile is a terminal spec error", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withTrustedConfigMapProfile()) + r, _ := setupUntrustedReconciler(t, m) // no ConfigMap seeded + + err := r.ensureTrustedEgressNetworkPolicy(t.Context(), m) + require.Error(t, err) + var specErr *SpecValidationError + require.ErrorAs(t, err, &specErr) + + badCM := trustedProfileConfigMap() + badCM.Data["profile.json"] = `{not json` + r2, _ := setupUntrustedReconciler(t, m, badCM) + err = r2.ensureTrustedEgressNetworkPolicy(t.Context(), m) + require.Error(t, err) + require.ErrorAs(t, err, &specErr) + }) + + //nolint:paralleltest // Swaps the package-level DNS lookup stub (held for the test lifetime). + t.Run("DNS failure is transient, never a terminal spec error", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withTrustedConfigMapProfile()) + // The strict stub fails every lookup; the fixture host is deliberately + // absent from the map so the failure assertion is pinned to the fixture. + stubEgressDNSStrict(t, map[string][]net.IP{}) + r, _ := setupUntrustedReconciler(t, m, trustedProfileConfigMapWith("unresolvable.invalid")) + + err := r.ensureTrustedEgressNetworkPolicy(t.Context(), m) + require.Error(t, err) + assert.ErrorIs(t, err, egressbroker.ErrDNSResolution) + var specErr *SpecValidationError + assert.False(t, errorAsSpecValidation(err, &specErr), + "a DNS blip must retry with backoff, not poison the workload: %v", err) + }) +} + +// TestRenderTrustedEgressNetworkPolicy pins the trusted render's shape and its +// blast-radius-only security invariant (the doc comment carries the invariant; +// the render carries no untrusted wiring). +func TestRenderTrustedEgressNetworkPolicy(t *testing.T) { + t.Parallel() + + m := v1beta1test.NewMCPServer("github-mcp", "default") + np := renderTrustedEgressNetworkPolicy(m, []string{"140.82.114.26/32"}, []int32{8443, 443}) + + assert.Equal(t, "github-mcp-egress", np.Name) + assert.Equal(t, labelsForMCPServer(m.Name), np.Spec.PodSelector.MatchLabels) + assert.Equal(t, labelsForMCPServer(m.Name), np.Labels) + + // loopback + DNS + destination + ports = 4 rules; no token-store rule. + require.Len(t, np.Spec.Egress, 4) + portRule := np.Spec.Egress[3] + require.Len(t, portRule.Ports, 2) + assert.Equal(t, int32(443), portRule.Ports[0].Port.IntVal, "ports must be sorted (deterministic render)") + assert.Equal(t, int32(8443), portRule.Ports[1].Port.IntVal) + assert.Empty(t, portRule.To) + + // No ports → no port rule. + np = renderTrustedEgressNetworkPolicy(m, []string{"140.82.114.26/32"}, nil) + require.Len(t, np.Spec.Egress, 3) +} diff --git a/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpservers.yaml b/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpservers.yaml index cd4c3ae980..7f75953ed9 100644 --- a/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpservers.yaml +++ b/deploy/charts/operator-crds/files/crds/toolhive.stacklok.dev_mcpservers.yaml @@ -244,7 +244,11 @@ spec: broker may inject each provider's credential. Required when Untrusted is true. When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar - Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. Only the + NetworkPolicy rendering machinery is shared between them. SECURITY INVARIANT: the + trusted-mode NetworkPolicy rendered from PermissionProfile is blast-radius reduction + only, never a credential boundary — the credential guarantee comes solely from this + field's broker + single-tenant pods in untrusted mode. properties: providers: description: |- @@ -1294,7 +1298,11 @@ spec: broker may inject each provider's credential. Required when Untrusted is true. When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar - Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. Only the + NetworkPolicy rendering machinery is shared between them. SECURITY INVARIANT: the + trusted-mode NetworkPolicy rendered from PermissionProfile is blast-radius reduction + only, never a credential boundary — the credential guarantee comes solely from this + field's broker + single-tenant pods in untrusted mode. properties: providers: description: |- diff --git a/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpservers.yaml b/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpservers.yaml index 2a7b8ca662..a6b02107d2 100644 --- a/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpservers.yaml +++ b/deploy/charts/operator-crds/templates/toolhive.stacklok.dev_mcpservers.yaml @@ -247,7 +247,11 @@ spec: broker may inject each provider's credential. Required when Untrusted is true. When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar - Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. Only the + NetworkPolicy rendering machinery is shared between them. SECURITY INVARIANT: the + trusted-mode NetworkPolicy rendered from PermissionProfile is blast-radius reduction + only, never a credential boundary — the credential guarantee comes solely from this + field's broker + single-tenant pods in untrusted mode. properties: providers: description: |- @@ -1297,7 +1301,11 @@ spec: broker may inject each provider's credential. Required when Untrusted is true. When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar - Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. + Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. Only the + NetworkPolicy rendering machinery is shared between them. SECURITY INVARIANT: the + trusted-mode NetworkPolicy rendered from PermissionProfile is blast-radius reduction + only, never a credential boundary — the credential guarantee comes solely from this + field's broker + single-tenant pods in untrusted mode. properties: providers: description: |- diff --git a/docs/arch/05-runconfig-and-permissions.md b/docs/arch/05-runconfig-and-permissions.md index 2ea013b676..fcfc76268a 100644 --- a/docs/arch/05-runconfig-and-permissions.md +++ b/docs/arch/05-runconfig-and-permissions.md @@ -744,20 +744,39 @@ Custom permission profiles can be defined in JSON files for reusable security po - RunConfig permission profile → Security context - Volume mounts → PersistentVolumeClaims or HostPath -**Network policies (untrusted mode):** - -NetworkPolicy generation is real — but scoped to **untrusted mode**, not -general permission profiles. For MCPServers with `spec.untrusted: true`, the -operator renders an egress-only NetworkPolicy from `spec.egressPolicy`: session -pods may reach only loopback (the in-pod Envoy/broker sidecar), cluster DNS -(kube-system / `k8s-app=kube-dns` pods only), and the CIDRs the policy's -`allowedHosts` resolve to. In that mode +**Network policies (trusted and untrusted modes):** + +NetworkPolicy generation is real for **both** egress dialects, which remain +intentionally separate CRD vocabularies sharing only the NetworkPolicy +rendering machinery (loopback + cluster-DNS + destination-CIDR rules). + +For **trusted** MCPServers (`spec.untrusted: false`, the default), the +operator renders an egress-only NetworkPolicy named `-egress` when +`spec.permissionProfile` resolves to a profile whose +`network.outbound` pins `allowHost` and/or `allowPort` — builtin `none`/`network` +or ConfigMap-backed profiles. Opt-in semantics: `insecureAllowAll: true` +(builtin `network`) or an empty `allowHost`/`allowPort` set (builtin `none`) +means **no** NetworkPolicy — the operator never auto-default-denies; deleting +the profile or turning it permissive deletes the policy. The policy selects +the workload's pods (the standard `mcpserver` labels) and permits loopback, +cluster DNS, the resolved `allowHost` CIDRs, and the `allowPort` ports. + +SECURITY INVARIANT: the trusted-mode NetworkPolicy is **blast-radius reduction +only, never a credential boundary**. Trusted workloads hold their own +credentials, so a compromise confined by this policy can still exfiltrate them +to any *allowed* destination. The credential-boundary guarantee exists only in +untrusted mode (broker + single-tenant pods). + +For **untrusted** MCPServers (`spec.untrusted: true`), the operator renders an +egress-only NetworkPolicy from `spec.egressPolicy`: session pods may reach only +loopback (the in-pod Envoy/broker sidecar), cluster DNS (kube-system / +`k8s-app=kube-dns` pods only), the CIDRs the policy's `allowedHosts` resolve +to, and the broker's token store. In that mode `permissionProfile.network.outbound` is **ignored** for the backend pod — the two are intentionally separate dialects: `permissionProfile` remains the Docker/Squid dialect (local mode, above), `egressPolicy` is the K8s -untrusted-mode dialect. General profile → NetworkPolicy conversion for trusted -workloads remains unimplemented; the load-bearing untrusted-mode controls are -the broker's destination binding and per-dial IP validation, with the +untrusted-mode dialect. The load-bearing untrusted-mode controls are the +broker's destination binding and per-dial IP validation, with the NetworkPolicy as defense-in-depth. See [Untrusted Mode](16-untrusted-mode.md) and [ADR-0001](adr/0001-untrusted-mcp-egress-broker.md) (D7). diff --git a/pkg/container/kubernetes/client.go b/pkg/container/kubernetes/client.go index 0b6d72cd09..c8d8ba45d2 100644 --- a/pkg/container/kubernetes/client.go +++ b/pkg/container/kubernetes/client.go @@ -357,7 +357,10 @@ func (c *Client) DeployWorkload(ctx context.Context, command []string, envVars map[string]string, containerLabels map[string]string, - _ *permissions.Profile, // TODO: Implement permission profile support for Kubernetes + // Permission profile egress is rendered by the thv-operator MCPServer + // controller (spec.permissionProfile → NetworkPolicy); this in-process + // client intentionally ignores it. + _ *permissions.Profile, transportType string, options *runtime.DeployWorkloadOptions, _ bool, From 905892fe02971e738658384c6f2d8fc8622a4200 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 23 Jul 2026 15:23:23 +0300 Subject: [PATCH 11/12] Gate untrusted mode behind an opt-in feature flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single-tenant untrusted mode (one backend pod per user session) is too costly to ship as a default. Make it opt-in via TOOLHIVE_ENABLE_UNTRUSTED_MODE (helm operator.features.untrustedMode, default false) so only operators who accept the risk and resource cost enable it. When the flag is off and spec.untrusted is true, the workload reconciles as a normal trusted workload: no single-tenant pod lifecycle, no bump CA, no untrusted NetworkPolicy, no sentinel injection, no vMCP pod cloning or reaper. A clear UntrustedMode=False/UntrustedModeDisabled condition and one-shot Warning event tell the operator the mode is disabled rather than silently ignored. The flag flows from helm through the operator Deployment env and is forwarded to every vMCP Deployment the operator creates, so one value controls both processes. The trusted-mode egress NetworkPolicy (permissionProfile → NetworkPolicy) stays enabled regardless — it carries no per-user cost. CRD fields and CEL validation are unaffected; the flag gates behavior, not schema. --- .../api/v1beta1/mcpserver_types.go | 18 + .../controllers/mcpserver_controller.go | 93 +++++- .../mcpserver_untrusted_env_test.go | 308 +++++++++++++++--- .../mcpserver_untrusted_resources_test.go | 93 +++++- .../virtualmcpserver_deployment.go | 19 ++ .../virtualmcpserver_rbac_untrusted_test.go | 48 ++- .../mcp-server-untrusted/suite_test.go | 9 + deploy/charts/operator/README.md | 3 +- .../charts/operator/templates/deployment.yaml | 2 + .../operator/tests/default_install_test.yaml | 1 + .../operator/tests/feature_flags_test.yaml | 13 +- deploy/charts/operator/values.yaml | 10 + docs/arch/16-untrusted-mode.md | 11 + docs/operator/untrusted-mode.md | 16 + pkg/vmcp/cli/untrusted.go | 13 +- pkg/vmcp/cli/untrusted_test.go | 53 ++- pkg/vmcp/session/untrusted/mode.go | 62 ++++ pkg/vmcp/session/untrusted/mode_test.go | 72 ++++ pkg/vmcp/workloads/k8s.go | 17 +- pkg/vmcp/workloads/k8s_test.go | 27 +- 20 files changed, 788 insertions(+), 100 deletions(-) create mode 100644 pkg/vmcp/session/untrusted/mode.go create mode 100644 pkg/vmcp/session/untrusted/mode_test.go diff --git a/cmd/thv-operator/api/v1beta1/mcpserver_types.go b/cmd/thv-operator/api/v1beta1/mcpserver_types.go index 864aa69927..4255758d64 100644 --- a/cmd/thv-operator/api/v1beta1/mcpserver_types.go +++ b/cmd/thv-operator/api/v1beta1/mcpserver_types.go @@ -19,6 +19,12 @@ const ( // ConditionPodTemplateValid indicates whether the PodTemplateSpec is valid ConditionPodTemplateValid = "PodTemplateValid" + + // ConditionTypeUntrustedMode reports the effective untrusted-mode posture + // when spec.untrusted is true: "false" with ReasonUntrustedModeDisabled + // when the operator runs with TOOLHIVE_ENABLE_UNTRUSTED_MODE off (the + // workload is reconciled as trusted), "true" otherwise. + ConditionTypeUntrustedMode = "UntrustedMode" ) const ( @@ -66,6 +72,18 @@ const ( // "wait for convergence". ConditionReasonUntrustedPolicyInvalid = "UntrustedEgressPolicyInvalid" + // ConditionReasonUntrustedModeDisabled indicates spec.untrusted is true but + // the operator runs with untrusted mode disabled + // (TOOLHIVE_ENABLE_UNTRUSTED_MODE unset/false): the workload is reconciled + // as a normal trusted workload — no per-session pods, no egress broker, + // and secretKeyRef backend env is admitted. + ConditionReasonUntrustedModeDisabled = "UntrustedModeDisabled" + + // ConditionReasonUntrustedModeEnabled indicates spec.untrusted is true and + // the operator runs with untrusted mode enabled; the workload gets the + // full single-tenant data plane. + ConditionReasonUntrustedModeEnabled = "UntrustedModeEnabled" + // ConditionReasonPermissionProfileInvalid indicates a terminal rejection of // a trusted workload's spec.permissionProfile for egress NetworkPolicy // rendering (unknown builtin name, missing ConfigMap/key, malformed profile diff --git a/cmd/thv-operator/controllers/mcpserver_controller.go b/cmd/thv-operator/controllers/mcpserver_controller.go index 35e1648f03..b1d1a58f59 100644 --- a/cmd/thv-operator/controllers/mcpserver_controller.go +++ b/cmd/thv-operator/controllers/mcpserver_controller.go @@ -46,6 +46,7 @@ import ( "github.com/stacklok/toolhive/pkg/container/kubernetes" "github.com/stacklok/toolhive/pkg/transport" "github.com/stacklok/toolhive/pkg/transport/session" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) // MCPServerReconciler reconciles a MCPServer object @@ -249,6 +250,11 @@ func (r *MCPServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( r.validateSessionStorageForReplicas(ctx, mcpServer) r.validateRateLimitConfig(ctx, mcpServer) + // Surface the effective untrusted-mode posture for spec.untrusted=true + // workloads (disabled ⇒ reconciled as trusted, with a Warning event). + // Advisory only; never blocks the reconcile. + r.surfaceUntrustedModeCondition(ctx, mcpServer) + // Validate PodTemplateSpec early - before other validations // This ensures we fail fast if the spec is invalid if !r.validateAndUpdatePodTemplateStatus(ctx, mcpServer) { @@ -1185,10 +1191,87 @@ func logOBOSecretEnvVarError(ctx context.Context, err error) { "see the referenced MCPExternalAuthConfig status for details") } -// isUntrusted reports whether the workload must be treated as untrusted. -// Wave 1: reads the spec field (the interim Wave-0 annotation is removed). +// isUntrusted reports whether the workload must be treated as untrusted: +// spec.untrusted is set AND untrusted mode is enabled for this operator +// process (untrusted.ModeEnabled, the TOOLHIVE_ENABLE_UNTRUSTED_MODE env +// gate). Single-tenant untrusted mode (one backend pod per user/session plus +// its sidecar data plane) is too costly to be the default, so it is opt-in. +// When the flag is off a spec.untrusted=true workload is reconciled as a +// normal trusted workload; the reconciler surfaces that degradation on the +// UntrustedMode condition (see surfaceUntrustedModeCondition). Every +// untrusted-mode behavior — data-plane resources, the env gate, sentinel +// injection, groupRef fronting validation — keys on this one check. func isUntrusted(m *mcpv1beta1.MCPServer) bool { - return m.Spec.Untrusted + return m.Spec.Untrusted && untrusted.ModeEnabled() +} + +// surfaceUntrustedModeCondition reports the effective untrusted-mode posture +// for spec.untrusted=true workloads on the UntrustedMode condition: "false" +// with ReasonUntrustedModeDisabled (plus a one-shot Warning event on entry +// into the disabled state) when the mode is off, "true" when it is on. It +// also clears the condition when spec.untrusted flips back to false, so the +// status never claims an untrusted posture for a trusted workload. Advisory +// only — the reconcile continues as trusted; this never fails the workload. +func (r *MCPServerReconciler) surfaceUntrustedModeCondition(ctx context.Context, mcpServer *mcpv1beta1.MCPServer) { + if !mcpServer.Spec.Untrusted { + if meta.FindStatusCondition(mcpServer.Status.Conditions, mcpv1beta1.ConditionTypeUntrustedMode) != nil { + meta.RemoveStatusCondition(&mcpServer.Status.Conditions, mcpv1beta1.ConditionTypeUntrustedMode) + if err := r.Status().Update(ctx, mcpServer); err != nil { + log.FromContext(ctx).Error(err, "Failed to clear UntrustedMode condition after untrusted→trusted flip") + } + } + return + } + + enabled := isUntrusted(mcpServer) + wasDisabled := meta.IsStatusConditionFalse(mcpServer.Status.Conditions, mcpv1beta1.ConditionTypeUntrustedMode) + status := func() metav1.ConditionStatus { + if enabled { + return metav1.ConditionTrue + } + return metav1.ConditionFalse + }() + reason := func() string { + if enabled { + return mcpv1beta1.ConditionReasonUntrustedModeEnabled + } + return mcpv1beta1.ConditionReasonUntrustedModeDisabled + }() + message := func() string { + if enabled { + return "spec.untrusted is true and untrusted mode is enabled; " + + "the workload gets the single-tenant data plane (per-session pods, egress broker)" + } + return "spec.untrusted is true but untrusted mode is disabled (" + + untrusted.EnvEnableUntrustedMode + " is not set on the operator); " + + "the workload is reconciled as a trusted workload: no per-session pods, no egress broker, " + + "and secretKeyRef backend env is admitted" + }() + meta.SetStatusCondition(&mcpServer.Status.Conditions, metav1.Condition{ + Type: mcpv1beta1.ConditionTypeUntrustedMode, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: mcpServer.Generation, + }) + // legacy r.Status().Update call site (see .claude/rules/operator.md + // "Status Writes") — consistent with the other advisory validators in this + // reconciler; SetStatusCondition's no-op-on-unchanged semantics make the + // write idempotent on re-observation. + if err := r.Status().Update(ctx, mcpServer); err != nil { + log.FromContext(ctx).Error(err, "Failed to update MCPServer status after untrusted-mode posture check") + return + } + + // Emit the Warning only on the transition into the disabled state, and + // only once the condition persisted — a failing status write would + // otherwise re-fire the event on every reconcile. + if !enabled && !wasDisabled && r.Recorder != nil { + r.Recorder.Eventf(mcpServer, nil, corev1.EventTypeWarning, + mcpv1beta1.ConditionReasonUntrustedModeDisabled, "UntrustedMode", + "spec.untrusted is true but untrusted mode is disabled on the operator "+ + "(%s); reconciling as a trusted workload", untrusted.EnvEnableUntrustedMode) + } } // untrustedGateSuffix documents the gate's semantics on every rejection @@ -1351,7 +1434,9 @@ func (r *MCPServerReconciler) deploymentForMCPServer( // credentialEnvName so token-requiring servers boot (ADR-0001). Literal values // pass the Wave-0 gate above by construction. Sentinel collision and forgery // rejections are terminal SpecValidationErrors, same pattern as the gate. - if m.Spec.Untrusted && finalPodTemplateSpec != nil { + // isUntrusted (not m.Spec.Untrusted): when the mode is disabled the workload + // is reconciled as trusted and no sentinels are injected. + if isUntrusted(m) && finalPodTemplateSpec != nil { if err := ctrlutil.InjectUntrustedSentinels(finalPodTemplateSpec, mcpContainerName, m.Spec.EgressPolicy); err != nil { return nil, &SpecValidationError{Message: err.Error()} } diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go index 2b88db8603..a53fb59d21 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go @@ -29,11 +29,17 @@ import ( "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1/v1beta1test" "github.com/stacklok/toolhive/cmd/thv-operator/internal/testutil" ctrlutil "github.com/stacklok/toolhive/cmd/thv-operator/pkg/controllerutil" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) // withUntrustedSpec marks the MCPServer fixture as untrusted via the -// Wave-1 spec.untrusted field. -func withUntrustedSpec() v1beta1test.MCPServerOption { +// Wave-1 spec.untrusted field and enables untrusted mode for the test +// process (TOOLHIVE_ENABLE_UNTRUSTED_MODE) — the mode is opt-in and +// isUntrusted requires both, so untrusted fixtures must also flip the flag. +// t.Setenv auto-restores, but it serializes the calling test (no t.Parallel). +func withUntrustedSpec(t *testing.T) v1beta1test.MCPServerOption { + t.Helper() + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") return func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true } @@ -41,10 +47,13 @@ func withUntrustedSpec() v1beta1test.MCPServerOption { // withUntrustedCompliantPolicy adds a minimal valid EgressPolicy (hostname // destination; reconcile-path tests stub untrustedDNSLookup so no real DNS -// resolution happens). Wave 3: untrusted servers without an EgressPolicy -// terminate at the egress-resource gate, so fixtures that must proceed down -// the normal reconcile path need this. -func withUntrustedCompliantPolicy() v1beta1test.MCPServerOption { +// resolution happens) and enables untrusted mode for the test. Wave 3: +// untrusted servers without an EgressPolicy terminate at the egress-resource +// gate, so fixtures that must proceed down the normal reconcile path need +// this. +func withUntrustedCompliantPolicy(t *testing.T) v1beta1test.MCPServerOption { + t.Helper() + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") return v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{ @@ -62,18 +71,39 @@ func withSecrets(secrets ...mcpv1beta1.SecretRef) v1beta1test.MCPServerOption { } } -// TestIsUntrusted pins the Wave-1 body of isUntrusted: it reads spec.untrusted -// and nothing else — in particular the interim Wave-0 annotation is inert. +// TestIsUntrusted pins the isUntrusted gate: spec.untrusted AND the +// TOOLHIVE_ENABLE_UNTRUSTED_MODE env flag must both be on — the mode is +// opt-in, so the spec field alone is inert while the operator runs with the +// mode disabled. The interim Wave-0 annotation is inert either way. +// t.Setenv serializes the subtests (no t.Parallel). +// +//nolint:paralleltest // t.Setenv modifies the process environment. func TestIsUntrusted(t *testing.T) { - t.Parallel() + t.Run("mode enabled", func(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") + + assert.False(t, isUntrusted(v1beta1test.NewMCPServer("plain", "default"))) + assert.True(t, isUntrusted(v1beta1test.NewMCPServer("flagged", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true })))) + + annotated := v1beta1test.NewMCPServer("annotated", "default", v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Annotations = map[string]string{"toolhive.stacklok.dev/untrusted": "true"} + })) + assert.False(t, isUntrusted(annotated), "the interim Wave-0 annotation must have no effect") + }) - assert.False(t, isUntrusted(v1beta1test.NewMCPServer("plain", "default"))) - assert.True(t, isUntrusted(v1beta1test.NewMCPServer("flagged", "default", withUntrustedSpec()))) + t.Run("mode disabled treats spec.untrusted=true as trusted", func(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + assert.False(t, isUntrusted(v1beta1test.NewMCPServer("flagged", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true })))) + }) - annotated := v1beta1test.NewMCPServer("annotated", "default", v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { - m.Annotations = map[string]string{"toolhive.stacklok.dev/untrusted": "true"} - })) - assert.False(t, isUntrusted(annotated), "the interim Wave-0 annotation must have no effect in Wave 1") + t.Run("mode unset defaults to disabled", func(t *testing.T) { + // No t.Setenv: the absent-env default is OFF. + assert.False(t, untrusted.ModeEnabled(), "fixture precondition: the env var must be unset") + assert.False(t, isUntrusted(v1beta1test.NewMCPServer("flagged", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true })))) + }) } // setupUntrustedReconciler builds a fake-client-backed reconciler for the @@ -134,11 +164,10 @@ func stubUntrustedDNSOnce(t *testing.T) { }) } +//nolint:paralleltest // withUntrustedSpec calls t.Setenv (serializes the test). func TestMCPServerReconciler_UntrustedSecretEnvRejected(t *testing.T) { - t.Parallel() - mcpServer := v1beta1test.NewMCPServer("untrusted-secrets", "default", - withUntrustedSpec(), + withUntrustedSpec(t), withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), ) @@ -252,12 +281,11 @@ func TestMCPServerReconciler_InterimAnnotationIsInert(t *testing.T) { "annotated-but-trusted reconcile should continue down the normal path") } +//nolint:paralleltest // withUntrustedSpec calls t.Setenv (serializes the test). func TestMCPServerReconciler_UntrustedRawTemplateSecretEnvRejected(t *testing.T) { - t.Parallel() - // #13: spec.untrusted + raw podTemplateSpec smuggling secretKeyRef onto the mcp container. mcpServer := v1beta1test.NewMCPServer("untrusted-raw-template", "default", - withUntrustedSpec(), + withUntrustedSpec(t), v1beta1test.WithPodTemplateSpec(&runtime.RawExtension{ Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"API_TOKEN","valueFrom":{"secretKeyRef":{"name":"smuggled-secret","key":"token"}}}]}]}}`), }), @@ -283,13 +311,12 @@ func TestMCPServerReconciler_UntrustedRawTemplateSecretEnvRejected(t *testing.T) }, 2*time.Second, 10*time.Millisecond, "expected one Warning event on the invalid transition") } +//nolint:paralleltest // withUntrustedCompliantPolicy calls t.Setenv (serializes the test). func TestMCPServerReconciler_UntrustedCompliantDeploys(t *testing.T) { - t.Parallel() - // #15: untrusted but compliant (literal env only) — // the gate passes and the reconcile proceeds normally. mcpServer := v1beta1test.NewMCPServer("untrusted-compliant", "default", - withUntrustedCompliantPolicy(), + withUntrustedCompliantPolicy(t), v1beta1test.WithEnv(mcpv1beta1.EnvVar{Name: "SENTINEL", Value: "literal-value"}), v1beta1test.WithPodTemplateSpec(&runtime.RawExtension{ Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"LITERAL","value":"ok"}]}]}}`), @@ -318,14 +345,14 @@ func TestMCPServerReconciler_UntrustedCompliantDeploys(t *testing.T) { // 3. Re-breaking the spec latches Valid=False again AND the one-shot Warning // fires a second time — proving the pass path genuinely un-poisons the // latch instead of leaving the workload permanently silenced. +// +//nolint:paralleltest // withUntrustedCompliantPolicy calls t.Setenv (serializes the test). func TestMCPServerReconciler_UntrustedLatchClearsAndWarningReArms(t *testing.T) { - t.Parallel() - ctx := log.IntoContext(t.Context(), log.Log) // Start REJECTED: untrusted + spec.secrets. mcpServer := v1beta1test.NewMCPServer("untrusted-latch", "default", - withUntrustedCompliantPolicy(), + withUntrustedCompliantPolicy(t), withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), ) r, recorder := setupUntrustedReconciler(t, mcpServer) @@ -373,13 +400,12 @@ func TestMCPServerReconciler_UntrustedLatchClearsAndWarningReArms(t *testing.T) "the Warning must fire on the second invalid transition (latch-poisoning case)") } +//nolint:paralleltest // withUntrustedSpec calls t.Setenv (serializes the test). func TestDeploymentForMCPServer_UntrustedSecretEnvRejectedAtBuildTime(t *testing.T) { - t.Parallel() - // Defense-in-depth: deploymentForMCPServer itself rejects the built patch // for untrusted workloads (spec.secrets seam)... mcpServer := v1beta1test.NewMCPServer("untrusted-build-gate", "default", - withUntrustedSpec(), + withUntrustedSpec(t), withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), ) @@ -411,11 +437,11 @@ func TestDeploymentForMCPServer_UntrustedSecretEnvRejectedAtBuildTime(t *testing // seam in deploymentForMCPServer: for an untrusted workload with a declared // credentialEnvName, the --k8s-pod-patch argument carries the literal sentinel // env var on the mcp container, and the pod patch still passes the Wave-0 gate. +// +//nolint:paralleltest // withUntrustedSpec calls t.Setenv (serializes the test). func TestDeploymentForMCPServer_UntrustedSentinelInjection(t *testing.T) { - t.Parallel() - mcpServer := v1beta1test.NewMCPServer("untrusted-sentinel", "default", - withUntrustedSpec(), + withUntrustedSpec(t), v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{{ Provider: "github", @@ -446,11 +472,11 @@ func TestDeploymentForMCPServer_UntrustedSentinelInjection(t *testing.T) { // TestDeploymentForMCPServer_UntrustedSentinelCollision pins the terminal rejection // when a declared credentialEnvName collides with user-declared env. +// +//nolint:paralleltest // withUntrustedSpec calls t.Setenv (serializes the test). func TestDeploymentForMCPServer_UntrustedSentinelCollision(t *testing.T) { - t.Parallel() - mcpServer := v1beta1test.NewMCPServer("untrusted-collision", "default", - withUntrustedSpec(), + withUntrustedSpec(t), v1beta1test.WithEnv(mcpv1beta1.EnvVar{Name: "GITHUB_TOKEN", Value: "user-value"}), v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.PodTemplateSpec = &runtime.RawExtension{ @@ -478,11 +504,11 @@ func TestDeploymentForMCPServer_UntrustedSentinelCollision(t *testing.T) { // TestDeploymentForMCPServer_UntrustedSentinelForgery pins the terminal rejection // of a user-forged sentinel literal in the raw pod template. +// +//nolint:paralleltest // withUntrustedSpec calls t.Setenv (serializes the test). func TestDeploymentForMCPServer_UntrustedSentinelForgery(t *testing.T) { - t.Parallel() - mcpServer := v1beta1test.NewMCPServer("untrusted-forgery", "default", - withUntrustedSpec(), + withUntrustedSpec(t), v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.PodTemplateSpec = &runtime.RawExtension{ Raw: []byte(`{"spec":{"containers":[{"name":"mcp","env":[{"name":"FORGED","value":"thv-untrusted-sentinel:attacker"}]}]}}`), @@ -562,8 +588,12 @@ func podPatchFromDeployment(t *testing.T, deployment *appsv1.Deployment) *corev1 // has no fronting VirtualMCPServer gets GroupRefValidated=False with the // dedicated reason; the same workload reports valid once a vMCP fronts the // group. Trusted workloads in the same un-fronted group are unaffected. +// +//nolint:paralleltest // t.Setenv at the parent serializes the subtests' env view. func TestMCPServerReconciler_UntrustedGroupRefNotVMCPFronted(t *testing.T) { - t.Parallel() + // Untrusted mode must be ON for the untrusted groupRef check to arm; set it + // once for the whole test (parent-level t.Setenv covers all subtests). + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") readyGroup := func(name string) *mcpv1beta1.MCPGroup { return &mcpv1beta1.MCPGroup{ @@ -573,15 +603,21 @@ func TestMCPServerReconciler_UntrustedGroupRefNotVMCPFronted(t *testing.T) { } untrustedInGroup := func(name, group string) *mcpv1beta1.MCPServer { return v1beta1test.NewMCPServer(name, "default", - withUntrustedCompliantPolicy(), v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.Untrusted = true + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{ + Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + }}, + } m.Spec.GroupRef = &mcpv1beta1.MCPGroupRef{Name: group} }), ) } t.Run("untrusted workload in an un-fronted group gets the dedicated condition", func(t *testing.T) { - t.Parallel() + group := readyGroup("lonely-group") mcpServer := untrustedInGroup("untrusted-no-front", "lonely-group") r, _ := setupUntrustedReconciler(t, group, mcpServer) @@ -599,7 +635,7 @@ func TestMCPServerReconciler_UntrustedGroupRefNotVMCPFronted(t *testing.T) { }) t.Run("untrusted workload in a vMCP-fronted group validates", func(t *testing.T) { - t.Parallel() + group := readyGroup("fronted-group") vmcp := v1beta1test.NewVirtualMCPServer("front", "default", v1beta1test.WithVMCPGroupRef("fronted-group")) mcpServer := untrustedInGroup("untrusted-fronted", "fronted-group") @@ -616,7 +652,7 @@ func TestMCPServerReconciler_UntrustedGroupRefNotVMCPFronted(t *testing.T) { }) t.Run("trusted workload in an un-fronted group is unaffected", func(t *testing.T) { - t.Parallel() + group := readyGroup("trusted-group") mcpServer := v1beta1test.NewMCPServer("trusted-no-front", "default", v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { @@ -635,3 +671,187 @@ func TestMCPServerReconciler_UntrustedGroupRefNotVMCPFronted(t *testing.T) { "the vMCP-front requirement applies to untrusted workloads only") }) } + +// TestMCPServerReconciler_UntrustedModeDisabledReconcilesAsTrusted pins the +// feature-flag behavior: with TOOLHIVE_ENABLE_UNTRUSTED_MODE off, a +// spec.untrusted=true workload reconciles as a normal trusted workload — the +// secretKeyRef env gate does NOT fire, no untrusted data-plane resources are +// created — and the degradation is surfaced on the UntrustedMode condition +// (False/UntrustedModeDisabled) plus a one-shot Warning event. +// +//nolint:paralleltest // t.Setenv modifies the process environment. +func TestMCPServerReconciler_UntrustedModeDisabledReconcilesAsTrusted(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + + mcpServer := v1beta1test.NewMCPServer("untrusted-flag-off", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.Untrusted = true + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{ + Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + }}, + } + }), + withSecrets(mcpv1beta1.SecretRef{Name: "backend-creds", Key: "token", TargetEnvName: "API_TOKEN"}), + ) + + r, recorder := setupUntrustedReconciler(t, mcpServer) + + // The reconcile must continue down the normal trusted path (requeue waiting + // on the runconfig ConfigMap) — it must NOT fail the workload. + result := reconcileOnce(t, r, mcpServer) + assert.False(t, result.IsZero(), "flag-off untrusted workload must reconcile as trusted, not terminate") + + updated := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), updated)) + + // The secretKeyRef gate must NOT have fired: the flag-off workload is + // trusted and trusted workloads may source backend env from Secrets. + assert.Nil(t, meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionTypeValid), + "the untrusted env gate must not arm while the mode is disabled") + assert.NotEqual(t, mcpv1beta1.MCPServerPhaseFailed, updated.Status.Phase) + + // The degradation is surfaced on the UntrustedMode condition. + cond := meta.FindStatusCondition(updated.Status.Conditions, mcpv1beta1.ConditionTypeUntrustedMode) + require.NotNil(t, cond, "UntrustedMode condition must be set") + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, mcpv1beta1.ConditionReasonUntrustedModeDisabled, cond.Reason) + assert.Equal(t, updated.Generation, cond.ObservedGeneration) + assert.Contains(t, cond.Message, untrusted.EnvEnableUntrustedMode) + + // ...and with a one-shot Warning event on the transition. + require.Eventually(t, func() bool { + return countContaining(drainEvents(recorder), mcpv1beta1.ConditionReasonUntrustedModeDisabled) == 1 + }, 2*time.Second, 10*time.Millisecond, "expected one Warning event on the disabled transition") + + // No untrusted data-plane resources may exist (the ensure call deletes them). + assert.Empty(t, listCASecrets(t, r, mcpServer), "no bump CA Secrets may be created while the mode is disabled") + + // Idempotency: a second reconcile must not churn the condition or re-fire + // the event. + result = reconcileOnce(t, r, mcpServer) + afterSecond := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(t.Context(), client.ObjectKeyFromObject(mcpServer), afterSecond)) + condAfter := meta.FindStatusCondition(afterSecond.Status.Conditions, mcpv1beta1.ConditionTypeUntrustedMode) + require.NotNil(t, condAfter) + assert.Equal(t, *cond, *condAfter, "UntrustedMode condition must not churn on re-observe") + assert.False(t, result.IsZero()) + time.Sleep(50 * time.Millisecond) // let any async event emission settle + assert.Empty(t, drainEvents(recorder), "no event expected while the mode stays disabled") +} + +// TestMCPServerReconciler_UntrustedModeDisabledSentinelsNotInjected pins that +// flag-off untrusted workloads build a Deployment without sentinel env: the +// sentinel seam keys on isUntrusted, which is false while the mode is off. +// +//nolint:paralleltest // t.Setenv modifies the process environment. +func TestMCPServerReconciler_UntrustedModeDisabledSentinelsNotInjected(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + + mcpServer := v1beta1test.NewMCPServer("untrusted-flag-off-build", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.Untrusted = true + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + CredentialEnvName: "GITHUB_TOKEN", + }}} + }), + ) + + r, _ := setupUntrustedReconciler(t, mcpServer) + ctx := log.IntoContext(t.Context(), log.Log) + + deployment, err := r.deploymentForMCPServer(ctx, mcpServer, "test-checksum") + require.NoError(t, err) + require.NotNil(t, deployment) + + patch := podPatchFromDeployment(t, deployment) + for _, c := range patch.Spec.Containers { + for _, env := range c.Env { + assert.NotContains(t, env.Value, "thv-untrusted-sentinel:", + "flag-off untrusted workloads must never receive sentinel env") + } + } +} + +// TestMCPServerReconciler_SurfaceUntrustedModeConditionLifecycle pins the +// condition lifecycle independent of the reconcile path: enabled → True, +// disabled → False + Warning, spec.untrusted=false → cleared. +// +//nolint:paralleltest // t.Setenv modifies the process environment. +func TestMCPServerReconciler_SurfaceUntrustedModeConditionLifecycle(t *testing.T) { + ctx := log.IntoContext(t.Context(), log.Log) + + find := func(r *MCPServerReconciler, key client.ObjectKey) *metav1.Condition { + m := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(ctx, key, m)) + return meta.FindStatusCondition(m.Status.Conditions, mcpv1beta1.ConditionTypeUntrustedMode) + } + + t.Run("enabled reports True", func(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") + mcpServer := v1beta1test.NewMCPServer("mode-on", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true })) + r, recorder := setupUntrustedReconciler(t, mcpServer) + + r.surfaceUntrustedModeCondition(ctx, mcpServer) + + cond := find(r, client.ObjectKeyFromObject(mcpServer)) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionTrue, cond.Status) + assert.Equal(t, mcpv1beta1.ConditionReasonUntrustedModeEnabled, cond.Reason) + time.Sleep(50 * time.Millisecond) + assert.Empty(t, drainEvents(recorder), "no Warning when the mode is enabled") + }) + + t.Run("spec.untrusted=false clears the condition", func(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + mcpServer := v1beta1test.NewMCPServer("mode-cleared", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true })) + r, _ := setupUntrustedReconciler(t, mcpServer) + + r.surfaceUntrustedModeCondition(ctx, mcpServer) + require.NotNil(t, find(r, client.ObjectKeyFromObject(mcpServer)), "precondition: condition latched") + + // Flip to trusted; the condition must be removed. + current := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(mcpServer), current)) + current.Spec.Untrusted = false + require.NoError(t, r.Update(ctx, current)) + r.surfaceUntrustedModeCondition(ctx, current) + + assert.Nil(t, find(r, client.ObjectKeyFromObject(mcpServer)), + "UntrustedMode condition must clear when spec.untrusted flips to false") + }) + + t.Run("warning re-arms after the condition clears", func(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + mcpServer := v1beta1test.NewMCPServer("mode-rearm", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true })) + r, recorder := setupUntrustedReconciler(t, mcpServer) + + r.surfaceUntrustedModeCondition(ctx, mcpServer) + require.Eventually(t, func() bool { + return countContaining(drainEvents(recorder), mcpv1beta1.ConditionReasonUntrustedModeDisabled) == 1 + }, 2*time.Second, 10*time.Millisecond) + + // untrusted→trusted→untrusted: the Warning must fire again on the + // second disabled transition. + current := &mcpv1beta1.MCPServer{} + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(mcpServer), current)) + current.Spec.Untrusted = false + require.NoError(t, r.Update(ctx, current)) + r.surfaceUntrustedModeCondition(ctx, current) + + require.NoError(t, r.Get(ctx, client.ObjectKeyFromObject(mcpServer), current)) + current.Spec.Untrusted = true + require.NoError(t, r.Update(ctx, current)) + r.surfaceUntrustedModeCondition(ctx, current) + + require.Eventually(t, func() bool { + return countContaining(drainEvents(recorder), mcpv1beta1.ConditionReasonUntrustedModeDisabled) == 1 + }, 2*time.Second, 10*time.Millisecond, "the Warning must fire on the second disabled transition") + }) +} diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go index 4889c65f19..af8a41e184 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go @@ -22,9 +22,13 @@ import ( mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1/v1beta1test" "github.com/stacklok/toolhive/pkg/egressbroker" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) func withEgressPolicy() v1beta1test.MCPServerOption { + // The calling test must enable untrusted mode (t.Setenv on + // untrusted.EnvEnableUntrustedMode) or isUntrusted treats the server as + // trusted and ensureUntrustedResources deletes instead of creates. return v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{ @@ -92,9 +96,12 @@ func listCASecrets(t *testing.T, r *MCPServerReconciler, m *mcpv1beta1.MCPServer return secrets.Items } -//nolint:tparallel // Subtests swap the package-level DNS lookup stub; they must run serially. +//nolint:paralleltest // t.Setenv + package-level DNS stub: the test and its subtests run serially. func TestEnsureUntrustedResources(t *testing.T) { - t.Parallel() + // Untrusted mode must be ON for ensureUntrustedResources to create instead + // of delete; set it once for the whole test (parent-level t.Setenv is + // inherited by all serial subtests). + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") //nolint:paralleltest // Swaps the package-level DNS lookup stub (restored after each ensure). t.Run("untrusted MCPServer gets generation-named CA Secret, bundle, policy ConfigMap, NetworkPolicy", func(t *testing.T) { @@ -232,7 +239,8 @@ func TestEnsureUntrustedResources(t *testing.T) { //nolint:paralleltest // Serial with sibling subtests that swap the DNS lookup stub. t.Run("missing EgressPolicy on untrusted server is a terminal spec error", func(t *testing.T) { - m := v1beta1test.NewMCPServer("github-mcp", "default", withUntrustedSpec()) + m := v1beta1test.NewMCPServer("github-mcp", "default", + v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { m.Spec.Untrusted = true })) r, _ := setupUntrustedReconciler(t, m) err := r.ensureUntrustedResources(t.Context(), m) @@ -440,16 +448,25 @@ func errorAsSpecValidation(err error, target **SpecValidationError) bool { // flows to the backend StatefulSet (DeployWorkload applies ContainerLabels), // which is how the vMCP pod lifecycle resolves the clone template by selector // (LabelMCPServerUID + toolhive=true). A trusted MCPServer must NOT carry it. +// +//nolint:paralleltest // t.Setenv at the parent serializes the subtests' env view. func TestCreateRunConfig_UntrustedUIDLabel(t *testing.T) { - t.Parallel() + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") newServer := func(opts ...v1beta1test.MCPServerOption) *mcpv1beta1.MCPServer { return v1beta1test.NewMCPServer("github-mcp", "default", opts...) } t.Run("untrusted server stamps the mcpserver-uid container label", func(t *testing.T) { - t.Parallel() - m := newServer(withUntrustedCompliantPolicy()) + m := newServer(v1beta1test.Mutate(func(m *mcpv1beta1.MCPServer) { + m.Spec.Untrusted = true + m.Spec.EgressPolicy = &mcpv1beta1.EgressPolicy{ + Providers: []mcpv1beta1.ProviderEgress{{ + Provider: "github", + AllowedHosts: []string{"api.github.com"}, + }}, + } + })) r, _ := setupUntrustedReconciler(t, m) rc, err := r.createRunConfigFromMCPServer(m) require.NoError(t, err) @@ -458,7 +475,6 @@ func TestCreateRunConfig_UntrustedUIDLabel(t *testing.T) { }) t.Run("trusted server does not stamp the label", func(t *testing.T) { - t.Parallel() m := newServer() r, _ := setupUntrustedReconciler(t, m) rc, err := r.createRunConfigFromMCPServer(m) @@ -513,8 +529,9 @@ func assertDNSRuleRestricted(t *testing.T, np *networkingv1.NetworkPolicy) { // operator-resolved dialAllowlist the broker's D7 guard enforces (the env // override THV_EGRESSBROKER_DIAL_ALLOWLIST is unset in the clone wiring). // -//nolint:paralleltest // Swaps the package-level DNS lookup stub (restored after the ensure call). +//nolint:paralleltest // t.Setenv + swaps the package-level DNS lookup stub (restored after the ensure call). func TestOperatorPolicyContractWithSidecar(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) stubEgressDNS(t, githubDNS) r, _ := setupUntrustedReconciler(t, m) @@ -808,3 +825,63 @@ func TestRenderTrustedEgressNetworkPolicy(t *testing.T) { np = renderTrustedEgressNetworkPolicy(m, []string{"140.82.114.26/32"}, nil) require.Len(t, np.Spec.Egress, 3) } + +// TestEnsureUntrustedResources_ModeDisabled pins the flag-off behavior of the +// data-plane gate: with TOOLHIVE_ENABLE_UNTRUSTED_MODE off, a +// spec.untrusted=true MCPServer gets NO untrusted resources (the ensure call +// takes the delete path), while the trusted-mode egress NetworkPolicy keeps +// working — it has no single-tenant cost and is independent of the flag. +// +//nolint:paralleltest // t.Setenv + DNS stub serialization. +func TestEnsureUntrustedResources_ModeDisabled(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + + t.Run("no untrusted resources are created while the mode is disabled", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + r, _ := setupUntrustedReconciler(t, m) + + require.NoError(t, r.ensureUntrustedResources(t.Context(), m)) + + assert.Empty(t, listCASecrets(t, r, m), "no bump CA Secrets while the mode is disabled") + policyCM := &corev1.ConfigMap{} + err := r.Get(t.Context(), types.NamespacedName{Name: "github-mcp-egress-policy", Namespace: "default"}, policyCM) + assert.True(t, apierrors.IsNotFound(err), "no egress-policy ConfigMap while the mode is disabled") + np := &networkingv1.NetworkPolicy{} + err = r.Get(t.Context(), types.NamespacedName{Name: "github-mcp-egress", Namespace: "default"}, np) + assert.True(t, apierrors.IsNotFound(err), "no untrusted NetworkPolicy while the mode is disabled") + }) + + t.Run("disabling the mode deletes previously-created untrusted resources", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withEgressPolicy()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m) + + // Create the resources with the mode ON, then flip the flag off: the + // next ensure must converge to trusted and delete them. + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") + require.NoError(t, r.ensureUntrustedResources(t.Context(), m)) + require.Len(t, listCASecrets(t, r, m), 1, "precondition: untrusted resources exist") + + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + require.NoError(t, r.ensureUntrustedResources(t.Context(), m)) + assert.Empty(t, listCASecrets(t, r, m), "flag-off ensure must delete the untrusted resources") + np := &networkingv1.NetworkPolicy{} + err := r.Get(t.Context(), types.NamespacedName{Name: "github-mcp-egress", Namespace: "default"}, np) + assert.True(t, apierrors.IsNotFound(err), "untrusted NetworkPolicy must be deleted on flag-off") + }) + + t.Run("trusted egress NetworkPolicy works with the mode disabled", func(t *testing.T) { + m := v1beta1test.NewMCPServer("github-mcp", "default", withTrustedConfigMapProfile()) + stubEgressDNS(t, githubDNS) + r, _ := setupUntrustedReconciler(t, m, trustedProfileConfigMap()) + + require.NoError(t, r.ensureTrustedEgressNetworkPolicy(t.Context(), m)) + + np := &networkingv1.NetworkPolicy{} + require.NoError(t, r.Get(t.Context(), types.NamespacedName{ + Name: "github-mcp-egress", Namespace: "default"}, np), + "the trusted-mode egress NetworkPolicy is independent of the untrusted-mode flag") + assert.Equal(t, labelsForMCPServer(m.Name), np.Spec.PodSelector.MatchLabels) + assert.NotContains(t, np.Labels, "toolhive.stacklok.dev/untrusted-resource") + }) +} diff --git a/cmd/thv-operator/controllers/virtualmcpserver_deployment.go b/cmd/thv-operator/controllers/virtualmcpserver_deployment.go index 0f714e7b3c..e9cf81566e 100644 --- a/cmd/thv-operator/controllers/virtualmcpserver_deployment.go +++ b/cmd/thv-operator/controllers/virtualmcpserver_deployment.go @@ -12,6 +12,7 @@ import ( "os" "path" "sort" + "strconv" "strings" appsv1 "k8s.io/api/apps/v1" @@ -31,6 +32,7 @@ import ( vmcptypes "github.com/stacklok/toolhive/pkg/vmcp" vmcpconfig "github.com/stacklok/toolhive/pkg/vmcp/config" "github.com/stacklok/toolhive/pkg/vmcp/headerforward/wirefmt" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" "github.com/stacklok/toolhive/pkg/vmcp/workloads" ) @@ -459,6 +461,16 @@ func (r *VirtualMCPServerReconciler) buildEnvVarsForVmcp( } env = append(env, tokenStoreEnv...) + // Forward the operator's untrusted-mode feature flag so one chart value + // (operator.features.untrustedMode) controls both processes. The vMCP's + // own TOOLHIVE_ENABLE_UNTRUSTED_MODE gates its untrusted stack + // (buildUntrustedStack + the discovery metadata stamp); the operator never + // stamps untrusted behavior the vMCP would then act on. + env = append(env, corev1.EnvVar{ + Name: untrusted.EnvEnableUntrustedMode, + Value: strconv.FormatBool(untrusted.ModeEnabled()), + }) + return ctrlutil.EnsureRequiredEnvVars(ctx, env), nil } @@ -477,6 +489,13 @@ func (r *VirtualMCPServerReconciler) buildUntrustedTokenStoreEnvVars( ctx context.Context, vmcp *mcpv1beta1.VirtualMCPServer, ) ([]corev1.EnvVar, error) { + // Untrusted mode disabled (the operator's own feature flag): the vMCP + // never provisions untrusted pods, so no sidecar ever consumes these + // coordinates — and the operator's isUntrusted gate means no backend + // carries untrusted-mode resources for them to reference. + if !untrusted.ModeEnabled() { + return nil, nil + } as := vmcp.Spec.AuthServerConfig if as == nil || as.Storage == nil || as.Storage.Type != mcpv1beta1.AuthServerStorageTypeRedis || diff --git a/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go b/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go index 48c70ab386..b396fa6d13 100644 --- a/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go +++ b/cmd/thv-operator/controllers/virtualmcpserver_rbac_untrusted_test.go @@ -18,6 +18,7 @@ import ( mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1/v1beta1test" "github.com/stacklok/toolhive/cmd/thv-operator/internal/testutil" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) // TestVmcpDiscoveredRBACRules_PodsRule pins the untrusted-mode pods grant: @@ -103,6 +104,10 @@ func untrustedRedisStorage(addr string, te *mcpv1beta1.TokenEncryptionConfig) *m // with objects. The status manager is nil: none of the address-only paths // touch it (only the Sentinel+tokenEncryption condition does, and that test // supplies a mock). +// +// The token-store env vars are untrusted-mode wiring, so every caller must +// enable the mode (t.Setenv) or buildUntrustedTokenStoreEnvVars short-circuits +// to empty. func newTokenStoreTestReconciler(t *testing.T, objects ...*corev1.Secret) *VirtualMCPServerReconciler { t.Helper() scheme := testutil.NewScheme(t) @@ -118,8 +123,10 @@ func newTokenStoreTestReconciler(t *testing.T, objects ...*corev1.Secret) *Virtu // address when the embedded auth server uses standalone/cluster Redis storage, // and nothing otherwise. The KEK values are never injected here — they stay // Secret-only and reach sidecars as SecretKeyRef envs resolved at clone time. +// +//nolint:paralleltest // t.Setenv at the parent serializes the subtests' env view. func TestBuildUntrustedTokenStoreEnvVars(t *testing.T) { - t.Parallel() + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") tests := []struct { name string @@ -159,7 +166,7 @@ func TestBuildUntrustedTokenStoreEnvVars(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - t.Parallel() + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", v1beta1test.WithVMCPAuthServerConfig(tc.authCfg)) r := newTokenStoreTestReconciler(t) @@ -188,8 +195,10 @@ func TestBuildUntrustedTokenStoreEnvVars(t *testing.T) { // seam: standalone Redis storage WITHOUT ACL password coordinates still // renders the address (the vMCP/broker fail loud downstream), but flags the // misconfiguration with a Warning event so it is never silently dropped. +// +//nolint:paralleltest // t.Setenv modifies the process environment. func TestBuildUntrustedTokenStoreEnvVars_PasswordMissing(t *testing.T) { - t.Parallel() + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") cfg := &mcpv1beta1.EmbeddedAuthServerConfig{ Storage: &mcpv1beta1.AuthServerStorageConfig{ @@ -227,15 +236,17 @@ func TestBuildUntrustedTokenStoreEnvVars_PasswordMissing(t *testing.T) { // read from the Secret) as plain literals — the vMCP turns them into one // SecretKeyRef env per key ID on every cloned sidecar, so the KEK values // themselves never appear in any pod spec. +// +//nolint:paralleltest // t.Setenv at the parent serializes the subtests' env view. func TestBuildUntrustedTokenStoreEnvVars_KEK(t *testing.T) { - t.Parallel() + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") redisStorage := func(te *mcpv1beta1.TokenEncryptionConfig) *mcpv1beta1.EmbeddedAuthServerConfig { return untrustedRedisStorage("redis.auth:6379", te) } t.Run("tokenEncryption set emits the KEK coordinate env vars (full key set)", func(t *testing.T) { - t.Parallel() + cfg := redisStorage(&mcpv1beta1.TokenEncryptionConfig{ ActiveKeyID: "kek-2", KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, @@ -270,7 +281,7 @@ func TestBuildUntrustedTokenStoreEnvVars_KEK(t *testing.T) { }) t.Run("tokenEncryption nil emits the address and password coordinate env vars", func(t *testing.T) { - t.Parallel() + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", v1beta1test.WithVMCPAuthServerConfig(redisStorage(nil))) r := newTokenStoreTestReconciler(t) @@ -281,7 +292,7 @@ func TestBuildUntrustedTokenStoreEnvVars_KEK(t *testing.T) { }) t.Run("memory storage with tokenEncryption emits nothing (CEL guards admission)", func(t *testing.T) { - t.Parallel() + // The CEL rule rejects this at admission; the builder must still not // emit KEK coordinates for non-Redis storage (defense in depth). cfg := &mcpv1beta1.EmbeddedAuthServerConfig{ @@ -302,7 +313,7 @@ func TestBuildUntrustedTokenStoreEnvVars_KEK(t *testing.T) { }) t.Run("unresolvable KEK Secret is an error (fail closed, coordinates dropped)", func(t *testing.T) { - t.Parallel() + cfg := redisStorage(&mcpv1beta1.TokenEncryptionConfig{ ActiveKeyID: "kek-1", KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, @@ -316,7 +327,7 @@ func TestBuildUntrustedTokenStoreEnvVars_KEK(t *testing.T) { }) t.Run("sentinel storage with tokenEncryption emits a Warning event and no env", func(t *testing.T) { - t.Parallel() + cfg := untrustedRedisStorage("", &mcpv1beta1.TokenEncryptionConfig{ ActiveKeyID: "kek-1", KeySecretRef: corev1.LocalObjectReference{Name: "my-vmcp-kek"}, @@ -344,3 +355,22 @@ func TestBuildUntrustedTokenStoreEnvVars_KEK(t *testing.T) { } }) } + +// TestBuildUntrustedTokenStoreEnvVars_ModeDisabled pins the flag-off behavior: +// with TOOLHIVE_ENABLE_UNTRUSTED_MODE off, no token-store coordinates reach +// the vMCP Deployment even when the embedded auth server uses standalone +// Redis storage — untrusted pods are never provisioned, so nothing consumes +// them. +// +//nolint:paralleltest // t.Setenv modifies the process environment. +func TestBuildUntrustedTokenStoreEnvVars_ModeDisabled(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + + vmcp := v1beta1test.NewVirtualMCPServer("test-vmcp", "default", + v1beta1test.WithVMCPAuthServerConfig(untrustedRedisStorage("redis.auth:6379", nil))) + r := newTokenStoreTestReconciler(t) + + env, err := r.buildUntrustedTokenStoreEnvVars(context.Background(), vmcp) + require.NoError(t, err) + assert.Empty(t, env, "token-store coordinates must not render while untrusted mode is disabled") +} diff --git a/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go b/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go index 2af3ea43cd..defb0152ba 100644 --- a/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go +++ b/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go @@ -7,6 +7,7 @@ package controllers import ( "context" + "os" "testing" . "github.com/onsi/ginkgo/v2" @@ -39,6 +40,14 @@ func TestMCPServerUntrusted(t *testing.T) { } var _ = BeforeSuite(func() { + // Untrusted mode is opt-in (TOOLHIVE_ENABLE_UNTRUSTED_MODE, default off); + // this suite exercises untrusted behavior end to end, so the operator + // process under test must run with the mode enabled. Ginkgo keeps the + // entire suite in one process, so setting the env var here (before the + // manager starts) is equivalent to the chart injecting it on the + // operator Deployment. + Expect(os.Setenv("TOOLHIVE_ENABLE_UNTRUSTED_MODE", "true")).To(Succeed()) + untrustedSuiteEnv = testutil.StartSuite(testutil.SuiteOptions{ RegisterGroupRefIndexers: true, }) diff --git a/deploy/charts/operator/README.md b/deploy/charts/operator/README.md index 456e8762a3..0f899796ff 100644 --- a/deploy/charts/operator/README.md +++ b/deploy/charts/operator/README.md @@ -46,7 +46,7 @@ The command removes all the Kubernetes components associated with the chart and |-----|------|---------|-------------| | fullnameOverride | string | `"toolhive-operator"` | Provide a fully-qualified name override for resources | | nameOverride | string | `""` | Override the name of the chart | -| operator | object | `{"affinity":{},"autoscaling":{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80},"containerSecurityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"defaultImagePullSecrets":[],"defaultRedis":{"addr":"","existingSecret":"","existingSecretKey":""},"env":[],"features":{"experimental":false,"storageVersionMigrator":true},"gc":{"gogc":75,"gomemlimit":"110MiB"},"image":"ghcr.io/stacklok/toolhive/operator:v0.40.1","imagePullPolicy":"IfNotPresent","imagePullSecrets":[],"leaderElectionRole":{"binding":{"name":"toolhive-operator-leader-election-rolebinding"},"name":"toolhive-operator-leader-election-role","rules":[{"apiGroups":[""],"resources":["configmaps"],"verbs":["get","list","watch","create","update","patch","delete"]},{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","list","watch","create","update","patch","delete"]},{"apiGroups":["events.k8s.io"],"resources":["events"],"verbs":["create","patch"]}]},"livenessProbe":{"httpGet":{"path":"/healthz","port":"health"},"initialDelaySeconds":15,"periodSeconds":20},"nodeSelector":{},"podAnnotations":{},"podLabels":{},"podSecurityContext":{"runAsNonRoot":true},"ports":[{"containerPort":8080,"name":"metrics","protocol":"TCP"},{"containerPort":8081,"name":"health","protocol":"TCP"}],"proxyHost":"0.0.0.0","rbac":{"allowedNamespaces":[],"scope":"cluster"},"readinessProbe":{"httpGet":{"path":"/readyz","port":"health"},"initialDelaySeconds":5,"periodSeconds":10},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"10m","memory":"64Mi"}},"serviceAccount":{"annotations":{},"automountServiceAccountToken":true,"create":true,"labels":{},"name":"toolhive-operator"},"tolerations":[],"toolhiveRunnerImage":"ghcr.io/stacklok/toolhive/proxyrunner:v0.40.1","vmcpImage":"ghcr.io/stacklok/toolhive/vmcp:v0.40.1","volumeMounts":[],"volumes":[]}` | All values for the operator deployment and associated resources | +| operator | object | `{"affinity":{},"autoscaling":{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80},"containerSecurityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"defaultImagePullSecrets":[],"defaultRedis":{"addr":"","existingSecret":"","existingSecretKey":""},"env":[],"features":{"experimental":false,"storageVersionMigrator":true,"untrustedMode":false},"gc":{"gogc":75,"gomemlimit":"110MiB"},"image":"ghcr.io/stacklok/toolhive/operator:v0.40.1","imagePullPolicy":"IfNotPresent","imagePullSecrets":[],"leaderElectionRole":{"binding":{"name":"toolhive-operator-leader-election-rolebinding"},"name":"toolhive-operator-leader-election-role","rules":[{"apiGroups":[""],"resources":["configmaps"],"verbs":["get","list","watch","create","update","patch","delete"]},{"apiGroups":["coordination.k8s.io"],"resources":["leases"],"verbs":["get","list","watch","create","update","patch","delete"]},{"apiGroups":["events.k8s.io"],"resources":["events"],"verbs":["create","patch"]}]},"livenessProbe":{"httpGet":{"path":"/healthz","port":"health"},"initialDelaySeconds":15,"periodSeconds":20},"nodeSelector":{},"podAnnotations":{},"podLabels":{},"podSecurityContext":{"runAsNonRoot":true},"ports":[{"containerPort":8080,"name":"metrics","protocol":"TCP"},{"containerPort":8081,"name":"health","protocol":"TCP"}],"proxyHost":"0.0.0.0","rbac":{"allowedNamespaces":[],"scope":"cluster"},"readinessProbe":{"httpGet":{"path":"/readyz","port":"health"},"initialDelaySeconds":5,"periodSeconds":10},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"10m","memory":"64Mi"}},"serviceAccount":{"annotations":{},"automountServiceAccountToken":true,"create":true,"labels":{},"name":"toolhive-operator"},"tolerations":[],"toolhiveRunnerImage":"ghcr.io/stacklok/toolhive/proxyrunner:v0.40.1","vmcpImage":"ghcr.io/stacklok/toolhive/vmcp:v0.40.1","volumeMounts":[],"volumes":[]}` | All values for the operator deployment and associated resources | | operator.affinity | object | `{}` | Affinity settings for the operator pod | | operator.autoscaling | object | `{"enabled":false,"maxReplicas":100,"minReplicas":1,"targetCPUUtilizationPercentage":80}` | Configuration for horizontal pod autoscaling | | operator.autoscaling.enabled | bool | `false` | Enable autoscaling for the operator | @@ -62,6 +62,7 @@ The command removes all the Kubernetes components associated with the chart and | operator.env | list | `[]` | Environment variables to set in the operator container. Supported toolhive-specific variables include: - TOOLHIVE_SKIP_UPDATE_CHECK: set to "true" to disable the operator's periodic update check against the ToolHive update API. Also disables the usage-metrics collection that is gated on the same check. | | operator.features.experimental | bool | `false` | Enable experimental features | | operator.features.storageVersionMigrator | bool | `true` | Enable the StorageVersionMigrator controller, which auto-cleans status.storedVersions on opted-in toolhive.stacklok.dev CRDs so a future release can drop deprecated versions (e.g. v1alpha1) without orphaning etcd objects in the cluster. Enabled by default; set to false to opt out and handle storage-version cleanup yourself. Sets TOOLHIVE_ENABLE_STORAGE_VERSION_MIGRATOR in the operator deployment. Requires `operator.rbac.scope=cluster` — the controller watches cluster-scoped CRDs and re-stores resources across all namespaces, so the chart rejects this being true when scope is namespace. | +| operator.features.untrustedMode | bool | `false` | Enable untrusted single-tenant MCP server mode (one backend pod per (user, session, untrusted MCPServer) with an egress-broker data plane). Disabled by default: the per-user pod-per-session tenancy model carries a significant resource cost, so only operators who accept that cost and risk should enable it. When disabled, MCPServers with spec.untrusted=true are reconciled as normal trusted workloads and an UntrustedMode status condition plus a Warning event surface the degradation. Sets TOOLHIVE_ENABLE_UNTRUSTED_MODE on the operator deployment, which the operator forwards to every vMCP Deployment it creates. | | operator.gc | object | `{"gogc":75,"gomemlimit":"110MiB"}` | Go memory limits and garbage collection percentage for the operator container | | operator.gc.gogc | int | `75` | Go garbage collection percentage for the operator container | | operator.gc.gomemlimit | string | `"110MiB"` | Go memory limits for the operator container | diff --git a/deploy/charts/operator/templates/deployment.yaml b/deploy/charts/operator/templates/deployment.yaml index c83ba99db0..bd1013880a 100644 --- a/deploy/charts/operator/templates/deployment.yaml +++ b/deploy/charts/operator/templates/deployment.yaml @@ -68,6 +68,8 @@ spec: value: "true" - name: ENABLE_EXPERIMENTAL_FEATURES value: {{ .Values.operator.features.experimental | quote }} + - name: TOOLHIVE_ENABLE_UNTRUSTED_MODE + value: {{ .Values.operator.features.untrustedMode | quote }} - name: TOOLHIVE_ENABLE_STORAGE_VERSION_MIGRATOR value: {{ .Values.operator.features.storageVersionMigrator | quote }} {{- if eq .Values.operator.rbac.scope "namespace" }} diff --git a/deploy/charts/operator/tests/default_install_test.yaml b/deploy/charts/operator/tests/default_install_test.yaml index 440c0e9f4e..274d2ebde2 100644 --- a/deploy/charts/operator/tests/default_install_test.yaml +++ b/deploy/charts/operator/tests/default_install_test.yaml @@ -52,6 +52,7 @@ tests: template: deployment.yaml asserts: - contains: { path: 'spec.template.spec.containers[0].env', content: { name: ENABLE_EXPERIMENTAL_FEATURES, value: "false" } } + - contains: { path: 'spec.template.spec.containers[0].env', content: { name: TOOLHIVE_ENABLE_UNTRUSTED_MODE, value: "false" } } - contains: { path: 'spec.template.spec.containers[0].env', content: { name: TOOLHIVE_ENABLE_STORAGE_VERSION_MIGRATOR, value: "true" } } - contains: { path: 'spec.template.spec.containers[0].env', content: { name: UNSTRUCTURED_LOGS, value: "false" } } - contains: { path: 'spec.template.spec.containers[0].env', content: { name: TOOLHIVE_USE_CONFIGMAP, value: "true" } } diff --git a/deploy/charts/operator/tests/feature_flags_test.yaml b/deploy/charts/operator/tests/feature_flags_test.yaml index 11c1cbced7..7f5b23da38 100644 --- a/deploy/charts/operator/tests/feature_flags_test.yaml +++ b/deploy/charts/operator/tests/feature_flags_test.yaml @@ -8,11 +8,12 @@ suite: opt-in feature flags templates: - deployment.yaml tests: - - it: flips both feature-flag env vars to the quoted string "true" + - it: flips all feature-flag env vars to the quoted string "true" template: deployment.yaml set: operator.features.experimental: true operator.features.storageVersionMigrator: true + operator.features.untrustedMode: true asserts: - contains: path: 'spec.template.spec.containers[0].env' @@ -20,6 +21,16 @@ tests: - contains: path: 'spec.template.spec.containers[0].env' content: { name: TOOLHIVE_ENABLE_STORAGE_VERSION_MIGRATOR, value: "true" } + - contains: + path: 'spec.template.spec.containers[0].env' + content: { name: TOOLHIVE_ENABLE_UNTRUSTED_MODE, value: "true" } + + - it: defaults untrusted mode to the quoted string "false" (opt-in only) + template: deployment.yaml + asserts: + - contains: + path: 'spec.template.spec.containers[0].env' + content: { name: TOOLHIVE_ENABLE_UNTRUSTED_MODE, value: "false" } - it: opts out of the storage version migrator when set to false template: deployment.yaml diff --git a/deploy/charts/operator/values.yaml b/deploy/charts/operator/values.yaml index 976648e890..5bba786999 100644 --- a/deploy/charts/operator/values.yaml +++ b/deploy/charts/operator/values.yaml @@ -8,6 +8,16 @@ operator: features: # -- Enable experimental features experimental: false + # -- Enable untrusted single-tenant MCP server mode (one backend pod per + # (user, session, untrusted MCPServer) with an egress-broker data plane). + # Disabled by default: the per-user pod-per-session tenancy model carries + # a significant resource cost, so only operators who accept that cost and + # risk should enable it. When disabled, MCPServers with spec.untrusted=true + # are reconciled as normal trusted workloads and an UntrustedMode status + # condition plus a Warning event surface the degradation. Sets + # TOOLHIVE_ENABLE_UNTRUSTED_MODE on the operator deployment, which the + # operator forwards to every vMCP Deployment it creates. + untrustedMode: false # -- Enable the StorageVersionMigrator controller, which auto-cleans # status.storedVersions on opted-in toolhive.stacklok.dev CRDs so a # future release can drop deprecated versions (e.g. v1alpha1) without diff --git a/docs/arch/16-untrusted-mode.md b/docs/arch/16-untrusted-mode.md index 39bb673850..e50af23da4 100644 --- a/docs/arch/16-untrusted-mode.md +++ b/docs/arch/16-untrusted-mode.md @@ -156,6 +156,17 @@ pod template. user touches. 100 users each driving 3 untrusted servers = 300 pods. This is the deliberate cost of attribution (ADR D2): pod identity *is* user identity. +**Opt-in gate.** Because of that cost, untrusted mode ships **disabled by +default**: the operator helm value `operator.features.untrustedMode` (default +`false`) sets `TOOLHIVE_ENABLE_UNTRUSTED_MODE` on the operator Deployment, and +the operator forwards its own value to every vMCP Deployment it creates — one +value controls both processes. With the flag off, `spec.untrusted: true` is +reconciled as a trusted workload (no per-session pods, no bump CA, no egress +lockdown, no egress broker, no secretKeyRef gate) and the operator reports the +degradation on the MCPServer's `UntrustedMode` status condition plus a Warning +event. The CRD fields, CEL validation, and the trusted-mode egress +NetworkPolicy from `spec.permissionProfile` work regardless of the flag. + **Caps** (admission gate, all fail closed when Redis is unreachable): | Control | Default | Tunable | diff --git a/docs/operator/untrusted-mode.md b/docs/operator/untrusted-mode.md index 5e8ba398f7..390ab37377 100644 --- a/docs/operator/untrusted-mode.md +++ b/docs/operator/untrusted-mode.md @@ -13,6 +13,22 @@ model, and the full security boundary: ## Prerequisites +- **Untrusted mode is opt-in and disabled by default.** Set the operator helm + value `operator.features.untrustedMode: true` (this sets + `TOOLHIVE_ENABLE_UNTRUSTED_MODE=true` on the operator Deployment, which the + operator forwards to every vMCP Deployment it creates — one value controls + both processes). Enable it only if you accept the cost: untrusted mode runs + one backend pod per (user, session, untrusted MCPServer) plus Envoy/broker + sidecars per pod. + - When the flag is off, an `MCPServer` with `spec.untrusted: true` is + reconciled as a normal **trusted** workload: no per-session pods, no bump + CA, no egress-lockdown NetworkPolicy, no egress broker, and the + secretKeyRef backend-env gate does not apply. The operator surfaces this + on the `UntrustedMode` status condition (`False` / + `UntrustedModeDisabled`) and emits a one-shot Warning event. + - The CRD fields (`spec.untrusted`, `spec.egressPolicy`), CEL validation, + and the trusted-mode egress `NetworkPolicy` from `spec.permissionProfile` + work regardless of the flag. - A `VirtualMCPServer` fronting an `MCPGroup` (untrusted workloads must be group members — enforced by CEL). - The vMCP's embedded auth server with the upstream provider(s) configured and diff --git a/pkg/vmcp/cli/untrusted.go b/pkg/vmcp/cli/untrusted.go index 14482280af..654bd04404 100644 --- a/pkg/vmcp/cli/untrusted.go +++ b/pkg/vmcp/cli/untrusted.go @@ -120,8 +120,14 @@ func groupHasUntrustedBackend(backends []vmcp.Backend) bool { // buildUntrustedStack wires the untrusted-mode stack for the vMCP serve path. // -// It returns (nil, nil) — untrusted mode off — unless the group contains at -// least one untrusted backend (the feature gate). When the gate is on it +// It returns (nil, nil) — untrusted mode off — when the mode is disabled for +// this process (untrusted.ModeEnabled, the TOOLHIVE_ENABLE_UNTRUSTED_MODE env +// gate) or when the group contains no untrusted backend (the feature gate). +// When the mode is disabled, untrusted backends are served through the +// trusted multi-tenant path — their untrusted metadata stamp is already +// suppressed at discovery (untrusted.MarkBackend), so groupHasUntrustedBackend +// never fires for them either; the env gate here is defense-in-depth. +// When the gate is on it // requires Redis-backed session storage (multi-pod admission counters and pod // leases are Redis state) and a resolvable vMCP namespace (untrusted mode is // Kubernetes-only); both are hard startup errors rather than silent @@ -147,6 +153,9 @@ func buildUntrustedStack( vmcpName string, meterProvider metric.MeterProvider, ) (*untrustedBundle, error) { + if !untrusted.ModeEnabled() { + return nil, nil + } if !groupHasUntrustedBackend(backends) { return nil, nil } diff --git a/pkg/vmcp/cli/untrusted_test.go b/pkg/vmcp/cli/untrusted_test.go index 6855bb799c..172773a375 100644 --- a/pkg/vmcp/cli/untrusted_test.go +++ b/pkg/vmcp/cli/untrusted_test.go @@ -15,10 +15,10 @@ import ( "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) -func untrustedTestBackend(id string) vmcp.Backend { +func untrustedTestBackend() vmcp.Backend { return vmcp.Backend{ - ID: id, - Name: id, + ID: "b", + Name: "b", Metadata: map[string]string{ untrusted.MetadataKeyUntrusted: "true", untrusted.MetadataKeyMCPServerUID: "uid-1", @@ -37,7 +37,7 @@ func TestGroupHasUntrustedBackend(t *testing.T) { assert.False(t, groupHasUntrustedBackend(nil), "no backends = off") assert.False(t, groupHasUntrustedBackend([]vmcp.Backend{trustedTestBackend()}), "trusted-only = off") assert.True(t, groupHasUntrustedBackend( - []vmcp.Backend{trustedTestBackend(), untrustedTestBackend("b")}), "any untrusted = on") + []vmcp.Backend{trustedTestBackend(), untrustedTestBackend()}), "any untrusted = on") } //nolint:paralleltest // t.Setenv modifies the process environment; subtests cannot run in parallel. @@ -216,11 +216,13 @@ func TestResolveUntrustedTunables(t *testing.T) { } // TestBuildUntrustedStack_Gating pins the startup feature gate: the stack is -// only wired when the group actually contains an untrusted backend, and the -// hard prerequisites (Redis session storage, resolvable namespace) fail loudly +// only wired when the group actually contains an untrusted backend AND +// untrusted mode is enabled (TOOLHIVE_ENABLE_UNTRUSTED_MODE), and the hard +// prerequisites (Redis session storage, resolvable namespace) fail loudly // rather than silently degrading. +// +//nolint:paralleltest // t.Setenv modifies the process environment. func TestBuildUntrustedStack_Gating(t *testing.T) { - t.Parallel() ctx := t.Context() redisCfg := &config.Config{ @@ -232,14 +234,14 @@ func TestBuildUntrustedStack_Gating(t *testing.T) { } t.Run("off when no untrusted backend (nil, no error)", func(t *testing.T) { - t.Parallel() + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") bundle, err := buildUntrustedStack(ctx, redisCfg, []vmcp.Backend{trustedTestBackend()}, "toolhive", "vmcp", nil) require.NoError(t, err) assert.Nil(t, bundle) }) t.Run("off ignores missing session storage when gate is closed", func(t *testing.T) { - t.Parallel() + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") noStorage := &config.Config{Group: "g"} bundle, err := buildUntrustedStack(ctx, noStorage, []vmcp.Backend{trustedTestBackend()}, "toolhive", "vmcp", nil) require.NoError(t, err) @@ -247,17 +249,42 @@ func TestBuildUntrustedStack_Gating(t *testing.T) { }) t.Run("untrusted backend + non-Redis storage is a hard error", func(t *testing.T) { - t.Parallel() + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") noStorage := &config.Config{Group: "g"} - _, err := buildUntrustedStack(ctx, noStorage, []vmcp.Backend{untrustedTestBackend("b")}, "toolhive", "vmcp", nil) + _, err := buildUntrustedStack(ctx, noStorage, []vmcp.Backend{untrustedTestBackend()}, "toolhive", "vmcp", nil) require.Error(t, err) assert.Contains(t, err.Error(), "sessionStorage.provider=redis") }) t.Run("untrusted backend + unresolvable namespace is a hard error", func(t *testing.T) { - t.Parallel() - _, err := buildUntrustedStack(ctx, redisCfg, []vmcp.Backend{untrustedTestBackend("b")}, "local", "vmcp", nil) + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") + _, err := buildUntrustedStack(ctx, redisCfg, []vmcp.Backend{untrustedTestBackend()}, "local", "vmcp", nil) require.Error(t, err) assert.Contains(t, err.Error(), "namespace") }) + + // The env gate is checked before any backend/prerequisite logic: with the + // flag off an untrusted backend must silently degrade to trusted service, + // not error — and must not reach the Redis prerequisite check. + t.Run("mode disabled returns nil even with an untrusted backend", func(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + bundle, err := buildUntrustedStack(ctx, redisCfg, []vmcp.Backend{untrustedTestBackend()}, "toolhive", "vmcp", nil) + require.NoError(t, err) + assert.Nil(t, bundle) + }) + + t.Run("mode disabled by default (env unset) returns nil", func(t *testing.T) { + // No t.Setenv: assert the absent-env default is OFF. + bundle, err := buildUntrustedStack(ctx, redisCfg, []vmcp.Backend{untrustedTestBackend()}, "toolhive", "vmcp", nil) + require.NoError(t, err) + assert.Nil(t, bundle) + }) + + t.Run("mode disabled ignores every prerequisite (no error path)", func(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "") + noStorage := &config.Config{Group: "g"} + bundle, err := buildUntrustedStack(ctx, noStorage, []vmcp.Backend{untrustedTestBackend()}, "local", "vmcp", nil) + require.NoError(t, err) + assert.Nil(t, bundle) + }) } diff --git a/pkg/vmcp/session/untrusted/mode.go b/pkg/vmcp/session/untrusted/mode.go new file mode 100644 index 0000000000..65f199310d --- /dev/null +++ b/pkg/vmcp/session/untrusted/mode.go @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "log/slog" + "os" + "sync" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" +) + +// EnvEnableUntrustedMode is the environment variable gating the untrusted +// single-tenant mode (one backend pod per (user, session, untrusted +// MCPServer), plus its bump CA, egress-lockdown NetworkPolicy, and sidecar +// data plane). The cost of that tenancy model is too high to be the default, +// so the mode is opt-in: unset or any value other than "true"/"1" means OFF. +// The operator helm chart surfaces it as operator.features.untrustedMode and +// the operator forwards its own value to the vMCP Deployments it creates, so +// one chart value controls both processes. +const EnvEnableUntrustedMode = "TOOLHIVE_ENABLE_UNTRUSTED_MODE" + +// MetadataKeyUntrustedOffWarned guards the one-per-process WARN emitted when a +// backend asks to run untrusted while the mode is disabled (the untrusted +// metadata stamp is suppressed and the workload runs trusted). Discovery runs +// per backend; logging per stamp would flood, and the suppressed condition is +// already surfaced per-CR by the operator (UntrustedMode condition + event). +var metadataKeyUntrustedOffWarned sync.Once + +// ModeEnabled reports whether untrusted mode is enabled for this process. +// Read per call (never cached): tests toggle the env var, and the variable is +// set at process start and never mutated in production, so the syscall is +// free and staleness is impossible. +func ModeEnabled() bool { + v := os.Getenv(EnvEnableUntrustedMode) + return v == "true" || v == "1" +} + +// MarkBackend stamps the untrusted identity metadata on a discovered vMCP +// backend when — and only when — the MCPServer opts in (spec.untrusted) AND +// the mode is enabled for this process. When the mode is disabled the stamp +// is suppressed: the resolver never provisions a per-session pod for the +// backend and it is served through the trusted shared StatefulSet. The stamp +// is centralised here because the metadata keys are consumed across package +// boundaries (cli gate, session manager, resolver) and must not be set +// anywhere else. +func MarkBackend(mcpServer *mcpv1beta1.MCPServer, metadata map[string]string) { + if !mcpServer.Spec.Untrusted { + return + } + if !ModeEnabled() { + metadataKeyUntrustedOffWarned.Do(func() { + slog.Warn("spec.untrusted=true but untrusted mode is disabled; "+ + "the workload is served through the trusted multi-tenant path (no per-session pods, no egress broker)", + "env_var", EnvEnableUntrustedMode, "mcpserver", mcpServer.Name) + }) + return + } + metadata[MetadataKeyUntrusted] = "true" + metadata[MetadataKeyMCPServerUID] = string(mcpServer.UID) +} diff --git a/pkg/vmcp/session/untrusted/mode_test.go b/pkg/vmcp/session/untrusted/mode_test.go new file mode 100644 index 0000000000..f6cf42d13f --- /dev/null +++ b/pkg/vmcp/session/untrusted/mode_test.go @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package untrusted + +import ( + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" +) + +//nolint:paralleltest // t.Setenv modifies the process environment. +func TestModeEnabled(t *testing.T) { + cases := []struct { + name string + value *string // nil = unset + want bool + }{ + {name: "unset defaults to off", value: nil, want: false}, + {name: "empty string is off", value: ptr(""), want: false}, + {name: `"true" is on`, value: ptr("true"), want: true}, + {name: `"1" is on`, value: ptr("1"), want: true}, + {name: `"false" is off`, value: ptr("false"), want: false}, + {name: `"TRUE" is off (exact match only)`, value: ptr("TRUE"), want: false}, + {name: `"yes" is off`, value: ptr("yes"), want: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if tc.value != nil { + t.Setenv(EnvEnableUntrustedMode, *tc.value) + } + assert.Equal(t, tc.want, ModeEnabled()) + }) + } +} + +//nolint:paralleltest // t.Setenv modifies the process environment. +func TestMarkBackend(t *testing.T) { + newServer := func(flagged bool) *mcpv1beta1.MCPServer { + return &mcpv1beta1.MCPServer{ + ObjectMeta: metav1.ObjectMeta{Name: "srv", UID: "uid-1"}, + Spec: mcpv1beta1.MCPServerSpec{Untrusted: flagged}, + } + } + + t.Run("flagged + mode on stamps the untrusted metadata", func(t *testing.T) { + t.Setenv(EnvEnableUntrustedMode, "true") + md := map[string]string{} + MarkBackend(newServer(true), md) + assert.Equal(t, "true", md[MetadataKeyUntrusted]) + assert.Equal(t, "uid-1", md[MetadataKeyMCPServerUID]) + }) + + t.Run("flagged + mode off stamps nothing", func(t *testing.T) { + t.Setenv(EnvEnableUntrustedMode, "false") + md := map[string]string{} + MarkBackend(newServer(true), md) + assert.Empty(t, md) + }) + + t.Run("unflagged stamps nothing regardless of the mode", func(t *testing.T) { + t.Setenv(EnvEnableUntrustedMode, "true") + md := map[string]string{} + MarkBackend(newServer(false), md) + assert.Empty(t, md) + }) +} + +func ptr(s string) *string { return &s } diff --git a/pkg/vmcp/workloads/k8s.go b/pkg/vmcp/workloads/k8s.go index 50b40907a5..4228852f06 100644 --- a/pkg/vmcp/workloads/k8s.go +++ b/pkg/vmcp/workloads/k8s.go @@ -23,6 +23,7 @@ import ( "github.com/stacklok/toolhive/pkg/vmcp" "github.com/stacklok/toolhive/pkg/vmcp/auth/converters" "github.com/stacklok/toolhive/pkg/vmcp/headerforward/wirefmt" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" "github.com/stacklok/toolhive/pkg/workloads/types" ) @@ -34,12 +35,6 @@ const ( metadataKeyWorkloadStatus = "workload_status" metadataKeyNamespace = "namespace" metadataKeyRemoteURL = "remote_url" - - // metadataKeyUntrusted mirrors untrusted.MetadataKeyUntrusted (the untrusted - // session package must not be imported here; the values must stay in sync). - metadataKeyUntrusted = "toolhive.stacklok.dev/untrusted" - // metadataKeyMCPServerUID mirrors untrusted.MetadataKeyMCPServerUID. - metadataKeyMCPServerUID = "toolhive.stacklok.dev/mcpserver-uid" ) // k8sDiscoverer is a direct implementation of Discoverer for Kubernetes workloads. @@ -294,11 +289,11 @@ func (d *k8sDiscoverer) mcpServerToBackend(ctx context.Context, mcpServer *mcpv1 // Mark untrusted backends for the per-session address resolver. The UID is // rename-safe identity: pods/StatefulSets are keyed on it (handoff §4), so - // an MCPServer rename cannot strand a session pod's ownership. - if mcpServer.Spec.Untrusted { - backend.Metadata[metadataKeyUntrusted] = "true" - backend.Metadata[metadataKeyMCPServerUID] = string(mcpServer.UID) - } + // an MCPServer rename cannot strand a session pod's ownership. MarkBackend + // suppresses the stamp when untrusted mode is disabled for this process + // (TOOLHIVE_ENABLE_UNTRUSTED_MODE): the backend is then served through the + // trusted shared StatefulSet. + untrusted.MarkBackend(mcpServer, backend.Metadata) // Discover and populate authentication configuration from MCPServer if err := d.discoverAuthConfig(ctx, mcpServer, backend); err != nil { diff --git a/pkg/vmcp/workloads/k8s_test.go b/pkg/vmcp/workloads/k8s_test.go index f0d1d703bc..da2fe61df3 100644 --- a/pkg/vmcp/workloads/k8s_test.go +++ b/pkg/vmcp/workloads/k8s_test.go @@ -19,6 +19,7 @@ import ( mcpv1beta1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1" "github.com/stacklok/toolhive/cmd/thv-operator/api/v1beta1/v1beta1test" "github.com/stacklok/toolhive/pkg/vmcp" + "github.com/stacklok/toolhive/pkg/vmcp/session/untrusted" ) const testNamespace = "test-namespace" @@ -376,12 +377,11 @@ func TestMCPServerToBackend_BasicFields(t *testing.T) { assert.Equal(t, namespace, backend.Metadata["namespace"]) } +//nolint:paralleltest // Subtests use t.Setenv (env mutation serializes them). func TestMCPServerToBackend_UntrustedMetadata(t *testing.T) { - t.Parallel() - namespace := testNamespace - newServer := func(untrusted bool) *mcpv1beta1.MCPServer { + newServer := func(untrustedFlag bool) *mcpv1beta1.MCPServer { return &mcpv1beta1.MCPServer{ ObjectMeta: metav1.ObjectMeta{ Name: "test-server", @@ -392,7 +392,7 @@ func TestMCPServerToBackend_UntrustedMetadata(t *testing.T) { Image: "test-image:latest", Transport: "streamable-http", ProxyPort: 8080, - Untrusted: untrusted, + Untrusted: untrustedFlag, }, Status: mcpv1beta1.MCPServerStatus{ Phase: mcpv1beta1.MCPServerPhaseReady, @@ -401,8 +401,9 @@ func TestMCPServerToBackend_UntrustedMetadata(t *testing.T) { } } - t.Run("untrusted server is marked with UID", func(t *testing.T) { - t.Parallel() + //nolint:paralleltest // t.Setenv modifies the process environment. + t.Run("untrusted server is marked with UID when the mode is enabled", func(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "true") srv := newServer(true) discoverer := NewK8SDiscovererWithClient(setupTestClient(t, srv), namespace).(*k8sDiscoverer) backend := discoverer.mcpServerToBackend(context.Background(), srv) @@ -411,8 +412,20 @@ func TestMCPServerToBackend_UntrustedMetadata(t *testing.T) { assert.Equal(t, "uid-abc-123", backend.Metadata["toolhive.stacklok.dev/mcpserver-uid"]) }) + //nolint:paralleltest // t.Setenv modifies the process environment. + t.Run("untrusted server carries no untrusted metadata when the mode is disabled", func(t *testing.T) { + t.Setenv(untrusted.EnvEnableUntrustedMode, "false") + srv := newServer(true) + discoverer := NewK8SDiscovererWithClient(setupTestClient(t, srv), namespace).(*k8sDiscoverer) + backend := discoverer.mcpServerToBackend(context.Background(), srv) + require.NotNil(t, backend) + _, hasUntrusted := backend.Metadata["toolhive.stacklok.dev/untrusted"] + _, hasUID := backend.Metadata["toolhive.stacklok.dev/mcpserver-uid"] + assert.False(t, hasUntrusted, "flag-off untrusted servers are served as trusted: no untrusted stamp") + assert.False(t, hasUID) + }) + t.Run("trusted server carries no untrusted metadata", func(t *testing.T) { - t.Parallel() srv := newServer(false) discoverer := NewK8SDiscovererWithClient(setupTestClient(t, srv), namespace).(*k8sDiscoverer) backend := discoverer.mcpServerToBackend(context.Background(), srv) From cb55a76796ec1b972e219acfbca29f1c73d92350 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 23 Jul 2026 17:22:33 +0300 Subject: [PATCH 12/12] Stabilize untrusted CA rotation and CI Fix a cluster of CI failures in the untrusted data plane, root-caused to shared state and read consistency rather than the rotation logic itself (which was verified correct). Mixed-read bug: the operator selected the current CA generation from the lagging informer cache while its GC read the same set via APIReader, so in the cache-lag window after a create/delete the GC could delete the retained previous generation. Read the CA set via APIReader in both, so one pass selects current, mints on empty, and hands the same set to GC. Wire APIReader into the MCPServerReconciler and the test reconcilers. Watch gap: the controller now Owns Secrets and ConfigMaps it creates, so CA deletion/change triggers reconcile instead of relying on requeue. Shared resource name: trusted and untrusted NetworkPolicies both used -egress, so the trusted ensure path deleted the untrusted policy it just created. Trusted now uses -egress-trusted. Rotation test: replace the non-atomic create-aged/delete-fresh with an atomic in-place CA update and assert the steady state (aged N-1 + fresh N, older GC'd) instead of a transient count that raced the cache. Also regenerate docs/operator/crd-api.md (authorizeUrl and the trusted blast-radius doc updates were stale relative to the Go types). --- cmd/thv-operator/app/app.go | 1 + .../controllers/mcpserver_controller.go | 8 ++ .../mcpserver_untrusted_env_test.go | 3 + .../mcpserver_untrusted_resources.go | 24 +++-- .../mcpserver_untrusted_resources_test.go | 20 ++-- .../reconciler_test_helpers_test.go | 2 + .../mcp-external-auth/suite_test.go | 1 + .../mcp-oidc-config/suite_test.go | 1 + ...erver_untrusted_egress_integration_test.go | 91 +++++++++++++------ .../mcp-server-untrusted/suite_test.go | 1 + .../test-integration/mcp-server/suite_test.go | 1 + .../mcp-toolconfig/suite_test.go | 1 + docs/operator/crd-api.md | 11 ++- 13 files changed, 114 insertions(+), 51 deletions(-) diff --git a/cmd/thv-operator/app/app.go b/cmd/thv-operator/app/app.go index 2aad8b882c..0f024e4e27 100644 --- a/cmd/thv-operator/app/app.go +++ b/cmd/thv-operator/app/app.go @@ -288,6 +288,7 @@ func setupServerControllers(mgr ctrl.Manager, imagePullSecretsDefaults imagepull Recorder: mgr.GetEventRecorder("mcpserver-controller"), PlatformDetector: ctrlutil.NewSharedPlatformDetector(), ImagePullSecretsDefaults: imagePullSecretsDefaults, + APIReader: mgr.GetAPIReader(), } if err := rec.SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create controller MCPServer: %w", err) diff --git a/cmd/thv-operator/controllers/mcpserver_controller.go b/cmd/thv-operator/controllers/mcpserver_controller.go index b1d1a58f59..0b7fa1a475 100644 --- a/cmd/thv-operator/controllers/mcpserver_controller.go +++ b/cmd/thv-operator/controllers/mcpserver_controller.go @@ -55,6 +55,12 @@ type MCPServerReconciler struct { Scheme *runtime.Scheme Recorder events.EventRecorder PlatformDetector *ctrlutil.SharedPlatformDetector + // APIReader bypasses the informer cache for reads where freshness is + // load-bearing. Used for bump-CA Secrets: rotation decisions made on a + // stale cached read can mint a duplicate generation or fail to rotate on + // time (the cache lags a create/delete by tens of milliseconds, which is + // exactly the rotation window the tests and real rotations race). + APIReader client.Reader // ImagePullSecretsDefaults are cluster-wide defaults sourced from the // operator chart that are merged with the per-CR imagePullSecrets when // constructing workloads. The zero value is a usable empty Defaults. @@ -3322,6 +3328,8 @@ func (r *MCPServerReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&mcpv1beta1.MCPServer{}). Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). + Owns(&corev1.ConfigMap{}). + Owns(&corev1.Secret{}). Watches(&mcpv1beta1.MCPExternalAuthConfig{}, externalAuthConfigHandler). Watches(&mcpv1beta1.MCPOIDCConfig{}, oidcConfigHandler). Watches(&mcpv1beta1.MCPAuthzConfig{}, authzConfigHandler). diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go index a53fb59d21..3182f56768 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_env_test.go @@ -125,6 +125,9 @@ func setupUntrustedReconciler(t *testing.T, objs ...client.Object) (*MCPServerRe Scheme: s, Recorder: eventRecorder, PlatformDetector: ctrlutil.NewSharedPlatformDetector(), + // The fake client reads its own writes, so it serves as the APIReader + // (read-your-write) the untrusted CA path needs. + APIReader: fakeClient, } return r, eventRecorder } diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go index ea6c466384..5f175fac4e 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources.go @@ -46,8 +46,14 @@ const ( // rename-safe UID selector. It mirrors // pkg/vmcp/session/untrusted.LabelMCPServerUID across the module boundary. untrustedMCPServerUIDLabel = "toolhive.stacklok.dev/mcpserver-uid" - // untrustedNetworkPolicySuffix names the egress-lockdown NetworkPolicy. + // untrustedNetworkPolicySuffix names the untrusted-mode egress-lockdown + // NetworkPolicy. untrustedNetworkPolicySuffix = "-egress" + // trustedNetworkPolicySuffix names the trusted-mode (blast-radius + // reduction) egress NetworkPolicy. It MUST differ from the untrusted + // suffix: the two ensure paths run back-to-back on the same MCPServer and + // would otherwise delete each other's policy. + trustedNetworkPolicySuffix = "-egress-trusted" // untrustedCAGenerationAnnotation is stamped on the backend StatefulSet's // pod template with the current bump-CA generation. The vMCP pod lifecycle @@ -284,8 +290,14 @@ func renderEgressPolicyYAML(policy *mcpv1beta1.EgressPolicy, dialAllowlist []str // in the Secret, mounted by the trusted sidecar container. Key material is // never logged. func (r *MCPServerReconciler) ensureUntrustedCA(ctx context.Context, m *mcpv1beta1.MCPServer) (string, error) { + // Read the CA set uncached (APIReader): this pass selects the current + // generation, mints one when the set is empty, and hands the SAME set to the + // GC below — it must observe its own writes. The informer cache is + // eventually consistent and can lag a Secret it does not own long enough to + // re-mint over an in-flight rotation. This is the documented APIReader + // exception (read-your-write), not a hot path (once per reconcile). secrets := &corev1.SecretList{} - if err := r.List(ctx, secrets, client.InNamespace(m.Namespace), + if err := r.APIReader.List(ctx, secrets, client.InNamespace(m.Namespace), client.MatchingLabels(untrustedResourceLabels(m))); err != nil { return "", fmt.Errorf("failed to list bump CA Secrets: %w", err) } @@ -421,7 +433,7 @@ func (r *MCPServerReconciler) createCASecretIfAbsent( // otherwise accumulate forever). func (r *MCPServerReconciler) gcUntrustedCAGenerations(ctx context.Context, m *mcpv1beta1.MCPServer, currentGen string) error { secrets := &corev1.SecretList{} - if err := r.List(ctx, secrets, client.InNamespace(m.Namespace), + if err := r.APIReader.List(ctx, secrets, client.InNamespace(m.Namespace), client.MatchingLabels(untrustedResourceLabels(m))); err != nil { return fmt.Errorf("failed to list bump CA Secrets for GC: %w", err) } @@ -442,7 +454,7 @@ func (r *MCPServerReconciler) gcUntrustedCAGenerations(ctx context.Context, m *m } bundles := &corev1.ConfigMapList{} - if err := r.List(ctx, bundles, client.InNamespace(m.Namespace), + if err := r.APIReader.List(ctx, bundles, client.InNamespace(m.Namespace), client.MatchingLabels(untrustedResourceLabels(m))); err != nil { return fmt.Errorf("failed to list bump CA bundles for GC: %w", err) } @@ -880,7 +892,7 @@ func renderTrustedEgressNetworkPolicy( egressRules = append(egressRules, rule) } return renderEgressNetworkPolicy( - m.Name+untrustedNetworkPolicySuffix, m.Namespace, labelsForMCPServer(m.Name), + m.Name+trustedNetworkPolicySuffix, m.Namespace, labelsForMCPServer(m.Name), metav1.LabelSelector{MatchLabels: labelsForMCPServer(m.Name)}, egressRules..., ) @@ -958,7 +970,7 @@ func (r *MCPServerReconciler) applyTrustedEgressNetworkPolicy( // untrusted ensure renders its own policy under the same name). func (r *MCPServerReconciler) deleteTrustedEgressNetworkPolicy(ctx context.Context, m *mcpv1beta1.MCPServer) error { return r.deleteIfExists(ctx, &networkingv1.NetworkPolicy{}, - m.Name+untrustedNetworkPolicySuffix, m.Namespace, "NetworkPolicy") + m.Name+trustedNetworkPolicySuffix, m.Namespace, "NetworkPolicy") } // resolvePermissionProfile resolves spec.permissionProfile to a toolhive-core diff --git a/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go index af8a41e184..feeb2d0b3c 100644 --- a/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go +++ b/cmd/thv-operator/controllers/mcpserver_untrusted_resources_test.go @@ -658,7 +658,7 @@ func TestEnsureTrustedEgressNetworkPolicy(t *testing.T) { np := &networkingv1.NetworkPolicy{} require.NoError(t, r.Get(ctx, types.NamespacedName{ - Name: "github-mcp-egress", Namespace: "default"}, np)) + Name: "github-mcp-egress-trusted", Namespace: "default"}, np)) assertOwnerRef(t, np.OwnerReferences, m) assert.Contains(t, np.Spec.PolicyTypes, networkingv1.PolicyTypeEgress) @@ -697,7 +697,7 @@ func TestEnsureTrustedEgressNetworkPolicy(t *testing.T) { require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) after := &networkingv1.NetworkPolicy{} require.NoError(t, r.Get(ctx, types.NamespacedName{ - Name: "github-mcp-egress", Namespace: "default"}, after)) + Name: "github-mcp-egress-trusted", Namespace: "default"}, after)) assert.Equal(t, before, after.ResourceVersion, "no-op reconcile must not rewrite the NetworkPolicy") }) @@ -713,7 +713,7 @@ func TestEnsureTrustedEgressNetworkPolicy(t *testing.T) { require.NoError(t, r.ensureTrustedEgressNetworkPolicy(t.Context(), m)) np := &networkingv1.NetworkPolicy{} err := r.Get(t.Context(), types.NamespacedName{ - Name: "github-mcp-egress", Namespace: "default"}, np) + Name: "github-mcp-egress-trusted", Namespace: "default"}, np) assert.True(t, apierrors.IsNotFound(err), "InsecureAllowAll must never render a policy") }) @@ -729,7 +729,7 @@ func TestEnsureTrustedEgressNetworkPolicy(t *testing.T) { require.NoError(t, r.ensureTrustedEgressNetworkPolicy(t.Context(), m)) np := &networkingv1.NetworkPolicy{} err := r.Get(t.Context(), types.NamespacedName{ - Name: "github-mcp-egress", Namespace: "default"}, np) + Name: "github-mcp-egress-trusted", Namespace: "default"}, np) assert.True(t, apierrors.IsNotFound(err), "empty allowHost/allowPort must never render a policy") }) @@ -743,14 +743,14 @@ func TestEnsureTrustedEgressNetworkPolicy(t *testing.T) { require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) np := &networkingv1.NetworkPolicy{} require.NoError(t, r.Get(ctx, types.NamespacedName{ - Name: "github-mcp-egress", Namespace: "default"}, np)) + Name: "github-mcp-egress-trusted", Namespace: "default"}, np)) // Profile turns permissive → policy deleted. cm := trustedProfileConfigMap() cm.Data["profile.json"] = `{"name":"p","network":{"outbound":{"insecure_allow_all":true}}}` require.NoError(t, r.Update(ctx, cm)) require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) - err := r.Get(ctx, types.NamespacedName{Name: "github-mcp-egress", Namespace: "default"}, np) + err := r.Get(ctx, types.NamespacedName{Name: "github-mcp-egress-trusted", Namespace: "default"}, np) assert.True(t, apierrors.IsNotFound(err), "a permissive profile must delete the policy") // Restrictive again, then the profile is removed → policy deleted. @@ -758,10 +758,10 @@ func TestEnsureTrustedEgressNetworkPolicy(t *testing.T) { require.NoError(t, r.Update(ctx, cm)) require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) require.NoError(t, r.Get(ctx, types.NamespacedName{ - Name: "github-mcp-egress", Namespace: "default"}, np)) + Name: "github-mcp-egress-trusted", Namespace: "default"}, np)) m.Spec.PermissionProfile = nil require.NoError(t, r.ensureTrustedEgressNetworkPolicy(ctx, m)) - err = r.Get(ctx, types.NamespacedName{Name: "github-mcp-egress", Namespace: "default"}, np) + err = r.Get(ctx, types.NamespacedName{Name: "github-mcp-egress-trusted", Namespace: "default"}, np) assert.True(t, apierrors.IsNotFound(err), "removing the profile must delete the policy") }) @@ -809,7 +809,7 @@ func TestRenderTrustedEgressNetworkPolicy(t *testing.T) { m := v1beta1test.NewMCPServer("github-mcp", "default") np := renderTrustedEgressNetworkPolicy(m, []string{"140.82.114.26/32"}, []int32{8443, 443}) - assert.Equal(t, "github-mcp-egress", np.Name) + assert.Equal(t, "github-mcp-egress-trusted", np.Name) assert.Equal(t, labelsForMCPServer(m.Name), np.Spec.PodSelector.MatchLabels) assert.Equal(t, labelsForMCPServer(m.Name), np.Labels) @@ -879,7 +879,7 @@ func TestEnsureUntrustedResources_ModeDisabled(t *testing.T) { np := &networkingv1.NetworkPolicy{} require.NoError(t, r.Get(t.Context(), types.NamespacedName{ - Name: "github-mcp-egress", Namespace: "default"}, np), + Name: "github-mcp-egress-trusted", Namespace: "default"}, np), "the trusted-mode egress NetworkPolicy is independent of the untrusted-mode flag") assert.Equal(t, labelsForMCPServer(m.Name), np.Spec.PodSelector.MatchLabels) assert.NotContains(t, np.Labels, "toolhive.stacklok.dev/untrusted-resource") diff --git a/cmd/thv-operator/controllers/reconciler_test_helpers_test.go b/cmd/thv-operator/controllers/reconciler_test_helpers_test.go index a643c8d74c..b5f084dfc3 100644 --- a/cmd/thv-operator/controllers/reconciler_test_helpers_test.go +++ b/cmd/thv-operator/controllers/reconciler_test_helpers_test.go @@ -63,6 +63,8 @@ func newTestMCPServerReconciler( Client: k8sClient, Scheme: scheme, PlatformDetector: ctrlutil.NewSharedPlatformDetectorWithDetector(mockDetector), + // The fake client reads its own writes, so it serves as the APIReader. + APIReader: k8sClient, } } diff --git a/cmd/thv-operator/test-integration/mcp-external-auth/suite_test.go b/cmd/thv-operator/test-integration/mcp-external-auth/suite_test.go index da97691d15..b8179472fe 100644 --- a/cmd/thv-operator/test-integration/mcp-external-auth/suite_test.go +++ b/cmd/thv-operator/test-integration/mcp-external-auth/suite_test.go @@ -56,6 +56,7 @@ var _ = BeforeSuite(func() { Client: suiteEnv.Manager.GetClient(), Scheme: suiteEnv.Manager.GetScheme(), PlatformDetector: ctrlutil.NewSharedPlatformDetector(), + APIReader: suiteEnv.Manager.GetAPIReader(), }).SetupWithManager(suiteEnv.Manager) Expect(err).ToNot(HaveOccurred()) diff --git a/cmd/thv-operator/test-integration/mcp-oidc-config/suite_test.go b/cmd/thv-operator/test-integration/mcp-oidc-config/suite_test.go index cc8e285fea..053ebeb0f5 100644 --- a/cmd/thv-operator/test-integration/mcp-oidc-config/suite_test.go +++ b/cmd/thv-operator/test-integration/mcp-oidc-config/suite_test.go @@ -58,6 +58,7 @@ var _ = BeforeSuite(func() { Client: suiteEnv.Manager.GetClient(), Scheme: suiteEnv.Manager.GetScheme(), PlatformDetector: ctrlutil.NewSharedPlatformDetector(), + APIReader: suiteEnv.Manager.GetAPIReader(), }).SetupWithManager(suiteEnv.Manager) Expect(err).ToNot(HaveOccurred()) diff --git a/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_egress_integration_test.go b/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_egress_integration_test.go index 0b8c832940..b8a5f8acc3 100644 --- a/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_egress_integration_test.go +++ b/cmd/thv-operator/test-integration/mcp-server-untrusted/mcpserver_untrusted_egress_integration_test.go @@ -128,48 +128,79 @@ var _ = Describe("MCPServer untrusted egress resources", Label("k8s", "untrusted }) It("Should rotate into a new generation, keep N-1, and GC older generations", func() { - // Force rotation: age the current Secret's CA into the rotation - // window by replacing its material with an aged CA (name follows - // the cert hash, so create the aged generation and delete the - // fresh one). + // Force rotation atomically: replace the CURRENT Secret's CA + // material in place with an aged CA (past the 50% rotation window). + // Updating data on the same object is a single apiserver write, so + // there is no empty-set window for a reconcile to misread as "no + // CA" and no third object for the GC to race. The rotation path + // keys on the cert's notAfter (not the Secret name), so a sole aged + // CA makes the next ensure rotate: mint N, retain the aged CA as + // N-1. current := &corev1.Secret{} Eventually(func() error { return getCurrentCASecret(current) }, timeout, interval).Should(Succeed()) agedCert, agedKey := mintAgedCAForRotation() - agedGen := egressbroker.CAGeneration(agedCert) - aged := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: egressbroker.ResourceNamesFor(serverName, agedGen).CASecret, - Namespace: "default", - Labels: current.Labels, - }, - Data: map[string][]byte{"ca.crt": agedCert, "ca.key": agedKey}, - } - Expect(untrustedK8sClient.Create(untrustedCtx, aged)).To(Succeed()) - Expect(untrustedK8sClient.Delete(untrustedCtx, current)).To(Succeed()) + freshName := current.Name + current.Data = map[string][]byte{"ca.crt": agedCert, "ca.key": agedKey} + Expect(untrustedK8sClient.Update(untrustedCtx, current)).To(Succeed()) - // The reconcile must mint a NEW generation alongside the aged one. - Eventually(func() int { + // The reconcile must mint a NEW generation alongside the aged one + // and settle to exactly N and N-1 (the aged in-place Secret is the + // retained previous; any older duplicate from the initial mint is + // GC'd). Poll the cert set, not a transient count. + listCA := func() []corev1.Secret { secrets := &corev1.SecretList{} if err := untrustedK8sClient.List(untrustedCtx, secrets, client.InNamespace("default"), client.MatchingLabels(current.Labels)); err != nil { - return 0 + return nil } - return len(secrets.Items) - }, timeout, interval).Should(Equal(2), "rotation keeps N and N-1 generations") + return secrets.Items + } + Eventually(func() bool { + items := listCA() + if len(items) != 2 { + return false + } + var sawAged, sawFresh bool + for i := range items { + if items[i].Name == freshName && string(items[i].Data["ca.crt"]) == string(agedCert) { + sawAged = true // the in-place aged CA, retained as N-1 + } + if items[i].Name != freshName { + if ca, err := egressbroker.ParseBumpCA(items[i].Data["ca.crt"], items[i].Data["ca.key"]); err == nil && + !ca.NeedsRotation(time.Now()) { + sawFresh = true // the freshly minted current generation + } + } + } + return sawAged && sawFresh + }, timeout, interval).Should(BeTrue(), + "rotation settles to the aged N-1 plus a fresh N, older generations GC'd") - // The aged generation's bundle converges on its own cert. - bundle := &corev1.ConfigMap{} - Eventually(func() error { - return untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ - Name: egressbroker.ResourceNamesFor(serverName, agedGen).CABundle, - Namespace: "default", - }, bundle) - }, timeout, interval).Should(Succeed()) - Expect(bundle.Data["ca.crt"]).To(Equal(string(agedCert)), - "the N-1 bundle must carry the N-1 cert (mid-rotation consistency)") + // Every retained generation's bundle converges on its own Secret's + // cert (mid-rotation consistency: N and N-1 pairs stay coherent). + Eventually(func() bool { + for _, s := range listCA() { + gen, ok := egressbroker.TrimGeneration(s.Name, egressbroker.BaseCASecretName(serverName)) + if !ok { + continue + } + b := &corev1.ConfigMap{} + if err := untrustedK8sClient.Get(untrustedCtx, types.NamespacedName{ + Name: egressbroker.ResourceNamesFor(serverName, gen).CABundle, + Namespace: "default", + }, b); err != nil { + return false + } + if b.Data["ca.crt"] != string(s.Data["ca.crt"]) { + return false + } + } + return true + }, timeout, interval).Should(BeTrue(), + "each retained generation's bundle must carry that generation's cert") }) It("Should render the egress policy ConfigMap from the CRD EgressPolicy", func() { diff --git a/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go b/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go index defb0152ba..ff3691392b 100644 --- a/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go +++ b/cmd/thv-operator/test-integration/mcp-server-untrusted/suite_test.go @@ -66,6 +66,7 @@ var _ = BeforeSuite(func() { Client: untrustedSuiteEnv.Manager.GetClient(), Scheme: untrustedSuiteEnv.Manager.GetScheme(), PlatformDetector: ctrlutil.NewSharedPlatformDetector(), + APIReader: untrustedSuiteEnv.Manager.GetAPIReader(), }).SetupWithManager(untrustedSuiteEnv.Manager) Expect(err).ToNot(HaveOccurred()) diff --git a/cmd/thv-operator/test-integration/mcp-server/suite_test.go b/cmd/thv-operator/test-integration/mcp-server/suite_test.go index 99c29d1086..087c873e11 100644 --- a/cmd/thv-operator/test-integration/mcp-server/suite_test.go +++ b/cmd/thv-operator/test-integration/mcp-server/suite_test.go @@ -60,6 +60,7 @@ var _ = BeforeSuite(func() { Client: suiteEnv.Manager.GetClient(), Scheme: suiteEnv.Manager.GetScheme(), PlatformDetector: ctrlutil.NewSharedPlatformDetector(), + APIReader: suiteEnv.Manager.GetAPIReader(), }).SetupWithManager(suiteEnv.Manager) Expect(err).ToNot(HaveOccurred()) diff --git a/cmd/thv-operator/test-integration/mcp-toolconfig/suite_test.go b/cmd/thv-operator/test-integration/mcp-toolconfig/suite_test.go index ba3c34e7be..33e08dbbbc 100644 --- a/cmd/thv-operator/test-integration/mcp-toolconfig/suite_test.go +++ b/cmd/thv-operator/test-integration/mcp-toolconfig/suite_test.go @@ -54,6 +54,7 @@ var _ = BeforeSuite(func() { Client: suiteEnv.Manager.GetClient(), Scheme: suiteEnv.Manager.GetScheme(), PlatformDetector: ctrlutil.NewSharedPlatformDetector(), + APIReader: suiteEnv.Manager.GetAPIReader(), }).SetupWithManager(suiteEnv.Manager) Expect(err).ToNot(HaveOccurred()) diff --git a/docs/operator/crd-api.md b/docs/operator/crd-api.md index 712b1a0ee3..9829159f4e 100644 --- a/docs/operator/crd-api.md +++ b/docs/operator/crd-api.md @@ -217,6 +217,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `providerName` _string_ | ProviderName is the name of the upstream provider configured in the
embedded authorization server. Must match an entry in AuthServer.Upstreams. | | | +| `authorizeUrl` _string_ | AuthorizeURL is the ToolHive authorization server's authorize-endpoint
URL (\{issuer\}/oauth/authorize). When set, it is carried in the
ConsentRequiredError returned when the provider token is absent, so
clients can direct the user to consent. The URL cannot be a complete
one-click link: the client must merge its own client_id, redirect_uri,
and PKCE parameters. Optional; when empty the error carries no URL. | | Optional: \{\}
| #### auth.types.XAAConfig @@ -3230,7 +3231,7 @@ _Appears in:_ | `endpointPrefix` _string_ | EndpointPrefix is the path prefix to prepend to SSE endpoint URLs.
This is used to handle path-based ingress routing scenarios where the ingress
strips a path prefix before forwarding to the backend. | | Optional: \{\}
| | `groupRef` _[api.v1beta1.MCPGroupRef](#apiv1beta1mcpgroupref)_ | GroupRef references the MCPGroup this server belongs to.
The referenced MCPGroup must be in the same namespace. | | Optional: \{\}
| | `untrusted` _boolean_ | Untrusted marks this MCP server as running untrusted code. When true, the operator
enforces the untrusted-mode invariants (ADR-0001): no Secret/ConfigMap-sourced env on
the backend container, single-tenant session-scoped backend pods, mandatory MCPGroup
membership behind a VirtualMCPServer, and egress only through the credential-broker
sidecar per EgressPolicy. K8s-only; ignored (and inert) in CLI/Docker mode. | false | Optional: \{\}
| -| `egressPolicy` _[api.v1beta1.EgressPolicy](#apiv1beta1egresspolicy)_ | EgressPolicy declares which upstream providers this server may call and where the
broker may inject each provider's credential. Required when Untrusted is true.
When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend
pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar
Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. | | Optional: \{\}
| +| `egressPolicy` _[api.v1beta1.EgressPolicy](#apiv1beta1egresspolicy)_ | EgressPolicy declares which upstream providers this server may call and where the
broker may inject each provider's credential. Required when Untrusted is true.
When Untrusted is true, PermissionProfile.Network.Outbound is IGNORED for the backend
pod — the untrusted NetworkPolicy is derived solely from EgressPolicy (+ DNS + sidecar
Docker-mode/Squid dialect, EgressPolicy is the K8s untrusted-mode dialect. Only the
NetworkPolicy rendering machinery is shared between them. SECURITY INVARIANT: the
trusted-mode NetworkPolicy rendered from PermissionProfile is blast-radius reduction
only, never a credential boundary — the credential guarantee comes solely from this
field's broker + single-tenant pods in untrusted mode. | | Optional: \{\}
| | `sessionAffinity` _string_ | SessionAffinity controls whether the Service routes repeated client connections to the same pod.
MCP protocols (SSE, streamable-http) are stateful, so ClientIP is the default.
Set to "None" for stateless servers or when using an external load balancer with its own affinity. | ClientIP | Enum: [ClientIP None]
Optional: \{\}
| | `replicas` _integer_ | Replicas is the desired number of proxy runner (thv run) pod replicas.
MCPServer creates two separate Deployments: one for the proxy runner and one
for the MCP server backend. This field controls the proxy runner Deployment.
When nil, the operator does not set Deployment.Spec.Replicas, leaving replica
management to an HPA or other external controller. | | Minimum: 0
Optional: \{\}
| | `backendReplicas` _integer_ | BackendReplicas is the desired number of MCP server backend pod replicas.
This controls the backend Deployment (the MCP server container itself),
independent of the proxy runner controlled by Replicas.
When nil, the operator does not set Deployment.Spec.Replicas, leaving replica
management to an HPA or other external controller. | | Minimum: 0
Optional: \{\}
| @@ -4166,10 +4167,10 @@ _Appears in:_ TokenEncryptionConfig configures AES-256-GCM envelope encryption of upstream OAuth token values stored in Redis. The Secret referenced by KeySecretRef holds one data entry per key ID (value = base64 32-byte KEK); -the operator mounts the active key as a SecretKeyRef env entry on the vMCP -container and clones the same reference into untrusted egress-broker -sidecars. Key material never appears in the CRD, a ConfigMap, or a pod env -literal. +the operator mounts every data key (active + retired) as a SecretKeyRef env +entry on the vMCP container and clones the same references into untrusted +egress-broker sidecars. Key material never appears in the CRD, a ConfigMap, +or a pod env literal.