diff --git a/README.md b/README.md index f93f9e3..980e406 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # RustFS Kubernetes Operator -A Kubernetes operator for [RustFS](https://rustfs.com/) object storage, written in Rust with [kube-rs](https://github.com/kube-rs/kube). It reconciles a **`Tenant` custom resource** (`rustfs.com/v1alpha1`), validates referenced credential and KMS Secrets, and applies RBAC, Services, and StatefulSets so RustFS runs inside your cluster, from single-node single-disk development tenants to erasure-coded distributed clusters. +A Kubernetes operator for [RustFS](https://rustfs.com/) object storage, written in Rust with [kube-rs](https://github.com/kube-rs/kube). It reconciles a **`Tenant` custom resource** (`rustfs.com/v1alpha1`), validates referenced credential and KMS Secrets, and applies ServiceAccounts, Services, and StatefulSets so RustFS runs inside your cluster, from single-node single-disk development tenants to erasure-coded distributed clusters. **Status:** v0.1.0 pre-release — under active development. diff --git a/deploy/k8s-dev/operator-rbac.yaml b/deploy/k8s-dev/operator-rbac.yaml index b710777..ab88c72 100755 --- a/deploy/k8s-dev/operator-rbac.yaml +++ b/deploy/k8s-dev/operator-rbac.yaml @@ -33,6 +33,8 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + # Legacy Tenant workload RBAC watch/cleanup. Write verbs remain during the + # security migration so an older binary can coexist during a rolling upgrade. - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] diff --git a/deploy/rustfs-operator/README.md b/deploy/rustfs-operator/README.md index 84893cd..9d3cc8f 100755 --- a/deploy/rustfs-operator/README.md +++ b/deploy/rustfs-operator/README.md @@ -372,6 +372,16 @@ spec: image: "registry.example.com/rustfs/rustfs@sha256:" ``` +The Operator also removes legacy Tenant workload Roles and RoleBindings and +disables automatic Kubernetes API token mounting for generated Tenant +ServiceAccounts. Existing default-ServiceAccount Tenants roll once to apply the +Pod template change. A custom image that calls the Kubernetes API must use a +user-owned ServiceAccount with the required token projection and a +least-privilege Role/RoleBinding under names other than the legacy +`{tenant}-role` and `{tenant}-role-binding`, then set `spec.serviceAccountName`. +Do not downgrade after reconciliation: an older Operator recreates the legacy +broad workload RBAC. + The annotation must change when the image reference changes and cannot override a known-incompatible official alpha or beta.1 through beta.8 reference that is not digest-qualified. For `tag@digest`, Kubernetes pulls by digest; after diff --git a/deploy/rustfs-operator/crds/tenant-crd.yaml b/deploy/rustfs-operator/crds/tenant-crd.yaml index 86fd0bd..0a9de80 100644 --- a/deploy/rustfs-operator/crds/tenant-crd.yaml +++ b/deploy/rustfs-operator/crds/tenant-crd.yaml @@ -165,6 +165,9 @@ spec: type: object type: object createServiceAccountRbac: + description: |- + Deprecated compatibility field. The operator never grants Kubernetes API permissions to + Tenant workloads. Configure a custom ServiceAccount and manage any required RBAC explicitly. nullable: true type: boolean credsSecret: diff --git a/deploy/rustfs-operator/crds/tenant.yaml b/deploy/rustfs-operator/crds/tenant.yaml index 86fd0bd..0a9de80 100755 --- a/deploy/rustfs-operator/crds/tenant.yaml +++ b/deploy/rustfs-operator/crds/tenant.yaml @@ -165,6 +165,9 @@ spec: type: object type: object createServiceAccountRbac: + description: |- + Deprecated compatibility field. The operator never grants Kubernetes API permissions to + Tenant workloads. Configure a custom ServiceAccount and manage any required RBAC explicitly. nullable: true type: boolean credsSecret: diff --git a/deploy/rustfs-operator/templates/clusterrole.yaml b/deploy/rustfs-operator/templates/clusterrole.yaml index bfe83cd..13a2b37 100755 --- a/deploy/rustfs-operator/templates/clusterrole.yaml +++ b/deploy/rustfs-operator/templates/clusterrole.yaml @@ -32,7 +32,8 @@ rules: resources: ["nodes"] verbs: ["get", "list", "watch"] - # RBAC resources created for tenants + # Legacy Tenant workload RBAC watch/cleanup. Write verbs remain during the + # security migration so an older binary can coexist during a rolling upgrade. - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index f830af7..40fa796 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -27,7 +27,7 @@ RustFS Operator manages RustFS object storage clusters on Kubernetes. Users desc The operator provides: - `Tenant` CRD (`rustfs.com/v1alpha1`) for declaring RustFS pools, persistence, scheduling, credentials, TLS, logging, encryption, and bootstrap provisioning. -- Controller reconciliation for Tenant-owned RBAC, Services, StatefulSets, PVC templates, status conditions, and Kubernetes Events. +- Controller reconciliation for Tenant ServiceAccounts, Services, StatefulSets, PVC templates, status conditions, and Kubernetes Events. - Helm chart under `deploy/rustfs-operator/` for production-style installation. - Operator Console API and UI for management workflows. - Optional operator STS endpoint for workload identity based temporary RustFS credentials. @@ -50,7 +50,8 @@ A `Tenant` is one RustFS cluster. A Tenant can contain one or more pools, but al When a Tenant is applied, the operator creates and owns: -- one ServiceAccount, Role, and RoleBinding when Tenant RBAC is enabled; +- one ServiceAccount when `serviceAccountName` is omitted; Tenant workloads receive no + operator-managed Kubernetes API permissions; - one headless Service named `{tenant}-hl` for StatefulSet peer DNS; - one S3 Service named `{tenant}-io` on port `9000`; - one Tenant Console Service named `{tenant}-console` on port `9001`; @@ -123,6 +124,17 @@ image-bound acknowledgement. Before upgrade, either pin a verified RustFS beta.9-or-later release tag, or verify the effective image and set `operator.rustfs.com/runtime-default-image-ack` to that exact image reference. +The Operator also removes legacy Tenant workload Roles and RoleBindings and +disables automatic Kubernetes API token mounting for its generated +ServiceAccounts. Existing default-ServiceAccount Tenants roll once to apply the +Pod template change. A custom image that calls the Kubernetes API must migrate +before upgrade: create a user-owned ServiceAccount with the required token +projection, bind a least-privilege Role under names other than the legacy +`{tenant}-role` and `{tenant}-role-binding`, and set `spec.serviceAccountName`. +Granting RBAC to the generated ServiceAccount is insufficient because its Pod +template disables token mounting. `createServiceAccountRbac` is retained only +as an ignored compatibility field. + The built-in RustFS image fallback also changes from the mutable `latest` tag to `rustfs/rustfs:1.0.0-beta.10`. A Tenant that omits `spec.image` and has no `TENANT_RUSTFS_IMAGE` Operator environment override will therefore roll to that @@ -132,9 +144,10 @@ control future RustFS upgrades independently of the Operator default. Treat this as a one-way workload security migration. After the new Operator has reconciled a Tenant, do not downgrade directly to an Operator version that predates these restricted defaults. An older controller omits the new seccomp -and container security fields: restricted admission rejects that update, while -a cluster without restricted admission can roll the workload back to weaker -settings. Recover by rolling forward to this version or a newer fixed version. +and container security fields and recreates the legacy broad workload RBAC: +restricted admission may reject the Pod update, while a cluster without it can +roll the workload back to weaker settings. Recover by rolling forward to this +version or a newer fixed version. Uninstall: @@ -413,7 +426,7 @@ Useful Tenant-level fields: | `scheduler` | Custom scheduler name. | | `env` | Additional RustFS container environment variables. Do not override operator-managed variables. | | `serviceAccountName` | Custom ServiceAccount for RustFS pods. | -| `createServiceAccountRbac` | Whether the operator should create Role/RoleBinding for the Tenant ServiceAccount. | +| `createServiceAccountRbac` | Deprecated compatibility field; ignored. Manage any custom ServiceAccount RBAC explicitly. | | `priorityClassName` | Tenant-level priority class. | | `lifecycle` | Kubernetes container lifecycle hooks. | | `podManagementPolicy` | StatefulSet pod management policy. | diff --git a/docs/operator-user-guide.zh-CN.md b/docs/operator-user-guide.zh-CN.md index e0e7060..2357dd6 100644 --- a/docs/operator-user-guide.zh-CN.md +++ b/docs/operator-user-guide.zh-CN.md @@ -27,7 +27,7 @@ RustFS Operator 用于在 Kubernetes 中管理 RustFS 对象存储集群。用 Operator 提供以下能力： - `Tenant` CRD（`rustfs.com/v1alpha1`）：声明 RustFS pool、持久化、调度、凭据、TLS、日志、加密和初始化 provisioning。 -- 控制器 reconciliation：维护 Tenant 相关的 RBAC、Service、StatefulSet、PVC 模板、状态条件和 Kubernetes Event。 +- 控制器 reconciliation：维护 Tenant ServiceAccount、Service、StatefulSet、PVC 模板、状态条件和 Kubernetes Event。 - Helm Chart：位于 `deploy/rustfs-operator/`，用于安装 Operator。 - Operator Console API 和 UI：用于 Operator 管理场景。 - 可选 Operator STS：基于 Kubernetes 工作负载身份签发临时 RustFS 凭据。 @@ -52,7 +52,8 @@ Operator 提供以下能力： 创建 Tenant 后，Operator 会创建并维护： -- Tenant RBAC 启用时的 ServiceAccount、Role、RoleBinding； +- 未配置 `serviceAccountName` 时创建一个 ServiceAccount；Operator 不会向 Tenant + workload 授予 Kubernetes API 权限； - headless Service：`{tenant}-hl`，用于 StatefulSet peer DNS； - S3 Service：`{tenant}-io`，端口 `9000`； - Tenant Console Service：`{tenant}-console`，端口 `9001`； @@ -121,6 +122,15 @@ StatefulSet template 如果尚未包含这些值，会在下一次 reconcile 时 升级前应固定到已验证的 RustFS beta.9 或更高 release tag；也可先验证实际生效的镜像， 再将 `operator.rustfs.com/runtime-default-image-ack` 设置为完全相同的镜像引用。 +Operator 还会删除旧版本为 Tenant workload 创建的 Role 和 RoleBinding，并禁止其 +自动挂载 Kubernetes API token。使用默认 ServiceAccount 的已有 Tenant 会因 Pod +template 变化发生一次滚动更新。若自定义镜像需要调用 Kubernetes API，必须在升级前 +创建用户自管的 ServiceAccount 并配置所需 token projection，以非旧版 +`{tenant}-role`、`{tenant}-role-binding` 的名称绑定最小权限 Role，再设置 +`spec.serviceAccountName`。只给默认生成的 ServiceAccount 增加 RBAC 不足以恢复访问， +因为其 Pod template 会禁用 token 挂载。`createServiceAccountRbac` 仅作为已忽略的 +兼容字段保留。 + 内置 RustFS 镜像 fallback 也会从可变的 `latest` 改为 `rustfs/rustfs:1.0.0-beta.10`。未设置 `spec.image`，且 Operator 没有配置 `TENANT_RUSTFS_IMAGE` 环境变量覆盖的 Tenant，会在 reconcile 时滚动到该固定版本。 @@ -128,9 +138,9 @@ StatefulSet template 如果尚未包含这些值，会在下一次 reconcile 时 应将此次变更视为单向 workload security migration。新版本 Operator 完成 Tenant reconcile 后，不要直接降级到尚未提供这些 restricted 默认值的旧版本。旧 Controller -会省略新增的 seccomp 和容器安全字段：restricted 准入会拒绝该更新；未启用 -restricted 准入的集群则可能把 workload 滚动回较弱配置。故障恢复应向前升级到当前 -版本或更新的修复版本。 +会省略新增的 seccomp 和容器安全字段，并重新创建旧版宽权限 workload RBAC： +restricted 准入可能拒绝 Pod 更新；未启用 restricted 准入的集群则可能把 workload +滚动回较弱配置。故障恢复应向前升级到当前版本或更新的修复版本。 卸载： @@ -406,7 +416,7 @@ spec: | `scheduler` | 自定义 scheduler 名称。 | | `env` | 额外 RustFS 容器环境变量。不要覆盖 Operator 自动管理的变量。 | | `serviceAccountName` | RustFS Pod 使用的自定义 ServiceAccount。 | -| `createServiceAccountRbac` | 是否由 Operator 为 Tenant ServiceAccount 创建 Role/RoleBinding。 | +| `createServiceAccountRbac` | 已废弃的兼容字段，不再生效；自定义 ServiceAccount 所需 RBAC 必须显式管理。 | | `priorityClassName` | Tenant 级 PriorityClass。 | | `lifecycle` | Kubernetes 容器 lifecycle hook。 | | `podManagementPolicy` | StatefulSet pod management policy。 | diff --git a/examples/README.md b/examples/README.md index 33f7c01..156b096 100755 --- a/examples/README.md +++ b/examples/README.md @@ -332,15 +332,14 @@ kubectl apply -f examples/spot-instance-tenant.yaml **Features demonstrated:** - Custom ServiceAccount usage (2 configurations) -- Manual RBAC management (Role + RoleBinding) -- Operator-managed RBAC with custom SA +- Resource-scoped, user-managed RBAC - Cloud workload identity integration (AWS/GCP/Azure) -- Additional permissions beyond defaults +- Explicit permissions for custom images - Security best practices **Two configurations:** -1. Custom SA without operator RBAC (you manage everything) -2. Custom SA with operator RBAC (operator creates Role/RoleBinding) +1. Custom SA with a user-managed, resource-scoped Role +2. Custom SA for cloud workload identity without Kubernetes API RBAC **Use case:** - Existing RBAC policies @@ -349,6 +348,10 @@ kubectl apply -f examples/spot-instance-tenant.yaml - Additional permissions needed **Deployment:** + +Replace `registry.example.com/rustfs-custom:1.0.0` and its matching acknowledgement +annotation with a verified custom image before applying the complete file. + ```bash kubectl apply -f examples/custom-rbac-tenant.yaml ``` @@ -456,10 +459,9 @@ spec: When you apply a Tenant, the operator creates: -1. **RBAC Resources** (conditional based on configuration) - - Role - - ServiceAccount - - RoleBinding +1. **ServiceAccount** (only when `serviceAccountName` is omitted) + - Kubernetes API token automount is disabled + - No workload Role or RoleBinding is created 2. **Services** - IO Service: `rustfs` (S3 API, port **9000**) diff --git a/examples/custom-rbac-tenant.yaml b/examples/custom-rbac-tenant.yaml index c6f33cd..62928ca 100755 --- a/examples/custom-rbac-tenant.yaml +++ b/examples/custom-rbac-tenant.yaml @@ -1,234 +1,120 @@ -# Custom RBAC Configuration Example +# Custom ServiceAccount and RBAC examples # -# This example demonstrates using a custom ServiceAccount with -# the RustFS operator, useful when: -# - Integrating with existing RBAC policies -# - Using external authentication/authorization systems -# - Applying organization-specific security policies -# - Need additional permissions beyond operator-managed Role -# -# Two configurations shown: -# 1. Custom SA without operator-managed RBAC (you manage everything) -# 2. Custom SA with operator-managed RBAC (operator creates Role/RoleBinding) +# RustFS Tenant workloads receive no operator-managed Kubernetes API permissions. +# Use a custom ServiceAccount and grant only the exact permissions required by a +# custom image or workload identity provider. --- -# Configuration 1: Custom SA without operator-managed RBAC -# You are responsible for all RBAC configuration - +# Configuration 1: user-managed, resource-scoped RBAC for a custom image that +# reads one ConfigMap from the Kubernetes API. apiVersion: v1 kind: ServiceAccount metadata: - name: my-custom-sa + name: custom-api-sa namespace: default - labels: - app: rustfs - annotations: - # Example: Workload Identity annotation for cloud providers - iam.gke.io/gcp-service-account: "rustfs@project.iam.gserviceaccount.com" +automountServiceAccountToken: true + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rustfs-runtime-config + namespace: default +data: + example: "value" --- -# Custom Role with additional permissions apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: custom-rustfs-role + name: rustfs-runtime-config-reader namespace: default rules: - # Standard RustFS permissions - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - - apiGroups: [""] - resources: ["services"] - verbs: ["create", "delete", "get"] - - - apiGroups: ["rustfs.com"] - resources: ["tenants"] - verbs: ["get", "list", "watch"] - - # Additional custom permissions - apiGroups: [""] resources: ["configmaps"] - verbs: ["get", "list", "watch"] - - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "create", "delete"] - - # Example: Access to monitoring resources - - apiGroups: ["monitoring.coreos.com"] - resources: ["servicemonitors"] - verbs: ["get", "create"] + resourceNames: ["rustfs-runtime-config"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: custom-rustfs-binding + name: rustfs-runtime-config-reader namespace: default subjects: - kind: ServiceAccount - name: my-custom-sa + name: custom-api-sa namespace: default roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: custom-rustfs-role + name: rustfs-runtime-config-reader --- apiVersion: rustfs.com/v1alpha1 kind: Tenant metadata: - name: custom-rbac-tenant-1 + name: custom-rbac-tenant namespace: default + annotations: + # Set only after verifying that this exact custom image supports the generated security profile. + operator.rustfs.com/runtime-default-image-ack: "registry.example.com/rustfs-custom:1.0.0" spec: - image: rustfs/rustfs:1.0.0-beta.10 - - # Use custom ServiceAccount - serviceAccountName: my-custom-sa - - # Do NOT create RBAC (we manage it ourselves above) - createServiceAccountRbac: false - + image: registry.example.com/rustfs-custom:1.0.0 + serviceAccountName: custom-api-sa pools: - name: pool-0 servers: 4 persistence: volumesPerServer: 4 - env: - - name: RUST_LOG - value: "info" - --- -# Configuration 2: Custom SA with operator-managed RBAC -# Operator creates Role and RoleBinding for your custom SA -# Useful when you want operator RBAC but need custom SA for other reasons -# (e.g., workload identity, external auth) - +# Configuration 2: custom ServiceAccount for cloud workload identity. The cloud +# provider webhook controls token projection; no Kubernetes Role is required. apiVersion: v1 kind: ServiceAccount metadata: name: cloud-identity-sa namespace: default annotations: - # Example: AWS IAM Role annotation eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/RustFSRole" - # Example: Azure Managed Identity + # GKE example: + # iam.gke.io/gcp-service-account: "rustfs@project.iam.gserviceaccount.com" + # Azure example: # azure.workload.identity/client-id: "12345678-1234-1234-1234-123456789012" --- apiVersion: rustfs.com/v1alpha1 kind: Tenant metadata: - name: custom-rbac-tenant-2 + name: cloud-identity-tenant namespace: default spec: image: rustfs/rustfs:1.0.0-beta.10 - - # Use custom ServiceAccount (with cloud identity annotations) serviceAccountName: cloud-identity-sa - - # Let operator create RBAC for this custom SA - createServiceAccountRbac: true - pools: - name: pool-0 servers: 4 persistence: volumesPerServer: 4 - env: - - name: RUST_LOG - value: "info" - - # Example: Cloud-specific configuration - name: AWS_REGION value: "us-east-1" --- -# Usage examples: - -# 1. Deploy custom SA without operator RBAC: -# kubectl apply -f custom-rbac-tenant.yaml -# # This creates: SA, Role, RoleBinding, and Tenant (custom-rbac-tenant-1) - -# 2. Verify RBAC: -# kubectl get sa my-custom-sa -# kubectl get role custom-rustfs-role -# kubectl get rolebinding custom-rustfs-binding - -# 3. Check ServiceAccount usage: -# kubectl get pods -l rustfs.tenant=custom-rbac-tenant-1 -o jsonpath='{.items[0].spec.serviceAccountName}' -# # Should output: my-custom-sa - -# 4. Test permissions: -# kubectl auth can-i get secrets --as=system:serviceaccount:default:my-custom-sa -# kubectl auth can-i create services --as=system:serviceaccount:default:my-custom-sa - -# 5. Deploy custom SA with operator RBAC: -# # Only apply the second ServiceAccount and Tenant -# kubectl apply -f - <( + &self, + name: &str, + namespace: &str, + patch: &P, + ) -> Result + where + T: Clone + DeserializeOwned + Debug + Resource, + ::DynamicType: Default, + P: Serialize + Debug, + { + let api: Api = Api::namespaced(self.client.clone(), namespace); + api.patch( + name, + &PatchParams::apply("rustfs-operator-security").force(), + &Patch::Apply(patch), + ) + .context(KubeSnafu) + .await + } + /// Validates that a credential Secret exists and contains required keys. /// /// This function only validates the Secret structure when `spec.credsSecret` is configured. diff --git a/src/lib.rs b/src/lib.rs index c83315b..39e3eb0 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,6 +29,7 @@ use hyper_util::server::conn::auto::Builder as HyperBuilder; use hyper_util::service::TowerToHyperService; use k8s_openapi::api::apps::v1 as appsv1; use k8s_openapi::api::core::v1 as corev1; +use k8s_openapi::api::rbac::v1 as rbacv1; use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; use kube::core::{ ApiResource, DynamicObject, GroupVersionKind, PartialObjectMeta, PartialObjectMetaExt, @@ -241,6 +242,16 @@ async fn run_controller( .watches_stream(secrets, move |secret| { tenant_refs_for_secret(secret, &secret_reference_index) }) + .watches( + Api::::all(client.clone()), + watcher::Config::default(), + tenant_refs_for_legacy_role, + ) + .watches( + Api::::all(client.clone()), + watcher::Config::default(), + tenant_refs_for_legacy_role_binding, + ) .owns( Api::::all(client.clone()), watcher::Config::default(), @@ -858,6 +869,32 @@ fn tenant_refs_for_config_map( deduplicate_tenant_refs(refs) } +fn tenant_refs_for_legacy_role(role: rbacv1::Role) -> Vec> { + tenant_refs_for_legacy_rbac(&role, "role") +} + +fn tenant_refs_for_legacy_role_binding( + role_binding: rbacv1::RoleBinding, +) -> Vec> { + tenant_refs_for_legacy_rbac(&role_binding, "role-binding") +} + +fn tenant_refs_for_legacy_rbac(resource: &K, suffix: &str) -> Vec> { + let metadata = resource.meta(); + let Some(resource_name) = metadata.name.as_deref() else { + return Vec::new(); + }; + + tenant_refs_from_metadata( + metadata.namespace.as_deref(), + metadata.owner_references.as_deref(), + metadata.labels.as_ref(), + ) + .into_iter() + .filter(|tenant| resource_name == format!("{}-{suffix}", tenant.name)) + .collect() +} + fn tenant_refs_for_pod(pod: corev1::Pod) -> Vec> { tenant_refs_from_metadata( pod.metadata.namespace.as_deref(), @@ -1468,6 +1505,86 @@ mod controller_watch_tests { assert_single_ref(&refs, "tenant-a", "storage"); } + #[test] + fn legacy_rbac_mapper_uses_tenant_owner_reference() { + let role = rbacv1::Role { + metadata: metav1::ObjectMeta { + name: Some("tenant-a-role".to_string()), + namespace: Some("storage".to_string()), + owner_references: Some(vec![tenant_owner_ref("tenant-a")]), + ..Default::default() + }, + ..Default::default() + }; + let role_binding = rbacv1::RoleBinding { + metadata: metav1::ObjectMeta { + name: Some("tenant-a-role-binding".to_string()), + namespace: Some("storage".to_string()), + owner_references: Some(vec![tenant_owner_ref("tenant-a")]), + ..Default::default() + }, + ..Default::default() + }; + + assert_single_ref(&tenant_refs_for_legacy_role(role), "tenant-a", "storage"); + assert_single_ref( + &tenant_refs_for_legacy_role_binding(role_binding), + "tenant-a", + "storage", + ); + } + + #[test] + fn legacy_rbac_mapper_uses_tenant_label_for_orphan() { + let role = rbacv1::Role { + metadata: metav1::ObjectMeta { + name: Some("tenant-a-role".to_string()), + namespace: Some("storage".to_string()), + labels: Some(BTreeMap::from([( + RUSTFS_TENANT_LABEL.to_string(), + "tenant-a".to_string(), + )])), + ..Default::default() + }, + ..Default::default() + }; + + assert_single_ref(&tenant_refs_for_legacy_role(role), "tenant-a", "storage"); + } + + #[test] + fn legacy_rbac_mapper_ignores_non_legacy_names() { + let role = rbacv1::Role { + metadata: metav1::ObjectMeta { + name: Some("tenant-a-custom-role".to_string()), + namespace: Some("storage".to_string()), + owner_references: Some(vec![tenant_owner_ref("tenant-a")]), + labels: Some(BTreeMap::from([( + RUSTFS_TENANT_LABEL.to_string(), + "tenant-a".to_string(), + )])), + ..Default::default() + }, + ..Default::default() + }; + let role_binding = rbacv1::RoleBinding { + metadata: metav1::ObjectMeta { + name: Some("tenant-a-role-binding-extra".to_string()), + namespace: Some("storage".to_string()), + owner_references: Some(vec![tenant_owner_ref("tenant-a")]), + labels: Some(BTreeMap::from([( + RUSTFS_TENANT_LABEL.to_string(), + "tenant-a".to_string(), + )])), + ..Default::default() + }, + ..Default::default() + }; + + assert!(tenant_refs_for_legacy_role(role).is_empty()); + assert!(tenant_refs_for_legacy_role_binding(role_binding).is_empty()); + } + #[test] fn cert_manager_certificate_mapper_uses_owner_reference_or_label() { let resource = cert_manager_certificate_api_resource(); diff --git a/src/reconcile.rs b/src/reconcile.rs index 4c42e40..b66f256 100755 --- a/src/reconcile.rs +++ b/src/reconcile.rs @@ -37,8 +37,9 @@ mod tls; const OUT_OF_SERVICE_TAINT_KEY: &str = "node.kubernetes.io/out-of-service"; use phases::{ - cleanup_removed_decommissioned_pool_statefulsets, finalize_tenant_status, - maybe_cleanup_terminating_pods, reconcile_pool_statefulsets, reconcile_rbac_resources, + cleanup_legacy_tenant_rbac, cleanup_removed_decommissioned_pool_statefulsets, + finalize_tenant_status, harden_existing_tenant_workload_identity, + maybe_cleanup_terminating_pods, reconcile_pool_statefulsets, reconcile_service_account, reconcile_services, validate_no_pool_rename, validate_tenant_prerequisites, }; use pool_lifecycle::reconcile_pool_lifecycle; @@ -62,7 +63,10 @@ pub async fn reconcile_rustfs(tenant: Arc, ctx: Arc) -> Result< let ns = tenant.namespace()?; let latest_tenant = ctx.get::(&tenant.name(), &ns).await?; + let cleanup_result = cleanup_legacy_tenant_rbac(&ctx, &latest_tenant, &ns).await; + if latest_tenant.metadata.deletion_timestamp.is_some() { + cleanup_result?; debug!( tenant = %tenant.name(), namespace = %ns, @@ -72,6 +76,11 @@ pub async fn reconcile_rustfs(tenant: Arc, ctx: Arc) -> Result< return Ok(Action::await_change()); } + let workload_identity_result = + harden_existing_tenant_workload_identity(&ctx, &latest_tenant, &ns).await; + context_result(cleanup_result, &ctx, &latest_tenant).await?; + context_result(workload_identity_result, &ctx, &latest_tenant).await?; + if should_mark_reconcile_started(&latest_tenant) { patch_reconcile_started(&ctx, &latest_tenant).await; } @@ -81,7 +90,7 @@ pub async fn reconcile_rustfs(tenant: Arc, ctx: Arc) -> Result< maybe_cleanup_terminating_pods(&ctx, &latest_tenant, &ns).await?; - reconcile_rbac_resources(&ctx, &latest_tenant, &ns).await?; + reconcile_service_account(&ctx, &latest_tenant, &ns).await?; reconcile_services(&ctx, &latest_tenant, &ns, &tls_plan).await?; @@ -110,11 +119,6 @@ pub async fn reconcile_rustfs(tenant: Arc, ctx: Arc) -> Result< finalize_tenant_status(&ctx, &latest_tenant, summary, tls_plan).await } -#[cfg(test)] -fn should_create_rbac(tenant: &Tenant) -> bool { - phases::should_create_rbac(tenant) -} - async fn context_result( result: Result, ctx: &Context, @@ -986,8 +990,8 @@ mod tests { NodePodDeletionSafety, PodCleanupDecision, cleanup_decision_for_pod, force_delete_requires_fencing, node_pod_deletion_safety, object_owned_by_tenant, pod_controller_owner_name_and_uid, pod_has_owner_kind, pod_matches_policy_controller_kind, - replicaset_matches_pod_controller_and_tenant, should_create_rbac, - should_mark_reconcile_started, statefulset_matches_pod_controller_and_tenant, + replicaset_matches_pod_controller_and_tenant, should_mark_reconcile_started, + statefulset_matches_pod_controller_and_tenant, }; use crate::types::v1alpha1::status::Status; use k8s_openapi::api::apps::v1 as appsv1; @@ -1214,39 +1218,7 @@ mod tests { assert!(should_mark_reconcile_started(&stale)); } - #[test] - fn test_should_create_rbac_default() { - let tenant = crate::tests::create_test_tenant(None, None); - - assert!(should_create_rbac(&tenant)); - } - - // Test 11: RBAC creation logic - custom SA with createServiceAccountRbac=true - #[test] - fn test_should_create_rbac_custom_sa_with_rbac() { - let tenant = crate::tests::create_test_tenant(Some("my-custom-sa".to_string()), Some(true)); - - assert!(should_create_rbac(&tenant)); - } - - // Test 12: RBAC creation logic - custom SA with createServiceAccountRbac=false - #[test] - fn test_should_skip_rbac_custom_sa_without_rbac() { - let tenant = - crate::tests::create_test_tenant(Some("my-custom-sa".to_string()), Some(false)); - - assert!(!should_create_rbac(&tenant)); - } - - // Test 13: RBAC creation logic - custom SA with createServiceAccountRbac=None (default) - #[test] - fn test_should_skip_rbac_custom_sa_default() { - let tenant = crate::tests::create_test_tenant(Some("my-custom-sa".to_string()), None); - - assert!(!should_create_rbac(&tenant)); - } - - // Test 14: Service account determination in reconcile logic + // Test: Service account determination in reconcile logic #[test] fn test_determine_sa_name_in_reconcile() { // Test default behavior diff --git a/src/reconcile/phases.rs b/src/reconcile/phases.rs index a378f20..d3013d6 100644 --- a/src/reconcile/phases.rs +++ b/src/reconcile/phases.rs @@ -23,13 +23,21 @@ use crate::status::{StatusBuilder, StatusError}; use crate::types; use crate::types::v1alpha1::status::pool::PoolLifecycleState; use crate::types::v1alpha1::status::{ConditionType, Reason}; -use crate::types::v1alpha1::tenant::Tenant; +use crate::types::v1alpha1::tenant::{ + RUSTFS_TENANT_LABEL, Tenant, uses_unpartitioned_rolling_update, +}; use crate::types::v1alpha1::tls::TlsPlan; -use kube::ResourceExt; -use kube::api::{DeleteParams, ListParams, PropagationPolicy}; +use k8s_openapi::NamespaceResourceScope; +use k8s_openapi::api::apps::v1::StatefulSet; +use k8s_openapi::api::core::v1::ServiceAccount; +use k8s_openapi::api::rbac::v1::{Role, RoleBinding}; +use kube::api::{DeleteParams, ListParams, Preconditions, PropagationPolicy}; use kube::runtime::controller::Action; use kube::runtime::events::EventType; +use kube::{Resource, ResourceExt}; +use serde::de::DeserializeOwned; use std::collections::HashSet; +use std::fmt::Debug; use std::time::Duration; use tracing::{debug, info, warn}; @@ -151,49 +159,310 @@ pub(super) async fn maybe_cleanup_terminating_pods( Ok(()) } -pub(super) fn should_create_rbac(tenant: &Tenant) -> bool { - let custom_sa = tenant.spec.service_account_name.is_some(); - let create_rbac = tenant.spec.create_service_account_rbac.unwrap_or(false); - !custom_sa || create_rbac +pub(super) async fn cleanup_legacy_tenant_rbac( + ctx: &Context, + tenant: &Tenant, + namespace: &str, +) -> Result<(), context::Error> { + // Attempt both deletions before returning an error. Deleting the Role first revokes every + // binding to the legacy policy; deleting the RoleBinding also revokes access if Role deletion + // fails transiently. + let role_name = tenant.legacy_role_name(); + let role_binding_name = tenant.legacy_role_binding_name(); + let role_result = + delete_owned_legacy_rbac_resource::(ctx, tenant, namespace, &role_name, "Role").await; + let role_binding_result = delete_owned_legacy_rbac_resource::( + ctx, + tenant, + namespace, + &role_binding_name, + "RoleBinding", + ) + .await; + + role_result?; + role_binding_result?; + Ok(()) } -pub(super) async fn reconcile_rbac_resources( +async fn delete_owned_legacy_rbac_resource( ctx: &Context, tenant: &Tenant, namespace: &str, -) -> Result<(), Error> { - if !should_create_rbac(tenant) { + name: &str, + kind: &str, +) -> Result<(), context::Error> +where + T: Clone + DeserializeOwned + Debug + Resource, + ::DynamicType: Default, +{ + let resource = match ctx.get::(name, namespace).await { + Ok(resource) => resource, + Err(error) if context::is_kube_not_found(&error) => return Ok(()), + Err(error) => return Err(error), + }; + + if !operator_resource_owned_by_tenant_or_predecessor(resource.meta(), tenant) { + warn!( + tenant = %tenant.name(), + namespace = %namespace, + resource_kind = kind, + resource = name, + "skipping legacy RBAC cleanup because the resource is not owned by this Tenant" + ); return Ok(()); } - let role = context_result(ctx.apply(&tenant.new_role(), namespace).await, ctx, tenant).await?; + let Some(uid) = resource.meta().uid.clone() else { + warn!( + tenant = %tenant.name(), + namespace = %namespace, + resource_kind = kind, + resource = name, + "skipping legacy RBAC cleanup because the resource UID is missing" + ); + return Ok(()); + }; + let delete_params = DeleteParams { + preconditions: Some(Preconditions { + uid: Some(uid), + resource_version: resource.meta().resource_version.clone(), + }), + ..DeleteParams::default() + }; + + match ctx + .delete_with_params::(name, namespace, &delete_params) + .await + { + Ok(()) => { + info!( + tenant = %tenant.name(), + namespace = %namespace, + resource_kind = kind, + resource = name, + "deleted legacy Tenant workload RBAC" + ); + Ok(()) + } + Err(error) if context::is_kube_not_found(&error) => Ok(()), + Err(error) => Err(error), + } +} +fn operator_resource_owned_by_tenant_or_predecessor( + metadata: &k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, + tenant: &Tenant, +) -> bool { + let labels_match = metadata.labels.as_ref().is_some_and(|labels| { + tenant + .common_labels() + .iter() + .all(|(key, value)| labels.get(key) == Some(value)) + }); + let owner_matches_tenant = + |owner: &k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference| { + owner.api_version == Tenant::api_version(&()) + && owner.kind == Tenant::kind(&()) + && owner.name == tenant.name() + && owner.controller == Some(true) + }; + let owner_scope_matches = metadata.owner_references.as_ref().is_none_or(|owners| { + owners.is_empty() + || (owners.iter().any(owner_matches_tenant) + && !owners + .iter() + .any(|owner| owner.controller == Some(true) && !owner_matches_tenant(owner))) + }); + let legacy_manager_matches = metadata.managed_fields.as_ref().is_some_and(|fields| { + fields + .iter() + .any(|field| field.manager.as_deref() == Some("rustfs-operator")) + }); + + labels_match && legacy_manager_matches && owner_scope_matches +} + +fn security_patch_metadata( + metadata: &k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, + name: &str, + namespace: &str, +) -> Option { + Some(serde_json::json!({ + "name": name, + "namespace": namespace, + "uid": metadata.uid.as_ref()?, + "resourceVersion": metadata.resource_version.as_ref()? + })) +} + +fn statefulset_uses_unpartitioned_rolling_update(statefulset: &StatefulSet) -> bool { + uses_unpartitioned_rolling_update( + statefulset + .spec + .as_ref() + .and_then(|spec| spec.update_strategy.as_ref()), + ) +} + +fn statefulset_disables_service_account_token(statefulset: &StatefulSet) -> bool { + statefulset + .spec + .as_ref() + .and_then(|spec| spec.template.spec.as_ref()) + .is_some_and(|spec| spec.automount_service_account_token == Some(false)) +} + +fn security_manager_owns_fields( + metadata: &k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, +) -> bool { + metadata.managed_fields.as_ref().is_some_and(|fields| { + fields + .iter() + .any(|field| field.manager.as_deref() == Some("rustfs-operator-security")) + }) +} + +async fn harden_existing_default_service_account( + ctx: &Context, + tenant: &Tenant, + namespace: &str, +) -> Result<(), context::Error> { if tenant.spec.service_account_name.is_some() { - let sa_name = tenant.service_account_name(); - context_result( - ctx.apply(&tenant.new_role_binding(&sa_name, &role), namespace) - .await, - ctx, - tenant, - ) - .await?; - } else { - let service_account = context_result( - ctx.apply(&tenant.new_service_account(), namespace).await, - ctx, - tenant, - ) + return Ok(()); + } + + let name = tenant.service_account_name(); + let service_account = match ctx.get::(&name, namespace).await { + Ok(service_account) => service_account, + Err(error) if context::is_kube_not_found(&error) => return Ok(()), + Err(error) => return Err(error), + }; + if service_account.automount_service_account_token == Some(false) + || !operator_resource_owned_by_tenant_or_predecessor(service_account.meta(), tenant) + { + return Ok(()); + } + let Some(metadata) = security_patch_metadata(service_account.meta(), &name, namespace) else { + warn!(tenant = %tenant.name(), namespace = %namespace, service_account = %name, "skipping ServiceAccount token hardening because resource identity is incomplete"); + return Ok(()); + }; + let patch = serde_json::json!({ + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": metadata, + "automountServiceAccountToken": false + }); + let _: ServiceAccount = ctx + .force_apply_security_fields(&name, namespace, &patch) .await?; - context_result( - ctx.apply( - &tenant.new_role_binding(&service_account.name_any(), &role), - namespace, - ) - .await, - ctx, - tenant, - ) + Ok(()) +} + +async fn harden_existing_statefulset_tokens( + ctx: &Context, + tenant: &Tenant, + namespace: &str, +) -> Result<(), context::Error> { + let selector = format!("{RUSTFS_TENANT_LABEL}={}", tenant.name()); + let statefulsets = ctx + .list_with_params::(namespace, &ListParams::default().labels(&selector)) .await?; + let default_service_account = tenant.spec.service_account_name.is_none(); + let mut first_error = None; + + for statefulset in statefulsets.items { + if !operator_resource_owned_by_tenant_or_predecessor(statefulset.meta(), tenant) { + continue; + } + let needs_hardening = default_service_account + && (!statefulset_disables_service_account_token(&statefulset) + || !statefulset_uses_unpartitioned_rolling_update(&statefulset)); + let needs_release = + !default_service_account && security_manager_owns_fields(statefulset.meta()); + if !needs_hardening && !needs_release { + continue; + } + + let name = statefulset.name_any(); + let Some(metadata) = security_patch_metadata(statefulset.meta(), &name, namespace) else { + warn!(tenant = %tenant.name(), namespace = %namespace, statefulset = %name, "skipping StatefulSet token hardening because resource identity is incomplete"); + continue; + }; + let patch = if needs_hardening { + serde_json::json!({ + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "metadata": metadata, + "spec": { + "updateStrategy": { + "type": "RollingUpdate", + "rollingUpdate": { "partition": 0 } + }, + "template": { + "spec": { "automountServiceAccountToken": false } + } + } + }) + } else { + serde_json::json!({ + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "metadata": metadata + }) + }; + if let Err(error) = ctx + .force_apply_security_fields::(&name, namespace, &patch) + .await + && first_error.is_none() + { + first_error = Some(error); + } + } + + first_error.map_or(Ok(()), Err) +} + +pub(super) async fn harden_existing_tenant_workload_identity( + ctx: &Context, + tenant: &Tenant, + namespace: &str, +) -> Result<(), context::Error> { + let service_account_result = + harden_existing_default_service_account(ctx, tenant, namespace).await; + let statefulset_result = harden_existing_statefulset_tokens(ctx, tenant, namespace).await; + + service_account_result?; + statefulset_result +} + +pub(super) async fn reconcile_service_account( + ctx: &Context, + tenant: &Tenant, + namespace: &str, +) -> Result<(), Error> { + if tenant.spec.service_account_name.is_none() { + let desired = tenant.new_service_account(); + let name = desired.name_any(); + let existing = match ctx.get::(&name, namespace).await { + Ok(existing) => Some(existing), + Err(error) if context::is_kube_not_found(&error) => None, + Err(error) => return context_result(Err(error), ctx, tenant).await, + }; + + match existing { + None => { + context_result(ctx.apply(&desired, namespace).await, ctx, tenant).await?; + } + Some(existing) + if operator_resource_owned_by_tenant_or_predecessor(existing.meta(), tenant) => + { + context_result(ctx.apply(&desired, namespace).await, ctx, tenant).await?; + } + Some(_) => { + warn!(tenant = %tenant.name(), namespace = %namespace, service_account = %name, "preserving same-name ServiceAccount because it is not operator-managed"); + } + } } Ok(()) @@ -1062,6 +1331,815 @@ pub(super) async fn finalize_tenant_status( #[cfg(test)] mod tests { use super::*; + use http::{Method, Request, Response, StatusCode}; + use k8s_openapi::api::rbac::v1::{Role, RoleBinding}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ManagedFieldsEntry, ObjectMeta}; + use kube::{Client, client::Body}; + use serde_json::{Value, json}; + use std::convert::Infallible; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use tower::service_fn; + + fn kube_response(status: StatusCode, body: Value) -> Response { + Response::builder() + .status(status) + .body(Body::from( + serde_json::to_vec(&body).expect("response should serialize"), + )) + .expect("response should build") + } + + fn operator_managed_fields() -> Vec { + vec![ManagedFieldsEntry { + manager: Some("rustfs-operator".to_string()), + ..Default::default() + }] + } + + fn legacy_rbac_metadata(tenant: &Tenant, name: &str, uid: &str, owned: bool) -> ObjectMeta { + ObjectMeta { + name: Some(name.to_string()), + namespace: Some("default".to_string()), + uid: Some(uid.to_string()), + resource_version: Some("7".to_string()), + labels: owned.then(|| tenant.common_labels()), + owner_references: owned.then(|| vec![tenant.new_owner_ref()]), + managed_fields: owned.then(operator_managed_fields), + ..Default::default() + } + } + + fn legacy_role(tenant: &Tenant, owned: bool) -> Role { + Role { + metadata: legacy_rbac_metadata( + tenant, + &tenant.legacy_role_name(), + "legacy-role-uid", + owned, + ), + ..Default::default() + } + } + + fn legacy_role_binding(tenant: &Tenant, owned: bool) -> RoleBinding { + RoleBinding { + metadata: legacy_rbac_metadata( + tenant, + &tenant.legacy_role_binding_name(), + "legacy-role-binding-uid", + owned, + ), + ..Default::default() + } + } + + #[test] + fn operator_ownership_recognizes_recreated_and_orphaned_tenants() { + let tenant = crate::tests::create_test_tenant(None, None); + let current_owned = ObjectMeta { + labels: Some(tenant.common_labels()), + owner_references: Some(vec![tenant.new_owner_ref()]), + managed_fields: Some(operator_managed_fields()), + ..Default::default() + }; + assert!(operator_resource_owned_by_tenant_or_predecessor( + ¤t_owned, + &tenant + )); + + let mut stale_owner = tenant.new_owner_ref(); + stale_owner.uid = "previous-tenant-uid".to_string(); + let stale_owned = ObjectMeta { + labels: Some(tenant.common_labels()), + owner_references: Some(vec![stale_owner]), + managed_fields: Some(operator_managed_fields()), + ..Default::default() + }; + assert!(operator_resource_owned_by_tenant_or_predecessor( + &stale_owned, + &tenant + )); + + let orphaned = ObjectMeta { + labels: Some(tenant.common_labels()), + managed_fields: Some(operator_managed_fields()), + ..Default::default() + }; + assert!(operator_resource_owned_by_tenant_or_predecessor( + &orphaned, &tenant + )); + + let user_managed = ObjectMeta { + labels: Some(tenant.common_labels()), + ..Default::default() + }; + assert!(!operator_resource_owned_by_tenant_or_predecessor( + &user_managed, + &tenant + )); + + let security_manager_only = ObjectMeta { + labels: Some(tenant.common_labels()), + owner_references: Some(vec![tenant.new_owner_ref()]), + managed_fields: Some(vec![ManagedFieldsEntry { + manager: Some("rustfs-operator-security".to_string()), + ..Default::default() + }]), + ..Default::default() + }; + assert!(!operator_resource_owned_by_tenant_or_predecessor( + &security_manager_only, + &tenant + )); + + let missing_labels = ObjectMeta { + owner_references: Some(vec![tenant.new_owner_ref()]), + managed_fields: Some(operator_managed_fields()), + ..Default::default() + }; + assert!(!operator_resource_owned_by_tenant_or_predecessor( + &missing_labels, + &tenant + )); + + let mut foreign_owner = tenant.new_owner_ref(); + foreign_owner.api_version = "v1".to_string(); + foreign_owner.kind = "ConfigMap".to_string(); + foreign_owner.name = "foreign-controller".to_string(); + let foreign_owned = ObjectMeta { + labels: Some(tenant.common_labels()), + owner_references: Some(vec![foreign_owner]), + managed_fields: Some(operator_managed_fields()), + ..Default::default() + }; + assert!(!operator_resource_owned_by_tenant_or_predecessor( + &foreign_owned, + &tenant + )); + } + + #[test] + fn operator_ownership_rejects_user_managed_resource_owned_by_current_tenant() { + let tenant = crate::tests::create_test_tenant(None, None); + let user_managed = ObjectMeta { + labels: Some(tenant.common_labels()), + owner_references: Some(vec![tenant.new_owner_ref()]), + ..Default::default() + }; + + assert!(!operator_resource_owned_by_tenant_or_predecessor( + &user_managed, + &tenant + )); + } + + #[tokio::test] + async fn legacy_rbac_cleanup_deletes_owned_resources_with_preconditions() { + let tenant = crate::tests::create_test_tenant(None, None); + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let tenant = tenant.clone(); + let request_count = Arc::clone(&request_count); + move |request: Request| { + let tenant = tenant.clone(); + let request_number = request_count.fetch_add(1, Ordering::SeqCst); + async move { + let expected = [ + ( + Method::GET, + "/apis/rbac.authorization.k8s.io/v1/namespaces/default/roles/test-tenant-role", + ), + ( + Method::DELETE, + "/apis/rbac.authorization.k8s.io/v1/namespaces/default/roles/test-tenant-role", + ), + ( + Method::GET, + "/apis/rbac.authorization.k8s.io/v1/namespaces/default/rolebindings/test-tenant-role-binding", + ), + ( + Method::DELETE, + "/apis/rbac.authorization.k8s.io/v1/namespaces/default/rolebindings/test-tenant-role-binding", + ), + ]; + assert_eq!( + (request.method().clone(), request.uri().path()), + ( + expected[request_number].0.clone(), + expected[request_number].1 + ) + ); + + let response = match request_number { + 0 => kube_response( + StatusCode::OK, + serde_json::to_value(legacy_role(&tenant, true)) + .expect("Role should serialize"), + ), + 1 => { + let body: Value = serde_json::from_slice( + &request + .into_body() + .collect_bytes() + .await + .expect("delete body should read"), + ) + .expect("delete body should be JSON"); + assert_eq!(body["preconditions"]["uid"], "legacy-role-uid"); + assert_eq!(body["preconditions"]["resourceVersion"], "7"); + kube_response( + StatusCode::OK, + json!({"apiVersion":"v1","kind":"Status","status":"Success"}), + ) + } + 2 => kube_response( + StatusCode::OK, + serde_json::to_value(legacy_role_binding(&tenant, true)) + .expect("RoleBinding should serialize"), + ), + 3 => { + let body: Value = serde_json::from_slice( + &request + .into_body() + .collect_bytes() + .await + .expect("delete body should read"), + ) + .expect("delete body should be JSON"); + assert_eq!(body["preconditions"]["uid"], "legacy-role-binding-uid"); + assert_eq!(body["preconditions"]["resourceVersion"], "7"); + kube_response( + StatusCode::OK, + json!({"apiVersion":"v1","kind":"Status","status":"Success"}), + ) + } + _ => unreachable!(), + }; + Ok::<_, Infallible>(response) + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + cleanup_legacy_tenant_rbac(&ctx, &tenant, "default") + .await + .expect("owned legacy RBAC should be deleted"); + + assert_eq!(request_count.load(Ordering::SeqCst), 4); + } + + #[tokio::test] + async fn legacy_rbac_cleanup_is_idempotent_when_resources_are_absent() { + let tenant = crate::tests::create_test_tenant(None, None); + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let request_count = Arc::clone(&request_count); + move |request: Request| { + request_count.fetch_add(1, Ordering::SeqCst); + async move { + assert_eq!(request.method(), Method::GET); + Ok::<_, Infallible>(kube_response( + StatusCode::NOT_FOUND, + json!({ + "apiVersion":"v1", + "kind":"Status", + "status":"Failure", + "reason":"NotFound", + "code":404 + }), + )) + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + cleanup_legacy_tenant_rbac(&ctx, &tenant, "default") + .await + .expect("missing legacy RBAC should be ignored"); + + assert_eq!(request_count.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn legacy_rbac_cleanup_ignores_delete_not_found_race() { + let tenant = crate::tests::create_test_tenant(None, None); + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let tenant = tenant.clone(); + let request_count = Arc::clone(&request_count); + move |request: Request| { + let tenant = tenant.clone(); + let request_number = request_count.fetch_add(1, Ordering::SeqCst); + async move { + let response = match request_number { + 0 => { + assert_eq!(request.method(), Method::GET); + kube_response( + StatusCode::OK, + serde_json::to_value(legacy_role(&tenant, true)) + .expect("Role should serialize"), + ) + } + 1 => { + assert_eq!(request.method(), Method::DELETE); + kube_response( + StatusCode::NOT_FOUND, + json!({"apiVersion":"v1","kind":"Status","status":"Failure","reason":"NotFound","code":404}), + ) + } + 2 => { + assert_eq!(request.method(), Method::GET); + kube_response( + StatusCode::NOT_FOUND, + json!({"apiVersion":"v1","kind":"Status","status":"Failure","reason":"NotFound","code":404}), + ) + } + _ => unreachable!(), + }; + Ok::<_, Infallible>(response) + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + cleanup_legacy_tenant_rbac(&ctx, &tenant, "default") + .await + .expect("a delete race should be idempotent"); + + assert_eq!(request_count.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn legacy_rbac_cleanup_preserves_user_managed_resources_owned_by_tenant() { + let tenant = crate::tests::create_test_tenant(None, None); + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let tenant = tenant.clone(); + let request_count = Arc::clone(&request_count); + move |request: Request| { + let tenant = tenant.clone(); + request_count.fetch_add(1, Ordering::SeqCst); + async move { + assert_eq!(request.method(), Method::GET); + let body = if request.uri().path().contains("rolebindings") { + let mut role_binding = legacy_role_binding(&tenant, false); + role_binding.metadata.labels = Some(tenant.common_labels()); + role_binding.metadata.owner_references = Some(vec![tenant.new_owner_ref()]); + serde_json::to_value(role_binding).expect("RoleBinding should serialize") + } else { + let mut role = legacy_role(&tenant, false); + role.metadata.labels = Some(tenant.common_labels()); + role.metadata.owner_references = Some(vec![tenant.new_owner_ref()]); + serde_json::to_value(role).expect("Role should serialize") + }; + Ok::<_, Infallible>(kube_response(StatusCode::OK, body)) + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + cleanup_legacy_tenant_rbac(&ctx, &tenant, "default") + .await + .expect("user-managed resources should be preserved"); + + assert_eq!(request_count.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn legacy_rbac_cleanup_attempts_both_resources_before_returning_error() { + let tenant = crate::tests::create_test_tenant(None, None); + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let request_count = Arc::clone(&request_count); + move |request: Request| { + let request_number = request_count.fetch_add(1, Ordering::SeqCst); + async move { + assert_eq!(request.method(), Method::GET); + let (status, code, reason) = match request_number { + 0 => { + assert!(request.uri().path().contains("/roles/")); + (StatusCode::INTERNAL_SERVER_ERROR, 500, "InternalError") + } + 1 => { + assert!(request.uri().path().contains("/rolebindings/")); + (StatusCode::NOT_FOUND, 404, "NotFound") + } + _ => unreachable!(), + }; + Ok::<_, Infallible>(kube_response( + status, + json!({ + "apiVersion":"v1", + "kind":"Status", + "status":"Failure", + "reason":reason, + "code":code + }), + )) + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + let error = cleanup_legacy_tenant_rbac(&ctx, &tenant, "default") + .await + .expect_err("non-404 cleanup errors should be returned"); + + assert!( + matches!( + error, + context::Error::Kube { + source: kube::Error::Api(response) + } if response.code == 500 + ), + "the first cleanup error should be preserved" + ); + assert_eq!(request_count.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn existing_default_service_account_is_force_hardened() { + let tenant = crate::tests::create_test_tenant(None, None); + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let tenant = tenant.clone(); + let request_count = Arc::clone(&request_count); + move |request: Request| { + let tenant = tenant.clone(); + let request_number = request_count.fetch_add(1, Ordering::SeqCst); + async move { + assert_eq!( + request.uri().path(), + "/api/v1/namespaces/default/serviceaccounts/test-tenant-sa" + ); + if request_number == 0 { + assert_eq!(request.method(), Method::GET); + let mut service_account = tenant.new_service_account(); + service_account.metadata.uid = Some("service-account-uid".to_string()); + service_account.metadata.resource_version = Some("11".to_string()); + service_account.metadata.managed_fields = Some(operator_managed_fields()); + service_account.automount_service_account_token = None; + return Ok::<_, Infallible>(kube_response( + StatusCode::OK, + serde_json::to_value(service_account) + .expect("ServiceAccount should serialize"), + )); + } + + assert_eq!(request.method(), Method::PATCH); + let query = request.uri().query().unwrap_or_default().to_string(); + let body: Value = serde_json::from_slice( + &request + .into_body() + .collect_bytes() + .await + .expect("apply body should read"), + ) + .expect("apply body should be JSON"); + assert_eq!(body["automountServiceAccountToken"], false); + assert_eq!(body["metadata"]["uid"], "service-account-uid"); + assert_eq!(body["metadata"]["resourceVersion"], "11"); + assert_eq!(request_number, 1); + assert!(query.contains("fieldManager=rustfs-operator-security")); + assert!(query.contains("force=true")); + Ok::<_, Infallible>(kube_response( + StatusCode::OK, + serde_json::to_value(tenant.new_service_account()) + .expect("ServiceAccount should serialize"), + )) + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + harden_existing_default_service_account(&ctx, &tenant, "default") + .await + .expect("default ServiceAccount should be hardened"); + + assert_eq!(request_count.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn existing_statefulset_is_hardened_before_normal_reconcile() { + let tenant = crate::tests::create_test_tenant(None, None); + let mut statefulset = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("StatefulSet should render"); + statefulset.metadata.uid = Some("statefulset-uid".to_string()); + statefulset.metadata.resource_version = Some("19".to_string()); + statefulset.metadata.managed_fields = Some(operator_managed_fields()); + let spec = statefulset + .spec + .as_mut() + .expect("StatefulSet should have spec"); + spec.update_strategy = Some(k8s_openapi::api::apps::v1::StatefulSetUpdateStrategy { + type_: Some("OnDelete".to_string()), + ..Default::default() + }); + spec.template + .spec + .as_mut() + .expect("Pod template should have spec") + .automount_service_account_token = None; + + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let statefulset = statefulset.clone(); + let request_count = Arc::clone(&request_count); + move |request: Request| { + let statefulset = statefulset.clone(); + let request_number = request_count.fetch_add(1, Ordering::SeqCst); + async move { + if request_number == 0 { + assert_eq!(request.method(), Method::GET); + assert_eq!( + request.uri().path(), + "/apis/apps/v1/namespaces/default/statefulsets" + ); + assert!( + request + .uri() + .query() + .unwrap_or_default() + .contains("labelSelector=rustfs.tenant%3Dtest-tenant") + ); + return Ok::<_, Infallible>(kube_response( + StatusCode::OK, + json!({ + "apiVersion": "apps/v1", + "kind": "StatefulSetList", + "metadata": {}, + "items": [statefulset] + }), + )); + } + + assert_eq!(request_number, 1); + assert_eq!(request.method(), Method::PATCH); + let query = request.uri().query().unwrap_or_default().to_string(); + let body: Value = serde_json::from_slice( + &request + .into_body() + .collect_bytes() + .await + .expect("apply body should read"), + ) + .expect("apply body should be JSON"); + assert!(query.contains("fieldManager=rustfs-operator-security")); + assert!(query.contains("force=true")); + assert_eq!(body["metadata"]["uid"], "statefulset-uid"); + assert_eq!(body["metadata"]["resourceVersion"], "19"); + assert_eq!(body["spec"]["updateStrategy"]["type"], "RollingUpdate"); + assert_eq!( + body["spec"]["updateStrategy"]["rollingUpdate"]["partition"], + 0 + ); + assert_eq!( + body["spec"]["template"]["spec"]["automountServiceAccountToken"], + false + ); + Ok::<_, Infallible>(kube_response( + StatusCode::OK, + serde_json::to_value(statefulset).expect("StatefulSet should serialize"), + )) + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + harden_existing_statefulset_tokens(&ctx, &tenant, "default") + .await + .expect("existing StatefulSet should be hardened independently"); + + assert_eq!(request_count.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn custom_service_account_releases_security_field_ownership_once() { + let tenant = crate::tests::create_test_tenant(Some("custom-api-sa".to_string()), None); + let mut statefulset = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("StatefulSet should render"); + statefulset.metadata.uid = Some("statefulset-uid".to_string()); + statefulset.metadata.resource_version = Some("29".to_string()); + let mut managed_fields = operator_managed_fields(); + managed_fields.push(ManagedFieldsEntry { + manager: Some("rustfs-operator-security".to_string()), + ..Default::default() + }); + statefulset.metadata.managed_fields = Some(managed_fields); + + let mut released_statefulset = statefulset.clone(); + released_statefulset.metadata.resource_version = Some("30".to_string()); + released_statefulset.metadata.managed_fields = Some(operator_managed_fields()); + + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let statefulset = statefulset.clone(); + let released_statefulset = released_statefulset.clone(); + let request_count = Arc::clone(&request_count); + move |request: Request| { + let statefulset = statefulset.clone(); + let released_statefulset = released_statefulset.clone(); + let request_number = request_count.fetch_add(1, Ordering::SeqCst); + async move { + match request_number { + 0 | 2 => { + assert_eq!(request.method(), Method::GET); + assert_eq!( + request.uri().path(), + "/apis/apps/v1/namespaces/default/statefulsets" + ); + assert!( + request + .uri() + .query() + .unwrap_or_default() + .contains("labelSelector=rustfs.tenant%3Dtest-tenant") + ); + let item = if request_number == 0 { + statefulset + } else { + released_statefulset + }; + Ok::<_, Infallible>(kube_response( + StatusCode::OK, + json!({ + "apiVersion": "apps/v1", + "kind": "StatefulSetList", + "metadata": {}, + "items": [item] + }), + )) + } + 1 => { + assert_eq!(request.method(), Method::PATCH); + assert_eq!( + request.uri().path(), + "/apis/apps/v1/namespaces/default/statefulsets/test-tenant-pool-0" + ); + let query = request.uri().query().unwrap_or_default().to_string(); + let body: Value = serde_json::from_slice( + &request + .into_body() + .collect_bytes() + .await + .expect("release body should read"), + ) + .expect("release body should be JSON"); + assert!(query.contains("fieldManager=rustfs-operator-security")); + assert!(query.contains("force=true")); + assert_eq!(body["apiVersion"], "apps/v1"); + assert_eq!(body["kind"], "StatefulSet"); + assert_eq!(body["metadata"]["uid"], "statefulset-uid"); + assert_eq!(body["metadata"]["resourceVersion"], "29"); + assert!(body.get("spec").is_none()); + Ok::<_, Infallible>(kube_response( + StatusCode::OK, + serde_json::to_value(released_statefulset) + .expect("StatefulSet should serialize"), + )) + } + _ => panic!("security ownership should be released exactly once"), + } + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + harden_existing_statefulset_tokens(&ctx, &tenant, "default") + .await + .expect("custom ServiceAccount should release security field ownership"); + harden_existing_statefulset_tokens(&ctx, &tenant, "default") + .await + .expect("released security ownership should be idempotent"); + + assert_eq!(request_count.load(Ordering::SeqCst), 3); + } + + #[tokio::test] + async fn same_name_user_managed_service_account_is_preserved() { + let tenant = crate::tests::create_test_tenant(None, None); + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let request_count = Arc::clone(&request_count); + move |request: Request| { + request_count.fetch_add(1, Ordering::SeqCst); + async move { + assert_eq!(request.method(), Method::GET); + Ok::<_, Infallible>(kube_response( + StatusCode::OK, + serde_json::to_value(ServiceAccount { + metadata: ObjectMeta { + name: Some("test-tenant-sa".to_string()), + namespace: Some("default".to_string()), + ..Default::default() + }, + automount_service_account_token: Some(true), + ..Default::default() + }) + .expect("ServiceAccount should serialize"), + )) + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + reconcile_service_account(&ctx, &tenant, "default") + .await + .expect("same-name user-managed ServiceAccount should be preserved"); + + assert_eq!(request_count.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn predecessor_service_account_is_adopted_by_current_tenant() { + let tenant = crate::tests::create_test_tenant(None, None); + let mut predecessor = tenant.new_service_account(); + predecessor + .metadata + .owner_references + .as_mut() + .expect("owner should exist")[0] + .uid = "previous-tenant-uid".to_string(); + predecessor.metadata.uid = Some("service-account-uid".to_string()); + predecessor.metadata.resource_version = Some("23".to_string()); + predecessor.metadata.managed_fields = Some(operator_managed_fields()); + + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let tenant = tenant.clone(); + let predecessor = predecessor.clone(); + let request_count = Arc::clone(&request_count); + move |request: Request| { + let tenant = tenant.clone(); + let predecessor = predecessor.clone(); + let request_number = request_count.fetch_add(1, Ordering::SeqCst); + async move { + if request_number == 0 { + assert_eq!(request.method(), Method::GET); + return Ok::<_, Infallible>(kube_response( + StatusCode::OK, + serde_json::to_value(predecessor) + .expect("ServiceAccount should serialize"), + )); + } + + assert_eq!(request_number, 1); + assert_eq!(request.method(), Method::PATCH); + let body: Value = serde_json::from_slice( + &request + .into_body() + .collect_bytes() + .await + .expect("apply body should read"), + ) + .expect("apply body should be JSON"); + assert_eq!( + body["metadata"]["ownerReferences"][0]["uid"], + "test-uid-123" + ); + assert_eq!(body["automountServiceAccountToken"], false); + Ok::<_, Infallible>(kube_response( + StatusCode::OK, + serde_json::to_value(tenant.new_service_account()) + .expect("ServiceAccount should serialize"), + )) + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + reconcile_service_account(&ctx, &tenant, "default") + .await + .expect("predecessor ServiceAccount should be adopted"); + + assert_eq!(request_count.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn custom_service_account_is_never_modified_by_operator() { + let tenant = + crate::tests::create_test_tenant(Some("cloud-identity-sa".to_string()), Some(true)); + let request_count = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let request_count = Arc::clone(&request_count); + move |_request: Request| { + request_count.fetch_add(1, Ordering::SeqCst); + async move { Ok::<_, Infallible>(kube_response(StatusCode::OK, json!({}))) } + } + }); + let ctx = Context::new(Client::new(service, "default")); + + reconcile_service_account(&ctx, &tenant, "default") + .await + .expect("custom ServiceAccount should remain user-managed"); + + assert_eq!(request_count.load(Ordering::SeqCst), 0); + } #[test] fn removed_pool_cleanup_marks_reconciling_and_requeues() { diff --git a/src/types/v1alpha1/tenant.rs b/src/types/v1alpha1/tenant.rs index 963f41a..50664ad 100755 --- a/src/types/v1alpha1/tenant.rs +++ b/src/types/v1alpha1/tenant.rs @@ -37,6 +37,7 @@ mod services; mod workloads; pub use workloads::RUNTIME_DEFAULT_IMAGE_ACK_ANNOTATION; +pub(crate) use workloads::uses_unpartitioned_rolling_update; pub(crate) const MAX_TENANT_POOLS: u32 = 32; pub(crate) const MAX_TENANT_POLICIES: u32 = 256; @@ -159,6 +160,8 @@ pub struct TenantSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub service_account_name: Option, + /// Deprecated compatibility field. The operator never grants Kubernetes API permissions to + /// Tenant workloads. Configure a custom ServiceAccount and manage any required RBAC explicitly. #[serde(default, skip_serializing_if = "Option::is_none")] pub create_service_account_rbac: Option, diff --git a/src/types/v1alpha1/tenant/rbac.rs b/src/types/v1alpha1/tenant/rbac.rs index 76469fb..77d84fe 100755 --- a/src/types/v1alpha1/tenant/rbac.rs +++ b/src/types/v1alpha1/tenant/rbac.rs @@ -13,74 +13,16 @@ // limitations under the License. use super::Tenant; -use k8s_openapi::Resource as _; use k8s_openapi::api::core::v1 as corev1; -use k8s_openapi::api::rbac::v1 as rbacv1; use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1; -use kube::{Resource, ResourceExt}; - -fn role_binding_name(tenant: &Tenant) -> String { - format!("{}-role-binding", tenant.name()) -} - -fn role_name(tenant: &Tenant) -> String { - format!("{}-role", tenant.name()) -} impl Tenant { - pub fn new_role_binding(&self, sa_name: &str, role: &rbacv1::Role) -> rbacv1::RoleBinding { - rbacv1::RoleBinding { - metadata: metav1::ObjectMeta { - name: Some(role_binding_name(self)), - namespace: self.namespace().ok(), - owner_references: Some(vec![self.new_owner_ref()]), - labels: Some(self.common_labels()), - ..Default::default() - }, - subjects: Some(vec![rbacv1::Subject { - kind: corev1::ServiceAccount::KIND.to_owned(), - namespace: self.namespace().ok(), - name: sa_name.to_owned(), - ..Default::default() - }]), - role_ref: rbacv1::RoleRef { - api_group: rbacv1::Role::GROUP.to_owned(), - kind: rbacv1::Role::KIND.to_owned(), - name: role.name_any(), - }, - } + pub(crate) fn legacy_role_binding_name(&self) -> String { + format!("{}-role-binding", self.name()) } - pub fn new_role(&self) -> rbacv1::Role { - rbacv1::Role { - metadata: metav1::ObjectMeta { - name: Some(role_name(self)), - namespace: self.namespace().ok(), - owner_references: Some(vec![self.new_owner_ref()]), - labels: Some(self.common_labels()), - ..Default::default() - }, - rules: Some(vec![ - rbacv1::PolicyRule { - api_groups: Some(vec![String::new()]), - resources: Some(vec!["secrets".to_owned()]), - verbs: vec!["get".to_owned(), "list".to_owned(), "watch".to_owned()], - ..Default::default() - }, - rbacv1::PolicyRule { - api_groups: Some(vec![String::new()]), - resources: Some(vec!["services".to_owned()]), - verbs: vec!["create".to_owned(), "delete".to_owned(), "get".to_owned()], - ..Default::default() - }, - rbacv1::PolicyRule { - api_groups: Some(vec![Self::group(&()).to_string()]), - resources: Some(vec![Self::plural(&()).to_string()]), - verbs: vec!["get".to_owned(), "list".to_owned(), "watch".to_owned()], - ..Default::default() - }, - ]), - } + pub(crate) fn legacy_role_name(&self) -> String { + format!("{}-role", self.name()) } pub fn new_service_account(&self) -> corev1::ServiceAccount { @@ -92,6 +34,7 @@ impl Tenant { labels: Some(self.common_labels()), ..Default::default() }, + automount_service_account_token: Some(false), ..Default::default() } } @@ -109,6 +52,7 @@ mod tests { // Verify metadata assert_eq!(sa.metadata.name, Some("test-tenant-sa".to_string())); assert_eq!(sa.metadata.namespace, Some("default".to_string())); + assert_eq!(sa.automount_service_account_token, Some(false)); // Verify owner reference exists if let Some(owner_refs) = &sa.metadata.owner_references { @@ -121,92 +65,14 @@ mod tests { } } - // Test: Role structure validation #[test] - fn test_new_role_structure() { + fn legacy_rbac_names_remain_stable_for_cleanup() { let tenant = crate::tests::create_test_tenant(None, None); - let role = tenant.new_role(); - - // Verify metadata - assert_eq!(role.metadata.name, Some("test-tenant-role".to_string())); - assert_eq!(role.metadata.namespace, Some("default".to_string())); - - // Verify rules - if let Some(rules) = &role.rules { - assert_eq!(rules.len(), 3, "Role should have 3 policy rules"); - - // Verify secrets rule - let secrets_rule = &rules[0]; - assert_eq!(secrets_rule.resources, Some(vec!["secrets".to_string()])); - assert!(secrets_rule.verbs.contains(&"get".to_string())); - assert!(secrets_rule.verbs.contains(&"list".to_string())); - assert!(secrets_rule.verbs.contains(&"watch".to_string())); - - // Verify services rule - let services_rule = &rules[1]; - assert_eq!(services_rule.resources, Some(vec!["services".to_string()])); - assert!(services_rule.verbs.contains(&"create".to_string())); - assert!(services_rule.verbs.contains(&"delete".to_string())); - assert!(services_rule.verbs.contains(&"get".to_string())); - - // Verify tenants rule - let tenants_rule = &rules[2]; - assert_eq!(tenants_rule.resources, Some(vec!["tenants".to_string()])); - assert!(tenants_rule.verbs.contains(&"get".to_string())); - } else { - panic!("Role should have rules"); - } - } - - // Test: RoleBinding with default SA - #[test] - fn test_new_role_binding_default_sa() { - let tenant = crate::tests::create_test_tenant(None, None); - let role = tenant.new_role(); - let sa_name = tenant.service_account_name(); - - let role_binding = tenant.new_role_binding(&sa_name, &role); - - // Verify metadata + assert_eq!(tenant.legacy_role_name(), "test-tenant-role"); assert_eq!( - role_binding.metadata.name, - Some("test-tenant-role-binding".to_string()) + tenant.legacy_role_binding_name(), + "test-tenant-role-binding" ); - - // Verify subject points to default SA - if let Some(subjects) = &role_binding.subjects { - assert_eq!(subjects.len(), 1); - assert_eq!(subjects[0].kind, "ServiceAccount"); - assert_eq!(subjects[0].name, "test-tenant-sa"); - assert_eq!(subjects[0].namespace, Some("default".to_string())); - } else { - panic!("RoleBinding should have subjects"); - } - - // Verify role ref - assert_eq!(role_binding.role_ref.kind, "Role"); - assert_eq!(role_binding.role_ref.name, "test-tenant-role"); - } - - // Test: RoleBinding with custom SA - #[test] - fn test_new_role_binding_custom_sa() { - let tenant = crate::tests::create_test_tenant(Some("my-custom-sa".to_string()), Some(true)); - let role = tenant.new_role(); - let sa_name = tenant.service_account_name(); - - let role_binding = tenant.new_role_binding(&sa_name, &role); - - // Verify subject points to custom SA - if let Some(subjects) = &role_binding.subjects { - assert_eq!(subjects.len(), 1); - assert_eq!( - subjects[0].name, "my-custom-sa", - "RoleBinding should reference custom service account" - ); - } else { - panic!("RoleBinding should have subjects"); - } } } diff --git a/src/types/v1alpha1/tenant/workloads.rs b/src/types/v1alpha1/tenant/workloads.rs index b98cf73..5b75d92 100755 --- a/src/types/v1alpha1/tenant/workloads.rs +++ b/src/types/v1alpha1/tenant/workloads.rs @@ -45,6 +45,20 @@ pub const RUNTIME_DEFAULT_IMAGE_ACK_ANNOTATION: &str = // Kubernetes caps Localhost AppArmor names at PATH_MAX - 1 bytes. const MAX_APP_ARMOR_LOCALHOST_PROFILE_LENGTH: usize = 4095; +pub(crate) fn uses_unpartitioned_rolling_update( + strategy: Option<&v1::StatefulSetUpdateStrategy>, +) -> bool { + let strategy_type = strategy + .and_then(|strategy| strategy.type_.as_deref()) + .unwrap_or("RollingUpdate"); + let partition = strategy + .and_then(|strategy| strategy.rolling_update.as_ref()) + .and_then(|rolling_update| rolling_update.partition) + .unwrap_or(0); + + strategy_type == "RollingUpdate" && partition == 0 +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ImageUnverifiableReason { CustomRepository, @@ -1249,6 +1263,15 @@ impl Tenant { match_labels: Some(selector_labels), ..Default::default() }, + update_strategy: self.spec.service_account_name.is_none().then(|| { + v1::StatefulSetUpdateStrategy { + type_: Some("RollingUpdate".to_string()), + rolling_update: Some(v1::RollingUpdateStatefulSetStrategy { + partition: Some(0), + ..Default::default() + }), + } + }), template: corev1::PodTemplateSpec { metadata: Some(metav1::ObjectMeta { labels: Some(labels), @@ -1258,6 +1281,11 @@ impl Tenant { }), spec: Some(corev1::PodSpec { service_account_name: Some(self.service_account_name()), + automount_service_account_token: self + .spec + .service_account_name + .is_none() + .then_some(false), containers: vec![container], security_context: Some(pod_security_context), volumes: Some(pod_volumes), @@ -1405,6 +1433,22 @@ impl Tenant { return Ok(true); } + // Operator-created ServiceAccounts do not require Kubernetes API access. Compare this + // field only for the default ServiceAccount so custom workload identity webhooks remain + // free to manage token projection without causing a reconcile loop. + if self.spec.service_account_name.is_none() + && existing_pod_spec.automount_service_account_token + != desired_pod_spec.automount_service_account_token + { + return Ok(true); + } + + if self.spec.service_account_name.is_none() + && !uses_unpartitioned_rolling_update(existing_spec.update_strategy.as_ref()) + { + return Ok(true); + } + // Check scheduler if existing_pod_spec.scheduler_name != desired_pod_spec.scheduler_name { return Ok(true); @@ -1741,7 +1785,8 @@ mod tests { use super::{ DEFAULT_FS_GROUP, DEFAULT_RUN_AS_GROUP, DEFAULT_RUN_AS_USER, MAX_APP_ARMOR_LOCALHOST_PROFILE_LENGTH, RUNTIME_DEFAULT_IMAGE_ACK_ANNOTATION, - validate_declared_app_armor_profile, validate_declared_seccomp_profile, + uses_unpartitioned_rolling_update, validate_declared_app_armor_profile, + validate_declared_seccomp_profile, }; use crate::types::v1alpha1::encryption::{ EncryptionConfig, KmsBackendType, LocalKmsConfig, LocalKmsMasterKeySecretRef, @@ -1750,6 +1795,7 @@ mod tests { use crate::types::v1alpha1::security_context::{MAX_KUBERNETES_ID, PodSecurityContextOverride}; use crate::types::v1alpha1::tenant::{RpcSecretRef, Tenant}; use crate::types::v1alpha1::tls::{SecretKeyReference, TlsPlan}; + use k8s_openapi::api::apps::v1; use k8s_openapi::api::core::v1 as corev1; fn image_pull_secret(name: &str) -> corev1::LocalObjectReference { @@ -3732,6 +3778,12 @@ mod tests { let statefulset = tenant .new_statefulset(pool) .expect("Should create StatefulSet"); + assert!(uses_unpartitioned_rolling_update( + statefulset + .spec + .as_ref() + .and_then(|spec| spec.update_strategy.as_ref()) + )); let pod_spec = statefulset .spec @@ -3745,6 +3797,7 @@ mod tests { Some("test-tenant-sa".to_string()), "Pod should use default service account" ); + assert_eq!(pod_spec.automount_service_account_token, Some(false)); } // Test: StatefulSet uses custom service account @@ -3756,6 +3809,13 @@ mod tests { let statefulset = tenant .new_statefulset(pool) .expect("Should create StatefulSet"); + assert_eq!( + statefulset + .spec + .as_ref() + .and_then(|spec| spec.update_strategy.as_ref()), + None + ); let pod_spec = statefulset .spec @@ -3769,6 +3829,108 @@ mod tests { Some("my-custom-sa".to_string()), "Pod should use custom service account" ); + assert_eq!(pod_spec.automount_service_account_token, None); + } + + #[test] + fn default_service_account_token_hardening_triggers_statefulset_update() { + let tenant = crate::tests::create_test_tenant(None, None); + let pool = &tenant.spec.pools[0]; + let mut statefulset = tenant + .new_statefulset(pool) + .expect("Should create StatefulSet"); + statefulset + .spec + .as_mut() + .expect("StatefulSet should have spec") + .template + .spec + .as_mut() + .expect("Pod template should have spec") + .automount_service_account_token = None; + + assert!( + tenant + .statefulset_needs_update(&statefulset, pool) + .expect("Should compare StatefulSet"), + "Legacy default ServiceAccount token automount should trigger a rollout" + ); + } + + #[test] + fn default_service_account_token_hardening_replaces_non_rolling_strategy() { + let tenant = crate::tests::create_test_tenant(None, None); + let pool = &tenant.spec.pools[0]; + let mut statefulset = tenant + .new_statefulset(pool) + .expect("Should create StatefulSet"); + statefulset + .spec + .as_mut() + .expect("StatefulSet should have spec") + .update_strategy = Some(v1::StatefulSetUpdateStrategy { + type_: Some("OnDelete".to_string()), + ..Default::default() + }); + + assert!( + tenant + .statefulset_needs_update(&statefulset, pool) + .expect("Should compare StatefulSet"), + "OnDelete must not leave legacy Pods with mounted API tokens" + ); + } + + #[test] + fn default_service_account_token_hardening_removes_rolling_partition() { + let tenant = crate::tests::create_test_tenant(None, None); + let pool = &tenant.spec.pools[0]; + let mut statefulset = tenant + .new_statefulset(pool) + .expect("Should create StatefulSet"); + statefulset + .spec + .as_mut() + .expect("StatefulSet should have spec") + .update_strategy = Some(v1::StatefulSetUpdateStrategy { + type_: Some("RollingUpdate".to_string()), + rolling_update: Some(v1::RollingUpdateStatefulSetStrategy { + partition: Some(1), + ..Default::default() + }), + }); + + assert!( + tenant + .statefulset_needs_update(&statefulset, pool) + .expect("Should compare StatefulSet"), + "a rolling partition must not leave legacy Pods with mounted API tokens" + ); + } + + #[test] + fn custom_service_account_token_projection_does_not_reconcile_loop() { + let tenant = crate::tests::create_test_tenant(Some("my-custom-sa".to_string()), None); + let pool = &tenant.spec.pools[0]; + let mut statefulset = tenant + .new_statefulset(pool) + .expect("Should create StatefulSet"); + statefulset + .spec + .as_mut() + .expect("StatefulSet should have spec") + .template + .spec + .as_mut() + .expect("Pod template should have spec") + .automount_service_account_token = Some(true); + + assert!( + !tenant + .statefulset_needs_update(&statefulset, pool) + .expect("Should compare StatefulSet"), + "Custom ServiceAccount token projection should remain user-managed" + ); } // Test: StatefulSet renders tenant-level image pull secret